commit aa5e262f3ab777845b031ea08728937ed5c86b12 Author: glass beadian Date: Sat Oct 10 11:56:42 2026 +0000 Tactical Media Broadcast Hub — initial release Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/. diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..87bebe6 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +# Build outputs that must never enter an image context (the helper binaries are rebuilt in-image). +**/node_modules +**/src/gate/rust/target +apps/media-worker/bin +# WP3 compiled images must build from source, without local outputs or data. +**/dist +**/.next +.git +tmp +infra/occupymedia/files diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..99580d0 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,9 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +indent_style = space +indent_size = 2 +trim_trailing_whitespace = true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..167c4e6 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,22 @@ +name: ci +on: + push: + branches: [main] + pull_request: + +jobs: + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 9.12.0 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + - run: pnpm install --frozen-lockfile + - run: pnpm lint + - run: pnpm typecheck + - run: pnpm build diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3df549e --- /dev/null +++ b/.gitignore @@ -0,0 +1,31 @@ +node_modules/ +dist/ +.next/ +.turbo/ +coverage/ +*.log +.env +.env.* +!.env.example +.DS_Store +runs/ +.omo/ +.codegraph/ +tmp/ +target/ +__pycache__/ +AGENTS.md + +# internal build-process artifacts — never release +audit-*.json +docs/reports/ +docs/research/ +docs/model-assignments.md +docs/audit-final.md +docs/execution-levels.md +docs/proposal.md +docs/work-breakdown.md +docs/frontend-deep-dive.md +docs/s2a-compositor-gate-report.md +infra/occupymedia/files/ +apps/media-worker/tmp-wp12/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d4089ef --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Tactical Media Broadcast Hub contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..8410019 --- /dev/null +++ b/README.md @@ -0,0 +1,63 @@ +# Tactical Media Broadcast Hub + +Self-hosted multi-tenant social / video / livestream management platform with a +generic connector system and a Virtual Channel / Auto Director subsystem. + +Built for collectives that need to coordinate media across platforms without +concentrating access in one person's hands. Every credential lives in an +encrypted vault. Every editorial decision lands in a tamper-evident audit log. +Every feature is multi-tenant with row-level security. + +## Features + +- **Publishing pipeline** — drafts → group approval → schedule → fenced dispatch → receipts. Multi-platform fan-out (Mastodon, Telegram, Bluesky, PeerTube, Facebook, Instagram, Threads, LinkedIn, Reddit, Twitch, Discord, Slack). DST-proof scheduling. +- **Column wall** — live multi-platform feed columns, TweetDeck-style. Safe rendering (hostile-HTML tested). Per-source cooldown + adaptive backoff. +- **Live Control Room** — browser multiview, TAKE/CUT, preview/program, lower-thirds, scrolling chyrons, audio mixing, tally. +- **Auto-Director** — priority rules switch sources automatically so the stream never dies mid-action. +- **Chat governance** — bridged chats (Telegram/Twitch/Discord) into one overlay. Polls with configurable quorum. Approval gates on every post. Machine-readable reason codes. +- **Distributed transcode workers** — laptops and GPUs join as render workers over Tailscale. Per-job sandboxed FFmpeg (UID/seccomp/no-network). +- **Archive** — content-addressed encrypted S3, dedup, full-text search, CSV/JSON export. Deleted platform content stays citable. +- **Analytics** — month-partitioned snapshots, current + previous month with deltas, per-provider breakdown. + +## Quickstart + +```bash +pnpm install +pnpm lint && pnpm typecheck && pnpm build + +# infrastructure +docker compose -f infra/docker-compose.yml up -d postgres redis temporal minio otel-collector prometheus grafana + +# everything incl. media stub + mock live source + mock RTMP sink +docker compose -f infra/docker-compose.yml --profile dev up -d + +# apps +pnpm dev # api :3000, web :3001(adjust if colliding), scheduler, media-worker :3100 +curl localhost:3000/healthz +curl localhost:3000/readyz +``` + +## Documentation + +| Section | Path | +|---|---| +| Engineering guide | [`docs/engineering-guide.md`](docs/engineering-guide.md) | +| Admin guide | [`docs/admin-guide.md`](docs/admin-guide.md) | +| User manual | [`docs/user-manual-v2.md`](docs/user-manual-v2.md) | +| Design docs | [`docs/design/`](docs/design/) | +| Streaming architecture | [`docs/design/streaming-architecture.md`](docs/design/streaming-architecture.md) | +| Deployment (occupymedia.cc) | [`infra/occupymedia/DEPLOY.md`](infra/occupymedia/DEPLOY.md) | + +## Deterministic live-source testing + +```bash +pnpm --filter @tactical-media-broadcast-hub/media-worker soak -- --hours 8 --out +``` + +## Security & trust model + +Self-hosted, end to end — no corporate cloud, no accounts needed to run the +system itself. Multi-tenant with row-level security; platform credentials in an +encrypted vault the database never sees in plaintext; every editorial action in +a hash-chained audit log; anonymous-infrastructure friendly (the host carries +boxes, not content — storage is ciphertext). diff --git a/apps/api/migrations/001_tenants.sql b/apps/api/migrations/001_tenants.sql new file mode 100644 index 0000000..90d5cd5 --- /dev/null +++ b/apps/api/migrations/001_tenants.sql @@ -0,0 +1,212 @@ +-- 001: tenant schema, RLS, audit chain, lifecycle triggers +CREATE EXTENSION IF NOT EXISTS pgcrypto; + +CREATE TABLE workspaces ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + name text NOT NULL, + slug text NOT NULL UNIQUE, + authz_epoch bigint NOT NULL DEFAULT 1, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE users ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + external_id text NOT NULL UNIQUE, + email text NOT NULL, + display_name text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE memberships ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + role text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, user_id) +); + +CREATE TABLE role_permissions ( + role text NOT NULL, + permission text NOT NULL, + PRIMARY KEY (role, permission) +); + +CREATE TABLE connections ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + provider text NOT NULL, + channel_id text NOT NULL, + fields jsonb NOT NULL DEFAULT '{}', + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE credentials ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + connection_id uuid REFERENCES connections(id) ON DELETE CASCADE, + kind text NOT NULL, + ciphertext bytea NOT NULL, + key_version int NOT NULL DEFAULT 1, + created_at timestamptz NOT NULL DEFAULT now(), + rotated_at timestamptz +); + +CREATE TABLE audit_log ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid REFERENCES workspaces(id) ON DELETE CASCADE, + actor_user_id uuid, + actor_role text, + action text NOT NULL, + target_type text NOT NULL, + target_id text NOT NULL, + diff jsonb, + at timestamptz NOT NULL DEFAULT now(), + seq bigint GENERATED ALWAYS AS IDENTITY, + prev_hash text, + hash text NOT NULL +); +CREATE INDEX audit_log_ws_chain ON audit_log (workspace_id, seq DESC) WHERE workspace_id IS NOT NULL; +CREATE INDEX audit_log_system_chain ON audit_log (seq DESC) WHERE workspace_id IS NULL; + +-- Tenancy guard for SECURITY DEFINER functions (audit finding #3): +-- every cross-tenant function takes an explicit workspace parameter and +-- asserts it against the session's tenant context. +CREATE FUNCTION fn_assert_tenant(p_workspace_id uuid) RETURNS void +LANGUAGE plpgsql STABLE AS $$ +DECLARE v_ctx text; +BEGIN + v_ctx := current_setting('app.workspace_id', true); + IF v_ctx IS NULL OR v_ctx = '' OR v_ctx::uuid IS DISTINCT FROM p_workspace_id THEN + RAISE EXCEPTION 'tenant context mismatch' USING ERRCODE = '42501'; + END IF; + IF NOT EXISTS (SELECT 1 FROM workspaces w WHERE w.id = p_workspace_id) THEN + RAISE EXCEPTION 'unknown workspace' USING ERRCODE = '42501'; + END IF; +END $$; + +-- Hash chain: tenant rows chain per workspace; system rows chain on the +-- workspace_id IS NULL branch. Branch selection is by TENANT CONTEXT +-- PRESENCE, never by role (audit findings #1/#6: sp_scheduler/vault callers +-- with an active tenant context must append to that tenant's chain). +CREATE FUNCTION audit_hash(p_prev text, p_actor uuid, p_action text, p_target_id text, p_at timestamptz) +RETURNS text LANGUAGE sql IMMUTABLE AS $$ + SELECT encode(digest( + coalesce(p_prev, '') || '|' || coalesce(p_actor::text, '') || '|' || + p_action || '|' || p_target_id || '|' || extract(epoch FROM p_at)::text, + 'sha256'), 'hex') +$$; + +CREATE FUNCTION audit_append( + p_workspace_id uuid, + p_actor_user_id uuid, + p_actor_role text, + p_action text, + p_target_type text, + p_target_id text, + p_diff jsonb DEFAULT NULL +) RETURNS uuid +LANGUAGE plpgsql SECURITY DEFINER SET search_path = public AS $$ +DECLARE + v_id uuid; v_prev text; v_hash text; v_now timestamptz := clock_timestamp(); +BEGIN + PERFORM fn_assert_tenant(p_workspace_id); + SELECT hash INTO v_prev FROM audit_log + WHERE workspace_id = p_workspace_id ORDER BY seq DESC LIMIT 1; + v_hash := audit_hash(v_prev, p_actor_user_id, p_action, p_target_id, v_now); + INSERT INTO audit_log (workspace_id, actor_user_id, actor_role, action, target_type, target_id, diff, at, prev_hash, hash) + VALUES (p_workspace_id, p_actor_user_id, p_actor_role, p_action, p_target_type, p_target_id, p_diff, v_now, v_prev, v_hash) + RETURNING id INTO v_id; + RETURN v_id; +END $$; + +CREATE FUNCTION audit_append_system( + p_actor_user_id uuid, + p_action text, + p_target_type text, + p_target_id text, + p_diff jsonb DEFAULT NULL +) RETURNS uuid +LANGUAGE plpgsql SECURITY DEFINER SET search_path = public AS $$ +DECLARE + v_id uuid; v_prev text; v_hash text; v_now timestamptz := clock_timestamp(); +BEGIN + IF coalesce(current_setting('app.workspace_id', true), '') <> '' THEN + RAISE EXCEPTION 'system append must not run inside a tenant context (use audit_append)'; + END IF; + SELECT hash INTO v_prev FROM audit_log + WHERE workspace_id IS NULL ORDER BY seq DESC LIMIT 1; + v_hash := audit_hash(v_prev, p_actor_user_id, p_action, p_target_id, v_now); + INSERT INTO audit_log (workspace_id, actor_user_id, action, target_type, target_id, diff, at, prev_hash, hash) + VALUES (NULL, p_actor_user_id, p_action, p_target_type, p_target_id, p_diff, v_now, v_prev, v_hash) + RETURNING id INTO v_id; + RETURN v_id; +END $$; + +-- Tamper detection: recompute the chain. +CREATE FUNCTION audit_verify_chain(p_workspace_id uuid DEFAULT NULL) RETURNS boolean +LANGUAGE plpgsql STABLE SECURITY DEFINER SET search_path = public AS $$ +DECLARE r record; v_prev text := NULL; +BEGIN + FOR r IN SELECT * FROM audit_log + WHERE workspace_id IS NOT DISTINCT FROM p_workspace_id + ORDER BY seq ASC LOOP + IF r.prev_hash IS DISTINCT FROM v_prev THEN RETURN false; END IF; + IF audit_hash(r.prev_hash, r.actor_user_id, r.action, r.target_id, r.at) <> r.hash THEN + RETURN false; + END IF; + v_prev := r.hash; + END LOOP; + RETURN true; +END $$; + +-- Lifecycle immutability (audit finding #8): tenant rows never change tenant. +CREATE FUNCTION reject_workspace_change() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + IF NEW.workspace_id IS DISTINCT FROM OLD.workspace_id THEN + RAISE EXCEPTION 'workspace_id is immutable' USING ERRCODE = '23514'; + END IF; + RETURN NEW; +END $$; + +CREATE TRIGGER trg_conn_ws_immutable BEFORE UPDATE ON connections + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_cred_ws_immutable BEFORE UPDATE ON credentials + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_mem_ws_immutable BEFORE UPDATE ON memberships + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +-- Continuous authz (audit finding #2): membership/role changes bump the +-- workspace authz epoch so live sessions fail closed on next check. +-- COALESCE matters: OLD is NULL-valued on INSERT, NEW on DELETE. +CREATE FUNCTION bump_authz_epoch() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + UPDATE workspaces SET authz_epoch = authz_epoch + 1 + WHERE id = COALESCE(NEW.workspace_id, OLD.workspace_id); + RETURN COALESCE(NEW, OLD); +END $$; + +CREATE TRIGGER trg_mem_bump_epoch AFTER INSERT OR UPDATE OR DELETE ON memberships + FOR EACH ROW EXECUTE FUNCTION bump_authz_epoch(); + +-- RLS: fail closed. Missing app.workspace_id => NULL => no tenant rows visible. +-- FORCE: applies even to the table owner, so the single-role dev pool cannot +-- silently bypass tenancy; tenant writes must go through withTenant(). +ALTER TABLE memberships ENABLE ROW LEVEL SECURITY; +ALTER TABLE connections ENABLE ROW LEVEL SECURITY; +ALTER TABLE credentials ENABLE ROW LEVEL SECURITY; +ALTER TABLE audit_log ENABLE ROW LEVEL SECURITY; +ALTER TABLE memberships FORCE ROW LEVEL SECURITY; +ALTER TABLE connections FORCE ROW LEVEL SECURITY; +ALTER TABLE credentials FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON memberships + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON connections + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON credentials + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_audit ON audit_log + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); diff --git a/apps/api/migrations/002_rbac_seed.sql b/apps/api/migrations/002_rbac_seed.sql new file mode 100644 index 0000000..f0a286c --- /dev/null +++ b/apps/api/migrations/002_rbac_seed.sql @@ -0,0 +1,90 @@ +-- 002: RBAC seed — 37 permissions across 10 roles +-- publication.delete is a legacy seed (including Publisher). Deprecated for moderation; +-- future hide/delete routes must use dedicated moderation permissions from 011. +BEGIN; + +INSERT INTO role_permissions (role, permission) +SELECT 'Owner', p FROM unnest(ARRAY[ + 'workspace.view','workspace.settings','workspace.members.manage', + 'connection.view','connection.manage','credential.manage', + 'draft.view','draft.create','draft.edit','approval.request','approval.grant', + 'publication.publish','publication.schedule','publication.delete', + 'media.view','media.manage','inbox.view','inbox.reply','stream.view', + 'analytics.view','audit.view', + 'virtualchannel.view','virtualchannel.create','virtualchannel.edit', + 'director.view','director.take','director.override_auto','director.change_layout','director.lock_program', + 'streamsource.manage','streamoutput.manage', + 'livestream.view','livestream.manage', + 'livechat.view','livechat.send','livechat.moderate' +]) AS p +ON CONFLICT DO NOTHING; + +INSERT INTO role_permissions (role, permission) +SELECT 'Admin', p FROM unnest(ARRAY[ + 'workspace.view','workspace.settings','workspace.members.manage', + 'connection.view','connection.manage','credential.manage', + 'draft.view','draft.create','draft.edit','approval.request','approval.grant', + 'publication.publish','publication.schedule','publication.delete', + 'media.view','media.manage','inbox.view','inbox.reply','stream.view', + 'analytics.view','audit.view', + 'virtualchannel.view','director.view','livestream.view','livechat.view' +]) AS p +ON CONFLICT DO NOTHING; + +INSERT INTO role_permissions (role, permission) +SELECT 'Publisher', p FROM unnest(ARRAY[ + 'workspace.view','connection.view','draft.view','draft.create','draft.edit', + 'approval.request','publication.publish','publication.schedule','publication.delete', + 'media.view','media.manage','inbox.view','inbox.reply','stream.view','analytics.view' +]) AS p +ON CONFLICT DO NOTHING; + +INSERT INTO role_permissions (role, permission) +SELECT 'Editor', p FROM unnest(ARRAY[ + 'workspace.view','connection.view','draft.view','draft.create','draft.edit', + 'media.view','media.manage','inbox.view','inbox.reply','stream.view' +]) AS p +ON CONFLICT DO NOTHING; + +INSERT INTO role_permissions (role, permission) +SELECT 'Approver', p FROM unnest(ARRAY[ + 'workspace.view','draft.view','approval.grant','analytics.view','audit.view' +]) AS p +ON CONFLICT DO NOTHING; + +INSERT INTO role_permissions (role, permission) +SELECT 'Analyst', p FROM unnest(ARRAY[ + 'workspace.view','analytics.view','stream.view','audit.view' +]) AS p +ON CONFLICT DO NOTHING; + +INSERT INTO role_permissions (role, permission) +SELECT 'Moderator', p FROM unnest(ARRAY[ + 'workspace.view','inbox.view','inbox.reply', + 'livechat.view','livechat.send','livechat.moderate' +]) AS p +ON CONFLICT DO NOTHING; + +INSERT INTO role_permissions (role, permission) +SELECT 'VideoPublisher', p FROM unnest(ARRAY[ + 'workspace.view','draft.view','draft.create','draft.edit', + 'publication.publish','publication.schedule','media.view','media.manage' +]) AS p +ON CONFLICT DO NOTHING; + +INSERT INTO role_permissions (role, permission) +SELECT 'StreamOperator', p FROM unnest(ARRAY[ + 'workspace.view','livestream.view','livestream.manage','stream.view' +]) AS p +ON CONFLICT DO NOTHING; + +INSERT INTO role_permissions (role, permission) +SELECT 'BroadcastOperator', p FROM unnest(ARRAY[ + 'workspace.view','virtualchannel.view', + 'director.view','director.take','director.override_auto','director.change_layout','director.lock_program', + 'streamsource.manage','livestream.view', + 'livechat.view','livechat.send','livechat.moderate' +]) AS p +ON CONFLICT DO NOTHING; + +COMMIT; diff --git a/apps/api/migrations/003_app_role.sql b/apps/api/migrations/003_app_role.sql new file mode 100644 index 0000000..d5cbacf --- /dev/null +++ b/apps/api/migrations/003_app_role.sql @@ -0,0 +1,19 @@ +-- 003: dedicated non-superuser app role for the runtime pool. +-- Migrations run as the table owner (superuser in dev); the application and +-- tests connect as sp_app — neither superuser nor owner — so FORCE ROW LEVEL +-- SECURITY binds for real. +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'sp_app') THEN + CREATE ROLE sp_app LOGIN PASSWORD 'sp_app_dev' NOSUPERUSER NOCREATEDB NOCREATEROLE; + END IF; +END $$; + +GRANT USAGE ON SCHEMA public TO sp_app; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO sp_app; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO sp_app; + +ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO sp_app; +ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT USAGE, SELECT ON SEQUENCES TO sp_app; diff --git a/apps/api/migrations/004_policy_nullif.sql b/apps/api/migrations/004_policy_nullif.sql new file mode 100644 index 0000000..0ce94c4 --- /dev/null +++ b/apps/api/migrations/004_policy_nullif.sql @@ -0,0 +1,16 @@ +-- 004: pooled sessions can carry an empty-string GUC placeholder after +-- SET LOCAL commits; NULLIF makes the RLS expressions fail closed cleanly +-- instead of raising a cast error. +DROP POLICY IF EXISTS tenant_isolation ON memberships; +DROP POLICY IF EXISTS tenant_isolation ON connections; +DROP POLICY IF EXISTS tenant_isolation ON credentials; +DROP POLICY IF EXISTS tenant_audit ON audit_log; + +CREATE POLICY tenant_isolation ON memberships + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON connections + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON credentials + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_audit ON audit_log + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); diff --git a/apps/api/migrations/005_epoch_fix.sql b/apps/api/migrations/005_epoch_fix.sql new file mode 100644 index 0000000..2965df9 --- /dev/null +++ b/apps/api/migrations/005_epoch_fix.sql @@ -0,0 +1,10 @@ +-- 005: fix epoch bump for INSERT (OLD is NULL on INSERT; the original +-- UPDATE matched zero rows silently). CREATE OR REPLACE also strips any +-- debug variant left in dev databases. +CREATE OR REPLACE FUNCTION bump_authz_epoch() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + UPDATE workspaces SET authz_epoch = authz_epoch + 1 + WHERE id = COALESCE(NEW.workspace_id, OLD.workspace_id); + RETURN COALESCE(NEW, OLD); +END $$; diff --git a/apps/api/migrations/006_publishing.sql b/apps/api/migrations/006_publishing.sql new file mode 100644 index 0000000..952fea2 --- /dev/null +++ b/apps/api/migrations/006_publishing.sql @@ -0,0 +1,89 @@ +-- 006: publishing pipeline (drafts -> approval -> scheduling -> publication) +CREATE TABLE drafts ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + author_user_id uuid NOT NULL REFERENCES users(id), + post jsonb NOT NULL, + state text NOT NULL DEFAULT 'editing' + CHECK (state IN ('editing', 'in_review', 'approved', 'rejected', 'scheduled', 'published')), + reviewer_user_id uuid REFERENCES users(id), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE scheduled_publications ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + draft_id uuid NOT NULL REFERENCES drafts(id) ON DELETE CASCADE, + connection_id uuid NOT NULL REFERENCES connections(id), + publish_at timestamptz NOT NULL, + idempotency_key text NOT NULL UNIQUE, + state text NOT NULL DEFAULT 'pending' + CHECK (state IN ('pending', 'in_flight', 'done', 'failed')), + error text, + UNIQUE (draft_id) +); + +CREATE TABLE publications ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + draft_id uuid NOT NULL REFERENCES drafts(id) ON DELETE CASCADE, + connection_id uuid NOT NULL REFERENCES connections(id), + idempotency_key text NOT NULL UNIQUE, + remote_id text NOT NULL, + remote_url text, + published_at timestamptz NOT NULL, + provider_metadata jsonb NOT NULL DEFAULT '{}' +); + +CREATE INDEX drafts_workspace_state ON drafts (workspace_id, state); +CREATE INDEX scheduled_publications_due ON scheduled_publications (publish_at) + WHERE state = 'pending'; +CREATE INDEX publications_draft ON publications (draft_id); + +CREATE TRIGGER trg_draft_ws_immutable BEFORE UPDATE ON drafts + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_scheduled_pub_ws_immutable BEFORE UPDATE ON scheduled_publications + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_publication_ws_immutable BEFORE UPDATE ON publications + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE drafts ENABLE ROW LEVEL SECURITY; +ALTER TABLE scheduled_publications ENABLE ROW LEVEL SECURITY; +ALTER TABLE publications ENABLE ROW LEVEL SECURITY; +ALTER TABLE drafts FORCE ROW LEVEL SECURITY; +ALTER TABLE scheduled_publications FORCE ROW LEVEL SECURITY; +ALTER TABLE publications FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON drafts + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON scheduled_publications + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON publications + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +GRANT SELECT, INSERT, UPDATE, DELETE ON drafts, scheduled_publications, publications TO sp_app; + +-- Narrow SECURITY DEFINER lookups let the scheduler discover tenant context +-- without weakening FORCE RLS for ordinary table access. +CREATE FUNCTION publishing_schedule_workspace(p_idempotency_key text) RETURNS uuid +LANGUAGE sql STABLE SECURITY DEFINER SET search_path = public AS $$ + SELECT workspace_id + FROM scheduled_publications + WHERE idempotency_key = p_idempotency_key +$$; + +CREATE FUNCTION publishing_due_schedules() RETURNS TABLE(idempotency_key text, publish_at timestamptz) +LANGUAGE sql STABLE SECURITY DEFINER SET search_path = public AS $$ + SELECT sp.idempotency_key, sp.publish_at + FROM scheduled_publications sp + WHERE sp.state = 'pending' AND sp.publish_at <= now() + ORDER BY sp.publish_at +$$; + +REVOKE ALL ON FUNCTION publishing_schedule_workspace(text) FROM PUBLIC; +REVOKE ALL ON FUNCTION publishing_due_schedules() FROM PUBLIC; +GRANT EXECUTE ON FUNCTION publishing_schedule_workspace(text), publishing_due_schedules() TO sp_app; diff --git a/apps/api/migrations/007_auth.sql b/apps/api/migrations/007_auth.sql new file mode 100644 index 0000000..31298c7 --- /dev/null +++ b/apps/api/migrations/007_auth.sql @@ -0,0 +1,30 @@ +-- 007: first-party auth (email + password) and membership invitations. +-- Passwords live in their own table (never in users, never in audit diffs); +-- hashes are scrypt with a per-user salt, produced by apps/api/src/auth/password.ts. +CREATE TABLE user_passwords ( + user_id uuid PRIMARY KEY REFERENCES users(id) ON DELETE CASCADE, + password_hash text NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now() +); + +-- Invited members exist as memberships before they first sign in. +ALTER TABLE memberships + ADD COLUMN status text NOT NULL DEFAULT 'active' + CHECK (status IN ('active', 'invited')); + +GRANT SELECT, INSERT, UPDATE, DELETE ON user_passwords TO sp_app; + +-- Login happens before any tenant context exists, and memberships is FORCE +-- RLS. This narrow SECURITY DEFINER lookup returns only the caller-supplied +-- user's own memberships so login can pick a workspace without weakening RLS. +CREATE FUNCTION auth_user_memberships(p_user_id uuid) +RETURNS TABLE(workspace_id uuid, role text, status text, created_at timestamptz) +LANGUAGE sql STABLE SECURITY DEFINER SET search_path = public AS $$ + SELECT m.workspace_id, m.role, m.status, m.created_at + FROM memberships m + WHERE m.user_id = p_user_id + ORDER BY m.created_at ASC +$$; + +REVOKE ALL ON FUNCTION auth_user_memberships(uuid) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION auth_user_memberships(uuid) TO sp_app; diff --git a/apps/api/migrations/008_groups.sql b/apps/api/migrations/008_groups.sql new file mode 100644 index 0000000..ac9ef3b --- /dev/null +++ b/apps/api/migrations/008_groups.sql @@ -0,0 +1,104 @@ +-- 008: groups — sub-workspace sharing scopes for connections and assets. +-- Group membership gates which connections a non-Owner/Admin can publish through. +-- +-- Child tables carry a denormalised workspace_id so that: +-- * RLS is the same direct column predicate as every other tenant table +-- (no subquery policies); +-- * composite FKs pin every row to ONE workspace — a group cannot hold a +-- connection or a member from another tenant, even via the owner pool; +-- * the epoch bump can read workspace_id from the row itself. Looking it up +-- in groups would miss cascaded deletes (DELETE FROM groups -> its +-- group_members rows), because the parent row is already gone when the +-- child's AFTER trigger runs — deleting a group would silently skip the bump. + +-- Composite FK targets. +ALTER TABLE connections ADD CONSTRAINT connections_id_workspace_key UNIQUE (id, workspace_id); + +CREATE TABLE groups ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + name text NOT NULL CHECK (btrim(name) <> ''), + description text NOT NULL DEFAULT '', + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, name), + UNIQUE (id, workspace_id) +); + +CREATE TABLE group_members ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL, + group_id uuid NOT NULL, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + added_by uuid REFERENCES users(id) ON DELETE SET NULL, + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (group_id, user_id), + FOREIGN KEY (group_id, workspace_id) REFERENCES groups(id, workspace_id) ON DELETE CASCADE, + -- Only workspace members can join a group; leaving the workspace leaves its groups. + FOREIGN KEY (workspace_id, user_id) REFERENCES memberships(workspace_id, user_id) ON DELETE CASCADE +); + +CREATE TABLE group_connections ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL, + group_id uuid NOT NULL, + connection_id uuid NOT NULL, + assigned_by uuid REFERENCES users(id) ON DELETE SET NULL, + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (group_id, connection_id), + FOREIGN KEY (group_id, workspace_id) REFERENCES groups(id, workspace_id) ON DELETE CASCADE, + FOREIGN KEY (connection_id, workspace_id) REFERENCES connections(id, workspace_id) ON DELETE CASCADE +); + +CREATE TABLE group_assets ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL, + group_id uuid NOT NULL, + asset_id text NOT NULL CHECK (btrim(asset_id) <> ''), -- media asset storage key or id + assigned_by uuid REFERENCES users(id) ON DELETE SET NULL, + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (group_id, asset_id), + FOREIGN KEY (group_id, workspace_id) REFERENCES groups(id, workspace_id) ON DELETE CASCADE +); + +CREATE INDEX group_members_user ON group_members (workspace_id, user_id); +CREATE INDEX group_connections_connection ON group_connections (connection_id); + +CREATE TRIGGER trg_group_ws_immutable BEFORE UPDATE ON groups + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_group_member_ws_immutable BEFORE UPDATE ON group_members + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_group_connection_ws_immutable BEFORE UPDATE ON group_connections + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_group_asset_ws_immutable BEFORE UPDATE ON group_assets + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +-- Continuous authz: group membership is an authorization input (it gates +-- publishing), so changes bump the workspace epoch exactly like memberships. +-- Reuses bump_authz_epoch() (001/005): it reads the row's own workspace_id, +-- which also covers cascades from DELETE FROM groups / memberships. +CREATE TRIGGER trg_group_member_bump_epoch AFTER INSERT OR UPDATE OR DELETE ON group_members + FOR EACH ROW EXECUTE FUNCTION bump_authz_epoch(); + +ALTER TABLE groups ENABLE ROW LEVEL SECURITY; +ALTER TABLE group_members ENABLE ROW LEVEL SECURITY; +ALTER TABLE group_connections ENABLE ROW LEVEL SECURITY; +ALTER TABLE group_assets ENABLE ROW LEVEL SECURITY; +ALTER TABLE groups FORCE ROW LEVEL SECURITY; +ALTER TABLE group_members FORCE ROW LEVEL SECURITY; +ALTER TABLE group_connections FORCE ROW LEVEL SECURITY; +ALTER TABLE group_assets FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON groups + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON group_members + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON group_connections + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON group_assets + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +GRANT SELECT, INSERT, UPDATE, DELETE ON groups, group_members, group_connections, group_assets TO sp_app; diff --git a/apps/api/migrations/009_video.sql b/apps/api/migrations/009_video.sql new file mode 100644 index 0000000..510ec73 --- /dev/null +++ b/apps/api/migrations/009_video.sql @@ -0,0 +1,52 @@ +-- 009: first-class video assets and their processing lifecycle +CREATE TABLE video_assets ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + storage_key text NOT NULL, + mime_type text NOT NULL, + size_bytes bigint NOT NULL, + width int, + height int, + duration_ms int, + frame_rate numeric, + codec text, + audio_codec text, + thumbnail_key text, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE video_processing_jobs ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + video_asset_id uuid NOT NULL REFERENCES video_assets(id) ON DELETE CASCADE, + state text NOT NULL CHECK (state IN ('queued', 'probing', 'transcoding', 'complete', 'failed')), + progress_percent int NOT NULL DEFAULT 0, + output_key text, + error text, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX video_assets_workspace_created + ON video_assets (workspace_id, created_at DESC, id DESC); +CREATE INDEX video_processing_jobs_asset_created + ON video_processing_jobs (video_asset_id, created_at DESC); + +CREATE TRIGGER trg_video_asset_ws_immutable BEFORE UPDATE ON video_assets + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_video_job_ws_immutable BEFORE UPDATE ON video_processing_jobs + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE video_assets ENABLE ROW LEVEL SECURITY; +ALTER TABLE video_processing_jobs ENABLE ROW LEVEL SECURITY; +ALTER TABLE video_assets FORCE ROW LEVEL SECURITY; +ALTER TABLE video_processing_jobs FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON video_assets + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON video_processing_jobs + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +GRANT SELECT, INSERT, UPDATE, DELETE ON video_assets, video_processing_jobs TO sp_app; diff --git a/apps/api/migrations/010_audit_hash_v2.sql b/apps/api/migrations/010_audit_hash_v2.sql new file mode 100644 index 0000000..455eef8 --- /dev/null +++ b/apps/api/migrations/010_audit_hash_v2.sql @@ -0,0 +1,80 @@ +-- 010: hash the audit diff and serialize appends to each chain. +BEGIN; + +ALTER TABLE audit_log ADD COLUMN hash_version smallint NOT NULL DEFAULT 1 + CHECK (hash_version IN (1, 2)); + +CREATE FUNCTION audit_hash_v2(p_prev text, p_actor uuid, p_action text, p_target_id text, + p_at timestamptz, p_diff jsonb) +RETURNS text LANGUAGE sql IMMUTABLE AS $$ + SELECT encode(digest( + coalesce(p_prev, '') || '|' || coalesce(p_actor::text, '') || '|' || + p_action || '|' || p_target_id || '|' || extract(epoch FROM p_at)::text || '|' || + coalesce(p_diff::text, ''), + 'sha256'), 'hex') +$$; + +CREATE OR REPLACE FUNCTION audit_append( + p_workspace_id uuid, p_actor_user_id uuid, p_actor_role text, p_action text, + p_target_type text, p_target_id text, p_diff jsonb DEFAULT NULL +) RETURNS uuid +LANGUAGE plpgsql SECURITY DEFINER SET search_path = public AS $$ +DECLARE + v_id uuid; v_prev text; v_hash text; v_now timestamptz := clock_timestamp(); +BEGIN + PERFORM fn_assert_tenant(p_workspace_id); + PERFORM pg_advisory_xact_lock(hashtext('audit:' || p_workspace_id::text)); + SELECT hash INTO v_prev FROM audit_log + WHERE workspace_id = p_workspace_id ORDER BY seq DESC LIMIT 1; + v_hash := audit_hash_v2(v_prev, p_actor_user_id, p_action, p_target_id, v_now, p_diff); + INSERT INTO audit_log (workspace_id, actor_user_id, actor_role, action, target_type, target_id, + diff, at, prev_hash, hash, hash_version) + VALUES (p_workspace_id, p_actor_user_id, p_actor_role, p_action, p_target_type, p_target_id, + p_diff, v_now, v_prev, v_hash, 2) + RETURNING id INTO v_id; + RETURN v_id; +END $$; + +CREATE OR REPLACE FUNCTION audit_append_system( + p_actor_user_id uuid, p_action text, p_target_type text, p_target_id text, + p_diff jsonb DEFAULT NULL +) RETURNS uuid +LANGUAGE plpgsql SECURITY DEFINER SET search_path = public AS $$ +DECLARE + v_id uuid; v_prev text; v_hash text; v_now timestamptz := clock_timestamp(); +BEGIN + IF coalesce(current_setting('app.workspace_id', true), '') <> '' THEN + RAISE EXCEPTION 'system append must not run inside a tenant context (use audit_append)'; + END IF; + PERFORM pg_advisory_xact_lock(hashtext('audit:system')); + SELECT hash INTO v_prev FROM audit_log + WHERE workspace_id IS NULL ORDER BY seq DESC LIMIT 1; + v_hash := audit_hash_v2(v_prev, p_actor_user_id, p_action, p_target_id, v_now, p_diff); + INSERT INTO audit_log (workspace_id, actor_user_id, action, target_type, target_id, + diff, at, prev_hash, hash, hash_version) + VALUES (NULL, p_actor_user_id, p_action, p_target_type, p_target_id, + p_diff, v_now, v_prev, v_hash, 2) + RETURNING id INTO v_id; + RETURN v_id; +END $$; + +CREATE OR REPLACE FUNCTION audit_verify_chain(p_workspace_id uuid DEFAULT NULL) RETURNS boolean +LANGUAGE plpgsql STABLE SECURITY DEFINER SET search_path = public AS $$ +DECLARE r record; v_prev text := NULL; v_expected text; +BEGIN + FOR r IN SELECT * FROM audit_log + WHERE workspace_id IS NOT DISTINCT FROM p_workspace_id + ORDER BY seq ASC LOOP + IF r.prev_hash IS DISTINCT FROM v_prev THEN RETURN false; END IF; + IF r.hash_version = 1 THEN + v_expected := audit_hash(r.prev_hash, r.actor_user_id, r.action, r.target_id, r.at); + ELSE + v_expected := audit_hash_v2(r.prev_hash, r.actor_user_id, r.action, r.target_id, r.at, r.diff); + END IF; + IF v_expected IS DISTINCT FROM r.hash THEN RETURN false; END IF; + v_prev := r.hash; + END LOOP; + RETURN true; +END $$; + +COMMIT; diff --git a/apps/api/migrations/012_live_sources.sql b/apps/api/migrations/012_live_sources.sql new file mode 100644 index 0000000..5d0ca0b --- /dev/null +++ b/apps/api/migrations/012_live_sources.sql @@ -0,0 +1,114 @@ +-- 012: WP8 Live Source Monitor — per-source reducer state + observation dedupe. +-- (010/011 are reserved for the WP7 egress package; the monitor consumes those +-- budgets later and deliberately carries no rate machinery of its own.) +-- +-- One row per monitored source identity (connection_id, remote_ref): the keyed +-- partition the MonitorObservation reducer folds into (proposal §4.4). Each +-- child row carries a denormalised workspace_id so RLS stays a direct column +-- predicate and composite FKs pin the row to ONE workspace's connection. +-- +-- `version` is the per-source compare-and-swap token: every applied mutation +-- bumps it, updates are guarded by `WHERE ... AND version = $n`, conflicting +-- writers reload and re-evaluate, and downstream consumers (WS fan-out, +-- outbox) drop non-increasing versions. + +CREATE TABLE source_monitor_states ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL, + connection_id uuid NOT NULL, + remote_ref text NOT NULL CHECK (btrim(remote_ref) <> ''), + state text NOT NULL DEFAULT 'OFFLINE' + CHECK (state IN ('OFFLINE', 'PROBING', 'LIVE', 'UNSTABLE', 'ERROR')), + version bigint NOT NULL DEFAULT 0 CHECK (version >= 0), + -- Highest *available* signal tier; NULL until the first fresh signal lands. + authoritative_tier text + CHECK (authoritative_tier IN ('media-health', 'hls-probe', 'webhook', 'api-poll')), + -- Last applied observedAt per signal tier. Staleness is judged WITHIN one + -- tier only (provider clocks are not comparable across signal sources). + last_media_health_at timestamptz, + last_hls_probe_at timestamptz, + last_webhook_at timestamptz, + last_api_poll_at timestamptz, + -- Remaining per-tier reducer bookkeeping: lastOutcome, lastProviderEventId, + -- deniedUntil, failStreak. jsonb keeps the reducer state shape in one place. + tier_status jsonb NOT NULL DEFAULT '{}'::jsonb, + ok_streak int NOT NULL DEFAULT 0 CHECK (ok_streak >= 0), + fail_streak int NOT NULL DEFAULT 0 CHECK (fail_streak >= 0), + state_entered_at timestamptz NOT NULL DEFAULT now(), + -- Hysteresis: min-hold keeps a freshly entered state from being cut down, + -- cooldown blocks the reverse transition (no rapid A -> B -> A). + hold_until timestamptz, + cooldown_until timestamptz, + previous_state text CHECK (previous_state IN ('OFFLINE', 'PROBING', 'LIVE', 'UNSTABLE', 'ERROR')), + last_transition_at timestamptz, + last_transition_reason text, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, connection_id, remote_ref), + UNIQUE (id, workspace_id), + FOREIGN KEY (connection_id, workspace_id) REFERENCES connections (id, workspace_id) ON DELETE CASCADE +); + +CREATE INDEX source_monitor_states_ws_created + ON source_monitor_states (workspace_id, created_at DESC, id DESC); +CREATE INDEX source_monitor_states_ws_state + ON source_monitor_states (workspace_id, state); + +-- Observation dedupe is enforced by the DB, not the app (proposal §4.4): a +-- replayed providerEventId can never apply twice, regardless of which process +-- or replica delivers it. Rows are journal entries, never mutated. +CREATE TABLE monitor_seen_events ( + workspace_id uuid NOT NULL, + provider_event_id text NOT NULL CHECK (btrim(provider_event_id) <> ''), + source_monitor_state_id uuid NOT NULL, + seen_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, provider_event_id), + FOREIGN KEY (source_monitor_state_id, workspace_id) + REFERENCES source_monitor_states (id, workspace_id) ON DELETE CASCADE +); +CREATE INDEX monitor_seen_events_seen_at ON monitor_seen_events (seen_at); + +CREATE TRIGGER trg_monitor_state_ws_immutable BEFORE UPDATE ON source_monitor_states + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_monitor_seen_event_ws_immutable BEFORE UPDATE ON monitor_seen_events + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE source_monitor_states ENABLE ROW LEVEL SECURITY; +ALTER TABLE monitor_seen_events ENABLE ROW LEVEL SECURITY; +ALTER TABLE source_monitor_states FORCE ROW LEVEL SECURITY; +ALTER TABLE monitor_seen_events FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON source_monitor_states + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON monitor_seen_events + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +GRANT SELECT, INSERT, UPDATE, DELETE ON source_monitor_states, monitor_seen_events TO sp_app; + +-- Narrow SECURITY DEFINER lookup: the monitor/scheduler processes discover the +-- tenant that owns a source without weakening FORCE RLS for ordinary access +-- (same standing as publishing_schedule_workspace in 006). +CREATE FUNCTION live_monitor_source_workspace(p_connection_id uuid, p_remote_ref text) RETURNS uuid +LANGUAGE sql STABLE SECURITY DEFINER SET search_path = public AS $$ + SELECT workspace_id + FROM source_monitor_states + WHERE connection_id = p_connection_id AND remote_ref = p_remote_ref + LIMIT 1 +$$; + +-- Retention for the dedupe journal. Runs in system context on purpose: pruned +-- rows span tenants, and a tenant that stops recording must not leak journal +-- rows forever. +CREATE FUNCTION monitor_prune_seen_events(p_before timestamptz) RETURNS integer +LANGUAGE sql SECURITY DEFINER SET search_path = public AS $$ + WITH removed AS ( + DELETE FROM monitor_seen_events WHERE seen_at < p_before RETURNING 1 + ) + SELECT count(*) FROM removed +$$; + +REVOKE ALL ON FUNCTION live_monitor_source_workspace(uuid, text) FROM PUBLIC; +REVOKE ALL ON FUNCTION monitor_prune_seen_events(timestamptz) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION live_monitor_source_workspace(uuid, text), monitor_prune_seen_events(timestamptz) TO sp_app; diff --git a/apps/api/migrations/013_production.sql b/apps/api/migrations/013_production.sql new file mode 100644 index 0000000..646b615 --- /dev/null +++ b/apps/api/migrations/013_production.sql @@ -0,0 +1,327 @@ +-- 013: Streaming slice S1 — production schema (docs/design/streaming-architecture.md +-- §2, §4, §6 S1; proposal §4.4 same-channel invariant). +-- +-- No virtual channel or channel-source table existed before this migration, so +-- 013 creates the minimal `virtual_channels` / `virtual_channel_sources` the +-- production tables hang off. `virtual_channel_sources.kind` is born with +-- `playlist` alongside `hls` (§5.1: a playlist is an ordinary ingest kind). +-- +-- Same-channel invariant (structural, not a handler check): every production +-- row carries `virtual_channel_id`, and every reference to a scene, source or +-- overlay goes through a composite FK that includes (workspace_id, +-- virtual_channel_id). A slot can only reference a source of its own channel, +-- a bus can only hold a scene of its own channel, an overlay revision can only +-- belong to an overlay slot of its own channel — a cross-channel write fails +-- at the constraint, whichever code path issued it. +-- +-- Guard FKs (a reference that must block a direct delete but not a purge) +-- are NO ACTION *and* DEFERRABLE INITIALLY DEFERRED. Plain NO ACTION is +-- checked when each cascade step's RI trigger fires, so a workspace purge that +-- reaches video_assets before it reaches playlist_items (deeper in the +-- workspace -> channel -> source -> item chain) aborts mid-cascade. Deferred, +-- the check runs at commit: a purge succeeds, a direct delete still fails. +-- +-- Geometry is integer basis points of the output frame (0..10000), matching +-- the desired-state v2 type layer (apps/api/src/media-control/production.ts). + +-- Video assets get the composite key that playlist items reference, so a +-- playlist can never name another tenant's asset by raw id. +ALTER TABLE video_assets ADD CONSTRAINT video_assets_id_workspace_key UNIQUE (id, workspace_id); + +-- Ownership columns (the row's parent aggregate) are immutable: moving a row +-- to another parent is delete + create. A column listed with a leading '?' is +-- an `ON DELETE SET NULL` reference: only its non-null -> NULL transition (the +-- FK action itself) is permitted. +CREATE FUNCTION reject_owner_change() RETURNS trigger +LANGUAGE plpgsql AS $$ +DECLARE + col text; + nullable boolean; + old_value text; + new_value text; +BEGIN + FOREACH col IN ARRAY TG_ARGV LOOP + nullable := left(col, 1) = '?'; + IF nullable THEN col := substr(col, 2); END IF; + old_value := to_jsonb(OLD) ->> col; + new_value := to_jsonb(NEW) ->> col; + IF new_value IS DISTINCT FROM old_value AND NOT (nullable AND new_value IS NULL) THEN + RAISE EXCEPTION '%.% is immutable', TG_TABLE_NAME, col USING ERRCODE = '23514'; + END IF; + END LOOP; + RETURN NEW; +END $$; + +CREATE TABLE virtual_channels ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + slug text NOT NULL CHECK (slug ~ '^[a-z0-9][a-z0-9-]{0,62}$'), + name text NOT NULL CHECK (btrim(name) <> '' AND length(name) <= 120), + desired_status text NOT NULL DEFAULT 'STOPPED' CHECK (desired_status IN ('STOPPED', 'RUNNING')), + observed_status text NOT NULL DEFAULT 'STOPPED' + CHECK (observed_status IN ('PENDING_CAPACITY', 'STOPPED', 'STARTING', 'RUNNING', 'DEGRADED', 'RESTARTING', 'FAILED', 'UNKNOWN')), + -- The single per-channel desiredVersion fence: every desired-state tx + -- (TAKE, scene edit, overlay swap, gain step) locks this row FOR UPDATE and + -- bumps it once. + desired_version bigint NOT NULL DEFAULT 0 CHECK (desired_version >= 0), + observed_version bigint NOT NULL DEFAULT 0 CHECK (observed_version >= 0), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, slug), + UNIQUE (workspace_id, id) +); + +CREATE TABLE virtual_channel_sources ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL, + virtual_channel_id uuid NOT NULL, + kind text NOT NULL CHECK (kind IN ('hls', 'playlist')), + label text NOT NULL CHECK (label ~ '^[[:alnum:]][[:alnum:] _.-]{0,63}$'), + priority int NOT NULL DEFAULT 0 CHECK (priority >= 0), + -- Connection the source was imported through (a candidate poller); NULL for + -- playlists and hand-entered URLs. + connection_id uuid, + -- Remote identity (import dedupe); a playlist is local and has none. + remote_ref text CHECK (remote_ref IS NULL OR btrim(remote_ref) <> ''), + hls_url text CHECK (hls_url IS NULL OR (hls_url ~ '^https://' AND length(hls_url) <= 2048)), + playlist_loop boolean, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CHECK ( + (kind = 'hls' AND hls_url IS NOT NULL AND playlist_loop IS NULL) OR + (kind = 'playlist' AND hls_url IS NULL AND remote_ref IS NULL AND connection_id IS NULL AND playlist_loop IS NOT NULL) + ), + UNIQUE (workspace_id, virtual_channel_id, remote_ref), + UNIQUE (workspace_id, id), + UNIQUE (workspace_id, virtual_channel_id, id), + FOREIGN KEY (workspace_id, virtual_channel_id) REFERENCES virtual_channels (workspace_id, id) + ON DELETE CASCADE ON UPDATE RESTRICT, + FOREIGN KEY (connection_id, workspace_id) REFERENCES connections (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT DEFERRABLE INITIALLY DEFERRED +); +CREATE INDEX virtual_channel_sources_priority ON virtual_channel_sources (workspace_id, virtual_channel_id, priority); +CREATE INDEX virtual_channel_sources_connection ON virtual_channel_sources (connection_id, workspace_id) WHERE connection_id IS NOT NULL; + +-- Ordered, pre-normalized assets of a playlist source (the worker's asset feeder walks these). +CREATE TABLE playlist_items ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL, + virtual_channel_id uuid NOT NULL, + source_id uuid NOT NULL, + position int NOT NULL CHECK (position >= 0 AND position < 500), + video_asset_id uuid NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, source_id, position), + UNIQUE (workspace_id, id), + FOREIGN KEY (workspace_id, virtual_channel_id, source_id) REFERENCES virtual_channel_sources (workspace_id, virtual_channel_id, id) + ON DELETE CASCADE ON UPDATE RESTRICT, + FOREIGN KEY (video_asset_id, workspace_id) REFERENCES video_assets (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT DEFERRABLE INITIALLY DEFERRED +); +CREATE INDEX playlist_items_asset ON playlist_items (video_asset_id, workspace_id); + +CREATE TABLE production_scenes ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL, + virtual_channel_id uuid NOT NULL, + label text NOT NULL CHECK (label ~ '^[[:alnum:]][[:alnum:] _.-]{0,63}$'), + layout text NOT NULL CHECK (layout IN ('single', 'pip', '2x1', '2x2', '3x3', '4x4', 'custom')), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, id), + UNIQUE (workspace_id, virtual_channel_id, id), + FOREIGN KEY (workspace_id, virtual_channel_id) REFERENCES virtual_channels (workspace_id, id) + ON DELETE CASCADE ON UPDATE RESTRICT +); +CREATE INDEX production_scenes_channel ON production_scenes (workspace_id, virtual_channel_id); + +CREATE TABLE scene_slots ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL, + virtual_channel_id uuid NOT NULL, + scene_id uuid NOT NULL, + slot_index int NOT NULL CHECK (slot_index >= 0 AND slot_index < 16), + -- NULL = empty slot (renders the fallback); also the ON DELETE SET NULL target. + source_id uuid, + -- Explicit rectangle for custom layouts and PIP insets; all NULL = the layout's own cell. + x int, y int, w int, h int, + fit text NOT NULL DEFAULT 'contain' CHECK (fit IN ('contain', 'cover', 'crop')), + audio_mode text NOT NULL DEFAULT 'off' CHECK (audio_mode IN ('off', 'on', 'primary')), + gain_db smallint NOT NULL DEFAULT 0 CHECK (gain_db BETWEEN -24 AND 0 AND gain_db % 3 = 0), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CHECK ((x IS NULL AND y IS NULL AND w IS NULL AND h IS NULL) OR + (x IS NOT NULL AND y IS NOT NULL AND w IS NOT NULL AND h IS NOT NULL)), + CHECK (x >= 0 AND y >= 0 AND w > 0 AND h > 0), + CHECK (x + w <= 10000 AND y + h <= 10000), + UNIQUE (workspace_id, scene_id, slot_index), + UNIQUE (workspace_id, id), + FOREIGN KEY (workspace_id, virtual_channel_id, scene_id) REFERENCES production_scenes (workspace_id, virtual_channel_id, id) + ON DELETE CASCADE ON UPDATE RESTRICT, + -- Column-list SET NULL (PostgreSQL >= 15): empties the slot, keeps tenant + channel set. + FOREIGN KEY (workspace_id, virtual_channel_id, source_id) REFERENCES virtual_channel_sources (workspace_id, virtual_channel_id, id) + ON DELETE SET NULL (source_id) ON UPDATE RESTRICT +); +-- At most one primary audio slot per scene (program mix default: primary-only). +CREATE UNIQUE INDEX scene_slots_one_primary ON scene_slots (workspace_id, scene_id) WHERE audio_mode = 'primary'; +CREATE INDEX scene_slots_source ON scene_slots (workspace_id, virtual_channel_id, source_id) WHERE source_id IS NOT NULL; +CREATE INDEX scene_slots_scene ON scene_slots (workspace_id, virtual_channel_id, scene_id); + +CREATE TABLE production_buses ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL, + virtual_channel_id uuid NOT NULL, + bus text NOT NULL CHECK (bus IN ('preview', 'program')), + -- Declared mutable reference: TAKE/CUT re-point it in place. NULL = empty bus. + -- NO ACTION: a scene on a bus cannot be deleted directly, but a channel + -- cascade that removes both succeeds (checked at end of statement). + scene_id uuid, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, virtual_channel_id, bus), + UNIQUE (workspace_id, id), + FOREIGN KEY (workspace_id, virtual_channel_id) REFERENCES virtual_channels (workspace_id, id) + ON DELETE CASCADE ON UPDATE RESTRICT, + FOREIGN KEY (workspace_id, virtual_channel_id, scene_id) REFERENCES production_scenes (workspace_id, virtual_channel_id, id) + ON DELETE NO ACTION ON UPDATE RESTRICT DEFERRABLE INITIALLY DEFERRED +); +CREATE INDEX production_buses_scene ON production_buses (workspace_id, virtual_channel_id, scene_id) WHERE scene_id IS NOT NULL; + +-- An overlay slot sits on one scene; toggling `enabled` arms it on PST and it +-- goes live with TAKE. Its pixels are always a pre-rendered asset (§4.2) +-- except the server-generated clock, whose format is a closed enum. +CREATE TABLE overlay_slots ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL, + virtual_channel_id uuid NOT NULL, + scene_id uuid NOT NULL, + kind text NOT NULL CHECK (kind IN ('lower_third', 'chyron', 'logo', 'clock')), + enabled boolean NOT NULL DEFAULT false, + layer smallint NOT NULL DEFAULT 0 CHECK (layer BETWEEN 0 AND 7), + pos_x int NOT NULL DEFAULT 0 CHECK (pos_x BETWEEN 0 AND 10000), + pos_y int NOT NULL DEFAULT 0 CHECK (pos_y BETWEEN 0 AND 10000), + chyron_speed_px_s smallint CHECK (chyron_speed_px_s BETWEEN 40 AND 400), + chyron_strip_width_px int CHECK (chyron_strip_width_px BETWEEN 1 AND 16384), + clock_format text CHECK (clock_format IN ('HH:MM', 'HH:MM:SS')), + -- Revision currently composited; NULL until the first render lands. + current_revision_id uuid, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CHECK ((kind = 'chyron') = (chyron_speed_px_s IS NOT NULL AND chyron_strip_width_px IS NOT NULL)), + CHECK (kind = 'chyron' OR (chyron_speed_px_s IS NULL AND chyron_strip_width_px IS NULL)), + CHECK ((kind = 'clock') = (clock_format IS NOT NULL)), + CHECK (kind <> 'clock' OR current_revision_id IS NULL), + UNIQUE (workspace_id, id), + UNIQUE (workspace_id, virtual_channel_id, id), + FOREIGN KEY (workspace_id, virtual_channel_id, scene_id) REFERENCES production_scenes (workspace_id, virtual_channel_id, id) + ON DELETE CASCADE ON UPDATE RESTRICT +); +CREATE INDEX overlay_slots_scene ON overlay_slots (workspace_id, virtual_channel_id, scene_id); + +-- Append-only history of an overlay slot's validated content and its rendered +-- asset. FFmpeg only ever sees `asset_id` (sha256 content id from StorageService). +CREATE TABLE overlay_revisions ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL, + virtual_channel_id uuid NOT NULL, + overlay_slot_id uuid NOT NULL, + template_id text NOT NULL CHECK (template_id ~ '^[a-z0-9][a-z0-9_-]{0,63}$'), + -- Typed field values (title <= 80, subtitle <= 120, style enum), validated in the app layer. + fields jsonb NOT NULL DEFAULT '{}'::jsonb CHECK (jsonb_typeof(fields) = 'object'), + render_state text NOT NULL DEFAULT 'pending' CHECK (render_state IN ('pending', 'rendered', 'failed')), + asset_id text CHECK (asset_id ~ '^[0-9a-f]{64}$'), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CHECK ((render_state = 'rendered') = (asset_id IS NOT NULL)), + UNIQUE (workspace_id, id), + UNIQUE (workspace_id, virtual_channel_id, overlay_slot_id, id), + FOREIGN KEY (workspace_id, virtual_channel_id, overlay_slot_id) REFERENCES overlay_slots (workspace_id, virtual_channel_id, id) + ON DELETE CASCADE ON UPDATE RESTRICT +); +CREATE INDEX overlay_revisions_slot ON overlay_revisions (workspace_id, virtual_channel_id, overlay_slot_id, created_at DESC); + +-- The current revision must be one of THIS slot's revisions (same channel, same slot). +ALTER TABLE overlay_slots ADD CONSTRAINT overlay_slots_current_revision_fkey + FOREIGN KEY (workspace_id, virtual_channel_id, id, current_revision_id) + REFERENCES overlay_revisions (workspace_id, virtual_channel_id, overlay_slot_id, id) + ON DELETE SET NULL (current_revision_id) ON UPDATE RESTRICT; +CREATE INDEX overlay_slots_current_revision ON overlay_slots (workspace_id, virtual_channel_id, id, current_revision_id) + WHERE current_revision_id IS NOT NULL; + +-- Immutability: tenant column everywhere, ownership columns per table. +CREATE TRIGGER trg_virtual_channel_ws_immutable BEFORE UPDATE ON virtual_channels + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_vc_source_ws_immutable BEFORE UPDATE ON virtual_channel_sources + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_vc_source_owner_immutable BEFORE UPDATE ON virtual_channel_sources + FOR EACH ROW EXECUTE FUNCTION reject_owner_change('virtual_channel_id', 'kind'); +CREATE TRIGGER trg_playlist_item_ws_immutable BEFORE UPDATE ON playlist_items + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_playlist_item_owner_immutable BEFORE UPDATE ON playlist_items + FOR EACH ROW EXECUTE FUNCTION reject_owner_change('virtual_channel_id', 'source_id'); +CREATE TRIGGER trg_scene_ws_immutable BEFORE UPDATE ON production_scenes + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_scene_owner_immutable BEFORE UPDATE ON production_scenes + FOR EACH ROW EXECUTE FUNCTION reject_owner_change('virtual_channel_id'); +CREATE TRIGGER trg_scene_slot_ws_immutable BEFORE UPDATE ON scene_slots + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_scene_slot_owner_immutable BEFORE UPDATE ON scene_slots + FOR EACH ROW EXECUTE FUNCTION reject_owner_change('virtual_channel_id', 'scene_id', '?source_id'); +CREATE TRIGGER trg_bus_ws_immutable BEFORE UPDATE ON production_buses + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_bus_owner_immutable BEFORE UPDATE ON production_buses + FOR EACH ROW EXECUTE FUNCTION reject_owner_change('virtual_channel_id', 'bus'); +CREATE TRIGGER trg_overlay_slot_ws_immutable BEFORE UPDATE ON overlay_slots + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_overlay_slot_owner_immutable BEFORE UPDATE ON overlay_slots + FOR EACH ROW EXECUTE FUNCTION reject_owner_change('virtual_channel_id', 'scene_id', 'kind'); +CREATE TRIGGER trg_overlay_revision_ws_immutable BEFORE UPDATE ON overlay_revisions + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_overlay_revision_owner_immutable BEFORE UPDATE ON overlay_revisions + FOR EACH ROW EXECUTE FUNCTION reject_owner_change('virtual_channel_id', 'overlay_slot_id', 'template_id', 'fields'); + +ALTER TABLE virtual_channels ENABLE ROW LEVEL SECURITY; +ALTER TABLE virtual_channel_sources ENABLE ROW LEVEL SECURITY; +ALTER TABLE playlist_items ENABLE ROW LEVEL SECURITY; +ALTER TABLE production_scenes ENABLE ROW LEVEL SECURITY; +ALTER TABLE scene_slots ENABLE ROW LEVEL SECURITY; +ALTER TABLE production_buses ENABLE ROW LEVEL SECURITY; +ALTER TABLE overlay_slots ENABLE ROW LEVEL SECURITY; +ALTER TABLE overlay_revisions ENABLE ROW LEVEL SECURITY; +ALTER TABLE virtual_channels FORCE ROW LEVEL SECURITY; +ALTER TABLE virtual_channel_sources FORCE ROW LEVEL SECURITY; +ALTER TABLE playlist_items FORCE ROW LEVEL SECURITY; +ALTER TABLE production_scenes FORCE ROW LEVEL SECURITY; +ALTER TABLE scene_slots FORCE ROW LEVEL SECURITY; +ALTER TABLE production_buses FORCE ROW LEVEL SECURITY; +ALTER TABLE overlay_slots FORCE ROW LEVEL SECURITY; +ALTER TABLE overlay_revisions FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON virtual_channels + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON virtual_channel_sources + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON playlist_items + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON production_scenes + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON scene_slots + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON production_buses + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON overlay_slots + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON overlay_revisions + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +GRANT SELECT, INSERT, UPDATE, DELETE ON virtual_channels, virtual_channel_sources, playlist_items, + production_scenes, scene_slots, production_buses, overlay_slots, overlay_revisions TO sp_app; diff --git a/apps/api/migrations/014_live_chat.sql b/apps/api/migrations/014_live_chat.sql new file mode 100644 index 0000000..a13cb29 --- /dev/null +++ b/apps/api/migrations/014_live_chat.sql @@ -0,0 +1,236 @@ +-- 014: Streaming slice S5 — chat hub + device pairing +-- (docs/design/streaming-architecture.md §3, §6 S5). +-- +-- Design notes: +-- * `live_chat_messages` is keyed by (workspace, channel) — the chat hub's +-- aggregate is a virtual channel's live chat, and the seq cursor a client +-- resumes with is per channel. Global provider dedupe uses a nonpartitioned +-- claim journal; the paging index is (workspace_id, channel_id, seq). +-- * Ordering is a hub-assigned `seq` under a row lock on the channel's +-- chat_seq row; provider timestamps stay display-only (clocks skew, §3.2). +-- * Monthly received_at partitions retain ingress ordering. PostgreSQL needs +-- the partition key in unique constraints, so global provider dedupe lives +-- in the nonpartitioned live_chat_seen_events journal, claimed in the same +-- transaction as the message insert. +-- * Devices and pairings carry their own workspace column and direct +-- workspace FK so revocation is workspace-scoped; user_id references +-- users(id) ON DELETE CASCADE so deleting a user removes their devices. +-- +-- RLS / NULLIF / sp_app grants follow 012/013 conventions. + +-- One row per (workspace, channel) holding the chat sequence. FOR UPDATE on +-- this row inside the ingest transaction is what makes seq assignment monotonic +-- per channel (§3.2 "monotonic seq assigned at hub ingress"). +CREATE TABLE live_chat_seq ( + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + channel_id uuid NOT NULL, + seq bigint NOT NULL DEFAULT 0 CHECK (seq >= 0), + updated_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, channel_id), + FOREIGN KEY (channel_id, workspace_id) REFERENCES virtual_channels (id, workspace_id) + ON DELETE CASCADE ON UPDATE RESTRICT +); + +CREATE TABLE live_chat_messages ( + id uuid NOT NULL DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL, + channel_id uuid NOT NULL, + -- Provider connection that produced the message (capability target). + connection_id uuid NOT NULL, + provider text NOT NULL CHECK (btrim(provider) <> ''), + -- Provider's own message id; the dedupe key with workspace + provider. + provider_msg_id text NOT NULL CHECK (btrim(provider_msg_id) <> ''), + -- Author + author_handle text NOT NULL CHECK (btrim(author_handle) <> ''), + author_display_name text, + author_avatar_url text, + author_badges jsonb NOT NULL DEFAULT '[]'::jsonb CHECK (jsonb_typeof(author_badges) = 'array'), + author_color text NOT NULL CHECK (author_color <> ''), + body text NOT NULL CHECK (length(body) <= 5000), + -- Reply threading (provider message id of the parent, nullable). + reply_to text, + -- Hub ingress + seq bigint NOT NULL CHECK (seq > 0), + sent_at timestamptz NOT NULL, + received_at timestamptz NOT NULL DEFAULT now(), + is_moderated boolean NOT NULL DEFAULT false, + source text NOT NULL DEFAULT 'live-chat' CHECK (source IN ('live-chat', 'social-reply', 'system')), + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (id, received_at), + UNIQUE (workspace_id, provider, provider_msg_id, received_at), + FOREIGN KEY (channel_id, workspace_id) REFERENCES virtual_channels (id, workspace_id) + ON DELETE CASCADE ON UPDATE RESTRICT, + FOREIGN KEY (connection_id, workspace_id) REFERENCES connections (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT DEFERRABLE INITIALLY DEFERRED +) PARTITION BY RANGE (received_at); + +-- Precreate monthly partitions; administrators run this before a new month. +CREATE FUNCTION live_chat_ensure_month(p_month date) RETURNS void +LANGUAGE plpgsql SET search_path = public AS $$ +DECLARE v_start timestamptz := date_trunc('month', p_month::timestamptz); + v_table text := 'live_chat_messages_' || to_char(v_start, 'YYYYMM'); +BEGIN + EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF live_chat_messages FOR VALUES FROM (%L) TO (%L)', + v_table, v_start, v_start + interval '1 month'); + EXECUTE format('ALTER TABLE %I ENABLE ROW LEVEL SECURITY', v_table); + EXECUTE format('ALTER TABLE %I FORCE ROW LEVEL SECURITY', v_table); + IF NOT EXISTS (SELECT 1 FROM pg_policies WHERE schemaname='public' AND tablename=v_table AND policyname='tenant_isolation') THEN + EXECUTE format('CREATE POLICY tenant_isolation ON %I USING (workspace_id = NULLIF(current_setting(''app.workspace_id'', true), '''')::uuid) WITH CHECK (workspace_id = NULLIF(current_setting(''app.workspace_id'', true), '''')::uuid)', v_table); + END IF; +END $$; +REVOKE ALL ON FUNCTION live_chat_ensure_month(date) FROM PUBLIC; +SELECT live_chat_ensure_month((current_date - interval '1 month')::date); +SELECT live_chat_ensure_month(current_date); +SELECT live_chat_ensure_month((current_date + interval '1 month')::date); +CREATE TABLE live_chat_messages_default PARTITION OF live_chat_messages DEFAULT; +ALTER TABLE live_chat_messages_default ENABLE ROW LEVEL SECURITY; +ALTER TABLE live_chat_messages_default FORCE ROW LEVEL SECURITY; +CREATE POLICY tenant_isolation ON live_chat_messages_default + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +CREATE INDEX live_chat_messages_channel_seq ON live_chat_messages (workspace_id, channel_id, seq); +CREATE INDEX live_chat_messages_author ON live_chat_messages (workspace_id, channel_id, author_handle); + +-- Global provider-message/event claim journal, mirroring +-- monitor_seen_events (ON CONFLICT DO NOTHING semantics come from the PK). +CREATE TABLE live_chat_seen_events ( + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + provider text NOT NULL CHECK (btrim(provider) <> ''), + provider_event_id text NOT NULL CHECK (btrim(provider_event_id) <> ''), + seen_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, provider, provider_event_id) +); + +CREATE TRIGGER trg_live_chat_seen_event_ws_immutable BEFORE UPDATE ON live_chat_seen_events + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +-- Devices: Opticast authenticates with a device token, not a session cookie +-- (§3.4). Epoch-gated: revoking bumps the epoch, tokens fail closed. +CREATE TABLE devices ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE ON UPDATE RESTRICT, + name text NOT NULL CHECK (btrim(name) <> '' AND length(name) <= 120), + scopes text[] NOT NULL DEFAULT '{livechat.view,stream.view}'::text[], + epoch bigint NOT NULL DEFAULT 1 CHECK (epoch >= 1), + authz_epoch bigint NOT NULL, + last_seen_at timestamptz, + revoked_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, id) +); +CREATE INDEX devices_ws_user ON devices (workspace_id, user_id); +CREATE INDEX devices_ws_revoked ON devices (workspace_id) WHERE revoked_at IS NULL; + +-- Single-use, TTL'd pairing codes. `redeemed_at` enforces single use; the +-- expiry is checked by the app (a code may expire unredeemed, and a purge of +-- expired rows is a plain DELETE). +CREATE TABLE device_pairings ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE ON UPDATE RESTRICT, + code text NOT NULL CHECK (code ~ '^[A-Z0-9]{4}-[A-Z0-9]{4}$'), + qr_payload text NOT NULL, + scopes text[] NOT NULL, + expires_at timestamptz NOT NULL, + redeemed_at timestamptz, + device_id uuid, + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (code), + UNIQUE (workspace_id, code), + FOREIGN KEY (device_id, workspace_id) REFERENCES devices (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT DEFERRABLE INITIALLY DEFERRED +); +CREATE INDEX device_pairings_ws_open ON device_pairings (workspace_id, code) WHERE redeemed_at IS NULL; + +CREATE TRIGGER trg_device_ws_immutable BEFORE UPDATE ON devices FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_device_pairing_ws_immutable BEFORE UPDATE ON device_pairings FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_live_chat_message_ws_immutable BEFORE UPDATE ON live_chat_messages FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_live_chat_seq_ws_immutable BEFORE UPDATE ON live_chat_seq FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE live_chat_seq ENABLE ROW LEVEL SECURITY; +ALTER TABLE live_chat_messages ENABLE ROW LEVEL SECURITY; +ALTER TABLE live_chat_seen_events ENABLE ROW LEVEL SECURITY; +ALTER TABLE devices ENABLE ROW LEVEL SECURITY; +ALTER TABLE device_pairings ENABLE ROW LEVEL SECURITY; +ALTER TABLE live_chat_seq FORCE ROW LEVEL SECURITY; +ALTER TABLE live_chat_messages FORCE ROW LEVEL SECURITY; +ALTER TABLE live_chat_seen_events FORCE ROW LEVEL SECURITY; +ALTER TABLE devices FORCE ROW LEVEL SECURITY; +ALTER TABLE device_pairings FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON live_chat_seq + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON live_chat_messages + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON live_chat_seen_events + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON devices + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON device_pairings + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +GRANT SELECT, INSERT, UPDATE, DELETE ON live_chat_seq, live_chat_messages, live_chat_seen_events, + devices, device_pairings TO sp_app; + +-- Retention (follow-on, noted here so the partition/retention row from the +-- work-breakdown finds its anchor): pruning live_chat_seen_events older than +-- the journal window, mirroring monitor_prune_seen_events from 012. +CREATE FUNCTION live_chat_prune_seen_events(p_before timestamptz) RETURNS integer +LANGUAGE sql SECURITY INVOKER SET search_path = public AS $$ + WITH removed AS ( + DELETE FROM live_chat_seen_events WHERE seen_at < p_before RETURNING 1 + ) + SELECT count(*) FROM removed +$$; +REVOKE ALL ON FUNCTION live_chat_prune_seen_events(timestamptz) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION live_chat_prune_seen_events(timestamptz) TO sp_app; + +-- Resolves only a valid short-lived bearer pairing code; no tenant rows exposed. +CREATE FUNCTION device_pairing_workspace(p_code text) RETURNS uuid +LANGUAGE sql SECURITY DEFINER SET search_path = public AS $$ + SELECT workspace_id FROM device_pairings WHERE code=p_code AND expires_at > now() +$$; +REVOKE ALL ON FUNCTION device_pairing_workspace(text) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION device_pairing_workspace(text) TO sp_app; + +-- Tenant-scoped retention for messages and their dedupe claims. +CREATE FUNCTION live_chat_prune(p_workspace uuid, p_before timestamptz) RETURNS integer +LANGUAGE plpgsql SECURITY DEFINER SET search_path = public AS $$ +DECLARE v_count integer; +BEGIN + PERFORM fn_assert_tenant(p_workspace); + DELETE FROM live_chat_messages WHERE workspace_id=p_workspace AND received_at < p_before; + GET DIAGNOSTICS v_count = ROW_COUNT; + DELETE FROM live_chat_seen_events WHERE workspace_id=p_workspace AND seen_at < p_before; + DELETE FROM device_pairings WHERE workspace_id=p_workspace AND expires_at < p_before; + RETURN v_count; +END $$; +REVOKE ALL ON FUNCTION live_chat_prune(uuid,timestamptz) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION live_chat_prune(uuid,timestamptz) TO sp_app; + +CREATE TABLE live_chat_requests ( + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + route text NOT NULL, + request_key text NOT NULL, + request_hash text NOT NULL, + status text NOT NULL DEFAULT 'pending' CHECK (status IN ('pending','complete','failed')), + response jsonb, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id,user_id,route,request_key) +); +ALTER TABLE live_chat_requests ENABLE ROW LEVEL SECURITY; +ALTER TABLE live_chat_requests FORCE ROW LEVEL SECURITY; +CREATE POLICY tenant_isolation ON live_chat_requests + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE TRIGGER trg_live_chat_request_ws_immutable BEFORE UPDATE ON live_chat_requests + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +GRANT SELECT,INSERT,UPDATE,DELETE ON live_chat_requests TO sp_app; diff --git a/apps/api/migrations/015_archive.sql b/apps/api/migrations/015_archive.sql new file mode 100644 index 0000000..3f3e30c --- /dev/null +++ b/apps/api/migrations/015_archive.sql @@ -0,0 +1,88 @@ +-- 015: WP15 Telegram live channel archive. All identities are tenant pinned. +CREATE TABLE feed_subscriptions ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL, + type text NOT NULL CHECK (type = 'telegram_channel'), + connection_id uuid NOT NULL, + channel_id text NOT NULL CHECK (btrim(channel_id) <> ''), + health text NOT NULL DEFAULT 'active' CHECK (health IN ('active', 'bot_removed', 'admin_required')), + health_checked_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, connection_id, channel_id), + UNIQUE (id, workspace_id), + FOREIGN KEY (connection_id, workspace_id) REFERENCES connections (id, workspace_id) ON DELETE CASCADE ON UPDATE RESTRICT +); +CREATE INDEX feed_subscriptions_channel ON feed_subscriptions (workspace_id, channel_id); + +CREATE TABLE archived_feed_items ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + remote_ref text NOT NULL CHECK (remote_ref ~ '^(-?[0-9]+|@[A-Za-z0-9_]+):[0-9]+$'), + channel_id text NOT NULL, + message_id bigint NOT NULL CHECK (message_id > 0), + media_group_id text, + text_content text NOT NULL DEFAULT '', + content_hash text NOT NULL CHECK (content_hash ~ '^[0-9a-f]{64}$'), + revision integer NOT NULL DEFAULT 1 CHECK (revision > 0), + media_refs jsonb NOT NULL DEFAULT '[]'::jsonb CHECK (jsonb_typeof(media_refs) = 'array'), + attribution jsonb NOT NULL DEFAULT '{}'::jsonb CHECK (jsonb_typeof(attribution) = 'object'), + protected_content boolean NOT NULL DEFAULT false, + posted_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, remote_ref), + UNIQUE (id, workspace_id), + CHECK (remote_ref = channel_id || ':' || message_id::text) +); +CREATE INDEX archived_feed_items_cursor ON archived_feed_items (workspace_id, posted_at DESC, id DESC); + +CREATE TABLE archived_feed_revisions ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL, + item_id uuid NOT NULL, + revision integer NOT NULL CHECK (revision > 0), + content_hash text NOT NULL CHECK (content_hash ~ '^[0-9a-f]{64}$'), + text_content text NOT NULL, + media_refs jsonb NOT NULL CHECK (jsonb_typeof(media_refs) = 'array'), + attribution jsonb NOT NULL CHECK (jsonb_typeof(attribution) = 'object'), + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, item_id, revision), + FOREIGN KEY (item_id, workspace_id) REFERENCES archived_feed_items (id, workspace_id) ON DELETE CASCADE ON UPDATE RESTRICT +); + +-- drafts predates tenant-pinned references. +ALTER TABLE drafts ADD CONSTRAINT drafts_id_workspace_key UNIQUE (id, workspace_id); + +CREATE TABLE archived_feed_drafts ( + workspace_id uuid NOT NULL, + item_id uuid NOT NULL, + draft_id uuid NOT NULL UNIQUE, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, item_id, draft_id), + FOREIGN KEY (item_id, workspace_id) REFERENCES archived_feed_items (id, workspace_id) ON DELETE CASCADE ON UPDATE RESTRICT, + FOREIGN KEY (draft_id, workspace_id) REFERENCES drafts (id, workspace_id) ON DELETE CASCADE ON UPDATE RESTRICT +); + +CREATE TRIGGER trg_feed_subscriptions_ws_immutable BEFORE UPDATE ON feed_subscriptions + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_archived_feed_items_ws_immutable BEFORE UPDATE ON archived_feed_items + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_archived_feed_revisions_ws_immutable BEFORE UPDATE ON archived_feed_revisions + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_archived_feed_drafts_ws_immutable BEFORE UPDATE ON archived_feed_drafts + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE feed_subscriptions ENABLE ROW LEVEL SECURITY; +ALTER TABLE archived_feed_items ENABLE ROW LEVEL SECURITY; +ALTER TABLE archived_feed_revisions ENABLE ROW LEVEL SECURITY; +ALTER TABLE archived_feed_drafts ENABLE ROW LEVEL SECURITY; +ALTER TABLE feed_subscriptions FORCE ROW LEVEL SECURITY; +ALTER TABLE archived_feed_items FORCE ROW LEVEL SECURITY; +ALTER TABLE archived_feed_revisions FORCE ROW LEVEL SECURITY; +ALTER TABLE archived_feed_drafts FORCE ROW LEVEL SECURITY; +CREATE POLICY tenant_isolation ON feed_subscriptions USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON archived_feed_items USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON archived_feed_revisions USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON archived_feed_drafts USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +GRANT SELECT, INSERT, UPDATE, DELETE ON feed_subscriptions, archived_feed_items, archived_feed_revisions, archived_feed_drafts TO sp_app; diff --git a/apps/api/migrations/016_username_auth.sql b/apps/api/migrations/016_username_auth.sql new file mode 100644 index 0000000..a8d1e2d --- /dev/null +++ b/apps/api/migrations/016_username_auth.sql @@ -0,0 +1,20 @@ +-- 016: username-based first-party auth. +-- Usernames are the login identity (external_id = local:); email is +-- optional metadata for future password recovery. Existing email-keyed users +-- get a username derived from their email local-part (or display name). +ALTER TABLE users ADD COLUMN username text; + +UPDATE users SET username = COALESCE( + NULLIF(regexp_replace(lower(split_part(coalesce(email, ''), '@', 1)), '[^a-z0-9_.-]', '', 'g'), ''), + NULLIF(regexp_replace(lower(display_name), '[^a-z0-9_.-]', '', 'g'), ''), + 'member' +); +-- Deterministic dedupe: colliding candidates get a short id suffix. +UPDATE users u SET username = u.username || '_' || substr(u.id::text, 1, 4) +WHERE EXISTS (SELECT 1 FROM users v WHERE v.username = u.username AND v.id < u.id); + +ALTER TABLE users ALTER COLUMN username SET NOT NULL; +CREATE UNIQUE INDEX users_username_key ON users (username); +ALTER TABLE users ALTER COLUMN email DROP NOT NULL; + +GRANT SELECT, INSERT, UPDATE, DELETE ON users TO sp_app; diff --git a/apps/api/migrations/017_invite_links.sql b/apps/api/migrations/017_invite_links.sql new file mode 100644 index 0000000..aa6c933 --- /dev/null +++ b/apps/api/migrations/017_invite_links.sql @@ -0,0 +1,64 @@ +-- 017: multi-use, expiring workspace invite links (Telegram-style). +-- One link, many claimants: anyone holding the token may join with the link's +-- role until the link expires, is revoked, or exhausts max_uses (null = +-- unlimited until expiry). Claimers arrive unauthenticated, so the claim path +-- is the SECURITY DEFINER analogue of login (see invite_link_for_claim below): +-- workspace_invite_links is FORCE RLS and no tenant context exists pre-join. +CREATE TABLE workspace_invite_links ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + role text NOT NULL, + token text NOT NULL UNIQUE, + created_by uuid NOT NULL REFERENCES users(id), + created_at timestamptz NOT NULL DEFAULT now(), + expires_at timestamptz NOT NULL, + max_uses int CHECK (max_uses IS NULL OR max_uses > 0), + use_count int NOT NULL DEFAULT 0 CHECK (use_count >= 0), + revoked_at timestamptz +); + +CREATE TRIGGER trg_invite_link_ws_immutable BEFORE UPDATE ON workspace_invite_links + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +-- Same fail-closed tenant isolation as every tenant table (001/006 style): +-- no app.workspace_id => NULL => no rows. FORCE binds sp_app too. +ALTER TABLE workspace_invite_links ENABLE ROW LEVEL SECURITY; +ALTER TABLE workspace_invite_links FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON workspace_invite_links + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +GRANT SELECT, INSERT, UPDATE, DELETE ON workspace_invite_links TO sp_app; + +-- Claim runs with NO tenant context (the claimer has no membership yet), and +-- the table is FORCE RLS: ordinary SELECTs see zero rows. Like +-- auth_user_memberships (007) and publishing_schedule_workspace (006), this +-- narrow SECURITY DEFINER lookup exposes exactly one row keyed by the secret +-- token — workspace_id discovery only; the caller enters the tenant context +-- afterwards and every subsequent statement is RLS-bound as sp_app. +CREATE FUNCTION invite_link_for_claim(p_token text) +RETURNS TABLE(id uuid, workspace_id uuid, role text, expires_at timestamptz, + max_uses int, use_count int, revoked_at timestamptz) +LANGUAGE sql STABLE SECURITY DEFINER SET search_path = public AS $$ + SELECT l.id, l.workspace_id, l.role, l.expires_at, l.max_uses, l.use_count, l.revoked_at + FROM workspace_invite_links l + WHERE l.token = p_token +$$; + +REVOKE ALL ON FUNCTION invite_link_for_claim(text) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION invite_link_for_claim(text) TO sp_app; + +-- Token lookup at claim time: point lookup served by the backing index of the +-- inline UNIQUE constraint on token (auto-named workspace_invite_links_token_key). +-- Statement for documentation/parity with 016's users_username_key — do NOT run twice: +-- CREATE UNIQUE INDEX workspace_invite_links_token_key ON workspace_invite_links (token); + +-- Workspace listing (admin GET /team/invite-links), newest first: +CREATE INDEX workspace_invite_links_ws_created + ON workspace_invite_links (workspace_id, created_at DESC); + +-- Expiry sweep / active-link diagnostics, mirroring the partial-index style of +-- audit_log (001) and scheduled_publications_due (006): +CREATE INDEX workspace_invite_links_expiry + ON workspace_invite_links (expires_at) WHERE revoked_at IS NULL; diff --git a/apps/api/migrations/018_publishing_execution.sql b/apps/api/migrations/018_publishing_execution.sql new file mode 100644 index 0000000..ef52838 --- /dev/null +++ b/apps/api/migrations/018_publishing_execution.sql @@ -0,0 +1,485 @@ +-- 018: WP3 publishing execution spine (fences, attempt journal, receipts). +-- Preserves 006: same drafts / scheduled_publications / publications tables, +-- same PKs, UNIQUE(draft_id) on schedules and global idempotency_key +-- uniqueness. Adds the scheduling principal, revision/approval proof, +-- credential generation/revocation, execution generation and the durable +-- attempt journal the scheduler claims, fences and finalizes against. +-- +-- The current runner (db/migrate.ts) wraps each file in ONE transaction, so +-- this file is written to be transactional end to end: no CREATE INDEX +-- CONCURRENTLY, bounded by lock_timeout, preflight fails closed on corrupt +-- tenant references instead of silently reparenting them. Quiesce the +-- publishing worker for the upgrade (plan §2 "Existing data rollout"). +SET LOCAL lock_timeout = '5s'; + +-- --------------------------------------------------------------------------- +-- Preflight: refuse to upgrade data that the tenant-composite FKs below would +-- reject anyway, with an operator-readable reason. Nothing is reparented. +-- --------------------------------------------------------------------------- +DO $$ +DECLARE v_bad bigint; +BEGIN + SELECT count(*) INTO v_bad FROM scheduled_publications sp + WHERE NOT EXISTS (SELECT 1 FROM drafts d WHERE d.id = sp.draft_id AND d.workspace_id = sp.workspace_id) + OR NOT EXISTS (SELECT 1 FROM connections c WHERE c.id = sp.connection_id AND c.workspace_id = sp.workspace_id); + IF v_bad > 0 THEN + RAISE EXCEPTION '018 preflight: % scheduled_publications rows reference a draft/connection in another workspace', v_bad; + END IF; + SELECT count(*) INTO v_bad FROM publications p + WHERE NOT EXISTS (SELECT 1 FROM drafts d WHERE d.id = p.draft_id AND d.workspace_id = p.workspace_id) + OR NOT EXISTS (SELECT 1 FROM connections c WHERE c.id = p.connection_id AND c.workspace_id = p.workspace_id); + IF v_bad > 0 THEN + RAISE EXCEPTION '018 preflight: % publications rows reference a draft/connection in another workspace', v_bad; + END IF; + SELECT count(*) INTO v_bad FROM credentials cr + WHERE cr.connection_id IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM connections c WHERE c.id = cr.connection_id AND c.workspace_id = cr.workspace_id); + IF v_bad > 0 THEN + RAISE EXCEPTION '018 preflight: % credentials rows reference a connection in another workspace', v_bad; + END IF; + SELECT count(*) INTO v_bad FROM ( + SELECT 1 FROM credentials WHERE kind = 'connector-oauth' AND connection_id IS NOT NULL + GROUP BY workspace_id, connection_id HAVING count(*) > 1 + ) dup; + IF v_bad > 0 THEN + RAISE EXCEPTION '018 preflight: % connections have more than one connector-oauth credential; reconcile before upgrading', v_bad; + END IF; +END $$; + +-- --------------------------------------------------------------------------- +-- Credentials: generation + revocation. key_version is NOT a token +-- generation (APV1 sealing writes version 1 on every rotation). +-- --------------------------------------------------------------------------- +ALTER TABLE credentials + ADD COLUMN generation bigint NOT NULL DEFAULT 1 CHECK (generation > 0), + ADD COLUMN revoked_at timestamptz; +ALTER TABLE credentials ADD CONSTRAINT credentials_id_workspace_key UNIQUE (id, workspace_id); +-- One live connector credential per connection; readers stop using LIMIT 1. +CREATE UNIQUE INDEX credentials_connector_oauth_unique + ON credentials (workspace_id, connection_id) WHERE kind = 'connector-oauth'; + +-- generation is DB-owned: any secret-material or lifecycle change rotates it, +-- and callers cannot rewind it. +CREATE FUNCTION credentials_bump_generation() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + IF NEW.ciphertext IS DISTINCT FROM OLD.ciphertext + OR NEW.key_version IS DISTINCT FROM OLD.key_version + OR NEW.rotated_at IS DISTINCT FROM OLD.rotated_at + OR NEW.revoked_at IS DISTINCT FROM OLD.revoked_at THEN + NEW.generation := OLD.generation + 1; + ELSE + NEW.generation := OLD.generation; + END IF; + RETURN NEW; +END $$; + +CREATE TRIGGER trg_cred_generation BEFORE UPDATE ON credentials + FOR EACH ROW EXECUTE FUNCTION credentials_bump_generation(); +-- Credentials are an execution-authorization input: changes bump the epoch +-- exactly like memberships (reuses bump_authz_epoch, 001/005). +CREATE TRIGGER trg_cred_bump_epoch + AFTER INSERT OR DELETE OR UPDATE OF ciphertext, key_version, rotated_at, revoked_at ON credentials + FOR EACH ROW EXECUTE FUNCTION bump_authz_epoch(); + +-- --------------------------------------------------------------------------- +-- Connections: tombstones keep journal evidence intact after removal. +-- --------------------------------------------------------------------------- +ALTER TABLE connections ADD COLUMN disconnected_at timestamptz; +CREATE TRIGGER trg_conn_disconnect_bump_epoch AFTER UPDATE OF disconnected_at ON connections + FOR EACH ROW WHEN (NEW.disconnected_at IS DISTINCT FROM OLD.disconnected_at) + EXECUTE FUNCTION bump_authz_epoch(); + +-- Group sharing gates userCanUseConnection, so it bumps the epoch too (008 +-- only covered group_members). +CREATE TRIGGER trg_group_connection_bump_epoch AFTER INSERT OR UPDATE OR DELETE ON group_connections + FOR EACH ROW EXECUTE FUNCTION bump_authz_epoch(); + +-- role_permissions is global. Every workspace's Owner/Admin/... roles come +-- from it, so a change bumps all workspaces once per statement, locked in +-- UUID order to avoid lock-order inversions between concurrent bumps. +CREATE FUNCTION bump_all_authz_epochs() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + PERFORM 1 FROM workspaces ORDER BY id FOR UPDATE; + UPDATE workspaces SET authz_epoch = authz_epoch + 1; + RETURN NULL; +END $$; +CREATE TRIGGER trg_role_permissions_bump_epoch AFTER INSERT OR UPDATE OR DELETE ON role_permissions + FOR EACH STATEMENT EXECUTE FUNCTION bump_all_authz_epochs(); + +-- --------------------------------------------------------------------------- +-- Drafts: revision + approval proof + business attempt counter. +-- --------------------------------------------------------------------------- +ALTER TABLE drafts + ADD COLUMN revision bigint NOT NULL DEFAULT 1 CHECK (revision > 0), + ADD COLUMN approved_revision bigint, + ADD COLUMN approved_hash text, + ADD COLUMN approved_at timestamptz, + ADD COLUMN review_reason text, + ADD COLUMN publication_attempt bigint NOT NULL DEFAULT 0 CHECK (publication_attempt >= 0); +-- drafts_id_workspace_key (id, workspace_id) already exists from 015. + +-- --------------------------------------------------------------------------- +-- Scheduled publications: principal/fence snapshot + execution generation. +-- --------------------------------------------------------------------------- +ALTER TABLE scheduled_publications + ADD COLUMN created_at timestamptz NOT NULL DEFAULT now(), + ADD COLUMN updated_at timestamptz NOT NULL DEFAULT now(), + ADD COLUMN scheduled_by_user_id uuid REFERENCES users(id) ON DELETE NO ACTION ON UPDATE RESTRICT, + ADD COLUMN required_permission text + CHECK (required_permission IN ('publication.schedule', 'publication.publish')), + ADD COLUMN authorized_epoch bigint, + ADD COLUMN approved_revision bigint, + ADD COLUMN post_hash text, + ADD COLUMN credential_id uuid, + ADD COLUMN credential_generation bigint, + ADD COLUMN generation bigint NOT NULL DEFAULT 1 CHECK (generation > 0), + ADD COLUMN attempt_no bigint NOT NULL DEFAULT 1 CHECK (attempt_no > 0), + ADD COLUMN execution_version smallint NOT NULL DEFAULT 0 CHECK (execution_version IN (0, 1)), + ADD COLUMN timezone text NOT NULL DEFAULT 'UTC', + ADD COLUMN local_publish_at timestamp, + ADD COLUMN tzdata_version text, + ADD COLUMN next_attempt_at timestamptz, + ADD COLUMN retry_count int NOT NULL DEFAULT 0 CHECK (retry_count >= 0), + ADD COLUMN error_code text; + +ALTER TABLE scheduled_publications DROP CONSTRAINT scheduled_publications_state_check; +ALTER TABLE scheduled_publications ADD CONSTRAINT scheduled_publications_state_check + CHECK (state IN ('pending', 'in_flight', 'done', 'failed', 'needs_reauthorization', 'cancelled')); + +-- Legacy rows: timestamps from the draft, due time from publish_at, UTC zone. +UPDATE scheduled_publications sp + SET created_at = d.created_at, updated_at = d.updated_at, next_attempt_at = sp.publish_at + FROM drafts d + WHERE d.id = sp.draft_id AND d.workspace_id = sp.workspace_id; + +-- Existing publications are authoritative: their schedule is done, draft published. +UPDATE scheduled_publications sp SET state = 'done', error = NULL, error_code = NULL + WHERE EXISTS (SELECT 1 FROM publications p + WHERE p.workspace_id = sp.workspace_id + AND (p.idempotency_key = sp.idempotency_key OR p.draft_id = sp.draft_id)); +UPDATE drafts d SET state = 'published' + WHERE d.state <> 'published' + AND EXISTS (SELECT 1 FROM publications p WHERE p.workspace_id = d.workspace_id AND p.draft_id = d.id); +-- No recorded principal: never infer one from author/audit history. +UPDATE scheduled_publications SET state = 'needs_reauthorization', error_code = 'LEGACY_AUTHORIZATION_MISSING' + WHERE state = 'pending'; +-- In flight without a publication: the send may have happened. +UPDATE scheduled_publications SET state = 'failed', error_code = 'PUBLISH_OUTCOME_UNKNOWN' + WHERE state = 'in_flight'; + +-- Upgraded (execution_version = 1) rows must carry every execution fence +-- while executable or done. Legacy rows (0) are readable, never executable: +-- every claim requires execution_version = 1. +ALTER TABLE scheduled_publications ADD CONSTRAINT scheduled_publications_fences_check CHECK ( + execution_version = 0 + OR state IN ('failed', 'needs_reauthorization', 'cancelled') + OR (scheduled_by_user_id IS NOT NULL AND required_permission IS NOT NULL + AND authorized_epoch IS NOT NULL AND approved_revision IS NOT NULL + AND post_hash IS NOT NULL AND credential_id IS NOT NULL + AND credential_generation IS NOT NULL AND next_attempt_at IS NOT NULL) +); + +ALTER TABLE scheduled_publications + ADD CONSTRAINT scheduled_publications_id_workspace_key UNIQUE (id, workspace_id); + +-- Tenant-composite replacements for 006's single-column FKs (same ON DELETE +-- semantics; journal tables below add NO ACTION guards). +ALTER TABLE scheduled_publications + DROP CONSTRAINT scheduled_publications_draft_id_fkey, + DROP CONSTRAINT scheduled_publications_connection_id_fkey, + ADD CONSTRAINT scheduled_publications_draft_ws_fkey FOREIGN KEY (draft_id, workspace_id) + REFERENCES drafts (id, workspace_id) ON DELETE CASCADE ON UPDATE RESTRICT, + ADD CONSTRAINT scheduled_publications_connection_ws_fkey FOREIGN KEY (connection_id, workspace_id) + REFERENCES connections (id, workspace_id) ON DELETE NO ACTION ON UPDATE RESTRICT, + ADD CONSTRAINT scheduled_publications_credential_ws_fkey FOREIGN KEY (credential_id, workspace_id) + REFERENCES credentials (id, workspace_id) ON DELETE NO ACTION ON UPDATE RESTRICT; + +ALTER TABLE publications + DROP CONSTRAINT publications_draft_id_fkey, + DROP CONSTRAINT publications_connection_id_fkey, + ADD CONSTRAINT publications_draft_ws_fkey FOREIGN KEY (draft_id, workspace_id) + REFERENCES drafts (id, workspace_id) ON DELETE CASCADE ON UPDATE RESTRICT, + ADD CONSTRAINT publications_connection_ws_fkey FOREIGN KEY (connection_id, workspace_id) + REFERENCES connections (id, workspace_id) ON DELETE NO ACTION ON UPDATE RESTRICT; + +-- Schedules: DB-owned updated_at; generation and execution_version only move forward. +CREATE FUNCTION scheduled_publications_guard() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + IF NEW.generation < OLD.generation THEN + RAISE EXCEPTION 'schedule generation cannot decrease' USING ERRCODE = '23514'; + END IF; + IF NEW.execution_version < OLD.execution_version THEN + RAISE EXCEPTION 'schedule execution_version cannot decrease' USING ERRCODE = '23514'; + END IF; + NEW.updated_at := now(); + RETURN NEW; +END $$; +CREATE TRIGGER trg_scheduled_pub_guard BEFORE UPDATE ON scheduled_publications + FOR EACH ROW EXECUTE FUNCTION scheduled_publications_guard(); + +-- --------------------------------------------------------------------------- +-- Execution journal: one row per business attempt. Snapshot columns are +-- immutable; mutable columns carry ownership (owner_token/claim_epoch/lease) +-- and the dispatch marker. +-- --------------------------------------------------------------------------- +CREATE TABLE publishing_attempts ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + schedule_id uuid NOT NULL, + draft_id uuid NOT NULL, + connection_id uuid NOT NULL, + attempt_no bigint NOT NULL CHECK (attempt_no > 0), + idempotency_key text NOT NULL CHECK (btrim(idempotency_key) <> ''), + schedule_generation bigint NOT NULL CHECK (schedule_generation > 0), + post_snapshot jsonb NOT NULL CHECK (jsonb_typeof(post_snapshot) = 'object'), + post_hash text NOT NULL CHECK (btrim(post_hash) <> ''), + scheduled_by_user_id uuid NOT NULL REFERENCES users(id) ON DELETE NO ACTION ON UPDATE RESTRICT, + required_permission text NOT NULL + CHECK (required_permission IN ('publication.schedule', 'publication.publish')), + approved_revision bigint NOT NULL CHECK (approved_revision > 0), + reviewer_user_id uuid NOT NULL REFERENCES users(id) ON DELETE NO ACTION ON UPDATE RESTRICT, + credential_id uuid NOT NULL, + credential_generation bigint NOT NULL CHECK (credential_generation > 0), + authorized_epoch bigint NOT NULL, + dispatch_count int NOT NULL DEFAULT 0 CHECK (dispatch_count >= 0 AND dispatch_count <= 5), + status text NOT NULL DEFAULT 'ready' + CHECK (status IN ('ready', 'in_progress', 'completed', 'failed', 'unknown', 'abandoned')), + claim_epoch bigint NOT NULL DEFAULT 0 CHECK (claim_epoch >= 0), + owner_token uuid, + lease_until timestamptz, + dispatch_started_at timestamptz, + provider_retry_deadline timestamptz, + error_code text, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + -- Ownership exists exactly while in progress. + CHECK ((status = 'in_progress') = (owner_token IS NOT NULL AND lease_until IS NOT NULL)), + CHECK (status = 'in_progress' OR (owner_token IS NULL AND lease_until IS NULL)), + -- A dispatch marker implies at least one counted dispatch. + CHECK (dispatch_started_at IS NULL OR dispatch_count > 0), + UNIQUE (workspace_id, id), + UNIQUE (workspace_id, draft_id, attempt_no), + UNIQUE (workspace_id, idempotency_key), + FOREIGN KEY (schedule_id, workspace_id) REFERENCES scheduled_publications (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT, + FOREIGN KEY (draft_id, workspace_id) REFERENCES drafts (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT, + FOREIGN KEY (connection_id, workspace_id) REFERENCES connections (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT, + FOREIGN KEY (credential_id, workspace_id) REFERENCES credentials (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT +); + +-- Snapshot immutability, terminal statuses, and DB-owned claim epoch: every +-- new owner token advances claim_epoch by exactly one, so a deposed owner's +-- (owner_token, claim_epoch) pair can never match again. +CREATE FUNCTION publishing_attempts_guard() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + IF (NEW.schedule_id, NEW.draft_id, NEW.connection_id, NEW.attempt_no, NEW.idempotency_key, + NEW.schedule_generation, NEW.post_snapshot, NEW.post_hash, NEW.scheduled_by_user_id, + NEW.required_permission, NEW.approved_revision, NEW.reviewer_user_id, + NEW.credential_id, NEW.credential_generation, NEW.created_at) + IS DISTINCT FROM + (OLD.schedule_id, OLD.draft_id, OLD.connection_id, OLD.attempt_no, OLD.idempotency_key, + OLD.schedule_generation, OLD.post_snapshot, OLD.post_hash, OLD.scheduled_by_user_id, + OLD.required_permission, OLD.approved_revision, OLD.reviewer_user_id, + OLD.credential_id, OLD.credential_generation, OLD.created_at) THEN + RAISE EXCEPTION 'publishing attempt snapshot is immutable' USING ERRCODE = '23514'; + END IF; + IF OLD.status IN ('completed', 'abandoned') AND NEW.status IS DISTINCT FROM OLD.status THEN + RAISE EXCEPTION 'publishing attempt % is terminal', OLD.status USING ERRCODE = '23514'; + END IF; + IF NEW.dispatch_count < OLD.dispatch_count THEN + RAISE EXCEPTION 'dispatch_count cannot decrease' USING ERRCODE = '23514'; + END IF; + IF NEW.provider_retry_deadline IS DISTINCT FROM OLD.provider_retry_deadline + AND OLD.provider_retry_deadline IS NOT NULL THEN + RAISE EXCEPTION 'provider_retry_deadline is fixed at first dispatch' USING ERRCODE = '23514'; + END IF; + IF NEW.owner_token IS NOT NULL AND NEW.owner_token IS DISTINCT FROM OLD.owner_token THEN + NEW.claim_epoch := OLD.claim_epoch + 1; + ELSE + NEW.claim_epoch := OLD.claim_epoch; + END IF; + NEW.updated_at := now(); + RETURN NEW; +END $$; + +CREATE TRIGGER trg_publishing_attempt_ws_immutable BEFORE UPDATE ON publishing_attempts + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_publishing_attempt_guard BEFORE UPDATE ON publishing_attempts + FOR EACH ROW EXECUTE FUNCTION publishing_attempts_guard(); + +-- Durable provider evidence, separate from the attempt so it survives activity +-- timeouts and stale-owner completion refusal. Append-only. +CREATE TABLE publishing_receipts ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + attempt_id uuid NOT NULL, + remote_id text NOT NULL CHECK (btrim(remote_id) <> '' AND length(remote_id) <= 512), + remote_url text CHECK (remote_url IS NULL OR length(remote_url) <= 2048), + published_at timestamptz, + provider_metadata jsonb NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(provider_metadata) = 'object'), + received_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, attempt_id), + FOREIGN KEY (attempt_id, workspace_id) REFERENCES publishing_attempts (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT +); + +-- Receipts are never rewritten. Deletion is only the workspace purge cascade +-- (the parent workspace row is already gone when the cascade reaches here). +CREATE FUNCTION publishing_receipts_append_only() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + IF TG_OP = 'DELETE' THEN + IF EXISTS (SELECT 1 FROM workspaces w WHERE w.id = OLD.workspace_id) THEN + RAISE EXCEPTION 'publishing receipts are append-only' USING ERRCODE = '23514'; + END IF; + RETURN OLD; + END IF; + RAISE EXCEPTION 'publishing receipts are append-only' USING ERRCODE = '23514'; +END $$; +CREATE TRIGGER trg_publishing_receipt_append_only BEFORE UPDATE OR DELETE ON publishing_receipts + FOR EACH ROW EXECUTE FUNCTION publishing_receipts_append_only(); + +-- Draft-creation request idempotency, committed atomically with the draft. +CREATE TABLE publishing_requests ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE ON UPDATE RESTRICT, + route text NOT NULL CHECK (btrim(route) <> ''), + request_key uuid NOT NULL, + request_hash text NOT NULL CHECK (btrim(request_hash) <> ''), + response jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, user_id, route, request_key) +); +CREATE TRIGGER trg_publishing_request_ws_immutable BEFORE UPDATE ON publishing_requests + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +-- --------------------------------------------------------------------------- +-- Draft fence: content changes rotate the revision and void approval; they +-- are refused while a dispatch is active or its outcome is unknown. +-- --------------------------------------------------------------------------- +CREATE FUNCTION drafts_revision_fence() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + IF NEW.publication_attempt < OLD.publication_attempt THEN + RAISE EXCEPTION 'publication_attempt cannot decrease' USING ERRCODE = '23514'; + END IF; + IF NEW.post IS DISTINCT FROM OLD.post THEN + IF EXISTS (SELECT 1 FROM publishing_attempts a + WHERE a.workspace_id = OLD.workspace_id AND a.draft_id = OLD.id + AND (a.status = 'unknown' + OR (a.status = 'in_progress' AND a.dispatch_started_at IS NOT NULL))) THEN + RAISE EXCEPTION 'draft content is locked while a dispatch is unsettled' USING ERRCODE = '55000'; + END IF; + NEW.revision := OLD.revision + 1; + NEW.approved_revision := NULL; + NEW.approved_hash := NULL; + NEW.approved_at := NULL; + NEW.reviewer_user_id := NULL; + ELSE + NEW.revision := OLD.revision; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER trg_draft_revision_fence BEFORE UPDATE ON drafts + FOR EACH ROW EXECUTE FUNCTION drafts_revision_fence(); + +-- --------------------------------------------------------------------------- +-- RLS + grants for the new tenant tables (001/006/017 style). +-- --------------------------------------------------------------------------- +ALTER TABLE publishing_attempts ENABLE ROW LEVEL SECURITY; +ALTER TABLE publishing_receipts ENABLE ROW LEVEL SECURITY; +ALTER TABLE publishing_requests ENABLE ROW LEVEL SECURITY; +ALTER TABLE publishing_attempts FORCE ROW LEVEL SECURITY; +ALTER TABLE publishing_receipts FORCE ROW LEVEL SECURITY; +ALTER TABLE publishing_requests FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON publishing_attempts + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON publishing_receipts + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +CREATE POLICY tenant_isolation ON publishing_requests + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +-- 003's default privileges grant all four verbs; narrow them explicitly: +-- the journal is never deleted by the app, receipts are append-only. +GRANT SELECT, INSERT, UPDATE ON publishing_attempts TO sp_app; +REVOKE DELETE ON publishing_attempts FROM sp_app; +GRANT SELECT, INSERT ON publishing_receipts TO sp_app; +REVOKE UPDATE, DELETE ON publishing_receipts FROM sp_app; +GRANT SELECT, INSERT, UPDATE, DELETE ON publishing_requests TO sp_app; + +-- --------------------------------------------------------------------------- +-- Indexes: keyset lists (no OFFSET), due/recovery discovery, FK coverage. +-- --------------------------------------------------------------------------- +CREATE INDEX drafts_ws_updated ON drafts (workspace_id, updated_at DESC, id DESC); +CREATE INDEX drafts_ws_state_updated ON drafts (workspace_id, state, updated_at DESC, id DESC); + +CREATE INDEX scheduled_publications_ws_publish ON scheduled_publications (workspace_id, publish_at DESC, id DESC); +CREATE INDEX scheduled_publications_ws_conn_publish + ON scheduled_publications (workspace_id, connection_id, publish_at DESC, id DESC); +CREATE INDEX scheduled_publications_next_due ON scheduled_publications (next_attempt_at, id) + WHERE state = 'pending'; +CREATE INDEX scheduled_publications_ws_credential ON scheduled_publications (workspace_id, credential_id) + WHERE credential_id IS NOT NULL; + +CREATE INDEX publications_ws_published ON publications (workspace_id, published_at DESC, id DESC); +CREATE INDEX publications_ws_conn_published + ON publications (workspace_id, connection_id, published_at DESC, id DESC); + +CREATE INDEX publishing_attempts_ws_schedule ON publishing_attempts (workspace_id, schedule_id, attempt_no DESC); +CREATE INDEX publishing_attempts_ws_connection ON publishing_attempts (workspace_id, connection_id); +CREATE INDEX publishing_attempts_ws_credential ON publishing_attempts (workspace_id, credential_id); +CREATE INDEX publishing_attempts_lease_recovery ON publishing_attempts (lease_until, id) + WHERE status = 'in_progress'; + +-- --------------------------------------------------------------------------- +-- Discovery (replaces 006). Identifiers/times only, bounded, fixed +-- search_path, PUBLIC revoked. Callers enter withTenant(workspace_id) and +-- re-verify key/generation ownership under RLS. +-- --------------------------------------------------------------------------- +DROP FUNCTION publishing_due_schedules(); + +CREATE FUNCTION publishing_due_schedules(p_limit integer DEFAULT 100) +RETURNS TABLE(workspace_id uuid, schedule_id uuid, idempotency_key text, generation bigint, publish_at timestamptz) +LANGUAGE sql STABLE SECURITY DEFINER SET search_path = public AS $$ + SELECT due.workspace_id, due.schedule_id, due.idempotency_key, due.generation, due.publish_at + FROM ( + SELECT sp.workspace_id, sp.id AS schedule_id, sp.idempotency_key, sp.generation, + sp.publish_at, sp.next_attempt_at AS due_at + FROM scheduled_publications sp + WHERE sp.state = 'pending' AND sp.execution_version = 1 AND sp.next_attempt_at <= now() + UNION ALL + SELECT a.workspace_id, a.schedule_id, a.idempotency_key, a.schedule_generation, + sp.publish_at, a.lease_until AS due_at + FROM publishing_attempts a + JOIN scheduled_publications sp ON sp.id = a.schedule_id AND sp.workspace_id = a.workspace_id + WHERE a.status = 'in_progress' AND a.lease_until < now() + ) due + ORDER BY due.due_at, due.schedule_id + LIMIT LEAST(GREATEST(COALESCE(p_limit, 100), 1), 100) +$$; + +-- Attempt journal first (every executable key), then legacy schedules/publications. +CREATE OR REPLACE FUNCTION publishing_schedule_workspace(p_idempotency_key text) RETURNS uuid +LANGUAGE sql STABLE SECURITY DEFINER SET search_path = public AS $$ + SELECT COALESCE( + (SELECT a.workspace_id FROM publishing_attempts a WHERE a.idempotency_key = p_idempotency_key LIMIT 1), + (SELECT sp.workspace_id FROM scheduled_publications sp WHERE sp.idempotency_key = p_idempotency_key), + (SELECT p.workspace_id FROM publications p WHERE p.idempotency_key = p_idempotency_key) + ) +$$; + +REVOKE ALL ON FUNCTION publishing_due_schedules(integer) FROM PUBLIC; +REVOKE ALL ON FUNCTION publishing_schedule_workspace(text) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION publishing_due_schedules(integer), publishing_schedule_workspace(text) TO sp_app; diff --git a/apps/api/migrations/019_publishing_integration.sql b/apps/api/migrations/019_publishing_integration.sql new file mode 100644 index 0000000..6865c10 --- /dev/null +++ b/apps/api/migrations/019_publishing_integration.sql @@ -0,0 +1,103 @@ +-- 018 is retained unchanged: local anp_publishing_it records its application. +-- These corrections use independently tracked resumable stages. +-- wp3:phase expand transaction +SET LOCAL lock_timeout = '5s'; +ALTER TABLE publishing_attempts ADD COLUMN first_dispatch_started_at timestamptz; +CREATE OR REPLACE FUNCTION publishing_attempts_guard() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + IF (NEW.schedule_id, NEW.draft_id, NEW.connection_id, NEW.attempt_no, NEW.idempotency_key, + NEW.schedule_generation, NEW.post_snapshot, NEW.post_hash, NEW.scheduled_by_user_id, + NEW.required_permission, NEW.approved_revision, NEW.reviewer_user_id, + NEW.credential_id, NEW.credential_generation, NEW.created_at) + IS DISTINCT FROM + (OLD.schedule_id, OLD.draft_id, OLD.connection_id, OLD.attempt_no, OLD.idempotency_key, + OLD.schedule_generation, OLD.post_snapshot, OLD.post_hash, OLD.scheduled_by_user_id, + OLD.required_permission, OLD.approved_revision, OLD.reviewer_user_id, + OLD.credential_id, OLD.credential_generation, OLD.created_at) THEN + RAISE EXCEPTION 'publishing attempt snapshot is immutable' USING ERRCODE = '23514'; + END IF; + IF OLD.status IN ('completed', 'abandoned') AND NEW.status IS DISTINCT FROM OLD.status THEN + RAISE EXCEPTION 'publishing attempt % is terminal', OLD.status USING ERRCODE = '23514'; + END IF; + IF OLD.first_dispatch_started_at IS NOT NULL AND NEW.first_dispatch_started_at IS DISTINCT FROM OLD.first_dispatch_started_at THEN + RAISE EXCEPTION 'first dispatch time is immutable' USING ERRCODE = '23514'; + END IF; + IF NEW.dispatch_count < OLD.dispatch_count THEN + RAISE EXCEPTION 'dispatch_count cannot decrease' USING ERRCODE = '23514'; + END IF; + IF NEW.provider_retry_deadline IS DISTINCT FROM OLD.provider_retry_deadline + AND OLD.provider_retry_deadline IS NOT NULL THEN + RAISE EXCEPTION 'provider_retry_deadline is fixed at first dispatch' USING ERRCODE = '23514'; + END IF; + IF NEW.owner_token IS NOT NULL AND NEW.owner_token IS DISTINCT FROM OLD.owner_token THEN + NEW.claim_epoch := OLD.claim_epoch + 1; + ELSE + NEW.claim_epoch := OLD.claim_epoch; + END IF; + NEW.updated_at := now(); + RETURN NEW; +END $$; + +CREATE OR REPLACE FUNCTION drafts_revision_fence() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + IF NEW.publication_attempt < OLD.publication_attempt THEN + RAISE EXCEPTION 'publication_attempt cannot decrease' USING ERRCODE = '23514'; + END IF; + IF NEW.post IS DISTINCT FROM OLD.post THEN + IF EXISTS (SELECT 1 FROM publishing_attempts a + WHERE a.workspace_id = OLD.workspace_id AND a.draft_id = OLD.id + AND (a.status = 'unknown' + OR (a.status IN ('ready', 'in_progress') AND a.dispatch_started_at IS NOT NULL))) THEN + RAISE EXCEPTION 'draft content is locked while a dispatch is unsettled' USING ERRCODE = '55000'; + END IF; + NEW.revision := OLD.revision + 1; + NEW.approved_revision := NULL; + NEW.approved_hash := NULL; + NEW.approved_at := NULL; + NEW.reviewer_user_id := NULL; + ELSE + NEW.revision := OLD.revision; + END IF; + RETURN NEW; +END $$; +-- wp3:phase dispatch_backfill batch +WITH batch AS ( + SELECT id FROM publishing_attempts WHERE first_dispatch_started_at IS NULL AND dispatch_count > 0 LIMIT 1000 FOR UPDATE +) +UPDATE publishing_attempts a SET first_dispatch_started_at = CASE WHEN a.dispatch_count > 1 OR a.dispatch_started_at IS NULL THEN a.created_at ELSE a.dispatch_started_at END FROM batch WHERE a.id = batch.id; +-- wp3:phase receipt_recovery_index concurrent +CREATE INDEX CONCURRENTLY publishing_receipts_recovery ON publishing_receipts (workspace_id, attempt_id, received_at); +-- wp3:phase cutover transaction +SET LOCAL lock_timeout = '5s'; +DROP FUNCTION publishing_due_schedules(integer); +CREATE FUNCTION publishing_due_schedules(p_limit integer DEFAULT 100) +RETURNS TABLE(workspace_id uuid, schedule_id uuid, idempotency_key text, generation bigint, publish_at timestamptz, evidence_only boolean) +LANGUAGE sql STABLE SECURITY DEFINER SET search_path = public AS $$ + SELECT due.workspace_id, due.schedule_id, due.idempotency_key, due.generation, due.publish_at, due.evidence_only + FROM ( + SELECT sp.workspace_id, sp.id AS schedule_id, sp.idempotency_key, sp.generation, + sp.publish_at, sp.next_attempt_at AS due_at, false AS evidence_only + FROM scheduled_publications sp + WHERE sp.state = 'pending' AND sp.execution_version = 1 AND sp.next_attempt_at <= now() AND NOT EXISTS (SELECT 1 FROM publishing_attempts a JOIN publishing_receipts r ON r.attempt_id = a.id AND r.workspace_id = a.workspace_id WHERE a.idempotency_key = sp.idempotency_key AND a.workspace_id = sp.workspace_id) + UNION ALL + SELECT a.workspace_id, a.schedule_id, a.idempotency_key, a.schedule_generation, + sp.publish_at, a.lease_until AS due_at, false AS evidence_only + FROM publishing_attempts a + JOIN scheduled_publications sp ON sp.id = a.schedule_id AND sp.workspace_id = a.workspace_id + WHERE a.status = 'in_progress' AND a.lease_until < now() AND NOT EXISTS (SELECT 1 FROM publishing_receipts r WHERE r.attempt_id = a.id AND r.workspace_id = a.workspace_id) + UNION ALL + SELECT a.workspace_id, a.schedule_id, a.idempotency_key, a.schedule_generation, + sp.publish_at, r.received_at AS due_at, true AS evidence_only + FROM publishing_attempts a + JOIN publishing_receipts r ON r.attempt_id = a.id AND r.workspace_id = a.workspace_id + JOIN scheduled_publications sp ON sp.id = a.schedule_id AND sp.workspace_id = a.workspace_id + WHERE (a.status IN ('ready', 'failed', 'unknown') OR (a.status = 'in_progress' AND a.lease_until < now())) + ) due + ORDER BY due.due_at, due.schedule_id + LIMIT LEAST(GREATEST(COALESCE(p_limit, 100), 1), 100) +$$; + +REVOKE ALL ON FUNCTION publishing_due_schedules(integer) FROM PUBLIC; +GRANT EXECUTE ON FUNCTION publishing_due_schedules(integer) TO sp_app; diff --git a/apps/api/migrations/020_analytics.sql b/apps/api/migrations/020_analytics.sql new file mode 100644 index 0000000..425cdec --- /dev/null +++ b/apps/api/migrations/020_analytics.sql @@ -0,0 +1,93 @@ +-- 020: WP3 analytics snapshots — monthly per-workspace rollups. +-- +-- One row per (workspace, month), refreshed in place by the snapshot writer +-- (apps/api/src/analytics/snapshot.service.ts) with set-based SQL and an +-- idempotent upsert on (workspace_id, month). Months are materialized as +-- range partitions of a `month date` key, mirroring 014_live_chat.sql. +-- +-- Creating future partitions: RLS on the partitioned parent does NOT apply to +-- direct leaf access, so every leaf carries its own ENABLE/FORCE RLS and +-- tenant_isolation policy. `analytics_ensure_month(date)` creates a month's +-- partition idempotently with those controls; this migration precreates the +-- previous, current and next month. Administrators run +-- SELECT analytics_ensure_month(date '2025-09-01'); +-- ahead of each new month (cron it monthly, one statement, no downtime). The +-- DEFAULT partition below catches any month whose partition was not created, +-- so a missed call never drops data — it only loses pruning until the +-- partition is created and the rows moved into it. +-- +-- Partitioned-leaf register entry (proposal §4.4, exception (a)): PK carries +-- the partition key, the table is never the target of an FK, and rows are +-- referenced only by (workspace_id, month) / id soft links. +-- +-- Query catalog (owning WP: WP3 — packages/db does not exist in this repo, so +-- the catalog entries live here and in snapshot.service.ts): +-- analytics.snapshot.refresh INSERT .. ON CONFLICT (workspace_id, month) — +-- served by the PK (workspace_id, month, id); write path, p95 <= 250 ms. +-- analytics.snapshot.list keyset page ORDER BY month DESC — same PK, +-- backward index scan, p95 <= 20 ms. +-- analytics.snapshot.summary two PK point lookups (current + previous +-- month), p95 <= 20 ms. +CREATE TABLE analytics_snapshots ( + id uuid NOT NULL DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + -- Always a month start; the CHECK keeps the value aligned with its partition. + month date NOT NULL CHECK (month = date_trunc('month', month)::date), + publications_total bigint NOT NULL DEFAULT 0 CHECK (publications_total >= 0), + -- {provider: count} for publications published in the month. + publications_by_provider jsonb NOT NULL DEFAULT '{}'::jsonb CHECK (jsonb_typeof(publications_by_provider) = 'object'), + posts_archived_total bigint NOT NULL DEFAULT 0 CHECK (posts_archived_total >= 0), + -- Point-in-time current state at snapshot time (not a monthly delta): + -- connections not disconnected when the snapshot was taken. + active_connections int NOT NULL DEFAULT 0 CHECK (active_connections >= 0), + drafts_created_total bigint NOT NULL DEFAULT 0 CHECK (drafts_created_total >= 0), + publishing_attempts_total bigint NOT NULL DEFAULT 0 CHECK (publishing_attempts_total >= 0), + -- Attempts whose terminal status is 'failed', counted by their creation month. + publish_failures_total bigint NOT NULL DEFAULT 0 CHECK (publish_failures_total >= 0), + snapshot_started_at timestamptz NOT NULL DEFAULT now(), + snapshot_completed_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, month, id), + UNIQUE (workspace_id, month) +) PARTITION BY RANGE (month); + +-- Idempotent: safe to re-run for the same month. +CREATE FUNCTION analytics_ensure_month(p_month date) RETURNS void +LANGUAGE plpgsql SET search_path = public AS $$ +DECLARE v_start date := date_trunc('month', p_month)::date; + v_table text := 'analytics_snapshots_' || to_char(v_start, 'YYYYMM'); +BEGIN + EXECUTE format('CREATE TABLE IF NOT EXISTS %I PARTITION OF analytics_snapshots FOR VALUES FROM (%L) TO (%L)', + v_table, v_start, (v_start + interval '1 month')::date); + EXECUTE format('ALTER TABLE %I ENABLE ROW LEVEL SECURITY', v_table); + EXECUTE format('ALTER TABLE %I FORCE ROW LEVEL SECURITY', v_table); + IF NOT EXISTS (SELECT 1 FROM pg_policies WHERE schemaname = 'public' AND tablename = v_table AND policyname = 'tenant_isolation') THEN + EXECUTE format('CREATE POLICY tenant_isolation ON %I USING (workspace_id = NULLIF(current_setting(''app.workspace_id'', true), '''')::uuid) WITH CHECK (workspace_id = NULLIF(current_setting(''app.workspace_id'', true), '''')::uuid)', v_table); + END IF; +END $$; +REVOKE ALL ON FUNCTION analytics_ensure_month(date) FROM PUBLIC; + +SELECT analytics_ensure_month((current_date - interval '1 month')::date); +SELECT analytics_ensure_month(current_date); +SELECT analytics_ensure_month((current_date + interval '1 month')::date); + +-- Safety net for months nobody pre-created (see header). +CREATE TABLE analytics_snapshots_default PARTITION OF analytics_snapshots DEFAULT; +ALTER TABLE analytics_snapshots_default ENABLE ROW LEVEL SECURITY; +ALTER TABLE analytics_snapshots_default FORCE ROW LEVEL SECURITY; +CREATE POLICY tenant_isolation ON analytics_snapshots_default + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +CREATE TRIGGER trg_analytics_snapshot_ws_immutable BEFORE UPDATE ON analytics_snapshots + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE analytics_snapshots ENABLE ROW LEVEL SECURITY; +ALTER TABLE analytics_snapshots FORCE ROW LEVEL SECURITY; +CREATE POLICY tenant_isolation ON analytics_snapshots + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +-- Grants on the parent cover all current and future partitions for queries +-- routed through it (014 style); direct leaf access also inherits from the +-- owner's default privileges (003). +GRANT SELECT, INSERT, UPDATE, DELETE ON analytics_snapshots TO sp_app; diff --git a/apps/api/migrations/021_stream_columns.sql b/apps/api/migrations/021_stream_columns.sql new file mode 100644 index 0000000..66bdbcf --- /dev/null +++ b/apps/api/migrations/021_stream_columns.sql @@ -0,0 +1,52 @@ +-- 021: editable multi-platform streams columns — column definitions (WP3 §1.2, P0a). +-- stream_columns only: the resolved-source / ingest / fan-out / budget tables +-- (column_sources, feed_items, feed_item_columns, provider_budget_ledger) land +-- in their own forward-only migrations so each slice stays independently +-- reviewable. Additive: no existing table is touched. +-- +-- `spec` and `refresh_policy` are JSONB blobs validated app-side by +-- @tactical-media-broadcast-hub/core (stream-columns.ts); invalid specs are rejected before +-- insert and never stored. The CHECKs here are only coarse backstops — the +-- authoritative typed validation lives in the app layer (017-style split). +CREATE TABLE stream_columns ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + type text NOT NULL CHECK (type IN ('platform_feed','keyword','account_list','saved_search')), + title text NOT NULL CHECK (char_length(title) <= 80), -- user-editable; app layer enforces too + position integer NOT NULL DEFAULT 0 CHECK (position >= 0), -- dense rank; reorder rewrites the tail + spec jsonb NOT NULL CHECK (jsonb_typeof(spec) = 'object'), + refresh_policy jsonb NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(refresh_policy) = 'object'), + created_by uuid NOT NULL REFERENCES users(id), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + deleted_at timestamptz -- soft delete keeps feed_items FK-stable (§1.2) +); + +CREATE TRIGGER trg_stream_columns_ws_immutable BEFORE UPDATE ON stream_columns + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +-- Same fail-closed tenant isolation as every tenant table (001/006/017 style): +-- no app.workspace_id => NULL => no rows. FORCE binds sp_app too. +ALTER TABLE stream_columns ENABLE ROW LEVEL SECURITY; +ALTER TABLE stream_columns FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON stream_columns + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +GRANT SELECT, INSERT, UPDATE, DELETE ON stream_columns TO sp_app; + +-- Dense column order per workspace (UI chips, seeding, reorder). Partial on +-- deleted_at IS NULL: soft-deleted columns keep their row (FK-stable items) +-- without reserving their slot, so reorder/seeding only fights live rows. +-- Named per design §1.2; lazy seeding relies on it for idempotence (§1.3). +CREATE UNIQUE INDEX stream_columns_ws_order_uq + ON stream_columns (workspace_id, position) + WHERE deleted_at IS NULL; + +-- Workspace listing (GET /feeds/columns), newest-position reads served by the +-- unique index above; this covers soft-deleted lookups and count queries in +-- the partial-index style of workspace_invite_links_expiry (017). +CREATE INDEX stream_columns_ws_idx + ON stream_columns (workspace_id) + WHERE deleted_at IS NULL; diff --git a/apps/api/migrations/022_live_destinations.sql b/apps/api/migrations/022_live_destinations.sql new file mode 100644 index 0000000..f51f648 --- /dev/null +++ b/apps/api/migrations/022_live_destinations.sql @@ -0,0 +1,136 @@ +-- 022: WP6 live-stream destinations + stream outputs (go-live slice). +-- +-- Aggregate boundary (proposal §4.4, WP6/WP9): a `live_stream_destinations` +-- row owns the platform-side live target — provider, ingest endpoint URL and +-- the vaulted ingest credential — and never references media outputs. A +-- `stream_outputs` row is a media-output fan-out from a virtual channel and +-- references EITHER a destination (credential resolved through the +-- destination at spawn) OR its own generic credential handle — never both +-- (CHECK below, mirrored in packages/core/src/vc.ts and API validation). +-- Enablement lives only on StreamOutput; the destination carries no +-- enabled flag of its own, so "disable one output" never mutates the shared +-- platform target. +-- +-- FK actions per the WP6 lifecycle rule: both cross-aggregate FKs are +-- ON DELETE NO ACTION ON UPDATE RESTRICT; deletion is app-driven — 409 while +-- a referencing output is enabled, otherwise (disabled output) the app +-- disables dependents, revokes the ingest credential, then deletes the row. +-- `credentials` gains the tenant-composite key this migration needs +-- (tenant-composite FKs everywhere, per proposal §4.4/§4.5). +-- +-- Query catalog (owning WP: WP6 — packages/db does not exist in this repo, so +-- the catalog lives here and in apps/api/src/live/live-destination.repository.ts): +-- live.destinations.list keyset page ORDER BY label — served by +-- live_destinations_ws_label; read path, p95 <= 20 ms. +-- live.destinations.usage point lookup of referencing outputs by +-- destination (tenant) — served by stream_outputs_destination partial +-- index; read path, p95 <= 20 ms. +-- live.outputs.channel channel fan-out list — served by +-- stream_outputs_channel; read path, p95 <= 20 ms. +BEGIN; + +-- 001 created credentials with only a surrogate PK; 018 added the +-- tenant-composite key the cross-aggregate FK below requires. +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'credentials_id_workspace_key') THEN + ALTER TABLE credentials ADD CONSTRAINT credentials_id_workspace_key UNIQUE (id, workspace_id); + END IF; +END $$; + +CREATE TABLE live_stream_destinations ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + label text NOT NULL CHECK (btrim(label) <> '' AND length(label) <= 120), + -- Provider of the platform-side target (manifest id of a connector); the + -- app resolves behaviour through the connector registry, never by branching + -- on this value. + provider text NOT NULL CHECK (btrim(provider) <> '' AND length(provider) <= 64), + -- Ingest endpoint (rtmp/rtmps/srt). Validated app-side by the shared + -- media-control endpoint shape validator before insert; the CHECKs here are + -- coarse backstops only. + endpoint_url text NOT NULL + CHECK (endpoint_url ~ '^(rtmp|rtmps|srt)://' AND length(endpoint_url) <= 2048), + -- Ingest credential (stream key etc.) vaulted in `credentials`; NULL only + -- for providers with no key. Tenant-composite FK; deleting the credential + -- row before the destination is a FK error by design (lifecycle revokes in + -- order: destination first, then its credential). + credential_id uuid, + -- Derived, explicitly-stale platform state (runtime truth is Redis obs:*). + last_platform_state text NOT NULL DEFAULT 'unknown' + CHECK (last_platform_state IN ('unknown', 'created', 'waiting', 'live', 'ended')), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, id), + FOREIGN KEY (credential_id, workspace_id) REFERENCES credentials (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT +); + +CREATE TRIGGER trg_live_dest_ws_immutable BEFORE UPDATE ON live_stream_destinations + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +-- Same fail-closed tenant isolation as every tenant table (001/021 style): +-- no app.workspace_id => NULL => no rows. FORCE binds sp_app too. +ALTER TABLE live_stream_destinations ENABLE ROW LEVEL SECURITY; +ALTER TABLE live_stream_destinations FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON live_stream_destinations + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +-- Per-workspace listing ordered by label (live.destinations.list). +CREATE INDEX live_destinations_ws_label ON live_stream_destinations (workspace_id, label); + +-- Media-output fan-out of a virtual channel onto one live target. +CREATE TABLE stream_outputs ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + virtual_channel_id uuid NOT NULL, + label text NOT NULL CHECK (btrim(label) <> '' AND length(label) <= 120), + -- Output profile reference; resolution is app-side. + profile text NOT NULL DEFAULT '720p30' + CHECK (profile IN ('720p30', '1080p30', '1080p60')), + mode text NOT NULL DEFAULT 'passthrough' CHECK (mode IN ('passthrough', 'normalize')), + -- Enablement lives ONLY here (WP9): a disabled output stops pushing but the + -- destination row and its credential survive untouched. + enabled boolean NOT NULL DEFAULT true, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + -- Exactly one of (destination_id, credential_handle_id): destination- + -- referenced outputs resolve the ingest credential through the destination + -- at spawn; generic outputs carry their own opaque vault handle. The + -- NOT-VALID backstop below is validated app-side (021-style split). + destination_id uuid, + credential_handle_id text, + CONSTRAINT stream_outputs_target_exclusive CHECK ( + (destination_id IS NULL) <> (credential_handle_id IS NULL) + ), + CONSTRAINT stream_outputs_handle_shape CHECK ( + credential_handle_id IS NULL OR credential_handle_id ~ '^vh_[A-Za-z0-9_-]{16,128}$' + ), + UNIQUE (workspace_id, id), + UNIQUE (workspace_id, virtual_channel_id, id), + FOREIGN KEY (workspace_id, virtual_channel_id) REFERENCES virtual_channels (workspace_id, id) + ON DELETE CASCADE ON UPDATE RESTRICT, + FOREIGN KEY (destination_id, workspace_id) REFERENCES live_stream_destinations (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT +); + +CREATE TRIGGER trg_stream_outputs_ws_immutable BEFORE UPDATE ON stream_outputs + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +ALTER TABLE stream_outputs ENABLE ROW LEVEL SECURITY; +ALTER TABLE stream_outputs FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON stream_outputs + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +-- References by destination (live.destinations.usage, 409 on delete). +CREATE INDEX stream_outputs_destination ON stream_outputs (workspace_id, destination_id) + WHERE destination_id IS NOT NULL; +-- Channel fan-out listing (live.outputs.channel). +CREATE INDEX stream_outputs_channel ON stream_outputs (workspace_id, virtual_channel_id); + +GRANT SELECT, INSERT, UPDATE, DELETE ON live_stream_destinations TO sp_app; +GRANT SELECT, INSERT, UPDATE, DELETE ON stream_outputs TO sp_app; + +COMMIT; diff --git a/apps/api/migrations/023_virtual_channel_import.sql b/apps/api/migrations/023_virtual_channel_import.sql new file mode 100644 index 0000000..7febd1a --- /dev/null +++ b/apps/api/migrations/023_virtual_channel_import.sql @@ -0,0 +1,160 @@ +-- 023: WP7 bulk source import + live-settings guard. +-- +-- Three additive pieces, one migration: +-- +-- (1) `virtual_channel_sources` (013) is widened to admit *imported* sources. +-- 013's CHECK only modelled hand-entered HLS URLs and local playlists; a +-- PeerTube channel/live imported in bulk carries `remote_ref` + +-- `connection_id` and has an HLS URL only once a live is resolvable. The +-- dedupe key UNIQUE (workspace_id, virtual_channel_id, remote_ref) from +-- 013 is unchanged and stays the only dedupe authority: imports write +-- INSERT ... ON CONFLICT DO NOTHING RETURNING id. +-- +-- (2) `virtual_channel_imports`: one row per bulk import request. The HTTP +-- request only parses + validates + creates the job row; resolution and +-- inserts run in the background, and the row is the progress/report +-- document (processed/total + imported/duplicate/invalid/failed counts +-- and a bounded per-entry outcome list). A `running` row whose heartbeat +-- (`updated_at`) went stale is reaped to `abandoned` on read, so a +-- process crash never leaves a job spinning forever. +-- +-- (3) `virtual_channel_live_settings`: the channel's live output settings +-- (profile/resolution, latency, replay policy) behind an optimistic +-- concurrency `version`. Writers take the channel row FOR UPDATE (one +-- writer per channel), then CAS on `version`; a stale version or a +-- started live is a 409 with nothing written. The CHECKs mirror the +-- app-side validator (WP7 acceptance: permanentLive + saveReplay and an +-- out-of-range latencyMode are rejected by the DB on a direct insert +-- too), and a trigger keeps `version` monotone. +-- +-- House style throughout: tenant-composite keys (id, workspace_id), composite +-- FKs that pin every child to one workspace's channel, reject_workspace_change +-- / reject_owner_change immutability, FORCE RLS with the NULLIF tenant policy, +-- grants to sp_app (003 adds the default-privilege side for future tables). + +SET LOCAL lock_timeout = '5s'; + +-- (1) Imported sources: kind 'hls' now requires an HLS URL *or* a remote_ref +-- (a channel imported now, playback attached when its live starts). The +-- playlist arm is untouched: local, connectionless, remote_ref-less. +ALTER TABLE virtual_channel_sources DROP CONSTRAINT virtual_channel_sources_check; +ALTER TABLE virtual_channel_sources ADD CONSTRAINT virtual_channel_sources_check CHECK ( + ( + kind = 'hls' AND playlist_loop IS NULL + AND (hls_url IS NOT NULL OR remote_ref IS NOT NULL) + ) OR ( + kind = 'playlist' AND hls_url IS NULL AND remote_ref IS NULL + AND connection_id IS NULL AND playlist_loop IS NOT NULL + ) +); + +-- (2) Import jobs. +CREATE TABLE virtual_channel_imports ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL, + virtual_channel_id uuid NOT NULL, + -- Connection the import resolves through (default host for bare handles). + connection_id uuid, + status text NOT NULL DEFAULT 'running' + CHECK (status IN ('running', 'completed', 'failed', 'abandoned')), + -- total counts every input row (valid or not); processed advances per row. + total int NOT NULL DEFAULT 0 CHECK (total >= 0 AND total <= 5000), + processed int NOT NULL DEFAULT 0 CHECK (processed >= 0 AND processed <= total), + imported int NOT NULL DEFAULT 0 CHECK (imported >= 0), + duplicate int NOT NULL DEFAULT 0 CHECK (duplicate >= 0), + invalid int NOT NULL DEFAULT 0 CHECK (invalid >= 0), + failed int NOT NULL DEFAULT 0 CHECK (failed >= 0), + -- Bounded per-entry report (first IMPORT_REPORT_ENTRY_LIMIT rows): what the + -- user pasted, what happened, why. Counts are always authoritative. + entries jsonb NOT NULL DEFAULT '[]'::jsonb CHECK (jsonb_typeof(entries) = 'array'), + error text, + created_by uuid, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + completed_at timestamptz, + UNIQUE (id, workspace_id), + FOREIGN KEY (workspace_id, virtual_channel_id) REFERENCES virtual_channels (workspace_id, id) + ON DELETE CASCADE ON UPDATE RESTRICT, + FOREIGN KEY (connection_id, workspace_id) REFERENCES connections (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT DEFERRABLE INITIALLY DEFERRED +); + +CREATE TRIGGER trg_vci_ws_immutable BEFORE UPDATE ON virtual_channel_imports + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_vci_owner_immutable BEFORE UPDATE ON virtual_channel_imports + FOR EACH ROW EXECUTE FUNCTION reject_owner_change('virtual_channel_id'); + +ALTER TABLE virtual_channel_imports ENABLE ROW LEVEL SECURITY; +ALTER TABLE virtual_channel_imports FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON virtual_channel_imports + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +CREATE INDEX virtual_channel_imports_channel + ON virtual_channel_imports (workspace_id, virtual_channel_id, created_at DESC); + +-- (3) Channel live settings, version-fenced. +CREATE TABLE virtual_channel_live_settings ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL, + virtual_channel_id uuid NOT NULL, + -- Output shape: a named preset, or an explicit triple (custom). + output_profile text NOT NULL DEFAULT '720p30' + CHECK (output_profile IN ('720p30', '1080p30', '1080p60', 'custom')), + width int CHECK (width IS NULL OR (width >= 16 AND width <= 7680 AND width % 2 = 0)), + height int CHECK (height IS NULL OR (height >= 16 AND height <= 4320 AND height % 2 = 0)), + fps int CHECK (fps IS NULL OR (fps >= 1 AND fps <= 120)), + -- LiveVideoLatencyMode: 1=Default, 2=High latency, 3=Small latency. + latency_mode int NOT NULL DEFAULT 1 CHECK (latency_mode IN (1, 2, 3)), + permanent_live boolean NOT NULL DEFAULT false, + save_replay boolean NOT NULL DEFAULT false, + -- Replay privacy etc.; only meaningful while save_replay is on. + replay_settings jsonb CHECK (replay_settings IS NULL OR jsonb_typeof(replay_settings) = 'object'), + -- Latch: once the channel's live has started, settings are frozen (409). + live_started_at timestamptz, + -- Optimistic concurrency: every accepted write bumps version exactly once. + version bigint NOT NULL DEFAULT 0 CHECK (version >= 0), + updated_by uuid, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, virtual_channel_id), + UNIQUE (id, workspace_id), + FOREIGN KEY (workspace_id, virtual_channel_id) REFERENCES virtual_channels (workspace_id, id) + ON DELETE CASCADE ON UPDATE RESTRICT, + CHECK ( + (output_profile <> 'custom' AND width IS NULL AND height IS NULL AND fps IS NULL) + OR (output_profile = 'custom' AND width IS NOT NULL AND height IS NOT NULL AND fps IS NOT NULL) + ), + -- PeerTube rejects saveReplay + permanentLive (400); we refuse it earlier + -- and the DB refuses it for direct inserts. + CHECK (NOT (permanent_live AND save_replay)), + CHECK (save_replay OR replay_settings IS NULL) +); + +CREATE TRIGGER trg_vcls_ws_immutable BEFORE UPDATE ON virtual_channel_live_settings + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +CREATE TRIGGER trg_vcls_owner_immutable BEFORE UPDATE ON virtual_channel_live_settings + FOR EACH ROW EXECUTE FUNCTION reject_owner_change('virtual_channel_id'); + +-- version never regresses; a stale CAS writer cannot rewind the fence. +CREATE FUNCTION reject_live_settings_version_regression() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + IF NEW.version < OLD.version THEN + RAISE EXCEPTION 'live settings version cannot decrease' USING ERRCODE = '23514'; + END IF; + RETURN NEW; +END $$; + +CREATE TRIGGER trg_vcls_version_monotone BEFORE UPDATE ON virtual_channel_live_settings + FOR EACH ROW EXECUTE FUNCTION reject_live_settings_version_regression(); + +ALTER TABLE virtual_channel_live_settings ENABLE ROW LEVEL SECURITY; +ALTER TABLE virtual_channel_live_settings FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON virtual_channel_live_settings + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +GRANT SELECT, INSERT, UPDATE, DELETE ON virtual_channel_imports, virtual_channel_live_settings TO sp_app; diff --git a/apps/api/migrations/024_chatgov.sql b/apps/api/migrations/024_chatgov.sql new file mode 100644 index 0000000..ec42f55 --- /dev/null +++ b/apps/api/migrations/024_chatgov.sql @@ -0,0 +1,477 @@ +-- 024: chat governance spine — chatgov design §2, P0 slice (glm). +-- +-- Tables: polls, poll_votes, posting_rules, posting_policy_snapshots, +-- topic_bindings, chat_command_audit, chat_identities, gate_decisions. +-- poll_mirrors (card edit bookkeeping) is deliberately NOT here: it belongs to +-- the P5 mirror slice and lands in its own forward migration. +-- +-- Design notes (chatgov-design.md §0/§1/§2): +-- * Polls are OURS, not Telegram's (D1): ballots live in poll_votes, the +-- Telegram card is only a mirror. `polls.rule` is the compiled §1a object +-- snapshotted at open and never re-resolved mid-flight; `policy_version` +-- is its deterministic hash pinned at open and at every decision. +-- * One open gate per draft (partial unique index) stops poll spam gating +-- the same draft twice; a revote is a NEW poll with parent_poll_id. +-- * posting_rules are data with precedence topic > content_type > account > +-- workspace (D4). Only ONE live rule per (scope, account, topic, +-- content_type) slot: the partial NULLS NOT DISTINCT index ignores +-- superseded rows so a revision frees its slot without losing lineage. +-- Effective policies are compiled at SAVE time into +-- posting_policy_snapshots (whole-block replacement, no field merges). +-- * topic_bindings are OUR registry of (connection, chat, thread) — the Bot +-- API cannot list topics, so what we created here is authoritative (D3). +-- `message_thread_id IS NULL` means the channel root / whole group, so the +-- slot key needs NULLS NOT DISTINCT. +-- * chat_identities key on the immutable numeric provider user id — never a +-- username (§6). /register creates `pending`; console approval flips to +-- `approved`; nothing self-elevates. +-- * chat_command_audit records EVERY command attempt including rejects +-- (rate_limited, unauthorized, unknown_command). UNIQUE (workspace, +-- provider, update_id) is the replay killer for Telegram updates. +-- * gate_decisions: one row per destination per gate evaluation (E1/E2/E3, +-- D8 partial fan-out is a first-class outcome). `inputs` carries the +-- quorum state at close; reason codes are machine-readable (§4). +-- +-- Retention classes (WP2 convention; no registry table exists yet, so this +-- comment is the registration): poll_votes, chat_command_audit and +-- gate_decisions are append-hot and are monthly-partition candidates once a +-- partition manager lands (014/020 style leaves). polls, posting_rules, +-- posting_policy_snapshots, topic_bindings and chat_identities stay +-- unpartitioned — they are FK parents / small current-state tables. +-- +-- RLS / NULLIF / reject_workspace_change / sp_app grants follow 001/014/021 +-- conventions: no app.workspace_id => NULL => no rows; FORCE binds sp_app too. + +-- --------------------------------------------------------------------------- +-- topic_bindings +-- --------------------------------------------------------------------------- +CREATE TABLE topic_bindings ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + -- Telegram connection (one bot token per connection). + connection_id uuid NOT NULL, + -- Numeric -100… chat id preferred (usernames drift). Text, not bigint: + -- provider-neutral and stable across platforms. + chat_id text NOT NULL CHECK (chat_id ~ '^-?[0-9]{1,32}$'), + -- Telegram message_thread_id; NULL = channel root / whole group. + message_thread_id text CHECK (message_thread_id IS NULL OR message_thread_id ~ '^[0-9]{1,32}$'), + binding_type text NOT NULL CHECK (binding_type IN ('announce', 'control', 'audit')), + -- Telegram topic name limit is 128. + name text NOT NULL CHECK (char_length(btrim(name)) BETWEEN 1 AND 128), + -- One of the 7 fixed Telegram icon colors (7322096, 16766592, 13338331, + -- 9367192, 16749490, 16478047, 8914859). + icon_color integer CHECK (icon_color IS NULL OR icon_color IN + (7322096, 16766592, 13338331, 9367192, 16749490, 16478047, 8914859)), + state text NOT NULL DEFAULT 'active' CHECK (state IN ('active', 'closed', 'archived')), + -- Accounts (connections) that post into this topic. Array members cannot be + -- FK-enforced in PostgreSQL; the app layer validates they belong to the same + -- workspace (chatgov policy-service). + account_ids uuid[] NOT NULL DEFAULT '{}'::uuid[], + -- Control bindings: the virtual channel the commands act on. + linked_channel_id uuid, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, id), + UNIQUE NULLS NOT DISTINCT (workspace_id, connection_id, chat_id, message_thread_id), + FOREIGN KEY (connection_id, workspace_id) REFERENCES connections (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT, + FOREIGN KEY (linked_channel_id, workspace_id) REFERENCES virtual_channels (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT +); + +CREATE TRIGGER trg_topic_bindings_ws_immutable BEFORE UPDATE ON topic_bindings + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE topic_bindings ENABLE ROW LEVEL SECURITY; +ALTER TABLE topic_bindings FORCE ROW LEVEL SECURITY; +CREATE POLICY tenant_isolation ON topic_bindings + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +GRANT SELECT, INSERT, UPDATE, DELETE ON topic_bindings TO sp_app; + +-- One control binding per virtual channel (design §2): two control topics +-- commanding the same channel would race the director. +CREATE UNIQUE INDEX topic_bindings_one_control + ON topic_bindings (workspace_id, linked_channel_id) + WHERE binding_type = 'control' AND linked_channel_id IS NOT NULL; +-- Topic screen grid + gateway binding resolution (chat_id + thread -> row). +CREATE INDEX topic_bindings_ws_conn ON topic_bindings (workspace_id, connection_id, state); + + +-- --------------------------------------------------------------------------- +-- polls +-- --------------------------------------------------------------------------- +CREATE TABLE polls ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + -- Live-chat channel the poll is scoped to (nullable: console-only poll). + channel_id uuid, + -- Vote scoped to a subchat when set (topic binding), else the whole chat. + topic_binding_id uuid, + subject_kind text NOT NULL CHECK (subject_kind IN ('draft', 'chat_post', 'none')), + -- Set when subject_kind = 'draft'. Drafts are the only gating subject (D5). + draft_id uuid, + question text NOT NULL CHECK (char_length(question) BETWEEN 1 AND 300), + -- Revote lineage: a re-vote is a NEW poll pointing at the closed one. + parent_poll_id uuid, + -- Compiled §1a pass-criteria object, snapshotted at open (never re-resolved). + rule jsonb NOT NULL CHECK (jsonb_typeof(rule) = 'object'), + -- Deterministic hash of `rule` at open (pinned on every decision too). + policy_version text NOT NULL CHECK (btrim(policy_version) <> ''), + -- named | anonymous, extracted from `rule` for queryability. Anonymous polls + -- store NO voter linkage: poll_votes rows are written for named mode only + -- (§6 privacy; anonymous sentiment rides native Telegram polls in P7). + anonymity text NOT NULL DEFAULT 'named' CHECK (anonymity IN ('named', 'anonymous')), + state text NOT NULL DEFAULT 'open' + CHECK (state IN ('open', 'passed', 'failed', 'vetoed', 'quorum_unreachable', 'cancelled')), + opens_at timestamptz NOT NULL DEFAULT now(), + -- opens_at + rule.timebox.duration. + closes_at timestamptz NOT NULL, + closed_at timestamptz, + close_reason text CHECK (close_reason IN ('quorum_early', 'timebox', 'veto', 'cancelled')), + -- Exactly-once close claim: the state flip and its downstream trigger share + -- one transaction guarded by this column (P2 poll-service claims it). + outbox_state text NOT NULL DEFAULT 'none' CHECK (outbox_state IN ('none', 'claimed', 'done')), + created_by uuid NOT NULL REFERENCES users(id) ON DELETE NO ACTION ON UPDATE RESTRICT, + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, id), + CHECK ((state = 'open') = (closed_at IS NULL)), + CHECK (close_reason IS NULL OR state <> 'open'), + FOREIGN KEY (channel_id, workspace_id) REFERENCES virtual_channels (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT, + FOREIGN KEY (topic_binding_id, workspace_id) REFERENCES topic_bindings (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT +); + +CREATE TRIGGER trg_polls_ws_immutable BEFORE UPDATE ON polls + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE polls ENABLE ROW LEVEL SECURITY; +ALTER TABLE polls FORCE ROW LEVEL SECURITY; +CREATE POLICY tenant_isolation ON polls + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +GRANT SELECT, INSERT, UPDATE, DELETE ON polls TO sp_app; + +-- One open gate per draft (design §2): a second open poll on the same draft is +-- refused by the index, not by application discipline. +CREATE UNIQUE INDEX polls_open_one_per_draft ON polls (workspace_id, draft_id) + WHERE state = 'open' AND draft_id IS NOT NULL; +-- Close sweep + overlay listing ("closing soon" chips). +CREATE INDEX polls_ws_state ON polls (workspace_id, state, closes_at); +-- Revote lineage walks (blocked queue "open re-vote" action). +CREATE INDEX polls_ws_parent ON polls (workspace_id, parent_poll_id) WHERE parent_poll_id IS NOT NULL; + +-- Self-reference after the table exists (avoids a forward declaration). +ALTER TABLE polls ADD CONSTRAINT polls_parent_fk + FOREIGN KEY (parent_poll_id, workspace_id) REFERENCES polls (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT; + +-- --------------------------------------------------------------------------- +-- poll_votes +-- --------------------------------------------------------------------------- +CREATE TABLE poll_votes ( + poll_id uuid NOT NULL, + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + voter_kind text NOT NULL CHECK (voter_kind IN ('user', 'chat_identity', 'system')), + -- Resolved workspace user, when known (named mode). + voter_user_id uuid REFERENCES users(id) ON DELETE NO ACTION ON UPDATE RESTRICT, + -- provider:from.id (numeric for Telegram). Idempotency key half. + voter_remote_id text NOT NULL CHECK (btrim(voter_remote_id) <> ''), + voter_handle text NOT NULL CHECK (btrim(voter_handle) <> ''), + ballot text NOT NULL CHECK (ballot IN ('yes', 'no', 'veto', 'abstain')), + -- Eligibility AT CAST (informational; eligibility is re-read at close). + eligible boolean NOT NULL DEFAULT true, + -- Ineligible votes are stored, never counted (D9 / GitLab pattern). + counted boolean NOT NULL DEFAULT true, + -- INELIGIBLE_ACTOR | SELF_APPROVAL_DENIED | ... + ineligible_reason text, + cast_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (poll_id, voter_remote_id), + FOREIGN KEY (poll_id, workspace_id) REFERENCES polls (id, workspace_id) + ON DELETE CASCADE ON UPDATE RESTRICT +); + +CREATE TRIGGER trg_poll_votes_ws_immutable BEFORE UPDATE ON poll_votes + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE poll_votes ENABLE ROW LEVEL SECURITY; +ALTER TABLE poll_votes FORCE ROW LEVEL SECURITY; +CREATE POLICY tenant_isolation ON poll_votes + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +GRANT SELECT, INSERT, UPDATE, DELETE ON poll_votes TO sp_app; + +-- Tally reads (quorum evaluation on every ballot) hit counted votes only. +CREATE INDEX poll_votes_poll ON poll_votes (poll_id, counted); + +-- --------------------------------------------------------------------------- +-- posting_rules +-- --------------------------------------------------------------------------- +CREATE TABLE posting_rules ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + -- Precedence (D4): topic > content_type > account > workspace. + scope text NOT NULL CHECK (scope IN ('workspace', 'account', 'content_type', 'topic')), + -- Connection id (the policy anchor is the ACCOUNT, platform is an attribute). + -- NULL for workspace-scope rules and workspace-wide content_type rules. + account_id uuid, + -- Set when scope = 'topic'. + topic_binding_id uuid, + -- 'alert' | 'video' | ... when scope = 'content_type'. + content_type text, + -- §1a-shaped rule object minus the subject; validated app-side by + -- apps/api/src/chatgov/policy.ts before insert (021-style split: the CHECKs + -- here are only coarse backstops). + policy jsonb NOT NULL CHECK (jsonb_typeof(policy) = 'object'), + -- Compiled snapshot hash at save (deterministic; identical content shares + -- a version, so a re-save of the same policy is a no-op version-wise). + policy_version text NOT NULL CHECK (btrim(policy_version) <> ''), + -- Shadow mode: evaluate + log, never block/fire (POLICY_DRY_RUN). + dry_run boolean NOT NULL DEFAULT false, + created_by uuid NOT NULL REFERENCES users(id) ON DELETE NO ACTION ON UPDATE RESTRICT, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + -- Revision chain hook (P6 rules editor): a superseded row no longer holds + -- its scope slot (see the partial unique index below) but keeps lineage. + superseded_by uuid, + UNIQUE (workspace_id, id), + FOREIGN KEY (account_id, workspace_id) REFERENCES connections (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT, + FOREIGN KEY (topic_binding_id, workspace_id) REFERENCES topic_bindings (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT +); + +CREATE TRIGGER trg_posting_rules_ws_immutable BEFORE UPDATE ON posting_rules + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE posting_rules ENABLE ROW LEVEL SECURITY; +ALTER TABLE posting_rules FORCE ROW LEVEL SECURITY; +CREATE POLICY tenant_isolation ON posting_rules + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +GRANT SELECT, INSERT, UPDATE, DELETE ON posting_rules TO sp_app; + +-- ONE live rule per scope slot. NULLS NOT DISTINCT because "no account" / +-- "no topic" / "no content type" is itself a slot value (workspace default, +-- channel root); a plain UNIQUE would allow unbounded duplicates there. +-- Partial on superseded_by IS NULL so a revision frees the slot. +CREATE UNIQUE INDEX posting_rules_live_uq + ON posting_rules (workspace_id, scope, account_id, topic_binding_id, content_type) + NULLS NOT DISTINCT + WHERE superseded_by IS NULL; +-- Rules-editor listing and per-destination resolution scans. +CREATE INDEX posting_rules_ws_scope ON posting_rules (workspace_id, scope, account_id, topic_binding_id) + WHERE superseded_by IS NULL; + +ALTER TABLE posting_rules ADD CONSTRAINT posting_rules_superseded_fk + FOREIGN KEY (superseded_by, workspace_id) REFERENCES posting_rules (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT; + +-- --------------------------------------------------------------------------- +-- chat_identities +-- --------------------------------------------------------------------------- +CREATE TABLE chat_identities ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + provider text NOT NULL CHECK (btrim(provider) <> ''), + -- Immutable numeric provider user id (Telegram from.id) — NEVER a username + -- (§6: usernames are never trusted for authorization). + remote_user_id text NOT NULL CHECK (remote_user_id ~ '^[0-9]{1,20}$'), + -- Linked workspace user once /register is approved in the console. + user_id uuid REFERENCES users(id) ON DELETE NO ACTION ON UPDATE RESTRICT, + display_name text, + -- §1a admins-arm eligibility: membership role ∈ {Owner, Admin} OR this + -- override ≥ that role (evaluated at CLOSE time — TOCTOU guard). NULL = the + -- linked membership role governs. Written only by chatgov.manage holders. + role_override text CHECK (role_override IS NULL OR role_override IN ('Moderator', 'Editor', 'Publisher', 'Approver', 'Admin', 'Owner')), + -- /register creates pending; approval is a console action (chatgov.manage), + -- never self-service. + status text NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'approved', 'rejected', 'banned')), + approved_by uuid REFERENCES users(id) ON DELETE NO ACTION ON UPDATE RESTRICT, + first_seen_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, provider, remote_user_id), + -- Approval must name its approver. + CHECK (status <> 'approved' OR approved_by IS NOT NULL) +); + +CREATE TRIGGER trg_chat_identities_ws_immutable BEFORE UPDATE ON chat_identities + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE chat_identities ENABLE ROW LEVEL SECURITY; +ALTER TABLE chat_identities FORCE ROW LEVEL SECURITY; +CREATE POLICY tenant_isolation ON chat_identities + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +GRANT SELECT, INSERT, UPDATE, DELETE ON chat_identities TO sp_app; + +-- Gateway authorization path: numeric from.id -> identity -> membership role. +CREATE UNIQUE INDEX chat_identities_ws_user + ON chat_identities (workspace_id, provider, user_id) WHERE user_id IS NOT NULL AND status = 'approved'; + +-- --------------------------------------------------------------------------- +-- chat_command_audit +-- --------------------------------------------------------------------------- +CREATE TABLE chat_command_audit ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + -- Telegram update id (or provider event id): the replay killer. + update_id bigint NOT NULL CHECK (update_id > 0), + -- Topic binding the command arrived on (NULL: unbound chat — itself a + -- reject reason, NO_CHANNEL_BOUND). + binding_id uuid, + provider text NOT NULL CHECK (btrim(provider) <> ''), + remote_user_id text NOT NULL CHECK (btrim(remote_user_id) <> ''), + -- Resolved identity's workspace user, if any. + user_id uuid REFERENCES users(id) ON DELETE NO ACTION ON UPDATE RESTRICT, + command text NOT NULL CHECK (btrim(command) <> ''), + args jsonb NOT NULL DEFAULT '{}'::jsonb CHECK (jsonb_typeof(args) = 'object'), + -- RBAC permission the command required (NULL for /help, /register). + permission text, + -- Every attempt is audited, including rejects (§6). + decision text NOT NULL CHECK (decision IN ('accepted', 'rejected', 'unauthorized', 'unknown_command', 'rate_limited')), + -- Machine code: INELIGIBLE_ACTOR | LOCKED_CHANNEL | CONFIRM_EXPIRED | NO_CHANNEL_BOUND | ... + reason text, + -- Confirmation nonce when confirm-gated (P4). + nonce text, + -- Chat nonce reused as the director Idempotency-Key (P3/P4). + idempotency_key text, + -- Receipt fingerprint / poll id / draft id. + result_ref jsonb CHECK (result_ref IS NULL OR jsonb_typeof(result_ref) = 'object'), + received_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, provider, update_id), + FOREIGN KEY (binding_id, workspace_id) REFERENCES topic_bindings (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT +); + +CREATE TRIGGER trg_chat_command_audit_ws_immutable BEFORE UPDATE ON chat_command_audit + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE chat_command_audit ENABLE ROW LEVEL SECURITY; +ALTER TABLE chat_command_audit FORCE ROW LEVEL SECURITY; +CREATE POLICY tenant_isolation ON chat_command_audit + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +GRANT SELECT, INSERT, UPDATE, DELETE ON chat_command_audit TO sp_app; + +CREATE INDEX chat_command_audit_ws_time ON chat_command_audit (workspace_id, received_at DESC); + +-- --------------------------------------------------------------------------- +-- posting_policy_snapshots — effective policy resolved at SAVE time (D4). +-- Evaluation reads ONLY snapshots; rules are re-compiled on every rule change. +-- --------------------------------------------------------------------------- +CREATE TABLE posting_policy_snapshots ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + destination_kind text NOT NULL CHECK (destination_kind IN ('account', 'topic')), + -- The account (connection) the destination posts through; topic snapshots + -- carry their owning connection here too. + account_id uuid NOT NULL, + -- NULL = the account root (whole account, no topic override). + topic_binding_id uuid, + -- '' = the destination's default (no content-type override); content_type + -- rules compile one snapshot per type so E3 can re-check by (account, + -- [topic], contentType) without re-resolving precedence at dispatch time. + content_type text NOT NULL DEFAULT '' CHECK (content_type ~ '^[a-z0-9_.-]{0,40}$'), + -- Compiled §1a object. Immutable per policy_version: a rule change writes a + -- NEW version; decisions pin the version they evaluated. + effective jsonb NOT NULL CHECK (jsonb_typeof(effective) = 'object'), + policy_version text NOT NULL CHECK (btrim(policy_version) <> ''), + -- Audit: which scopes/rules contributed to this resolution. + resolved_from jsonb NOT NULL CHECK (jsonb_typeof(resolved_from) = 'object'), + updated_at timestamptz NOT NULL DEFAULT now(), + -- NULLS NOT DISTINCT: topic_binding_id NULL is its own slot (account root). + -- The design sketch's PK omitted content_type; the column is kept in the + -- slot key so a content_type rule cannot silently overwrite the account's + -- default snapshot (whole-block replacement, one row per resolved slot). + UNIQUE NULLS NOT DISTINCT (workspace_id, destination_kind, account_id, topic_binding_id, content_type), + FOREIGN KEY (account_id, workspace_id) REFERENCES connections (id, workspace_id) + ON DELETE CASCADE ON UPDATE RESTRICT, + FOREIGN KEY (topic_binding_id, workspace_id) REFERENCES topic_bindings (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT +); + +CREATE TRIGGER trg_posting_policy_snapshots_ws_immutable BEFORE UPDATE ON posting_policy_snapshots + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE posting_policy_snapshots ENABLE ROW LEVEL SECURITY; +ALTER TABLE posting_policy_snapshots FORCE ROW LEVEL SECURITY; +CREATE POLICY tenant_isolation ON posting_policy_snapshots + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +GRANT SELECT, INSERT, UPDATE, DELETE ON posting_policy_snapshots TO sp_app; + +-- E3 pre-dispatch re-check reads by (workspace, account[, topic][, contentType]). +CREATE INDEX posting_policy_snapshots_ws_acct + ON posting_policy_snapshots (workspace_id, account_id, updated_at DESC); + +-- --------------------------------------------------------------------------- +-- gate_decisions — one row per destination per gate evaluation (D8). +-- --------------------------------------------------------------------------- +CREATE TABLE gate_decisions ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + draft_id uuid NOT NULL, + -- {platform, accountId, topicBindingId?} — the destination this row is about. + destination jsonb NOT NULL CHECK (jsonb_typeof(destination) = 'object'), + gate text NOT NULL CHECK (gate IN ('E1', 'E2', 'E3')), + -- Which posting rule fired (NULL when the fail-closed default applied). + rule_id uuid, + policy_version text NOT NULL CHECK (btrim(policy_version) <> ''), + outcome text NOT NULL CHECK (outcome IN ('fired', 'blocked', 'pending')), + -- Approvals, poll tally, quorum state at close (arm-by-arm got/needed). + inputs jsonb NOT NULL CHECK (jsonb_typeof(inputs) = 'object'), + -- INSUFFICIENT_QUORUM | INELIGIBLE_ACTOR | SELF_APPROVAL_DENIED | VETOED | + -- TIMEBOX_EXPIRED_AUTO_REJECT | RESET_ON_CHANGE | PENDING_POLL | + -- OUTSIDE_POLICY | POLICY_DRY_RUN | OVERRIDDEN_BY_OWNER | ... + reason_codes text[] NOT NULL DEFAULT '{}'::text[], + -- 'system:gate-e3' | 'user:' | 'chat_poll:' + actor text NOT NULL CHECK (btrim(actor) <> ''), + -- E2 rows carry the poll whose outcome this decision persists. + poll_id uuid, + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, id), + FOREIGN KEY (draft_id, workspace_id) REFERENCES drafts (id, workspace_id) + ON DELETE CASCADE ON UPDATE RESTRICT, + FOREIGN KEY (rule_id, workspace_id) REFERENCES posting_rules (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT, + FOREIGN KEY (poll_id, workspace_id) REFERENCES polls (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT +); + +CREATE TRIGGER trg_gate_decisions_ws_immutable BEFORE UPDATE ON gate_decisions + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE gate_decisions ENABLE ROW LEVEL SECURITY; +ALTER TABLE gate_decisions FORCE ROW LEVEL SECURITY; +CREATE POLICY tenant_isolation ON gate_decisions + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +GRANT SELECT, INSERT, UPDATE, DELETE ON gate_decisions TO sp_app; + +-- Blocked queue + draft timeline (P1) read by (workspace, draft, gate). +CREATE INDEX gate_decisions_draft ON gate_decisions (workspace_id, draft_id, gate); +-- Poll outcome lookup from the blocked queue's "open re-vote" action. +CREATE INDEX gate_decisions_poll ON gate_decisions (workspace_id, poll_id) WHERE poll_id IS NOT NULL; + +-- --------------------------------------------------------------------------- +-- RBAC: chatgov permissions, seeded in the 002 pattern (ON CONFLICT DO NOTHING). +-- chatgov.manage = rules editor, topic registry, identity approval, policy +-- compile — Owner/Admin. chatgov.vote = open/cast governance votes — the +-- content + broadcast roles. One statement => one authz-epoch bump (018). +-- --------------------------------------------------------------------------- +INSERT INTO role_permissions (role, permission) +SELECT v.role, v.permission +FROM (VALUES + ('Owner', 'chatgov.manage'), + ('Owner', 'chatgov.vote'), + ('Admin', 'chatgov.manage'), + ('Admin', 'chatgov.vote'), + ('Publisher', 'chatgov.vote'), + ('Editor', 'chatgov.vote'), + ('Approver', 'chatgov.vote'), + ('Moderator', 'chatgov.vote'), + ('BroadcastOperator', 'chatgov.vote') +) AS v(role, permission) +ON CONFLICT DO NOTHING; diff --git a/apps/api/migrations/025_archive_search.sql b/apps/api/migrations/025_archive_search.sql new file mode 100644 index 0000000..ff1ae7b --- /dev/null +++ b/apps/api/migrations/025_archive_search.sql @@ -0,0 +1,55 @@ +-- 025: WP15 archive full-text search (firehose research recommendation: +-- Postgres FTS phase-1) over archived_feed_items. +-- +-- Search surface: +-- * tsvector over text_content + channel/attribution metadata, kept in sync +-- by trigger (the normalizer writes text_content on insert AND on edits, +-- so a plain trigger beats a generated column: 015's reject_workspace_change +-- trigger guards workspace_id, and edits must reindex text). +-- * GIN indexes: tsvector lookup + pg_trgm similarity for typo tolerance. +-- * The API ORs websearch_to_tsquery matching with pg_trgm similarity so a +-- misspelled term still matches via trigram distance (similarity >= 0.3). +-- * RLS is untouched: archived_feed_items keeps its tenant_isolation policy +-- from 015; search runs inside withTenant() so FORCE RLS binds honestly. +-- +-- CREATE EXTENSION requires privileges sp_app lacks; this migration runs as +-- the table owner only (015 grants are preserved below for completeness). +CREATE EXTENSION IF NOT EXISTS pg_trgm; + +ALTER TABLE archived_feed_items ADD COLUMN search_vector tsvector NOT NULL DEFAULT ''; + +-- String-valued attribution keys worth indexing as metadata (normalizer maps +-- Telegram author/origin fields here). Kept narrow on purpose. +CREATE FUNCTION archived_attribution_text(attribution jsonb) +RETURNS text LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$ + SELECT coalesce(string_agg(value, ' '), '') + FROM jsonb_each_text(attribution) + WHERE key IN ('channelUsername', 'channelTitle', 'author', 'authorSignature', + 'forwardFromChat', 'forwardFromUser', 'signature') +$$; + +CREATE FUNCTION archived_feed_items_search_vector() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + NEW.search_vector := + setweight(to_tsvector('simple', coalesce(NEW.channel_id, '')), 'A') + || setweight(to_tsvector('simple', archived_attribution_text(NEW.attribution)), 'B') + || to_tsvector('simple', coalesce(NEW.text_content, '')); + RETURN NEW; +END $$; + +CREATE TRIGGER trg_archived_feed_items_search + BEFORE INSERT OR UPDATE OF text_content, channel_id, attribution + ON archived_feed_items FOR EACH ROW + EXECUTE FUNCTION archived_feed_items_search_vector(); + +-- Exact/lexical lookups. 'simple' config on purpose: the corpus is a +-- multilingual firehose, no stemming dictionary is "right", so phase-1 is +-- exact term matching plus trigram fuzziness. +CREATE INDEX archived_feed_items_search ON archived_feed_items USING gin (search_vector); + +-- Fuzzy path (typo tolerance) + unindexed-corpus ILIKE support on text. +CREATE INDEX archived_feed_items_text_trgm ON archived_feed_items USING gin (text_content gin_trgm_ops); +CREATE INDEX archived_feed_items_channel_trgm ON archived_feed_items USING gin (channel_id gin_trgm_ops); + +GRANT SELECT ON archived_feed_items TO sp_app; diff --git a/apps/api/migrations/025_chatgov_p2.sql b/apps/api/migrations/025_chatgov_p2.sql new file mode 100644 index 0000000..27d2131 --- /dev/null +++ b/apps/api/migrations/025_chatgov_p2.sql @@ -0,0 +1,72 @@ +-- P2 competitive short-post variants. 022/023/024 exist. The runner keys +-- migrations by filename; 025_archive_search.sql belongs to another slice. +ALTER TABLE polls + ADD COLUMN kind text NOT NULL DEFAULT 'yes_no' CHECK (kind IN ('yes_no', 'variant_selection')), + ADD COLUMN winner_option_id uuid, + ADD COLUMN draft_revision bigint, + ADD COLUMN content_kind text NOT NULL DEFAULT 'short_post' CHECK (content_kind IN ('short_post', 'document')), + ADD COLUMN eligible_user_ids uuid[]; +-- document is a schema seam only; P2 accepts short_post exclusively. +ALTER TABLE polls DROP CONSTRAINT polls_state_check; +ALTER TABLE polls ADD CONSTRAINT polls_state_check CHECK + (state IN ('open', 'passed', 'failed', 'vetoed', 'quorum_unreachable', 'cancelled', 'stale')); +ALTER TABLE polls DROP CONSTRAINT polls_close_reason_check; +ALTER TABLE polls ADD CONSTRAINT polls_close_reason_check CHECK + (close_reason IN ('quorum_early', 'timebox', 'veto', 'cancelled', 'quorum_unreachable', 'reset_on_change')); + +CREATE TABLE poll_options ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + poll_id uuid NOT NULL, + seq integer NOT NULL CHECK (seq > 0), + -- Future document revision chains can anchor here. Short posts use the + -- poll's numeric draft_revision plus this immutable text snapshot. + draft_revision_id uuid, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 5000), + submitted_by uuid NOT NULL REFERENCES users(id) ON DELETE NO ACTION ON UPDATE RESTRICT, + submitted_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE (poll_id, seq), + UNIQUE (id, poll_id, workspace_id), + FOREIGN KEY (poll_id, workspace_id) REFERENCES polls(id, workspace_id) ON DELETE CASCADE ON UPDATE RESTRICT +); +CREATE TRIGGER trg_poll_options_ws_immutable BEFORE UPDATE ON poll_options + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); +-- Artifacts cannot be edited or individually removed, even by SQL callers. +-- Cascading workspace/poll retention remains possible. +CREATE FUNCTION chatgov_option_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF TG_OP = 'UPDATE' OR pg_trigger_depth() = 1 THEN + RAISE EXCEPTION 'poll options are immutable' USING ERRCODE = '55000'; + END IF; + RETURN OLD; +END $$; +CREATE TRIGGER trg_poll_options_immutable BEFORE UPDATE OR DELETE ON poll_options + FOR EACH ROW EXECUTE FUNCTION chatgov_option_immutable(); +ALTER TABLE poll_options ENABLE ROW LEVEL SECURITY; +ALTER TABLE poll_options FORCE ROW LEVEL SECURITY; +CREATE POLICY tenant_isolation ON poll_options + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); +GRANT SELECT, INSERT, UPDATE, DELETE ON poll_options TO sp_app; + +ALTER TABLE poll_votes ADD COLUMN option_id uuid; +ALTER TABLE poll_votes ADD CONSTRAINT poll_votes_option_fk + FOREIGN KEY (option_id, poll_id, workspace_id) REFERENCES poll_options(id, poll_id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT; +ALTER TABLE poll_votes ADD CONSTRAINT poll_votes_option_ballot_check CHECK (option_id IS NULL OR ballot = 'yes'); +ALTER TABLE polls ADD CONSTRAINT polls_winner_option_fk + FOREIGN KEY (winner_option_id, id, workspace_id) REFERENCES poll_options(id, poll_id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT; +ALTER TABLE polls ADD CONSTRAINT polls_winner_state_check CHECK + (winner_option_id IS NULL OR (kind = 'variant_selection' AND state = 'passed')); +ALTER TABLE poll_votes ADD COLUMN voter_identity_id uuid REFERENCES chat_identities(id) ON DELETE NO ACTION ON UPDATE RESTRICT; +ALTER TABLE polls ADD COLUMN final_evaluation jsonb; +-- Narrow expiry discovery follows the publishing due-schedule house pattern: +-- returns references only, never ballots/content. Mutations still use RLS. +CREATE FUNCTION chatgov_due_polls() RETURNS TABLE(workspace_id uuid, id uuid) +LANGUAGE sql STABLE SECURITY DEFINER SET search_path = public AS $$ + SELECT p.workspace_id, p.id FROM polls p WHERE p.state = 'open' AND p.closes_at <= now() + ORDER BY p.closes_at LIMIT 500 +$$; +REVOKE ALL ON FUNCTION chatgov_due_polls() FROM PUBLIC; +GRANT EXECUTE ON FUNCTION chatgov_due_polls() TO sp_app; diff --git a/apps/api/migrations/026_columns_p1.sql b/apps/api/migrations/026_columns_p1.sql new file mode 100644 index 0000000..8f44de9 --- /dev/null +++ b/apps/api/migrations/026_columns_p1.sql @@ -0,0 +1,230 @@ +-- 026: columns P1 — persistence + RBAC (WP3 §1.2/§3, §6 P1 row). +-- +-- (025 was taken by WP15 archive search; this file is the columns slice.) +-- +-- Three things land here, in 024's many-tables-one-file style: +-- +-- 1. The resolved-source / ingest / fan-out tables 021 deliberately left +-- for their own slice: column_sources, feed_items, feed_item_columns. +-- (provider_budget_ledger is P2b/P5 territory — no ledger caller exists +-- yet, so the table is not created here.) +-- 2. host_budgets — the shared per-host egress budget table owned by +-- @tactical-media-broadcast-hub/egress (its migrations/001_host_budgets.sql, applied +-- verbatim with credit; the egress package never migrates at runtime and +-- expects the application to apply and ledger it). The columns scheduler +-- is the first app-side caller of HostBudgets that does not go through +-- the whole egressFetch client, so the table must exist before P1's +-- poll loop draws from it. IF NOT EXISTS keeps this idempotent if a +-- later egress-integration migration also carries it. +-- 3. The `streams.columns.manage` RBAC seed (design §1.4/§0.5): write +-- permission on columns, granted to Owner/Admin/Editor by default. +-- Reads stay on `draft.view`. +-- +-- RLS / NULLIF / reject_workspace_change / sp_app grants follow the +-- 001/014/021/024 conventions: no app.workspace_id => NULL => no rows; FORCE +-- binds sp_app too. `spec`/`refresh_policy` stay validated app-side by +-- @tactical-media-broadcast-hub/core (021 note); the CHECKs here are coarse backstops. + +-- --------------------------------------------------------------------------- +-- column_sources — what the scheduler actually polls for a column (§1.2) +-- --------------------------------------------------------------------------- +CREATE TABLE column_sources ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + column_id uuid NOT NULL REFERENCES stream_columns(id) ON DELETE CASCADE ON UPDATE RESTRICT, + -- Identity of the thing polled (manifest id of a connector). + provider text NOT NULL CHECK (btrim(provider) <> '' AND length(provider) <= 64), + -- NULL for keyless sources (YouTube RSS, public Bluesky/Mastodon) — those + -- draw from the per-host budget only (§1.2 "Keyless sources"). + connection_id uuid, + source_kind text NOT NULL CHECK (source_kind IN ('home','mentions','search','author')), + -- Platform-adapted query (post-adaptation, what gets sent); NULL for + -- home/mentions/author sources. + query text, + -- Canonical account ref for source_kind='author': '{provider}:{handle@host}'. + actor_ref text, + state text NOT NULL DEFAULT 'pending' + CHECK (state IN ('pending','ready','stale','error','unsupported','rate_limited','budget_paused','disabled')), + last_error text, + last_polled_at timestamptz, + next_poll_at timestamptz, + consecutive_failures integer NOT NULL DEFAULT 0 CHECK (consecutive_failures >= 0), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, id), + FOREIGN KEY (connection_id, workspace_id) REFERENCES connections (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT, + -- One source row per (column, polled identity); NULLs equal NULLs so a + -- keyless search source dedupes against itself (024 NULLS NOT DISTINCT + -- style). The SAME identity in two different columns is intentional (§3.3: + -- following the same account in two columns costs one poll) — the due + -- source scan collapses those rows into one poll and fans items out to + -- every column sharing the identity. + UNIQUE NULLS NOT DISTINCT (workspace_id, column_id, provider, source_kind, connection_id, query, actor_ref) +); + +CREATE TRIGGER trg_column_sources_ws_immutable BEFORE UPDATE ON column_sources + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE column_sources ENABLE ROW LEVEL SECURITY; +ALTER TABLE column_sources FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON column_sources + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +GRANT SELECT, INSERT, UPDATE, DELETE ON column_sources TO sp_app; + +-- Due-source scan (§3.4): workspace + next_poll_at with the disabled rows +-- partial-indexed away. Design §1.2 names this column_sources_due_idx on +-- (next_poll_at); workspace leads here because every scan is tenant-scoped +-- under RLS and the composite keeps each workspace's due window contiguous. +CREATE INDEX column_sources_due_idx + ON column_sources (workspace_id, next_poll_at) + WHERE state <> 'disabled'; + +-- Column fan-out listing (ColumnView.sources) and per-column refresh. +CREATE INDEX column_sources_column_idx ON column_sources (column_id); + +-- Connection → sources (the degenerate platform_feed lookup in the scheduler +-- and the /feeds per-connection store fallback). +CREATE INDEX column_sources_connection_idx + ON column_sources (connection_id) + WHERE connection_id IS NOT NULL; + +-- --------------------------------------------------------------------------- +-- feed_items — ingested items (§1.2): persisted depth for keyset pagination +-- and cross-column dedup; Redis stays the hot read-through cache in front. +-- --------------------------------------------------------------------------- +CREATE TABLE feed_items ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + -- sha256(workspace_id | provider | canonical_remote_id) — see §3.3; the + -- canonicalization rule is app-side, the key is opaque to the DB. + dedup_key text NOT NULL CHECK (dedup_key <> ''), + provider text NOT NULL CHECK (btrim(provider) <> '' AND length(provider) <= 64), + remote_id text NOT NULL CHECK (remote_id <> ''), + -- Connector-reported FeedItem.id; first writer wins, kept stable so a + -- cursor minted from the hot cache still tie-breaks identically when the + -- same page is later served from the store. + item_id text NOT NULL CHECK (item_id <> ''), + author_handle text NOT NULL CHECK (author_handle <> ''), + -- Canonical '{provider}:{handle@host}' when resolvable (P0c feedAuthorRef). + author_ref text, + -- Provider permalink; absolute credential-free http(s) only (P0c parseSafeHttpUrl). + url text CHECK (url IS NULL OR url ~ '^https?://'), + -- Provider timestamp — the deterministic keyset sort key. + created_at_remote timestamptz NOT NULL, + fetched_at timestamptz NOT NULL DEFAULT now(), + -- safe-content typed rich-text AST. NEVER raw HTML (§5.1) — the app layer + -- produces this through normalizeFeedItem; the CHECK only backstops shape. + content jsonb NOT NULL CHECK (jsonb_typeof(content) = 'object'), + -- Derived flat text (astToPlainText) for saved-search filtering/previews. + plain_text text NOT NULL, + -- [{kind:'image'|'video', proxyUrl, ...}] — proxy-minted only (§5.3). + media jsonb NOT NULL DEFAULT '[]'::jsonb CHECK (jsonb_typeof(media) = 'array'), + -- Finite numbers only (validated at ingest). + metrics jsonb NOT NULL DEFAULT '{}'::jsonb CHECK (jsonb_typeof(metrics) = 'object'), + source_id uuid NOT NULL, + -- ToS expiry (YouTube Data-API items: fetched_at + 30d); sweeper is P2c. + expires_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, dedup_key), + UNIQUE (id, workspace_id), + FOREIGN KEY (source_id, workspace_id) REFERENCES column_sources (id, workspace_id) + ON DELETE NO ACTION ON UPDATE RESTRICT +); + +CREATE TRIGGER trg_feed_items_ws_immutable BEFORE UPDATE ON feed_items + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE feed_items ENABLE ROW LEVEL SECURITY; +ALTER TABLE feed_items FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON feed_items + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +GRANT SELECT, INSERT, UPDATE, DELETE ON feed_items TO sp_app; + +-- Keyset serving: GET /feeds/columns/:id/items pages (created_at_remote DESC, +-- dedup_key DESC) over this index (§1.4 "Cursor rule"). +CREATE INDEX feed_items_ws_time_idx + ON feed_items (workspace_id, created_at_remote DESC, dedup_key DESC); + +-- Store backfill for /feeds per connection and per-source item listing. +CREATE INDEX feed_items_source_idx ON feed_items (source_id, created_at_remote DESC); + +-- Expiry sweeper (P2c) pre-indexed here so the later slice adds no DDL. +CREATE INDEX feed_items_expiry_idx ON feed_items (expires_at) WHERE expires_at IS NOT NULL; + +-- --------------------------------------------------------------------------- +-- feed_item_columns — which columns show an item (§1.2 table 4). An item +-- ingested for a source identity is visible to every column sharing that +-- identity; rows are written by poll-time fan-out with ON CONFLICT DO NOTHING. +-- --------------------------------------------------------------------------- +-- stream_columns (021) has only its surrogate PK; migrations < 025 are frozen, +-- so the tenant-composite keys the FKs below need are added here — additive +-- unique indexes on supersets of existing keys, so no uniqueness changes. +CREATE UNIQUE INDEX IF NOT EXISTS stream_columns_id_workspace_key + ON stream_columns (id, workspace_id); + +CREATE TABLE feed_item_columns ( + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + item_id uuid NOT NULL, + column_id uuid NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (item_id, column_id), + FOREIGN KEY (item_id, workspace_id) REFERENCES feed_items (id, workspace_id) + ON DELETE CASCADE ON UPDATE RESTRICT, + FOREIGN KEY (column_id, workspace_id) REFERENCES stream_columns (id, workspace_id) + ON DELETE CASCADE ON UPDATE RESTRICT +); + +CREATE TRIGGER trg_feed_item_columns_ws_immutable BEFORE UPDATE ON feed_item_columns + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE feed_item_columns ENABLE ROW LEVEL SECURITY; +ALTER TABLE feed_item_columns FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON feed_item_columns + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +GRANT SELECT, INSERT, DELETE ON feed_item_columns TO sp_app; + +-- Serving index: the items endpoint joins column_id → item_id before the +-- keyset filter; the PK alone would force an item_id-ordered probe. +CREATE INDEX feed_item_columns_column_idx ON feed_item_columns (column_id, item_id); + +-- --------------------------------------------------------------------------- +-- host_budgets — applied verbatim from @tactical-media-broadcast-hub/egress +-- migrations/001_host_budgets.sql (WP7/WP8). Infrastructure state shared by +-- all egress callers; deliberately NOT tenant/RLS scoped (one IDNA lowercase +-- host:port key across callers and host classes — the whole point is that +-- monitor, feeds and any other caller of the same host share one bucket). +-- --------------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS host_budgets ( + host text PRIMARY KEY CHECK (host <> '' AND host = lower(host)), + capacity double precision NOT NULL CHECK (capacity >= 1 AND capacity < 'Infinity'::double precision), + -- Tokens per fixed 10-second interval (fractional tokens support continuous refill). + refill_per_interval double precision NOT NULL CHECK (refill_per_interval > 0 AND refill_per_interval < 'Infinity'::double precision), + tokens double precision NOT NULL CHECK (tokens >= 0 AND tokens <= capacity), + -- A future timestamp parks the bucket: no refill/debit until this deadline. + updated_at timestamptz NOT NULL DEFAULT clock_timestamp() CHECK (isfinite(updated_at)) +); + +COMMENT ON TABLE host_budgets IS 'Infrastructure egress budgets; one IDNA lowercase host:effective-port key across callers and host classes. Retain while host is in use.'; + +GRANT SELECT, INSERT, UPDATE ON host_budgets TO sp_app; + +-- --------------------------------------------------------------------------- +-- RBAC seed: streams.columns.manage (design §1.4 — granted to Owner/Admin/ +-- Editor by default; reads stay on draft.view). 002's unnest + ON CONFLICT +-- pattern so the seed is re-runnable and never fights a later role edit. +-- --------------------------------------------------------------------------- +INSERT INTO role_permissions (role, permission) +SELECT 'Owner', 'streams.columns.manage' +UNION ALL SELECT 'Admin', 'streams.columns.manage' +UNION ALL SELECT 'Editor', 'streams.columns.manage' +ON CONFLICT DO NOTHING; diff --git a/apps/api/migrations/027_intake.sql b/apps/api/migrations/027_intake.sql new file mode 100644 index 0000000..185b444 --- /dev/null +++ b/apps/api/migrations/027_intake.sql @@ -0,0 +1,186 @@ +-- 027: content intake firehose phase 1 (WP15+ firehose research §1/§2/§4/§5.3, +-- decision record docs/decisions/2026-10-09-intake-posture.md — Tier-1 +-- sanctioned sources only, metadata-first). +-- +-- NOTE on the number: this slice was specced as "026_intake.sql (next free)" +-- but 025_archive_search and 026_columns_p1 already exist, so intake lands as +-- 027 — the next free slot at authoring time. +-- +-- Two tables: +-- +-- intake_sources the per-workspace subscription registry (§4.4 sketch, +-- tenant-scoped per the phase-1 dedup note in §2.4: dedup +-- happens WITHIN a workspace; the global item registry + +-- claims split is the phase-1.5 upgrade). +-- intake_items the harvested rows. Dedup is CONTENT-ADDRESSED: +-- UNIQUE (workspace_id, content_hash) where content_hash is +-- sha256(normalized plain text + media origin identity), so +-- the same statement arriving via RSS and a Mastodon boost +-- (or the same URL watched twice) stores once. Storage-level +-- dedup stays in the S3 content-addressed layer. +-- +-- `content` is the P0c safe-content rich-text AST — the ONLY render source; +-- adapters never hand raw provider HTML through (apps/api/src/feeds/ +-- feed-normalize.ts funnels every item). `text` is the derived flat text +-- (astToPlainText) that FTS + pg_trgm index. +-- +-- FTS mirrors 025's design: 'simple' config (multilingual firehose, no +-- stemming dictionary is "right"), websearch_to_tsquery in the API ORed with +-- pg_trgm similarity for typo tolerance; tsvector maintained by trigger +-- because edits/telegram revisions must reindex. +-- +-- RLS / NULLIF / reject_workspace_change / sp_app grants follow the +-- 001/015/026 conventions: no app.workspace_id => NULL => no rows; FORCE +-- binds sp_app too. CREATE EXTENSION requires privileges sp_app lacks; this +-- migration runs as the table owner only (025 note). + +CREATE EXTENSION IF NOT EXISTS pg_trgm; + +-- --------------------------------------------------------------------------- +-- intake_sources +-- --------------------------------------------------------------------------- +CREATE TABLE intake_sources ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + -- Tier-1 sanctioned intake kinds (decision record). bluesky_keyword is a + -- stub seam in this slice — the Jetstream consumer is a later slice, so the + -- scheduler parks those rows in health='unsupported' rather than polling. + kind text NOT NULL CHECK (kind IN ('rss','mastodon_tag','mastodon_account','bluesky_keyword', + 'telegram_channel','peertube_channel','youtube_channel','web_watch')), + -- Per-kind identity: url (rss/web_watch), tag (mastodon_tag), instance + + -- accountId (mastodon_account), keywords[] (bluesky_keyword), channelId + -- (telegram_channel / youtube_channel), handle + instance (peertube_channel). + -- Validated app-side per kind; the DB only backstops "is an object". + params jsonb NOT NULL CHECK (jsonb_typeof(params) = 'object'), + -- sha256 of the canonically serialized params — the stable identity half of + -- UNIQUE (workspace_id, kind, params_hash). Canonicalization is app-side. + params_hash text NOT NULL CHECK (params_hash ~ '^[0-9a-f]{64}$'), + -- 0 = push-only (telegram webhook bridge, bluesky Jetstream): never due. + poll_interval_sec integer NOT NULL DEFAULT 300 CHECK (poll_interval_sec >= 0), + -- Adapter carry: etag / lastModified / lastContentHash / lastRemoteRef. + state jsonb NOT NULL DEFAULT '{}'::jsonb CHECK (jsonb_typeof(state) = 'object'), + health text NOT NULL DEFAULT 'active' + CHECK (health IN ('active','backoff','error','paused','unsupported')), + last_error text, + last_polled_at timestamptz, + next_poll_at timestamptz, + consecutive_failures integer NOT NULL DEFAULT 0 CHECK (consecutive_failures >= 0), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (workspace_id, id), + UNIQUE (workspace_id, kind, params_hash) +); + +CREATE TRIGGER trg_intake_sources_ws_immutable BEFORE UPDATE ON intake_sources + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE intake_sources ENABLE ROW LEVEL SECURITY; +ALTER TABLE intake_sources FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON intake_sources + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +GRANT SELECT, INSERT, UPDATE, DELETE ON intake_sources TO sp_app; + +-- Due-source scan (the scheduler's loop): workspace + next_poll_at with the +-- non-pollable rows partial-indexed away. +CREATE INDEX intake_sources_due_idx + ON intake_sources (workspace_id, next_poll_at) + WHERE health NOT IN ('paused','unsupported'); + +-- Listing / "which sources feed this item". +CREATE INDEX intake_sources_kind_idx ON intake_sources (workspace_id, kind); + +-- --------------------------------------------------------------------------- +-- intake_items +-- --------------------------------------------------------------------------- +CREATE TABLE intake_items ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT, + source_id uuid NOT NULL, + -- Platform-native id for the source (guid, status id, yt:video:…, tg chat:id, + -- url#hash for web_watch diffs). + remote_ref text NOT NULL CHECK (btrim(remote_ref) <> '' AND length(remote_ref) <= 2048), + -- sha256(NFKC + whitespace-collapsed AST plain text + sorted media origin + -- identity). THE dedup key: UNIQUE (workspace_id, content_hash). + content_hash text NOT NULL CHECK (content_hash ~ '^[0-9a-f]{64}$'), + published_at timestamptz NOT NULL, + -- Flat text derived from `content` at ingest (astToPlainText) — the FTS and + -- pg_trgm corpus. `text` is non-reserved; kept as the specced column name. + text text NOT NULL DEFAULT '', + -- P0c safe-content rich-text AST (doc/paragraph/inline). NEVER raw HTML. + content jsonb NOT NULL CHECK (jsonb_typeof(content) = 'object'), + -- [{kind, proxyUrl}…] for pull adapters (feed-normalize proxied media) or + -- [{kind, storageKey, …}] for the Telegram archive bridge (content-addressed + -- S3 keys already harvested by apps/api/src/archive). + media_refs jsonb NOT NULL DEFAULT '[]'::jsonb CHECK (jsonb_typeof(media_refs) = 'array'), + -- {authorHandle, authorRef, url, title, provider, …kind extras} + attribution jsonb NOT NULL DEFAULT '{}'::jsonb CHECK (jsonb_typeof(attribution) = 'object'), + fetched_at timestamptz NOT NULL DEFAULT now(), + created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (id, workspace_id), + -- Content-addressed dedup (spec §2.4 layer 2): first writer wins, every + -- later delivery of the same content no-ops. + UNIQUE (workspace_id, content_hash), + FOREIGN KEY (source_id, workspace_id) REFERENCES intake_sources (id, workspace_id) + ON DELETE CASCADE ON UPDATE RESTRICT +); + +-- Maintained by trigger below; declared here so the column exists for the +-- index + trigger DDL in one place. +ALTER TABLE intake_items ADD COLUMN search_vector tsvector NOT NULL DEFAULT ''; + +CREATE TRIGGER trg_intake_items_ws_immutable BEFORE UPDATE ON intake_items + FOR EACH ROW EXECUTE FUNCTION reject_workspace_change(); + +ALTER TABLE intake_items ENABLE ROW LEVEL SECURITY; +ALTER TABLE intake_items FORCE ROW LEVEL SECURITY; + +CREATE POLICY tenant_isolation ON intake_items + USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid) + WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid); + +GRANT SELECT, INSERT, UPDATE, DELETE ON intake_items TO sp_app; + +-- Per-source listing (GET /intake/items?sourceId=) keyset order. +CREATE INDEX intake_items_source_idx + ON intake_items (workspace_id, source_id, published_at DESC, id DESC); + +-- Whole-workspace newest-first (GET /intake/items without a source filter). +CREATE INDEX intake_items_time_idx + ON intake_items (workspace_id, published_at DESC, id DESC); + +-- --------------------------------------------------------------------------- +-- FTS (025 pattern) +-- --------------------------------------------------------------------------- +-- String-valued attribution keys worth indexing as metadata. +CREATE FUNCTION intake_attribution_text(attribution jsonb) +RETURNS text LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$ + SELECT coalesce(string_agg(value, ' '), '') + FROM jsonb_each_text(attribution) + WHERE key IN ('authorHandle','authorRef','title','channelTitle','channelUsername', + 'author','authorSignature','forwardFromChat','forwardFromUser','handle') +$$; + +CREATE FUNCTION intake_items_search_vector() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + NEW.search_vector := + setweight(to_tsvector('simple', coalesce(NEW.attribution->>'title', '')), 'A') + || setweight(to_tsvector('simple', intake_attribution_text(NEW.attribution)), 'B') + || to_tsvector('simple', coalesce(NEW.text, '')); + RETURN NEW; +END $$; + +CREATE TRIGGER trg_intake_items_search + BEFORE INSERT OR UPDATE OF text, attribution + ON intake_items FOR EACH ROW + EXECUTE FUNCTION intake_items_search_vector(); + +-- Exact/lexical path. fastupdate stays on explicitly: GIN insert cost is the +-- watch item at firehose write rates (research §2.7). +CREATE INDEX intake_items_fts ON intake_items USING gin (search_vector) WITH (fastupdate = on); + +-- Fuzzy path (typo tolerance) + ILIKE support on the text corpus. +CREATE INDEX intake_items_text_trgm ON intake_items USING gin (text gin_trgm_ops); diff --git a/apps/api/migrations/028_instance_settings.sql b/apps/api/migrations/028_instance_settings.sql new file mode 100644 index 0000000..635d40f --- /dev/null +++ b/apps/api/migrations/028_instance_settings.sql @@ -0,0 +1,79 @@ +-- 028: instance-level admin settings — content-policy posture for intake. +-- +-- The intake-posture decision record +-- (docs/decisions/2026-10-09-intake-posture.md) grades the intake tiers by +-- legal/ToS risk: +-- +-- YouTube channel RSS + metadata = safe (keyless, no quota, no ToS risk) +-- YouTube full harvest (yt-dlp) = ToS risk — per-source opt-in ONLY +-- (027's DEFAULT_MEDIA_POLICY already +-- parks youtube_channel at 'metadata'); +-- this switch is the instance-wide gate +-- the per-source opt-in is subordinate to +-- YouTube broadcast playlists = CC-only (Creative Commons licensed +-- content only) — safe to rebroadcast +-- Locked platforms (X/IG/TikTok) = metadata-only via sanctioned +-- connectors; a bounded read-only pool is +-- the decision record's Tier-2 "small +-- bounded pool, later" option — opt-in +-- +-- Those posture choices are INSTANCE-level (one deployment = one collective, +-- one egress IP, one legal exposure), not per-workspace: no workspace admin +-- may widen them for everyone. Hence this table is a system table: +-- +-- * no workspace_id — deliberately. It is not a tenant row. +-- * no RLS — there is nothing to isolate BY. Access control is at the API +-- seam (SettingsService runs as sp_app; only Owner/Admin roles reach the +-- setters through the controller's role check). NOT a grant-surface for +-- tenant sessions: the app pool is the only grantee. +-- +-- key-value shape (key TEXT PK, value TEXT NOT NULL) per task spec: settings +-- are few, admin-facing and read as a whole — typed decoding lives in +-- apps/api/src/settings/settings.service.ts (the ONE place that interprets +-- values; nothing else parses these strings), values are CHECK-constrained +-- to the exact literal the service layer understands so a stray UPDATE can +-- never park an undecodable row in the table. + +BEGIN; + +CREATE TABLE instance_settings ( + key text PRIMARY KEY, + value text NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now(), + updated_by uuid REFERENCES users(id) ON DELETE SET NULL, + -- Legal-exposure switches must never hold anything but the literals the + -- service layer decodes; unknown keys are equally rejected so a typo can't + -- fork the settings surface. + CONSTRAINT instance_settings_key_allowed CHECK (key IN ( + 'youtube_full_harvest', 'youtube_cc_only_broadcast', + 'scrape_metadata_only', 'scrape_bounded_pilot', 'intake_posture' + )), + CONSTRAINT instance_settings_bool_value CHECK ( + key NOT IN ('youtube_full_harvest', 'youtube_cc_only_broadcast', + 'scrape_metadata_only', 'scrape_bounded_pilot') + OR value IN ('true', 'false') + ), + CONSTRAINT instance_settings_posture_value CHECK ( + key <> 'intake_posture' + OR value IN ('metadata_only', 'community_submissions', 'bounded_pilot') + ) +); + +-- --------------------------------------------------------------------------- +-- Seed defaults (the safe posture — matches the 2026-10-09 decision record: +-- "metadata-only + community submissions + Tier-1 sanctioned sources"). +-- ON CONFLICT DO NOTHING keeps re-runs and later re-seeds idempotent. +-- --------------------------------------------------------------------------- +INSERT INTO instance_settings (key, value) VALUES + ('youtube_full_harvest', 'false'), -- yt-dlp OFF at the gate + ('youtube_cc_only_broadcast', 'true'), -- CC-only broadcast filter + ('scrape_metadata_only', 'true'), -- locked platforms: metadata + ('scrape_bounded_pilot', 'false'), -- no bounded X pool + ('intake_posture', 'metadata_only') -- overall posture +ON CONFLICT (key) DO NOTHING; + +-- System table: sp_app is the ONLY grantee (003's default privileges already +-- cover it, but the explicit grant documents intent the way 006/008/009 do). +GRANT SELECT, INSERT, UPDATE, DELETE ON instance_settings TO sp_app; + +COMMIT; diff --git a/apps/api/package.json b/apps/api/package.json new file mode 100644 index 0000000..2fe9eef --- /dev/null +++ b/apps/api/package.json @@ -0,0 +1,47 @@ +{ + "name": "@tactical-media-broadcast-hub/api", + "private": true, + "version": "0.1.0", + "scripts": { + "dev": "tsx watch src/main.ts", + "build": "tsc -p tsconfig.json", + "lint": "eslint src", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "vitest run", + "migrate": "tsx src/db/migrate.cli.ts" + }, + "dependencies": { + "@tactical-media-broadcast-hub/connector-sdk": "workspace:*", + "@tactical-media-broadcast-hub/connector-telegram": "workspace:*", + "@tactical-media-broadcast-hub/core": "workspace:*", + "@aws-sdk/client-s3": "^3.1146.0", + "@aws-sdk/s3-request-presigner": "^3.1146.0", + "@nestjs/common": "^10.4.1", + "@nestjs/core": "^10.4.1", + "@nestjs/platform-express": "^10.4.1", + "@opentelemetry/api": "^1.9.1", + "@opentelemetry/auto-instrumentations-node": "^0.80.0", + "@opentelemetry/exporter-trace-otlp-http": "^0.222.0", + "@opentelemetry/sdk-node": "^0.222.0", + "jose": "^6.2.12", + "pg": "^8.23.1", + "pino": "^9.3.2", + "reflect-metadata": "^0.2.2", + "rxjs": "^7.8.1", + "@tactical-media-broadcast-hub/publishing-execution": "workspace:*", + "@tactical-media-broadcast-hub/connector-runtime": "workspace:*", + "@tactical-media-broadcast-hub/connector-mastodon": "workspace:*", + "@temporalio/client": "1.24.0" + }, + "devDependencies": { + "@tactical-media-broadcast-hub/connector-mock-livestream": "workspace:^", + "@types/express": "^4.17.21", + "@types/node": "^22.5.0", + "@types/pg": "^8.23.1", + "tsx": "^4.17.0", + "vitest": "^2.0.5" + }, + "engines": { + "node": ">=22.13.0" + } +} diff --git a/apps/api/src/analytics/analytics.controller.ts b/apps/api/src/analytics/analytics.controller.ts new file mode 100644 index 0000000..337eb9e --- /dev/null +++ b/apps/api/src/analytics/analytics.controller.ts @@ -0,0 +1,62 @@ +import { BadRequestException, Body, Controller, Get, Post, Query, Req, UseGuards } from "@nestjs/common"; +import type { Request } from "express"; +import { parseLimit, requireSession } from "../http/validation.js"; +import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import { SnapshotService, SnapshotRecord, monthStart, parseMonth } from "./snapshot.service.js"; + +/** + * POST /analytics/snapshot is the manual trigger (analytics.view is held by + * Owner/Admin/Approver/Analyst — the closest thing to an admin/owner gate in + * the 002 permission set). The scheduler calls `writeMonthlySnapshot` + * (exported from snapshot.service.ts) directly later: same set-based SQL and + * idempotent upsert, no HTTP hop. + */ +@Controller("analytics") +@UseGuards(RbacGuard) +export class AnalyticsController { + constructor(private readonly snapshots: SnapshotService) {} + + @Post("snapshot") + @RequirePermission("analytics.view") + async snapshot( + @Req() request: Request & AuthenticatedRequest, + @Body() body: unknown, + ): Promise { + const input = typeof body === "object" && body !== null ? (body as Record) : {}; + const month = input.month === undefined || input.month === null || input.month === "" + ? monthStart() + : parseMonth(String(input.month)); + const session = requireSession(request); + return this.snapshots.refresh(session.workspaceId, month); + } + + @Get("snapshots") + @RequirePermission("analytics.view") + async list( + @Req() request: Request & AuthenticatedRequest, + @Query("cursor") cursor: string | undefined, + @Query("limit") limit: string | undefined, + ): Promise<{ items: SnapshotRecord[]; nextCursor: string | null }> { + if (cursor !== undefined && (cursor.length > 256 || !/^[A-Za-z0-9_-]+$/.test(cursor))) { + throw new BadRequestException("invalid cursor"); + } + const session = requireSession(request); + return this.snapshots.list(session.workspaceId, { + ...(cursor === undefined || cursor === "" ? {} : { cursor }), + limit: limit === undefined || limit === "" ? undefined : parseLimit(limit, 12, 24), + }); + } + + @Get("summary") + @RequirePermission("analytics.view") + async summary(@Req() request: Request & AuthenticatedRequest): Promise<{ + currentMonth: string; + previousMonth: string; + current: SnapshotRecord | null; + previous: SnapshotRecord | null; + deltas: ReturnType | null; + }> { + const session = requireSession(request); + return this.snapshots.summary(session.workspaceId); + } +} diff --git a/apps/api/src/analytics/analytics.integration.test.ts b/apps/api/src/analytics/analytics.integration.test.ts new file mode 100644 index 0000000..a6b4941 --- /dev/null +++ b/apps/api/src/analytics/analytics.integration.test.ts @@ -0,0 +1,358 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { join } from "node:path"; +import { Pool, type PoolClient } from "pg"; + +/** + * WP3 analytics snapshots against a live PostgreSQL, as sp_app (FORCE RLS + * binds). Uses its OWN database (ANALYTICS_IT_DATABASE_URL, plus + * ANALYTICS_IT_SUPER_DATABASE_URL for migrations/seeding) for the same reason + * as groups.integration.test.ts: tenancy.integration.test.ts wipes every + * workspace in the shared DB and vitest runs files in parallel. + * Skipped cleanly when either env var is unset. + */ +const appUrl = process.env.ANALYTICS_IT_DATABASE_URL; +const ownerUrl = process.env.ANALYTICS_IT_SUPER_DATABASE_URL; + +type Modules = { + service: typeof import("./snapshot.service.js"); + controller: typeof import("./analytics.controller.js"); + guard: typeof import("../rbac/rbac.guard.js"); + session: typeof import("../auth/session.js"); + db: typeof import("../db/client.js"); +}; + +describe.skipIf(appUrl === undefined || ownerUrl === undefined)("analytics snapshots (live DB)", () => { + let m: Modules; + let ownerPool: Pool; + const suffix = `${Date.now()}${Math.floor(Math.random() * 1e6)}`; + const currentMonth = new Date().toISOString().slice(0, 7) + "-01"; + const prevDate = new Date(`${currentMonth}T00:00:00Z`); + prevDate.setUTCMonth(prevDate.getUTCMonth() - 1); + const previousMonth = prevDate.toISOString().slice(0, 10); + let wsA: string; + let wsB: string; + let user: string; + + async function asOwner(fn: (client: PoolClient) => Promise): Promise { + const client = await ownerPool.connect(); + try { + return await fn(client); + } finally { + client.release(); + } + } + + /** Seed tenant rows honestly: FORCE RLS applies to the owner too. */ + async function inTenant(client: PoolClient, wsId: string, fn: () => Promise): Promise { + await client.query("BEGIN"); + await client.query("SELECT set_config('app.workspace_id', $1, true)", [wsId]); + await fn(); + await client.query("COMMIT"); + } + + beforeAll(async () => { + // The shared pool reads DATABASE_URL at import time: point it at this + // isolated DB first, then import the modules under test. + process.env.DATABASE_URL = appUrl; + process.env.SUPER_DATABASE_URL = ownerUrl; + const { runMigrations } = await import("../db/migrate.js"); + await runMigrations(join(process.cwd(), "migrations")); + m = { + service: await import("./snapshot.service.js"), + controller: await import("./analytics.controller.js"), + guard: await import("../rbac/rbac.guard.js"), + session: await import("../auth/session.js"), + db: await import("../db/client.js"), + }; + ownerPool = new Pool({ connectionString: ownerUrl }); + + await asOwner(async (c) => { + const ws = async (name: string): Promise => + (await c.query("INSERT INTO workspaces (name, slug) VALUES ($1, $2) RETURNING id", [name, `${name}-${suffix}`])).rows[0].id as string; + wsA = await ws("analytics-a"); + wsB = await ws("analytics-b"); + user = (await c.query( + "INSERT INTO users (external_id, username, email, display_name) VALUES ($1, $2, $3, $4) RETURNING id", + [`analytics:owner:${suffix}`, `analytics-owner-${suffix}`, `owner-${suffix}@example.com`, "owner"], + )).rows[0].id as string; + + const seed = async (wsId: string, archivedCount: number): Promise => { + await inTenant(c, wsId, async () => { + await c.query("INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1, $2, 'Owner')", [wsId, user]); + const tag = wsId.slice(0, 8); + // 3 connections (mastodon, bluesky, mastodon) + one draft each. + const drafts: string[] = []; + for (const [i, provider] of ["mastodon", "bluesky", "mastodon"].entries()) { + const conn = (await c.query( + "INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, $2, $3) RETURNING id", + [wsId, provider, `ch-${tag}-${i}`], + )).rows[0].id as string; + const draft = (await c.query( + "INSERT INTO drafts (workspace_id, author_user_id, post) VALUES ($1, $2, $3::jsonb) RETURNING id", + [wsId, user, JSON.stringify({ body: "x" })], + )).rows[0].id as string; + drafts.push(draft); + await c.query( + `INSERT INTO publications (workspace_id, draft_id, connection_id, idempotency_key, remote_id, published_at) + VALUES ($1, $2, $3, $4, $5, $6::date + $7::int)`, + [wsId, draft, conn, `pub-${tag}-${i}`, `r-${tag}-${i}`, currentMonth, i], + ); + } + // One scheduled publication (draft_id is UNIQUE there) carrying + // three attempts on the same draft: attempt_no 1..3. + const schedule = (await c.query( + `INSERT INTO scheduled_publications (workspace_id, draft_id, connection_id, publish_at, idempotency_key) + VALUES ($1, $2, (SELECT id FROM connections WHERE workspace_id = $1 LIMIT 1), $3::date, $4) RETURNING id`, + [wsId, drafts[0]!, currentMonth, `sch-${tag}`], + )).rows[0].id as string; + const credential = (await c.query( + "INSERT INTO credentials (workspace_id, kind, ciphertext) VALUES ($1, 'test', gen_random_bytes(16)) RETURNING id", + [wsId], + )).rows[0].id as string; + for (const n of [1, 2, 3]) { + await c.query( + `INSERT INTO publishing_attempts + (workspace_id, schedule_id, draft_id, connection_id, attempt_no, idempotency_key, schedule_generation, + post_snapshot, post_hash, scheduled_by_user_id, required_permission, approved_revision, + reviewer_user_id, credential_id, credential_generation, authorized_epoch, status) + VALUES ($1, $2, $3, (SELECT id FROM connections WHERE workspace_id = $1 LIMIT 1), $4, $5, 1, + '{}'::jsonb, $6, $7, 'publication.publish', 1, $7, $8, 1, 0, $9)`, + [wsId, schedule, drafts[0]!, n, `att-${tag}-${n}`, `hash-${tag}-${n}`, user, credential, n === 3 ? "completed" : "failed"], + ); + } + // Archived feed items: `archivedCount` in the current month, one in + // the previous month, one in the far past (outside both windows). + // remote_ref must equal channel_id || ':' || message_id. + // remote_ref must match (@handle|numeric):message_id AND equal + // channel_id || ':' || message_id; content_hash is sha256 hex (64). + for (let i = 0; i < archivedCount; i += 1) { + await c.query( + `INSERT INTO archived_feed_items (workspace_id, remote_ref, channel_id, message_id, content_hash, posted_at) + VALUES ($1, '@ch' || $2::text || ':' || $3::bigint::text, '@ch' || $2::text, $3::bigint, + encode(sha256($4::bytea), 'hex'), $5::date + ($6::int * interval '1 hour'))`, + [wsId, wsId.replace(/-/g, "").slice(0, 9), 1000 + i, `${tag}-${i}`, currentMonth, i], + ); + } + await c.query( + `INSERT INTO archived_feed_items (workspace_id, remote_ref, channel_id, message_id, content_hash, posted_at) + VALUES ($1, '@ch' || $2::text || ':9001', '@ch' || $2::text, 9001, + encode(sha256($3::bytea), 'hex'), $4::date)`, + [wsId, wsId.replace(/-/g, "").slice(0, 9), `${tag}-prev`, previousMonth], + ); + await c.query( + `INSERT INTO archived_feed_items (workspace_id, remote_ref, channel_id, message_id, content_hash, posted_at) + VALUES ($1, '@ch' || $2::text || ':9002', '@ch' || $2::text, 9002, + encode(sha256($3::bytea), 'hex'), '2020-01-01')`, + [wsId, wsId.replace(/-/g, "").slice(0, 9), `${tag}-old`], + ); + }); + }; + + await seed(wsA, 5); + await seed(wsB, 10); + }); + }); + + afterAll(async () => { + if (ownerPool !== undefined) { + await asOwner(async (c) => { + await c.query("DELETE FROM workspaces WHERE id = ANY($1::uuid[])", [[wsA, wsB]]); + await c.query("DELETE FROM users WHERE external_id LIKE $1", [`analytics:%:${suffix}`]); + }); + await ownerPool.end(); + } + await m?.db.pool.end(); + }); + + it("computes per-workspace monthly metrics from fixtures (set-based, no row loops)", async () => { + const service = new m.service.SnapshotService(); + const a = await service.refresh(wsA, currentMonth); + expect(a.month).toBe(currentMonth.slice(0, 7)); + expect(a.publicationsTotal).toBe(3); + expect(a.publicationsByProvider).toEqual({ mastodon: 2, bluesky: 1 }); + expect(a.postsArchivedTotal).toBe(5); + expect(a.activeConnections).toBe(3); + expect(a.draftsCreatedTotal).toBe(3); + expect(a.publishingAttemptsTotal).toBe(3); + expect(a.publishFailuresTotal).toBe(2); + + const b = await service.refresh(wsB, currentMonth); + expect(b.postsArchivedTotal).toBe(10); + expect(b.activeConnections).toBe(3); + // Disconnect wsB's connections as sp_app so RLS scopes the update to wsB + // (the owner pool is a superuser and bypasses RLS entirely). + await m.db.withTenant(wsB, (client) => client.query("UPDATE connections SET disconnected_at = now()")); + const bAfter = await service.refresh(wsB, currentMonth); + expect(bAfter.activeConnections).toBe(0); + }); + + it("is idempotent: running twice leaves exactly one row, same id and metrics", async () => { + const service = new m.service.SnapshotService(); + const first = await service.refresh(wsA, currentMonth); + const second = await service.refresh(wsA, currentMonth); + expect(second.id).toBe(first.id); + // Metrics are identical; only the snapshot timestamps move. + const stripTimes = (r: typeof first): Record => { + const { snapshotStartedAt, snapshotCompletedAt, ...metrics } = r; + void snapshotStartedAt; void snapshotCompletedAt; + return metrics; + }; + expect(stripTimes(second)).toEqual(stripTimes(first)); + expect(second.snapshotCompletedAt >= first.snapshotCompletedAt).toBe(true); + // Count through the app pool so FORCE RLS scopes the check to wsA (the + // owner pool is a superuser and would see every workspace's rows). + const rows = await m.db + .withTenant(wsA, (client) => client.query("SELECT id, publications_total FROM analytics_snapshots WHERE month = $1::date", [currentMonth])) + .then((r) => r.rows); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ id: first.id, publications_total: "3" }); + }); + + it("partition pruning: a single-month predicate plans a scan of exactly one leaf", async () => { + // Plan-time pruning needs a literal (a $n param defers pruning to run + // time); the value is a validated YYYY-MM-01 so inlining is safe. + if (!/^\d{4}-\d{2}-01$/.test(currentMonth)) throw new Error("bad month literal"); + const plan = await asOwner(async (c) => { + await c.query("BEGIN"); + await c.query("SELECT set_config('app.workspace_id', $1, true)", [wsA]); + const explained = await c.query<{ "QUERY PLAN": string }>( + `EXPLAIN (COSTS OFF) SELECT count(*) FROM analytics_snapshots WHERE month = DATE '${currentMonth}'`, + ); + await c.query("COMMIT"); + return explained.rows.map((r) => r["QUERY PLAN"]).join("\n"); + }); + const scanned = [...plan.matchAll(/Scan on (analytics_snapshots\S*)/g)].map((match) => match[1]!); + expect(scanned.filter((r) => /^analytics_snapshots_[0-9]{6}$/.test(r)).length).toBe(1); + for (const relation of scanned) expect(relation).not.toContain("default"); + }); + + it("cross-tenant denial: RLS hides other workspaces' snapshots (parent and every leaf)", async () => { + const service = new m.service.SnapshotService(); + await service.refresh(wsB, currentMonth); + const aList = await service.list(wsA, { limit: 24 }); + expect(aList.items.length).toBeGreaterThan(0); + expect(aList.items.every((item) => item.workspaceId === wsA)).toBe(true); + + // A raw read as the runtime role under B's context sees only B's rows. + const leaked = await m.db.withTenant(wsB, async (client) => + client.query("SELECT workspace_id FROM analytics_snapshots")); + expect(leaked.rows.length).toBeGreaterThan(0); + expect(new Set(leaked.rows.map((r) => r.workspace_id))).toEqual(new Set([wsB])); + + // Direct leaf access (parent RLS does not cascade to leaves): zero of + // A's rows visible from B's context in every partition. + const partitions = await asOwner((c) => + c.query<{ relname: string }>( + "SELECT c.relname FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'analytics_snapshots'::regclass", + )); + expect(partitions.rows.length).toBeGreaterThan(0); + for (const row of partitions.rows) { + if (!/^analytics_snapshots_(default|[0-9]{6})$/.test(row.relname)) throw new Error(`unexpected partition ${row.relname}`); + const seen = await m.db.withTenant(wsB, (client) => client.query(`SELECT workspace_id FROM ${row.relname}`)); + expect(seen.rows.every((r) => r.workspace_id === wsB)).toBe(true); + } + + // And a write attempt for another workspace is refused by WITH CHECK. + await expect( + m.db.withTenant(wsA, async (client) => + client.query( + `INSERT INTO analytics_snapshots (workspace_id, month) VALUES ($1::uuid, DATE '${currentMonth}')`, + [wsB], + )), + ).rejects.toThrow(/row-level security/i); + }); + + it("keyset pagination: newest first, cursor continues without overlap", async () => { + const service = new m.service.SnapshotService(); + await service.refresh(wsA, previousMonth); + const page1 = await service.list(wsA, { limit: 1 }); + expect(page1.items.map((i) => i.month)).toEqual([currentMonth.slice(0, 7)]); + expect(page1.nextCursor).not.toBeNull(); + const page2 = await service.list(wsA, { cursor: page1.nextCursor!, limit: 1 }); + expect(page2.items.map((i) => i.month)).toEqual([previousMonth.slice(0, 7)]); + expect(page2.nextCursor).toBeNull(); + await expect(service.list(wsA, { cursor: "!!!not-a-cursor" })).rejects.toMatchObject({ status: 400 }); + }); + + it("summary: current month + previous month with correct deltas", async () => { + const service = new m.service.SnapshotService(); + await service.refresh(wsB, previousMonth); + const summary = await service.summary(wsB); + expect(summary.currentMonth).toBe(currentMonth.slice(0, 7)); + expect(summary.previousMonth).toBe(previousMonth.slice(0, 7)); + expect(summary.previous).not.toBeNull(); + // previous month: only the one archived item; connections already + // disconnected when that snapshot was taken. + expect(summary.previous!.postsArchivedTotal).toBe(1); + expect(summary.previous!.publicationsTotal).toBe(0); + expect(summary.current!.postsArchivedTotal).toBe(10); + expect(summary.deltas).toMatchObject({ + publicationsTotal: 3, + postsArchivedTotal: 9, + activeConnections: 0, + publishFailuresTotal: 2, + }); + }); + + it("invalid month input is rejected before any query", async () => { + await expect(m.service.writeMonthlySnapshot({} as never, "2025-13")).rejects.toMatchObject({ status: 400 }); + await expect(new m.service.SnapshotService().refresh(wsA, "2025-1")).rejects.toMatchObject({ status: 400 }); + }); + + it("controller + RBAC guard chain: analytics.view required, wrong-workspace token is refused", async () => { + const controller = new m.controller.AnalyticsController(new m.service.SnapshotService()); + const user = (await asOwner((c) => c.query<{ id: string }>("SELECT id FROM users WHERE external_id = $1", [`analytics:owner:${suffix}`]))).rows[0]!.id; + + // Permission gate: a role without analytics.view is forbidden by the + // guard. The guard resolves the role from the MEMBERSHIP, not the token, + // so an Editor member of wsA is the honest deny-path fixture. + const editor = (await asOwner(async (c) => { + const id = (await c.query( + "INSERT INTO users (external_id, username, email, display_name) VALUES ($1, $2, $3, $4) RETURNING id", + [`analytics:editor:${suffix}`, `analytics-editor-${suffix}`, `editor-${suffix}@example.com`, "editor"], + )).rows[0].id as string; + await c.query("BEGIN"); + await c.query("SELECT set_config('app.workspace_id', $1, true)", [wsA]); + await c.query("INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1, $2, 'Editor')", [wsA, id]); + await c.query("COMMIT"); + return id; + })); + // The membership insert above bumps authz_epoch (continuous authz), so + // tokens are minted AFTER it — mirroring a fresh login per epoch. + const epochNow = await asOwner(async (c) => + Number((await c.query("SELECT authz_epoch FROM workspaces WHERE id = $1", [wsA])).rows[0]?.authz_epoch)); + const token = (userId: string, workspaceId: string, role: string): string => + m.session.issueSession({ userId, workspaceId, role, authzEpoch: epochNow }); + const request = (t: string): { headers: Record; session?: unknown } => ({ + headers: { authorization: `Bearer ${t}` }, + // In production RbacGuard.canActivate attaches this before the + // controller runs; do the same here after verifying the token. + session: m.session.verifySession(t) ?? undefined, + }); + + // Owner of wsA refreshes wsA's snapshot through the real controller. + const refreshed = await controller.snapshot(request(token(user, wsA, "Owner")) as never, {}); + expect(refreshed.workspaceId).toBe(wsA); + expect(refreshed.publicationsTotal).toBe(3); + + // A valid token for wsB does not leak wsA: wsB's refresh writes wsB's row. + const forB = await controller.snapshot(request(token(user, wsB, "Owner")) as never, {}); + expect(forB.workspaceId).toBe(wsB); + + const { Reflector } = await import("@nestjs/core"); + const guard = new m.guard.RbacGuard(new Reflector()); + const handler = (): void => undefined; + Reflect.defineMetadata(m.guard.PERMISSION_KEY, "analytics.view", handler); + const ctx = { + getHandler: () => handler, + switchToHttp: () => ({ getRequest: () => ({ headers: { authorization: `Bearer ${token(editor, wsA, "Editor")}` }, params: {} }) }), + } as unknown as import("@nestjs/common").ExecutionContext; + await expect(guard.canActivate(ctx)).rejects.toMatchObject({ status: 403 }); + // ...while the Owner's token passes the same gate. + const ownerCtx = { + getHandler: () => handler, + switchToHttp: () => ({ getRequest: () => ({ headers: { authorization: `Bearer ${token(user, wsA, "Owner")}` }, params: {} }) }), + } as unknown as import("@nestjs/common").ExecutionContext; + await expect(guard.canActivate(ownerCtx)).resolves.toBe(true); + }); +}); diff --git a/apps/api/src/analytics/snapshot.service.ts b/apps/api/src/analytics/snapshot.service.ts new file mode 100644 index 0000000..d63d58a --- /dev/null +++ b/apps/api/src/analytics/snapshot.service.ts @@ -0,0 +1,309 @@ +import { BadRequestException, Injectable, NotFoundException } from "@nestjs/common"; +import type { PoolClient } from "pg"; +import { withTenant } from "../db/client.js"; + +/** + * WP3 analytics snapshots: one rollup row per (workspace, month), recomputed + * from source tables with set-based SQL (no row loops) and upserted so a + * re-run is idempotent. Partitioned by month (020_analytics.sql). + * + * Exported as plain functions so the scheduler can call them later without + * going through Nest: + * import { writeMonthlySnapshot } from "./snapshot.service.js"; + * await withTenant(workspaceId, (client) => writeMonthlySnapshot(client, month)); + */ + +/** + * Month keys are accepted as `YYYY-MM` (API form) or `YYYY-MM-DD` where the + * day is 01 (the partition-key form used internally). Both denote the same + * month; anything else is a 400. + */ +export const MONTH_PATTERN = /^\d{4}-(0[1-9]|1[0-2])(-01)?$/; + +export function parseMonth(value: string, field = "month"): string { + if (!MONTH_PATTERN.test(value)) throw new BadRequestException(`${field} must be YYYY-MM`); + return value; +} + +/** First day (UTC) of the month containing `at`, as `YYYY-MM-DD`. */ +export function monthStart(at: Date = new Date()): string { + return at.toISOString().slice(0, 7) + "-01"; +} + +export interface SnapshotMetrics { + publicationsTotal: number; + publicationsByProvider: Record; + postsArchivedTotal: number; + activeConnections: number; + draftsCreatedTotal: number; + publishingAttemptsTotal: number; + publishFailuresTotal: number; +} + +export interface SnapshotRecord extends SnapshotMetrics { + id: string; + workspaceId: string; + month: string; + snapshotStartedAt: string; + snapshotCompletedAt: string; +} + +interface MetricsRow { + publications_total: string; + publications_by_provider: Record; + posts_archived_total: string; + active_connections: number; + drafts_created_total: string; + publishing_attempts_total: string; + publish_failures_total: string; +} + +interface SnapshotRow { + id: string; + workspace_id: string; + month: string | Date; // pg returns `date` as a UTC Date + publications_total: string; + publications_by_provider: Record; + posts_archived_total: string; + active_connections: number; + drafts_created_total: string; + publishing_attempts_total: string; + publish_failures_total: string; + snapshot_started_at: Date; + snapshot_completed_at: Date; +} + +/** + * Set-based metric computation for one (workspace, month), run inside a tenant + * transaction. Every source table is scanned once (the failure count is + * derived from the attempts pass, not a second scan). Must run inside RLS + * tenant context opened by the caller — no workspace_id literal is ever + * embedded in the SQL. + */ +export async function computeMonthlyMetrics(client: PoolClient, month: string): Promise { + const result = await client.query( + `SELECT + pub.publications_total, + COALESCE(pub.by_provider, '{}'::jsonb) AS publications_by_provider, + arch.posts_archived_total, + conn.active_connections, + dr.drafts_created_total, + att.publishing_attempts_total, + COALESCE(att.failed_total, 0) AS publish_failures_total + FROM (SELECT 1) AS anchor + LEFT JOIN LATERAL ( + -- one row per provider, so the total is SUM(n) — count(*) here would + -- count provider groups, not publications. + SELECT COALESCE(sum(c.n), 0)::bigint AS publications_total, + COALESCE(jsonb_object_agg(c.provider, c.n) FILTER (WHERE c.provider IS NOT NULL), '{}'::jsonb) AS by_provider + FROM ( + SELECT c.provider, count(*)::bigint AS n + FROM publications p + JOIN connections c ON c.id = p.connection_id + WHERE p.published_at >= $1::date + AND p.published_at < $1::date + interval '1 month' + GROUP BY c.provider + ) c + ) pub ON true + LEFT JOIN LATERAL ( + SELECT count(*)::bigint AS posts_archived_total + FROM archived_feed_items a + WHERE a.posted_at >= $1::date + AND a.posted_at < $1::date + interval '1 month' + ) arch ON true + LEFT JOIN LATERAL ( + SELECT count(*)::int AS active_connections + FROM connections c + WHERE c.disconnected_at IS NULL + ) conn ON true + LEFT JOIN LATERAL ( + SELECT count(*)::bigint AS drafts_created_total + FROM drafts d + WHERE d.created_at >= $1::date + AND d.created_at < $1::date + interval '1 month' + ) dr ON true + LEFT JOIN LATERAL ( + SELECT count(*)::bigint AS publishing_attempts_total, + count(*) FILTER (WHERE a.status = 'failed')::bigint AS failed_total + FROM publishing_attempts a + WHERE a.created_at >= $1::date + AND a.created_at < $1::date + interval '1 month' + ) att ON true`, + [month], + ); + return result.rows[0]!; +} + +/** + * Idempotent refresh of one workspace's snapshot for one month: INSERT on + * first run, UPDATE in place (same row id) on every re-run. Returns the + * persisted row. Runs under RLS, so a wrong workspace context simply yields + * no row (surfaced as 404 by the controller). + */ +export async function writeMonthlySnapshot(client: PoolClient, month: string): Promise { + // Normalize YYYY-MM / YYYY-MM-01 to the partition-key form. + const normalized = `${parseMonth(month).slice(0, 7)}-01`; + const metrics = await computeMonthlyMetrics(client, normalized); + const result = await client.query( + `INSERT INTO analytics_snapshots + (workspace_id, month, publications_total, publications_by_provider, + posts_archived_total, active_connections, drafts_created_total, + publishing_attempts_total, publish_failures_total, + snapshot_started_at, snapshot_completed_at) + VALUES (NULLIF(current_setting('app.workspace_id', true), '')::uuid, $1::date, + $2, $3::jsonb, $4, $5, $6, $7, $8, clock_timestamp(), clock_timestamp()) + ON CONFLICT (workspace_id, month) DO UPDATE SET + publications_total = EXCLUDED.publications_total, + publications_by_provider = EXCLUDED.publications_by_provider, + posts_archived_total = EXCLUDED.posts_archived_total, + active_connections = EXCLUDED.active_connections, + drafts_created_total = EXCLUDED.drafts_created_total, + publishing_attempts_total = EXCLUDED.publishing_attempts_total, + publish_failures_total = EXCLUDED.publish_failures_total, + snapshot_started_at = EXCLUDED.snapshot_started_at, + snapshot_completed_at = EXCLUDED.snapshot_completed_at + RETURNING *`, + [ + normalized, + metrics.publications_total, + JSON.stringify(metrics.publications_by_provider), + metrics.posts_archived_total, + metrics.active_connections, + metrics.drafts_created_total, + metrics.publishing_attempts_total, + metrics.publish_failures_total, + ], + ); + if (result.rows[0] === undefined) throw new NotFoundException("workspace not found for snapshot"); + return result.rows[0]; +} + +/** `pg` returns `date` columns as JS Dates (UTC midnight) — normalize to `YYYY-MM`. */ +function monthKey(value: string | Date): string { + return (typeof value === "string" ? value : value.toISOString().slice(0, 10)).slice(0, 7); +} + +function recordFromRow(row: SnapshotRow): SnapshotRecord { + return { + id: row.id, + workspaceId: row.workspace_id, + month: monthKey(row.month), + publicationsTotal: Number(row.publications_total), + publicationsByProvider: row.publications_by_provider, + postsArchivedTotal: Number(row.posts_archived_total), + activeConnections: row.active_connections, + draftsCreatedTotal: Number(row.drafts_created_total), + publishingAttemptsTotal: Number(row.publishing_attempts_total), + publishFailuresTotal: Number(row.publish_failures_total), + snapshotStartedAt: row.snapshot_started_at.toISOString(), + snapshotCompletedAt: row.snapshot_completed_at.toISOString(), + }; +} + +/** Month keyset cursor: the last emitted month, opaque base64url. */ +function encodeCursor(month: string): string { + return Buffer.from(JSON.stringify({ m: month }), "utf8").toString("base64url"); +} + +function decodeCursor(cursor: string): string { + try { + const value: unknown = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8")); + const month = (value as { m?: unknown } | null)?.m; + if (typeof month !== "string") throw new Error("shape"); + return `${parseMonth(month, "cursor").slice(0, 7)}-01`; + } catch { + throw new BadRequestException("invalid cursor"); + } +} + +@Injectable() +export class SnapshotService { + /** Refresh a month's snapshot for the caller's workspace (admin/owner gated at the controller). */ + async refresh(workspaceId: string, month: string): Promise { + return withTenant(workspaceId, async (client) => recordFromRow(await writeMonthlySnapshot(client, month))); + } + + /** Keyset page over months, newest first. Cursor = last emitted month. */ + async list( + workspaceId: string, + options: { cursor?: string; limit?: number } = {}, + ): Promise<{ items: SnapshotRecord[]; nextCursor: string | null }> { + const cursor = options.cursor === undefined ? null : decodeCursor(options.cursor); + const limit = Math.min(Math.max(options.limit ?? 12, 1), 24); + return withTenant(workspaceId, async (client) => { + const result = await client.query( + `SELECT * FROM analytics_snapshots + WHERE ($1::date IS NULL OR month < $1::date) + ORDER BY month DESC + LIMIT $2`, + [cursor, limit + 1], + ); + const rows = result.rows.slice(0, limit); + const last = rows.at(-1); + return { + items: rows.map(recordFromRow), + nextCursor: result.rows.length > limit && last !== undefined ? encodeCursor(monthKey(last.month)) : null, + }; + }); + } + + /** Current month vs previous month with deltas (missing months are null). */ + async summary(workspaceId: string): Promise<{ + currentMonth: string; + previousMonth: string; + current: SnapshotRecord | null; + previous: SnapshotRecord | null; + deltas: Record> | null; + }> { + const month = monthStart(); + const previous = previousMonthStart(month); + return withTenant(workspaceId, async (client) => { + const result = await client.query( + `SELECT * FROM analytics_snapshots + WHERE month IN ($1::date, ($1::date - interval '1 month')::date)`, + [month], + ); + const byMonth = new Map(result.rows.map((row) => [monthKey(row.month), recordFromRow(row)])); + const current = byMonth.get(month.slice(0, 7)) ?? null; + const previousRecord = byMonth.get(previous.slice(0, 7)) ?? null; + return { + currentMonth: month.slice(0, 7), + previousMonth: previous.slice(0, 7), + current, + previous: previousRecord, + deltas: current !== null && previousRecord !== null ? monthDeltas(previousRecord, current) : null, + }; + }); + } +} + +/** `YYYY-MM(-01)` of the month before `month`, same form as the input. */ +function previousMonthStart(month: string): string { + const date = new Date(`${month.slice(0, 7)}-01T00:00:00Z`); + date.setUTCMonth(date.getUTCMonth() - 1); + const key = date.toISOString().slice(0, 7); + return month.length === 7 ? key : `${key}-01`; +} + +/** Delta math for the summary endpoint (pure, unit-testable). */ +export function monthDeltas( + previous: SnapshotMetrics, + current: SnapshotMetrics, +): Record> { + const prevProviders = previous.publicationsByProvider; + const curProviders = current.publicationsByProvider; + const providers = new Set([...Object.keys(prevProviders), ...Object.keys(curProviders)]); + const providerDeltas: Record = {}; + for (const provider of providers) { + providerDeltas[provider] = (curProviders[provider] ?? 0) - (prevProviders[provider] ?? 0); + } + return { + publicationsTotal: current.publicationsTotal - previous.publicationsTotal, + publicationsByProvider: providerDeltas, + postsArchivedTotal: current.postsArchivedTotal - previous.postsArchivedTotal, + activeConnections: current.activeConnections - previous.activeConnections, + draftsCreatedTotal: current.draftsCreatedTotal - previous.draftsCreatedTotal, + publishingAttemptsTotal: current.publishingAttemptsTotal - previous.publishingAttemptsTotal, + publishFailuresTotal: current.publishFailuresTotal - previous.publishFailuresTotal, + }; +} diff --git a/apps/api/src/analytics/snapshot.test.ts b/apps/api/src/analytics/snapshot.test.ts new file mode 100644 index 0000000..2ab5d7d --- /dev/null +++ b/apps/api/src/analytics/snapshot.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "vitest"; +import { MONTH_PATTERN, monthDeltas, monthStart, parseMonth } from "./snapshot.service.js"; + +describe("analytics snapshot pure helpers", () => { + it("monthStart returns the UTC month start", () => { + expect(monthStart(new Date("2025-07-14T23:59:59.999Z"))).toBe("2025-07-01"); + expect(monthStart(new Date("2024-02-29T00:00:00Z"))).toBe("2024-02-01"); + }); + + it("parseMonth accepts YYYY-MM and month-start YYYY-MM-01, rejects everything else", () => { + expect(parseMonth("2025-01")).toBe("2025-01"); + expect(parseMonth("2025-01-01")).toBe("2025-01-01"); + expect(() => parseMonth("2025-13")).toThrow(); + expect(() => parseMonth("2025-1")).toThrow(); + expect(() => parseMonth("25-01")).toThrow(); + expect(() => parseMonth("2025-01-02")).toThrow(); + expect(MONTH_PATTERN.test("2025-12")).toBe(true); + expect(MONTH_PATTERN.test("2025-00")).toBe(false); + }); + + it("monthDeltas computes per-metric and per-provider deltas vs fixtures", () => { + const previous = { + publicationsTotal: 10, + publicationsByProvider: { mastodon: 4, bluesky: 6 }, + postsArchivedTotal: 100, + activeConnections: 3, + draftsCreatedTotal: 20, + publishingAttemptsTotal: 12, + publishFailuresTotal: 2, + }; + const current = { + publicationsTotal: 13, + publicationsByProvider: { mastodon: 4, bluesky: 7, reddit: 2 }, + postsArchivedTotal: 90, + activeConnections: 4, + draftsCreatedTotal: 20, + publishingAttemptsTotal: 15, + publishFailuresTotal: 1, + }; + expect(monthDeltas(previous, current)).toEqual({ + publicationsTotal: 3, + publicationsByProvider: { mastodon: 0, bluesky: 1, reddit: 2 }, + postsArchivedTotal: -10, + activeConnections: 1, + draftsCreatedTotal: 0, + publishingAttemptsTotal: 3, + publishFailuresTotal: -1, + }); + }); + + it("monthDeltas covers providers that vanished this month", () => { + const deltas = monthDeltas( + { publicationsTotal: 5, publicationsByProvider: { mastodon: 5 }, postsArchivedTotal: 0, activeConnections: 0, draftsCreatedTotal: 0, publishingAttemptsTotal: 0, publishFailuresTotal: 0 }, + { publicationsTotal: 1, publicationsByProvider: { bluesky: 1 }, postsArchivedTotal: 0, activeConnections: 0, draftsCreatedTotal: 0, publishingAttemptsTotal: 0, publishFailuresTotal: 0 }, + ); + expect(deltas.publicationsByProvider).toEqual({ mastodon: -5, bluesky: 1 }); + expect(deltas.publicationsTotal).toBe(-4); + }); +}); diff --git a/apps/api/src/app.module.ts b/apps/api/src/app.module.ts new file mode 100644 index 0000000..fe8a937 --- /dev/null +++ b/apps/api/src/app.module.ts @@ -0,0 +1,61 @@ +import { Module } from "@nestjs/common"; +import { HealthModule } from "./health/health.module"; +import { DemoController } from "./demo/demo.controller"; +import { PublishingController } from "./publishing/publishing.controller"; +import { pool, withTenant } from "./db/client"; +import { PublishingExecutionRepository, assertPublishingSchema } from "@tactical-media-broadcast-hub/publishing-execution"; +import { ApprovalGateService } from "./publishing/approval-gate.service.js"; +import { PUBLISHING_DEPENDENCIES, TemporalWorkflowStarter, PublishingService } from "./publishing/publishing.service"; +import { AuthController } from "./auth/auth.controller"; +import { AuthService } from "./auth/auth.service"; +import { ConnectionsController } from "./connections/connections.controller"; +import { ConnectionsService } from "./connections/connections.service"; +import { TeamController } from "./team/team.controller"; +import { TeamService } from "./team/team.service"; +import { InviteLinkController, InviteClaimController } from "./team/invite-link.controller"; +import { InviteLinkService } from "./team/invite-link.service"; +import { AuditController } from "./audit/audit.controller"; +import { AuditQueryService } from "./audit/audit.query"; +import { GroupsController } from "./groups/groups.controller"; +import { GroupsService } from "./groups/groups.service"; +import { VideoController } from "./video/video.controller"; +import { VideoService } from "./video/video.service"; +import { LiveMonitorModule } from "./live-monitor/live-monitor.module"; +import { MediaControlModule } from "./media-control/media-control.module"; +import { ProductionModule } from "./production/production.module"; +import { RealtimeModule } from "./realtime/realtime.module"; +import { LiveChatModule } from "./live-chat/live-chat.module"; +import { LiveModule } from "./live/live.module.js"; +import { ChatgovModule } from "./chatgov/chatgov.module.js"; +import { IntakeModule } from "./intake/intake.module.js"; +import { SettingsModule } from "./settings/settings.module.js"; +import { SettingsController } from "./settings/settings.controller.js"; +import { SettingsService } from "./settings/settings.service.js"; +import { VirtualChannelsModule } from "./virtual-channels/virtual-channels.module.js"; +import { DirectorModule } from "./director/director.module.js"; +import { AnalyticsController } from "./analytics/analytics.controller"; +import { SnapshotService } from "./analytics/snapshot.service"; + +@Module({ + imports: [HealthModule, LiveMonitorModule, MediaControlModule, ProductionModule, RealtimeModule, LiveChatModule, LiveModule, VirtualChannelsModule, ChatgovModule, IntakeModule, SettingsModule], + controllers: [ + DemoController, + PublishingController, + AuthController, + ConnectionsController, + TeamController, + InviteLinkController, + InviteClaimController, + AuditController, + VideoController, + GroupsController, + AnalyticsController, + SettingsController, + ], + providers: [ + { provide: PublishingExecutionRepository, useFactory: () => new PublishingExecutionRepository(pool) }, + TemporalWorkflowStarter, + { provide: PUBLISHING_DEPENDENCIES, inject: [PublishingExecutionRepository, TemporalWorkflowStarter, ApprovalGateService], useFactory: (repository: PublishingExecutionRepository, workflowStarter: TemporalWorkflowStarter, approvalGate: ApprovalGateService) => ({ repository, workflowStarter, approvalGate, tenantTransaction: withTenant, assertReady: () => assertPublishingSchema(pool) }) }, + PublishingService, ApprovalGateService, AuthService, ConnectionsService, TeamService, InviteLinkService, AuditQueryService, VideoService, GroupsService, SnapshotService, SettingsService], +}) +export class AppModule {} diff --git a/apps/api/src/archive/archive-search.integration.test.ts b/apps/api/src/archive/archive-search.integration.test.ts new file mode 100644 index 0000000..2971900 --- /dev/null +++ b/apps/api/src/archive/archive-search.integration.test.ts @@ -0,0 +1,256 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { join } from "node:path"; +import { Pool, type PoolClient } from "pg"; + +/** + * WP15 archive search + export against a live PostgreSQL, as sp_app (FORCE + * RLS binds). Uses its OWN database (ARCHIVE_SEARCH_IT_DATABASE_URL, plus + * ARCHIVE_SEARCH_IT_SUPER_DATABASE_URL for migrations/seeding) per the + * local-db-sol pattern: tenancy.integration.test.ts wipes every workspace in + * the shared DB and vitest runs files in parallel. Skipped when unset. + */ +const appUrl = process.env.ARCHIVE_SEARCH_IT_DATABASE_URL; +const ownerUrl = process.env.ARCHIVE_SEARCH_IT_SUPER_DATABASE_URL; + +type Modules = { + service: typeof import("./archive.service.js"); + "search-helpers": typeof import("./archive-search.js"); + db: typeof import("../db/client.js"); +}; + +describe.skipIf(appUrl === undefined || ownerUrl === undefined)("archive search + export (live DB)", () => { + let m: Modules; + let ownerPool: Pool; + const suffix = `${Date.now()}${Math.floor(Math.random() * 1e6)}`; + let wsA: string; + let wsB: string; + let chA = ""; + const chB = `@chB${suffix}`; + + async function asOwner(fn: (client: PoolClient) => Promise): Promise { + const client = await ownerPool.connect(); + try { + return await fn(client); + } finally { + client.release(); + } + } + + /** Seed tenant rows honestly: FORCE RLS applies to the owner too. */ + async function inTenant(client: PoolClient, wsId: string, fn: () => Promise): Promise { + await client.query("BEGIN"); + await client.query("SELECT set_config('app.workspace_id', $1, true)", [wsId]); + await fn(); + await client.query("COMMIT"); + } + + let seedSeq = 0; + /** remote_ref must equal channel_id || ':' || message_id; content_hash is sha256 hex. */ + async function seedItem(client: PoolClient, wsId: string, item: { + channel: string; message: number; text: string; postedAt: Date; attribution?: Record; + }): Promise { + seedSeq += 1; + await client.query( + `INSERT INTO archived_feed_items + (workspace_id, remote_ref, channel_id, message_id, text_content, content_hash, posted_at, attribution) + VALUES ($1, $2 || ':' || $3::bigint::text, $2, $3::bigint, $4, encode(sha256($5::bytea), 'hex'), $6, $7::jsonb)`, + [wsId, item.channel, item.message, item.text, `seed-${suffix}-${seedSeq}`, item.postedAt, + JSON.stringify(item.attribution ?? {})], + ); + } + + beforeAll(async () => { + process.env.DATABASE_URL = appUrl; + process.env.SUPER_DATABASE_URL = ownerUrl; + const { runMigrations } = await import("../db/migrate.js"); + await runMigrations(join(process.cwd(), "migrations")); + m = { + service: await import("./archive.service.js"), + "search-helpers": await import("./archive-search.js"), + db: await import("../db/client.js"), + }; + ownerPool = new Pool({ connectionString: ownerUrl }); + + await asOwner(async (c) => { + const ws = async (name: string): Promise => + (await c.query("INSERT INTO workspaces (name, slug) VALUES ($1, $2) RETURNING id", + [name, `${name}-${suffix}`])).rows[0].id as string; + wsA = await ws("search-a"); + wsB = await ws("search-b"); + }); + + // Tenant data: distinct texts per tenant, overlap only inside wsA. + chA = `@chA${suffix}`; + await asOwner((c) => inTenant(c, wsA, async () => { + await seedItem(c, wsA, { + channel: chA, message: 1001, + text: "Water rights are a community responsibility, not a commodity for markets.", + postedAt: new Date("2026-01-10T09:00:00Z"), + attribution: { channelUsername: chA.slice(1), author: "Water Working Group" }, + }); + await seedItem(c, wsA, { + channel: chA, message: 1002, + text: "Totally unrelated post about bicycle repair cooperatives and tools.", + postedAt: new Date("2026-01-11T09:00:00Z"), + }); + await seedItem(c, wsA, { + channel: chA, message: 1003, + text: "Water rights update: the hearing is postponed to February.", + postedAt: new Date("2026-01-12T09:00:00Z"), + }); + })); + await asOwner((c) => inTenant(c, wsB, async () => { + await seedItem(c, wsB, { + channel: chB, message: 2001, + text: "Water rights are a community responsibility, not a commodity for markets.", + postedAt: new Date("2026-01-05T09:00:00Z"), + }); + })); + }); + + afterAll(async () => { + if (ownerPool !== undefined) { + await asOwner(async (c) => { + await c.query("DELETE FROM workspaces WHERE id = ANY($1::uuid[])", [[wsA, wsB]]); + }); + await ownerPool.end(); + } + await m?.db.pool.end(); + }); + + it("full-text search matches archived text (websearch_to_tsquery)", async () => { + const service = new m.service.ArchiveService(null as never); + const result = await service.search(wsA, { q: "water rights" }); + expect(result.items).toHaveLength(2); + for (const item of result.items) { + expect(String(item.text)).toMatch(/water/i); + expect(item.remoteRef).toBeDefined(); + } + // Ranked path: both matches, the denser one first is NOT guaranteed, but + // the update (1003) must not shadow the older match. + expect(result.items.map((i) => i.messageId).sort()).toEqual([1001, 1003]); + }); + + it("fuzzy typo tolerance: misspelled token still matches via pg_trgm", async () => { + const service = new m.service.ArchiveService(null as never); + const result = await service.search(wsA, { q: "watter rghts" }); + expect(result.items.length).toBeGreaterThanOrEqual(1); + expect(result.items.map((i) => i.messageId).sort()).toEqual([1001, 1003]); + }); + + it("metadata search: channel id and attribution text are searchable", async () => { + const service = new m.service.ArchiveService(null as never); + const byChannel = await service.search(wsA, { q: chA }); + expect(byChannel.items).toHaveLength(3); + const byAuthor = await service.search(wsA, { q: "Working Group" }); + expect(byAuthor.items.map((i) => i.messageId)).toEqual([1001]); + }); + + it("RLS: search never crosses tenants even with shared text", async () => { + const service = new m.service.ArchiveService(null as never); + const a = await service.search(wsA, { q: "water rights" }); + const b = await service.search(wsB, { q: "water rights" }); + expect(a.items).toHaveLength(2); + expect(b.items).toHaveLength(1); + expect(b.items.every((i) => String(i.remoteRef).startsWith(chB))).toBe(true); + // Direct DB-level cross-tenant denial as sp_app. + const leaked = await m.db.withTenant(wsB, (client) => client.query( + "SELECT count(*)::int AS n FROM archived_feed_items WHERE workspace_id = $1", [wsA])); + expect(leaked.rows[0]?.n).toBe(0); + }); + + it("date + channel filters narrow the result set", async () => { + const service = new m.service.ArchiveService(null as never); + const from = await service.search(wsA, { q: "water", from: "2026-01-12" }); + expect(from.items.map((i) => i.messageId)).toEqual([1003]); + const to = await service.search(wsA, { q: "water", to: "2026-01-10T23:59:59Z" }); + expect(to.items.map((i) => i.messageId)).toEqual([1001]); + const channel = await service.search(wsA, { q: "bicycle", channelId: chA }); + expect(channel.items.map((i) => i.messageId)).toEqual([1002]); + const none = await service.search(wsA, { q: "bicycle", channelId: `@nonesuch${suffix}` }); + expect(none.items).toHaveLength(0); + }); + + it("keyset pagination walks the ranked result set without repeats or gaps", async () => { + const service = new m.service.ArchiveService(null as never); + const page1 = await service.search(wsA, { q: "water rights", limit: "1" }); + expect(page1.items).toHaveLength(1); + expect(page1.nextCursor).not.toBeNull(); + const page2 = await service.search(wsA, { q: "water rights", limit: "1", cursor: page1.nextCursor ?? "" }); + expect(page2.items).toHaveLength(1); + expect(page2.nextCursor).toBeNull(); + const seen = [...page1.items, ...page2.items].map((i) => i.messageId).sort(); + expect(seen).toEqual([1001, 1003]); + // Bogus cursor is rejected, not silently ignored. + await expect(service.search(wsA, { q: "water", cursor: "bogus-cursor" })) + .rejects.toThrow(/invalid cursor/); + }); + + it("export streams every matching item with faithful content (json + csv)", async () => { + const service = new m.service.ArchiveService(null as never); + const json: string[] = []; + const format = await service.exportItems(wsA, { q: "water rights" }, (chunk) => json.push(chunk)); + expect(format).toBe("json"); + const parsed = JSON.parse(json.join("")) as Array>; + expect(parsed).toHaveLength(2); + expect(parsed.map((i) => i.messageId).sort()).toEqual([1001, 1003]); + expect(parsed.every((i) => typeof i.postedAt === "string" && typeof i.remoteRef === "string")).toBe(true); + + const csv: string[] = []; + const csvFormat = await service.exportItems(wsA, { q: "water", format: "csv" }, (chunk) => csv.push(chunk)); + expect(csvFormat).toBe("csv"); + const csvText = csv.join(""); + const lines = csvText.split("\r\n").filter((line) => line !== ""); + expect(lines).toHaveLength(3); // header + 2 rows + const header = lines[0]!.split(","); + expect(header.slice(0, 4)).toEqual(["id", "remote_ref", "channel_id", "message_id"]); + const rows = lines.slice(1).map((line) => line.split(",")); + for (const row of rows) { + expect(row).toHaveLength(header.length); + expect(UUID_SHAPE.test(row[0]!)).toBe(true); + expect(row[1]).toMatch(new RegExp(`^${chA}:[0-9]+$`)); + } + const textCol = header.indexOf("text"); + const texts = rows.map((row) => row[textCol]!).sort(); + expect(texts[1]).toContain("hearing is postponed"); + }); + + it("export CSV escaping: commas, quotes and newlines round-trip", async () => { + const service = new m.service.ArchiveService(null as never); + const hostile = 'He said "water, rights", then\nleft'; + await asOwner((c) => inTenant(c, wsA, async () => { + await seedItem(c, wsA, { + channel: chA, message: 1004, text: hostile, postedAt: new Date("2026-01-13T09:00:00Z"), + }); + })); + const csv: string[] = []; + await service.exportItems(wsA, { channelId: chA, format: "csv" }, (chunk) => csv.push(chunk)); + const lines = csv.join("").split("\r\n").filter((l) => l !== ""); + const header = lines[0]!.split(","); + const textCol = header.indexOf("text"); + const rows = lines.slice(1).map((line) => line.split(",")); + const hostileRow = rows.find((row) => row[textCol] === hostile); + expect(hostileRow).toBeDefined(); + expect(hostileRow![textCol]).toBe(hostile); + expect(rows.every((row) => row.length === header.length)).toBe(true); + }); + + it("filter validation rejects garbage bounds, limits and formats", async () => { + const service = new m.service.ArchiveService(null as never); + await expect(service.search(wsA, { q: "water", from: "not-a-date" })).rejects.toThrow(/ISO-8601/); + await expect(service.search(wsA, { q: "water", from: "2026-02-01", to: "2026-01-01" })).rejects.toThrow(/after/); + await expect(service.search(wsA, { q: "water", limit: "0" })).rejects.toThrow(/between 1 and/); + await expect(service.search(wsA, { q: "water", limit: "1000" })).rejects.toThrow(/between 1 and/); + await expect(service.search(wsA, { q: "water", cursor: "bogus" })).rejects.toThrow(/invalid cursor/); + await expect(service.exportItems(wsA, { format: "xml" }, () => undefined)).rejects.toThrow(/csv or json/); + }); + + it("unindexed helper behavior: tokenization + CSV header shape", () => { + const helpers = m["search-helpers"]; + expect(helpers.searchTokens("Watter, Rghts! @channel123")).toEqual(["watter", "rghts", "@channel123"]); + expect(helpers.searchTokens("ab xx ")).toEqual([]); + expect(helpers.ARCHIVE_CSV_HEADER).toMatch(/^id,remote_ref,channel_id,message_id,posted_at,revision,protected_content,text,storage_keys,attribution\r\n$/); + }); +}); + +const UUID_SHAPE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; diff --git a/apps/api/src/archive/archive-search.ts b/apps/api/src/archive/archive-search.ts new file mode 100644 index 0000000..0cda3e2 --- /dev/null +++ b/apps/api/src/archive/archive-search.ts @@ -0,0 +1,280 @@ +import { BadRequestException } from "@nestjs/common"; +import { withTenant } from "../db/client.js"; +import type { StorageService } from "../storage/storage.js"; +import type { TelegramMedia } from "./normalizer.js"; + +/** + * WP15 archive full-text search (Postgres FTS phase-1, firehose research + * recommendation). Every query runs through withTenant() so the + * archived_feed_items tenant_isolation RLS policy (migration 015, FORCE) + * binds — there is deliberately no system/superuser path here. + * + * Matching is `websearch_to_tsquery` lexical OR pg_trgm similarity per token, + * so a misspelled token still matches via trigram distance (typo tolerance). + * Ranking: FTS ts_rank dominates, trigram similarity breaks ties. Keyset + * pagination follows the house pattern: base64url JSON cursor, fetch N+1. + */ + +export interface ArchiveSearchFilters { + q: string | null; + channelId: string | null; + from: string | null; + to: string | null; + cursor: string | undefined; + limit: number; +} + +export const SEARCH_DEFAULT_LIMIT = 20; +export const SEARCH_MAX_LIMIT = 100; +export const ARCHIVE_EXPORT_MAX_ROWS = 50_000; +const EXPORT_BATCH_SIZE = 500; + +export interface ArchiveItemRow { + id: string; + remote_ref: string; + channel_id: string; + message_id: string; + text_content: string; + media_refs: TelegramMedia[]; + attribution: Record; + protected_content: boolean; + posted_at: Date; + revision: number; + rank?: number; +} + +const ISO_TIMESTAMP = /^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?$/; +const UUID_SHAPE = /^[0-9a-f-]{36}$/i; + +function parseDateBound(value: unknown, field: string): string | null { + if (value === undefined || value === null || value === "") return null; + if (typeof value !== "string" || value.length > 32 || !ISO_TIMESTAMP.test(value) + || !Number.isFinite(Date.parse(value))) { + throw new BadRequestException(`${field} must be an ISO-8601 date or timestamp`); + } + return value; +} + +/** Shared filter/cursor-shape parsing for GET /archive/search and GET /archive/export. */ +export function parseArchiveSearchQuery(query: Record): ArchiveSearchFilters { + const rawQ = query.q; + const q = typeof rawQ === "string" && rawQ.trim() !== "" ? rawQ.trim() : null; + if (q !== null && q.length > 256) throw new BadRequestException("q must be at most 256 characters"); + const rawChannel = query.channelId; + const channelId = typeof rawChannel === "string" && rawChannel.trim() !== "" ? rawChannel.trim() : null; + if (channelId !== null && channelId.length > 128) throw new BadRequestException("channelId is too long"); + const from = parseDateBound(query.from, "from"); + const to = parseDateBound(query.to, "to"); + if (from !== null && to !== null && Date.parse(from) > Date.parse(to)) { + throw new BadRequestException("from must not be after to"); + } + let limit = SEARCH_DEFAULT_LIMIT; + const rawLimit = query.limit; + if (rawLimit !== undefined && rawLimit !== "") { + const n = typeof rawLimit === "string" ? Number(rawLimit) : NaN; + if (!Number.isInteger(n) || n < 1 || n > SEARCH_MAX_LIMIT) { + throw new BadRequestException(`limit must be an integer between 1 and ${SEARCH_MAX_LIMIT}`); + } + limit = n; + } + const rawCursor = query.cursor; + const cursor = typeof rawCursor === "string" && rawCursor !== "" ? rawCursor : undefined; + if (cursor !== undefined && (cursor.length > 256 || !/^[A-Za-z0-9_-]+$/.test(cursor))) { + throw new BadRequestException("invalid cursor"); + } + return { q, channelId, from, to, cursor, limit }; +} + +/** Lowercase word tokens (>= 3 chars, pg_trgm minimum), deduped and capped. */ +export function searchTokens(q: string): string[] { + const matches = q.toLowerCase().match(/[\p{L}\p{N}_@]+/gu) ?? []; + return [...new Set(matches.filter((token) => token.length >= 3))].slice(0, 8); +} + +/** Positional bind helper: keeps parameter indices stable while SQL fragments compose. */ +class Params { + readonly values: unknown[] = []; + bind(value: unknown): number { + this.values.push(value); + return this.values.length; + } +} + +function decodeCursor(cursor: string | undefined, what: string, + shape: (v: Record) => T): T { + if (cursor === undefined) throw new BadRequestException(`invalid cursor: ${what}`); + let value: unknown; + try { + value = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8")); + } catch { + throw new BadRequestException(`invalid cursor: ${what}`); + } + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new BadRequestException(`invalid cursor: ${what}`); + } + return shape(value as Record); +} + +/** + * The rank expression, bound once so SELECT, WHERE (keyset cursor) and + * ORDER BY share identical parameter indices. + */ +function rankExpression(params: Params, q: string, tokens: string[]): string { + const qi = params.bind(q); + const textSims = tokens.map((token) => `similarity(text_content, $${params.bind(token)})`); + const chanSims = tokens.map((token) => `similarity(channel_id, $${params.bind(token)})`); + return `(ts_rank(search_vector, websearch_to_tsquery('simple', $${qi})) * 2` + + (textSims.length > 0 ? ` + GREATEST(${textSims.join(", ")})` : "") + + (chanSims.length > 0 ? ` + GREATEST(${chanSims.join(", ")}) * 0.5` : "") + + `)`; +} + +function filterConds(params: Params, filters: ArchiveSearchFilters): string[] { + const conds: string[] = []; + if (filters.channelId !== null) conds.push(`channel_id = $${params.bind(filters.channelId)}`); + if (filters.from !== null) conds.push(`posted_at >= $${params.bind(filters.from)}::timestamptz`); + if (filters.to !== null) conds.push(`posted_at <= $${params.bind(filters.to)}::timestamptz`); + return conds; +} + +/** WHERE + ORDER BY for the ranked search path (q present). */ +function searchConds(params: Params, filters: ArchiveSearchFilters, tokens: string[]): [string, string] { + const rank = rankExpression(params, filters.q ?? "", tokens); + const conds = filterConds(params, filters); + const fuzzy = tokens.length === 0 ? "false" + : tokens.map((token) => { + const ti = params.bind(token); + return `(text_content % $${ti} OR channel_id % $${ti})`; + }).join(" OR "); + conds.push(`(search_vector @@ websearch_to_tsquery('simple', $${params.bind(filters.q ?? "")}) OR ${fuzzy})`); + if (filters.cursor !== undefined) { + const { rk, id } = decodeCursor<{ rk: number; id: string }>(filters.cursor, "rank", (v) => { + if (typeof v.rk !== "number" || !Number.isFinite(v.rk) + || typeof v.id !== "string" || !UUID_SHAPE.test(v.id)) throw new BadRequestException("invalid cursor: rank"); + return { rk: v.rk, id: v.id }; + }); + conds.push(`(${rank}, id) < ($${params.bind(rk)}::real, $${params.bind(id)}::uuid)`); + } + return [conds.join(" AND "), "ranked.rank DESC, ranked.id ASC"]; +} + +/** WHERE + ORDER BY for the export / no-query path (stable posted_at keyset). */ +function unrankedConds(params: Params, filters: ArchiveSearchFilters, + cursor: { at: string; id: string } | null): [string, string] { + const conds = filterConds(params, filters); + if (cursor) conds.push(`(posted_at, id) < ($${params.bind(cursor.at)}::timestamptz, $${params.bind(cursor.id)}::uuid)`); + return [conds.length > 0 ? conds.join(" AND ") : "true", "posted_at DESC, id DESC"]; +} + +const ROW_FIELDS = "id, remote_ref, channel_id, message_id, text_content, media_refs, attribution, protected_content, posted_at, revision"; + +export function itemFromRow(row: ArchiveItemRow): Record { + return { + id: row.id, + remoteRef: row.remote_ref, + channelId: row.channel_id, + messageId: Number(row.message_id), + text: row.text_content, + media: row.media_refs, + attribution: row.attribution, + protectedContent: row.protected_content, + postedAt: row.posted_at.toISOString(), + revision: row.revision, + }; +} + +export async function searchArchive(workspaceId: string, filters: ArchiveSearchFilters, + storage: StorageService): Promise<{ items: Record[]; nextCursor: string | null }> { + const params = new Params(); + const tokens = filters.q !== null ? searchTokens(filters.q) : []; + let sql: string; + if (filters.q !== null) { + const [where, orderBy] = searchConds(params, filters, tokens); + const rank = rankExpression(params, filters.q, tokens); + sql = `SELECT ${ROW_FIELDS}, rank FROM ( + SELECT ${ROW_FIELDS}, ${rank} AS rank FROM archived_feed_items WHERE ${where} + ) ranked ORDER BY ${orderBy} LIMIT $${params.bind(filters.limit + 1)}`; + } else { + const cursor = filters.cursor === undefined ? null + : decodeCursor<{ at: string; id: string }>(filters.cursor, "posted", (v) => { + if (typeof v.at !== "string" || !Number.isFinite(Date.parse(v.at)) + || typeof v.id !== "string" || !UUID_SHAPE.test(v.id)) throw new BadRequestException("invalid cursor: posted"); + return { at: v.at, id: v.id }; + }); + const [where, orderBy] = unrankedConds(params, filters, cursor); + sql = `SELECT ${ROW_FIELDS} FROM archived_feed_items WHERE ${where} + ORDER BY ${orderBy} LIMIT $${params.bind(filters.limit + 1)}`; + } + const result = await withTenant(workspaceId, (client) => client.query(sql, params.values)); + return paginate(result.rows, filters, storage); +} + +async function paginate(rows: ArchiveItemRow[], filters: ArchiveSearchFilters, + storage: StorageService): Promise<{ items: Record[]; nextCursor: string | null }> { + const page = rows.slice(0, filters.limit); + const last = page.at(-1); + const items = await Promise.all(page.map(async (row) => { + const item = itemFromRow(row); + if (filters.q !== null) item.rank = row.rank ?? 0; + item.media = await Promise.all(row.media_refs.map(async (media) => ({ + ...media, thumbUrl: media.storageKey ? await storage.presign(media.storageKey, 900) : null, + }))); + return item; + })); + const nextCursor = rows.length > filters.limit && last + ? Buffer.from(JSON.stringify(filters.q !== null + ? { rk: last.rank ?? 0, id: last.id } + : { at: last.posted_at.toISOString(), id: last.id })).toString("base64url") + : null; + return { items, nextCursor }; +} + +/** + * Streamed export: yields pages of plain item objects in posted_at keyset + * order. Exports never mint presigned URLs — media travels by storage key. + */ +export async function* exportArchiveItems(workspaceId: string, + filters: ArchiveSearchFilters): AsyncGenerator[]> { + let cursor: { at: string; id: string } | null = null; + let sent = 0; + while (sent < ARCHIVE_EXPORT_MAX_ROWS) { + const params = new Params(); + const remaining = Math.min(EXPORT_BATCH_SIZE, ARCHIVE_EXPORT_MAX_ROWS - sent); + const [where, orderBy] = unrankedConds(params, filters, cursor); + const result = await withTenant(workspaceId, (client) => client.query( + `SELECT ${ROW_FIELDS} FROM archived_feed_items WHERE ${where} + ORDER BY ${orderBy} LIMIT $${params.bind(remaining + 1)}`, params.values)); + const page = result.rows.slice(0, remaining); + if (page.length === 0) return; + sent += page.length; + const lastRow = page[page.length - 1]!; + cursor = { at: lastRow.posted_at.toISOString(), id: lastRow.id }; + yield page.map(itemFromRow); + if (result.rows.length <= remaining) return; + } +} + +const CSV_COLUMNS: Array<[string, (item: Record) => string]> = [ + ["id", (i) => String(i.id)], + ["remote_ref", (i) => String(i.remoteRef)], + ["channel_id", (i) => String(i.channelId)], + ["message_id", (i) => String(i.messageId)], + ["posted_at", (i) => String(i.postedAt)], + ["revision", (i) => String(i.revision)], + ["protected_content", (i) => (i.protectedContent === true ? "true" : "false")], + ["text", (i) => String(i.text)], + ["storage_keys", (i) => (Array.isArray(i.media) + ? (i.media as Array<{ storageKey?: string }>).map((m) => m.storageKey ?? "").filter((k) => k !== "").join(";") + : "")], + ["attribution", (i) => JSON.stringify(i.attribution ?? {})], +]; + +function csvCell(value: string): string { + return /[",\r\n]/.test(value) ? `"${value.replace(/"/g, '""')}"` : value; +} + +export const ARCHIVE_CSV_HEADER = CSV_COLUMNS.map(([name]) => name).join(",") + "\r\n"; + +export function csvItemRow(item: Record): string { + return CSV_COLUMNS.map(([, get]) => csvCell(get(item))).join(",") + "\r\n"; +} diff --git a/apps/api/src/archive/archive.controller.ts b/apps/api/src/archive/archive.controller.ts new file mode 100644 index 0000000..ec2880f --- /dev/null +++ b/apps/api/src/archive/archive.controller.ts @@ -0,0 +1,86 @@ +import { BadRequestException, Body, Controller, Get, Headers, Param, Post, Query, Req, Res, UseGuards } from "@nestjs/common"; +import type { Request, Response } from "express"; +import { parseId, requireObject, requireSession, requireString } from "../http/validation.js"; +import { RbacGuard, RequireGroupAccess, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import { ArchiveService } from "./archive.service.js"; + +@Controller("archive") +export class ArchiveController { + constructor(private readonly archive: ArchiveService) {} + + @Post("webhook/:workspaceId") + async webhook(@Param("workspaceId") workspaceId: string, + // eslint-disable-next-line no-restricted-syntax -- Fixed webhook protocol header, not provider branching. + @Headers("x-telegram-bot-api-secret-token") secret: string | undefined, + @Body() body: unknown): Promise<{ ok: true }> { + await this.archive.receive(parseId(workspaceId), secret, body); + return { ok: true }; + } + + @Post("subscriptions") + @UseGuards(RbacGuard) + @RequirePermission("connection.manage") + @RequireGroupAccess("body") + async subscribe(@Req() request: Request & AuthenticatedRequest, @Body() body: unknown): Promise<{ id: string }> { + const input = requireObject(body, "body"); + const workspaceId = requireSession(request).workspaceId; + return { id: await this.archive.subscribe(workspaceId, parseId(requireString(input.connectionId, "connectionId")), + requireString(input.channelId, "channelId")) }; + } + + @Get("items") + @UseGuards(RbacGuard) + @RequirePermission("draft.view") + async list(@Req() request: Request & AuthenticatedRequest, + @Query("cursor") cursor?: string): Promise<{ items: Record[]; nextCursor: string | null }> { + if (cursor && (cursor.length > 256 || !/^[A-Za-z0-9_-]+$/.test(cursor))) throw new BadRequestException("invalid cursor"); + return this.archive.list(requireSession(request).workspaceId, cursor); + } + + @Post("items/:itemId/drafts") + @UseGuards(RbacGuard) + @RequirePermission("draft.create") + async createDraft(@Req() request: Request & AuthenticatedRequest, + @Param("itemId") itemId: string): Promise<{ id: string; state: string }> { + const session = requireSession(request); + return this.archive.createDraft(session.workspaceId, { userId: session.userId, role: session.role }, parseId(itemId)); + } + + @Post("items/:itemId/amplify") + @UseGuards(RbacGuard) + @RequirePermission("publication.schedule") + @RequireGroupAccess("body") + async amplify(@Req() request: Request & AuthenticatedRequest, @Param("itemId") itemId: string, + @Body() body: unknown): Promise { + const input = requireObject(body, "body"); + if (input.attribution !== undefined && typeof input.attribution !== "boolean") throw new BadRequestException("attribution must be boolean"); + return this.archive.amplify(requireSession(request).workspaceId, parseId(itemId), + parseId(requireString(input.draftId, "draftId")), parseId(requireString(input.connectionId, "connectionId")), + input.attribution !== false); + } + + @Get("search") + @UseGuards(RbacGuard) + @RequirePermission("draft.view") + async search(@Req() request: Request & AuthenticatedRequest, + @Query() query: Record): Promise<{ items: Record[]; nextCursor: string | null }> { + return this.archive.search(requireSession(request).workspaceId, query); + } + + @Get("export") + @UseGuards(RbacGuard) + @RequirePermission("draft.view") + async exportAll(@Req() request: Request & AuthenticatedRequest, @Res() response: Response, + @Query() query: Record): Promise { + const workspaceId = requireSession(request).workspaceId; + const format = query.format === "csv" ? "csv" : query.format === undefined || query.format === "" ? "json" : null; + if (format === null) throw new BadRequestException("format must be csv or json"); + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + response.setHeader("Content-Type", format === "csv" ? "text/csv; charset=utf-8" : "application/json; charset=utf-8"); + response.setHeader("Content-Disposition", `attachment; filename="archive-export-${stamp}.${format}"`); + response.setHeader("Cache-Control", "no-store"); + response.setHeader("X-Content-Type-Options", "nosniff"); + await this.archive.exportItems(workspaceId, query, (chunk) => response.write(chunk)); + response.end(); + } +} diff --git a/apps/api/src/archive/archive.module.ts b/apps/api/src/archive/archive.module.ts new file mode 100644 index 0000000..ed0fe14 --- /dev/null +++ b/apps/api/src/archive/archive.module.ts @@ -0,0 +1,7 @@ +import { Module } from "@nestjs/common"; +import { PublishingService } from "../publishing/publishing.service.js"; +import { ArchiveController } from "./archive.controller.js"; +import { ArchiveService } from "./archive.service.js"; + +@Module({ controllers: [ArchiveController], providers: [ArchiveService, PublishingService] }) +export class ArchiveModule {} diff --git a/apps/api/src/archive/archive.service.ts b/apps/api/src/archive/archive.service.ts new file mode 100644 index 0000000..2b8a4fc --- /dev/null +++ b/apps/api/src/archive/archive.service.ts @@ -0,0 +1,239 @@ +import { BadRequestException, ForbiddenException, Injectable, NotFoundException } from "@nestjs/common"; +import { timingSafeEqual } from "node:crypto"; +// eslint-disable-next-line no-restricted-syntax -- Importing the connector adapter is the provider boundary, not provider branching. +import { createTelegramConnector } from "@tactical-media-broadcast-hub/connector-telegram"; +import { ConnectorError } from "@tactical-media-broadcast-hub/connector-sdk"; +import type { ConnectorConnection } from "@tactical-media-broadcast-hub/core"; +import { withTenant } from "../db/client.js"; +import { PublishingService, type Actor } from "../publishing/publishing.service.js"; +import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js"; +import { + ARCHIVE_CSV_HEADER, + csvItemRow, + exportArchiveItems, + parseArchiveSearchQuery, + searchArchive, + type ArchiveSearchFilters, +} from "./archive-search.js"; +import { TelegramAlbumDebouncer, type ArchiveItemInput, type TelegramMedia } from "./normalizer.js"; + +const ARCHIVE_PROVIDER = createTelegramConnector().manifest().id; + +interface ItemRow { + id: string; remote_ref: string; channel_id: string; message_id: string; + text_content: string; media_refs: TelegramMedia[]; attribution: Record; + protected_content: boolean; posted_at: Date; revision: number; +} +function object(value: unknown): Record | null { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : null; +} +function equalSecret(actual: string, expected: string): boolean { + const a = Buffer.from(actual); const b = Buffer.from(expected); + return a.length === b.length && timingSafeEqual(a, b); +} +function itemFromRow(row: ItemRow): Record { + return { id: row.id, remoteRef: row.remote_ref, channelId: row.channel_id, + messageId: Number(row.message_id), text: row.text_content, media: row.media_refs, + attribution: row.attribution, protectedContent: row.protected_content, + postedAt: row.posted_at.toISOString(), revision: row.revision }; +} + +@Injectable() +export class ArchiveService { + readonly albums: TelegramAlbumDebouncer; + private readonly storage: StorageService; + constructor(private readonly publishing: PublishingService) { + this.storage = new S3Storage(s3ConfigFromEnv()); + this.albums = new TelegramAlbumDebouncer((item) => this.save(item)); + } + + async subscribe(workspaceId: string, connectionId: string, channelId: string): Promise { + if (!/^-?\d{3,}$/.test(channelId)) throw new BadRequestException("channelId must be a numeric Telegram chat id"); + return withTenant(workspaceId, async (client) => { + const result = await client.query( + `INSERT INTO feed_subscriptions (workspace_id, type, connection_id, channel_id) + SELECT $1, 'telegram_channel', id, $3 FROM connections + WHERE id = $2 AND provider = 'telegram' + ON CONFLICT (workspace_id, connection_id, channel_id) DO UPDATE SET updated_at = now() + RETURNING id`, [workspaceId, connectionId, channelId]); + if (!result.rows[0]) throw new NotFoundException("Telegram connection not found"); + return String(result.rows[0].id); + }); + } + + async receive(workspaceId: string, secret: string | undefined, payload: unknown): Promise { + const expected = process.env.TELEGRAM_WEBHOOK_SECRET; + if (!expected || !secret || !equalSecret(secret, expected)) throw new ForbiddenException("webhook secret mismatch"); + const update = object(payload); + const post = object(update?.channel_post ?? update?.edited_channel_post); + const membership = object(update?.my_chat_member); + const chat = object(post?.chat ?? membership?.chat); + const channelId = chat?.id === undefined ? null : String(chat.id); + if (!channelId) return; + const subscriptions = await withTenant(workspaceId, (client) => client.query( + "SELECT id FROM feed_subscriptions WHERE channel_id = $1", [channelId])); + if (subscriptions.rowCount === 0) return; + if (membership) { + const next = object(membership.new_chat_member); + const status = next?.status; + const health = status === "administrator" || status === "creator" ? "active" + : status === "member" ? "admin_required" : "bot_removed"; + await withTenant(workspaceId, (client) => client.query( + "UPDATE feed_subscriptions SET health = $2, health_checked_at = now(), updated_at = now() WHERE channel_id = $1", + [channelId, health])); + return; + } + await this.albums.ingest(workspaceId, payload); + } + + private async harvest(media: TelegramMedia): Promise { + if (media.status !== "pending") return media; + const token = process.env.TELEGRAM_BOT_TOKEN; + if (!token) return media; + const response = await fetch(`https://api.telegram.org/bot${token}/getFile?file_id=${encodeURIComponent(media.fileId)}`); + const body: unknown = await response.json(); + const file = object(object(body)?.result); + const size = typeof file?.file_size === "number" ? file.file_size : media.sizeBytes; + if (size !== null && size > 20_000_000) return { ...media, sizeBytes: size, status: "Bot API cannot fetch" }; + const path = file?.file_path; + if (!response.ok || typeof path !== "string" || path.includes("..") || path.startsWith("/")) return { ...media, status: "failed" }; + const downloaded = await fetch(`https://api.telegram.org/file/bot${token}/${path}`); + if (!downloaded.ok) return { ...media, status: "failed" }; + const bytes = Buffer.from(await downloaded.arrayBuffer()); + if (bytes.length > 20_000_000) return { ...media, sizeBytes: bytes.length, status: "Bot API cannot fetch" }; + const stored = await this.storage.putContent(bytes, media.contentType); + return { ...media, sizeBytes: bytes.length, status: "stored", storageKey: stored.key }; + } + + async save(item: ArchiveItemInput): Promise { + const media = await Promise.all(item.mediaRefs.map((entry) => this.harvest(entry))); + await withTenant(item.workspaceId, async (client) => { + const existing = await client.query( + "SELECT * FROM archived_feed_items WHERE remote_ref = $1 FOR UPDATE", [item.remoteRef]); + const row = existing.rows[0]; + if (row?.content_hash === item.contentHash) return; + if (row && !item.edited) return; + if (row) { + const next = row.revision + 1; + await client.query(`UPDATE archived_feed_items SET text_content = $2, content_hash = $3, + media_refs = $4, attribution = $5, protected_content = $6, revision = $7, updated_at = now() WHERE id = $1`, + [row.id, item.textContent, item.contentHash, JSON.stringify(media), JSON.stringify(item.attribution), item.protectedContent, next]); + await client.query(`INSERT INTO archived_feed_revisions + (workspace_id, item_id, revision, content_hash, text_content, media_refs, attribution) + VALUES ($1, $2, $3, $4, $5, $6, $7)`, + [item.workspaceId, row.id, next, item.contentHash, item.textContent, JSON.stringify(media), JSON.stringify(item.attribution)]); + return; + } + const inserted = await client.query<{ id: string }>(`INSERT INTO archived_feed_items + (workspace_id, remote_ref, channel_id, message_id, media_group_id, text_content, content_hash, + media_refs, attribution, protected_content, posted_at) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11) + ON CONFLICT (workspace_id, remote_ref) DO NOTHING RETURNING id`, + [item.workspaceId, item.remoteRef, item.channelId, item.messageId, item.mediaGroupId, + item.textContent, item.contentHash, JSON.stringify(media), JSON.stringify(item.attribution), + item.protectedContent, item.postedAt]); + const id = inserted.rows[0]?.id; + if (id) await client.query(`INSERT INTO archived_feed_revisions + (workspace_id, item_id, revision, content_hash, text_content, media_refs, attribution) + VALUES ($1,$2,1,$3,$4,$5,$6)`, + [item.workspaceId, id, item.contentHash, item.textContent, JSON.stringify(media), JSON.stringify(item.attribution)]); + }); + } + + async list(workspaceId: string, cursor?: string): Promise<{ items: Record[]; nextCursor: string | null }> { + let parsed: { at: string; id: string } | null = null; + if (cursor) { + try { + const value: unknown = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8")); + const shape = object(value); + if (typeof shape?.at !== "string" || !Number.isFinite(Date.parse(shape.at)) + || typeof shape.id !== "string" || !/^[0-9a-f-]{36}$/i.test(shape.id)) throw new Error("shape"); + parsed = { at: shape.at, id: shape.id }; + } catch { throw new BadRequestException("invalid cursor"); } + } + const result = await withTenant(workspaceId, (client) => client.query( + `SELECT * FROM archived_feed_items WHERE ($1::timestamptz IS NULL OR (posted_at, id) < ($1::timestamptz, $2::uuid)) + ORDER BY posted_at DESC, id DESC LIMIT 21`, [parsed?.at ?? null, parsed?.id ?? null])); + const rows = result.rows.slice(0, 20); + const last = rows.at(-1); + const items = await Promise.all(rows.map(async (row) => { + const item = itemFromRow(row); + item.media = await Promise.all(row.media_refs.map(async (media) => ({ + ...media, thumbUrl: media.storageKey ? await this.storage.presign(media.storageKey, 900) : null, + }))); + return item; + })); + return { items, nextCursor: result.rows.length > 20 && last + ? Buffer.from(JSON.stringify({ at: last.posted_at.toISOString(), id: last.id })).toString("base64url") : null }; + } + + async createDraft(workspaceId: string, actor: Actor, itemId: string): Promise<{ id: string; state: string }> { + const found = await withTenant(workspaceId, (client) => client.query( + "SELECT * FROM archived_feed_items WHERE id = $1", [itemId])); + const item = found.rows[0]; + if (!item) throw new NotFoundException("archive item not found"); + const draft = await this.publishing.createDraft(workspaceId, actor, { + text: item.text_content, mediaAssetIds: [], visibility: "provider_default", + providerMetadata: { archiveItemId: itemId, sourceRemoteRef: item.remote_ref, + attribution: item.attribution, attributionEnabled: true }, + }); + await withTenant(workspaceId, (client) => client.query( + "INSERT INTO archived_feed_drafts (workspace_id, item_id, draft_id) VALUES ($1,$2,$3)", + [workspaceId, itemId, draft.id])); + return { id: draft.id, state: draft.state }; + } + + async amplify(workspaceId: string, itemId: string, draftId: string, connectionId: string, + attribution = true): Promise { + const found = await withTenant(workspaceId, (client) => client.query( + `SELECT i.* FROM archived_feed_items i JOIN archived_feed_drafts b + ON b.item_id = i.id AND b.workspace_id = i.workspace_id + JOIN drafts d ON d.id = b.draft_id AND d.workspace_id = b.workspace_id + WHERE i.id = $1 AND d.id = $2 AND d.state = 'approved'`, [itemId, draftId])); + const item = found.rows[0]; + if (!item) throw new ForbiddenException("an approved archive draft is required"); + if (item.protected_content) throw new ConnectorError(`${ARCHIVE_PROVIDER}: protected content cannot be forwarded`, ARCHIVE_PROVIDER); + const target = await withTenant(workspaceId, (client) => client.query<{ channel_id: string; fields: Record }>( + "SELECT channel_id, fields FROM connections WHERE id = $1 AND provider = $2", [connectionId, ARCHIVE_PROVIDER])); + const connectionRow = target.rows[0]; + if (!connectionRow) throw new NotFoundException("Telegram target connection not found"); + const token = process.env.TELEGRAM_BOT_TOKEN; + if (!token) throw new BadRequestException("Telegram bot token is not configured"); + const connection: ConnectorConnection = { id: connectionId, workspaceId, provider: ARCHIVE_PROVIDER, + channelId: connectionRow.channel_id, credentialId: { kind: "credential-ref", id: connectionId }, + fields: { ...connectionRow.fields, botToken: token }, createdAt: new Date() }; + const connector = createTelegramConnector(); + if (!connector.amplify) throw new ConnectorError(`${ARCHIVE_PROVIDER}: amplify unavailable`, ARCHIVE_PROVIDER); + const origin = item.attribution.channelUsername; + const commentary = attribution && typeof origin === "string" && origin !== "" + ? `Source: https://t.me/${origin}/${item.message_id}` : undefined; + return connector.amplify(connection, { kind: "share", targetRemoteId: item.remote_ref, + ...(commentary ? { commentary } : {}) }); + } + + async search(workspaceId: string, query: Record): Promise<{ items: Record[]; nextCursor: string | null }> { + const filters: ArchiveSearchFilters = parseArchiveSearchQuery(query); + return searchArchive(workspaceId, filters, this.storage); + } + + async exportItems(workspaceId: string, query: Record, + write: (chunk: string) => void): Promise<"csv" | "json"> { + const filters = parseArchiveSearchQuery(query); + const format = query.format === "csv" ? "csv" : query.format === undefined || query.format === "" ? "json" : null; + if (format === null) throw new BadRequestException("format must be csv or json"); + if (format === "csv") write(ARCHIVE_CSV_HEADER); + else write("["); + let first = true; + for await (const page of exportArchiveItems(workspaceId, filters)) { + for (const item of page) { + if (format === "csv") write(csvItemRow(item)); + else { + write((first ? "" : ",") + JSON.stringify(item)); + first = false; + } + } + } + write(format === "csv" ? "" : "]"); + return format; + } +} diff --git a/apps/api/src/archive/normalizer.ts b/apps/api/src/archive/normalizer.ts new file mode 100644 index 0000000..46d4a33 --- /dev/null +++ b/apps/api/src/archive/normalizer.ts @@ -0,0 +1,109 @@ +import { createHash } from "node:crypto"; + +export interface TelegramMedia { + fileId: string; + fileUniqueId: string; + sizeBytes: number | null; + contentType: string; + status: "pending" | "stored" | "Bot API cannot fetch" | "failed"; + storageKey?: string; +} +export interface ArchiveItemInput { + workspaceId: string; + channelId: string; + messageId: number; + remoteRef: string; + mediaGroupId: string | null; + textContent: string; + contentHash: string; + mediaRefs: TelegramMedia[]; + attribution: Record; + protectedContent: boolean; + postedAt: Date; + edited: boolean; +} +interface TelegramFile { file_id?: unknown; file_unique_id?: unknown; file_size?: unknown; mime_type?: unknown } +interface TelegramPost { + message_id: number; date: number; + chat: { id: number | string; title?: string; username?: string }; + text?: string; caption?: string; media_group_id?: string; author_signature?: string; + forward_origin?: Record; is_automatic_forward?: boolean; + has_protected_content?: boolean; + photo?: TelegramFile[]; video?: TelegramFile; document?: TelegramFile; +} +function record(value: unknown): Record | null { + return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : null; +} +export function parseTelegramPost(value: unknown): TelegramPost | null { + const post = record(value); + const chat = record(post?.chat); + if (!post || !chat || !Number.isSafeInteger(post.message_id) || !Number.isSafeInteger(post.date) + || (typeof chat.id !== "number" && typeof chat.id !== "string")) return null; + return post as unknown as TelegramPost; +} +function mediaOf(post: TelegramPost): TelegramMedia[] { + const photo = Array.isArray(post.photo) ? post.photo.at(-1) : undefined; + const entries: Array<[TelegramFile | undefined, string]> = [[photo, "image/jpeg"], [post.video, "video/mp4"], [post.document, "application/octet-stream"]]; + return entries.flatMap(([file, fallback]) => typeof file?.file_id === "string" && typeof file.file_unique_id === "string" + ? [{ fileId: file.file_id, fileUniqueId: file.file_unique_id, + sizeBytes: typeof file.file_size === "number" ? file.file_size : null, + contentType: typeof file.mime_type === "string" ? file.mime_type : fallback, + status: file.file_size !== undefined && typeof file.file_size === "number" && file.file_size > 20_000_000 + ? "Bot API cannot fetch" as const : "pending" as const }] + : []); +} +export function normalizePosts(workspaceId: string, posts: TelegramPost[], edited = false): ArchiveItemInput { + const ordered = [...posts].sort((a, b) => a.message_id - b.message_id); + const first = ordered[0]; + if (!first) throw new Error("empty Telegram post batch"); + const textContent = ordered.map((p) => p.text ?? p.caption ?? "").find((s) => s !== "") ?? ""; + const mediaRefs = ordered.flatMap(mediaOf); + const attribution: Record = { + channelTitle: first.chat.title ?? null, channelUsername: first.chat.username ?? null, + authorSignature: first.author_signature ?? null, forwardOrigin: first.forward_origin ?? null, + isAutomaticForward: first.is_automatic_forward === true, + }; + const channelId = String(first.chat.id); + const contentHash = createHash("sha256").update(JSON.stringify({ textContent, media: mediaRefs.map((m) => m.fileUniqueId), attribution })).digest("hex"); + return { workspaceId, channelId, messageId: first.message_id, remoteRef: `${channelId}:${first.message_id}`, + mediaGroupId: first.media_group_id ?? null, textContent, contentHash, mediaRefs, attribution, + protectedContent: ordered.some((p) => p.has_protected_content === true), + postedAt: new Date(first.date * 1000), edited }; +} +export interface AlbumTimers { + set(callback: () => void, ms: number): ReturnType; + clear(handle: ReturnType): void; +} +const defaultTimers: AlbumTimers = { set: (cb, ms) => setTimeout(cb, ms), clear: clearTimeout }; +/** A keyed 2s debounce. Every album update shares one durable archive identity. */ +export class TelegramAlbumDebouncer { + private readonly pending = new Map }>(); + constructor(private readonly save: (item: ArchiveItemInput) => Promise, private readonly timers: AlbumTimers = defaultTimers) {} + async ingest(workspaceId: string, update: unknown): Promise { + const envelope = record(update); + const edited = envelope?.edited_channel_post !== undefined; + const post = parseTelegramPost(envelope?.[edited ? "edited_channel_post" : "channel_post"]); + if (!post) return; + const group = post.media_group_id; + if (!group || edited) { await this.save(normalizePosts(workspaceId, [post], edited)); return; } + const key = `${workspaceId}:${post.chat.id}:${group}`; + const current = this.pending.get(key); + if (current) this.timers.clear(current.timer); + const posts = [...(current?.posts ?? []), post]; + const timer = this.timers.set(() => { void this.flush(key, workspaceId); }, 2_000); + this.pending.set(key, { posts, timer }); + } + async flush(key: string, workspaceId: string): Promise { + const group = this.pending.get(key); + if (!group) return; + this.pending.delete(key); + this.timers.clear(group.timer); + await this.save(normalizePosts(workspaceId, group.posts)); + } + async flushAll(): Promise { + for (const key of [...this.pending.keys()]) { + const workspaceId = key.slice(0, key.indexOf(":")); + await this.flush(key, workspaceId); + } + } +} diff --git a/apps/api/src/audit/audit.controller.ts b/apps/api/src/audit/audit.controller.ts new file mode 100644 index 0000000..e846d2c --- /dev/null +++ b/apps/api/src/audit/audit.controller.ts @@ -0,0 +1,35 @@ +import { BadRequestException, Controller, Get, Inject, Query, Req, UseGuards } from "@nestjs/common"; +import type { Request } from "express"; +import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import { isUuid, parseLimit, requireSession } from "../http/validation.js"; +import { AuditQueryService, type AuditPage } from "./audit.query.js"; + +@Controller("audit") +@UseGuards(RbacGuard) +export class AuditController { + constructor(@Inject(AuditQueryService) private readonly audit: AuditQueryService) {} + + @Get() + @RequirePermission("audit.view") + async list( + @Req() request: Request & AuthenticatedRequest, + @Query("action") action: string | undefined, + @Query("actor") actor: string | undefined, + @Query("before") before: string | undefined, + @Query("limit") limit: string | undefined, + ): Promise { + const session = requireSession(request); + if (actor !== undefined && actor !== "" && actor !== "system" && !isUuid(actor)) { + throw new BadRequestException("actor must be a user UUID or \"system\""); + } + if (before !== undefined && before !== "" && !/^\d{1,18}$/.test(before)) { + throw new BadRequestException("before must be a sequence cursor"); + } + return this.audit.page(session.workspaceId, { + ...(action === undefined || action === "" ? {} : { action }), + ...(actor === undefined || actor === "" ? {} : { actor }), + ...(before === undefined || before === "" ? {} : { before }), + limit: parseLimit(limit, 50, 200), + }); + } +} diff --git a/apps/api/src/audit/audit.query.ts b/apps/api/src/audit/audit.query.ts new file mode 100644 index 0000000..0a4fe21 --- /dev/null +++ b/apps/api/src/audit/audit.query.ts @@ -0,0 +1,113 @@ +import { Injectable } from "@nestjs/common"; +import { withTenant } from "../db/client.js"; + +export interface AuditEntryDto { + id: string; + seq: string; + at: string; + actorUserId: string | null; + actorEmail: string | null; + actorRole: string | null; + action: string; + targetType: string; + targetId: string; + diff: Record | null; +} + +export interface AuditActorFacet { + /** null = system / service actor. */ + id: string | null; + email: string | null; +} + +export interface AuditPage { + entries: AuditEntryDto[]; + /** Pass back as `before` to fetch the next (older) page; null when exhausted. */ + nextCursor: string | null; + facets: { actions: string[]; actors: AuditActorFacet[] }; +} + +export interface AuditFilter { + action?: string; + /** User UUID, or "system" for entries without a human actor. */ + actor?: string; + before?: string; + limit: number; +} + +interface AuditRow { + id: string; + seq: string; + at: Date; + actor_user_id: string | null; + actor_email: string | null; + actor_role: string | null; + action: string; + target_type: string; + target_id: string; + diff: Record | null; +} + +/** Read side of the tenant audit chain. RLS (tenant_audit policy) scopes rows to the session's workspace. */ +@Injectable() +export class AuditQueryService { + async page(workspaceId: string, filter: AuditFilter): Promise { + return withTenant(workspaceId, async (client) => { + const actorIsSystem = filter.actor === "system"; + const rows = await client.query( + `SELECT a.id, a.seq::text AS seq, a.at, a.actor_user_id, u.email AS actor_email, a.actor_role, + a.action, a.target_type, a.target_id, a.diff + FROM audit_log a + LEFT JOIN users u ON u.id = a.actor_user_id + WHERE a.workspace_id = $1 + AND ($2::text IS NULL OR a.action = $2) + AND ($3::uuid IS NULL OR a.actor_user_id = $3) + AND (NOT $4::boolean OR a.actor_user_id IS NULL) + AND ($5::bigint IS NULL OR a.seq < $5) + ORDER BY a.seq DESC + LIMIT $6`, + [ + workspaceId, + filter.action ?? null, + actorIsSystem ? null : (filter.actor ?? null), + actorIsSystem, + filter.before ?? null, + filter.limit + 1, + ], + ); + const all = rows.rows as AuditRow[]; + const hasMore = all.length > filter.limit; + const entries = all.slice(0, filter.limit).map((r) => ({ + id: r.id, + seq: r.seq, + at: new Date(r.at).toISOString(), + actorUserId: r.actor_user_id, + actorEmail: r.actor_email, + actorRole: r.actor_role, + action: r.action, + targetType: r.target_type, + targetId: r.target_id, + diff: r.diff, + })); + + const actions = await client.query( + "SELECT DISTINCT action FROM audit_log WHERE workspace_id = $1 ORDER BY action", + [workspaceId], + ); + const actors = await client.query( + `SELECT DISTINCT a.actor_user_id AS id, u.email + FROM audit_log a LEFT JOIN users u ON u.id = a.actor_user_id + WHERE a.workspace_id = $1 ORDER BY u.email NULLS LAST`, + [workspaceId], + ); + return { + entries, + nextCursor: hasMore ? (entries[entries.length - 1]?.seq ?? null) : null, + facets: { + actions: (actions.rows as Array<{ action: string }>).map((r) => r.action), + actors: actors.rows as AuditActorFacet[], + }, + }; + }); + } +} diff --git a/apps/api/src/audit/audit.scrub.test.ts b/apps/api/src/audit/audit.scrub.test.ts new file mode 100644 index 0000000..dcf29cb --- /dev/null +++ b/apps/api/src/audit/audit.scrub.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vitest"; +import { scrub } from "./audit.service.js"; + +describe("audit diff scrubbing", () => { + it("redacts secret-shaped keys (spec: secrets never enter audit payloads)", () => { + const clean = scrub({ password: "hunter2", stream_key: "sk-1", note: "hello", count: 3 }); + expect(clean).toEqual({ password: "[REDACTED]", stream_key: "[REDACTED]", note: "hello", count: 3 }); + }); + + it("passes null through", () => { + expect(scrub(null)).toBeNull(); + }); +}); diff --git a/apps/api/src/audit/audit.service.ts b/apps/api/src/audit/audit.service.ts new file mode 100644 index 0000000..7577916 --- /dev/null +++ b/apps/api/src/audit/audit.service.ts @@ -0,0 +1,38 @@ +import { withTenant } from "../db/client.js"; + +/** + * Audit service: hash-chained, secrets never enter payloads. + * Branch selection (findings #1/#6): tenant context present => tenant chain + * (audit_append), even for service roles; no tenant context => system chain. + */ + +const SECRET_KEYS = /(password|secret|token|key|credential|stream_key|refresh)/i; + +export function scrub(diff: Record | null): Record | null { + if (diff === null) return null; + const clean: Record = {}; + for (const [k, v] of Object.entries(diff)) { + clean[k] = SECRET_KEYS.test(k) ? "[REDACTED]" : v; + } + return clean; +} + +export interface AuditEvent { + actorUserId: string | null; + actorRole: string; + action: string; + targetType: string; + targetId: string; + diff?: Record | null; +} + +export async function appendTenantAudit(workspaceId: string, event: AuditEvent): Promise { + const res = await withTenant(workspaceId, async (client) => { + const result = await client.query( + "SELECT audit_append($1, $2, $3, $4, $5, $6, $7) AS id", + [workspaceId, event.actorUserId, event.actorRole, event.action, event.targetType, event.targetId, scrub(event.diff ?? null)], + ); + return result.rows[0]?.id as string; + }); + return res; +} diff --git a/apps/api/src/auth/auth.controller.ts b/apps/api/src/auth/auth.controller.ts new file mode 100644 index 0000000..0c28e89 --- /dev/null +++ b/apps/api/src/auth/auth.controller.ts @@ -0,0 +1,70 @@ +import { Body, Controller, Get, HttpCode, Inject, Post, Req, UnauthorizedException, UseGuards } from "@nestjs/common"; +import type { Request } from "express"; +import { RbacGuard, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import { optionalEmail, optionalString, parseId, requireObject, requireString, requireUsername, requireSession } from "../http/validation.js"; +import { AuthService, type AuthResult, type LoginInput, type MeResponse, type RegisterInput } from "./auth.service.js"; +import { verifySession } from "./session.js"; + +@Controller("auth") +export class AuthController { + constructor(@Inject(AuthService) private readonly auth: AuthService) {} + + @Post("register") + async register(@Body() body: unknown): Promise { + return this.auth.register(parseRegister(body)); + } + + @Post("login") + @HttpCode(200) + async login(@Body() body: unknown): Promise { + return this.auth.login(parseLogin(body)); + } + + /** Any valid, epoch-current session may read itself — no permission gate beyond authentication. */ + @Get("me") + @UseGuards(RbacGuard) + async me(@Req() request: Request & AuthenticatedRequest): Promise { + return this.auth.me(requireSession(request)); + } + + /** A signed, unexpired session can refresh its epoch and live role. */ + @Post("reissue") + @HttpCode(200) + async reissue(@Req() request: Request): Promise { + const authorization = request.headers.authorization; + if (authorization === undefined || !authorization.startsWith("Bearer ")) throw new UnauthorizedException(); + const session = verifySession(authorization.slice("Bearer ".length)); + if (session === null) throw new UnauthorizedException(); + return this.auth.reissue(session); + } + + /** Stateless tokens: logout is client-side (drop the token). Revocation-by-epoch covers role changes. */ + @Post("logout") + @HttpCode(200) + logout(): { ok: true } { + return { ok: true }; + } +} + +function parseRegister(value: unknown): RegisterInput { + const body = requireObject(value, "body"); + const username = requireUsername(body.username); + const email = optionalEmail(body.email); + return { + workspaceName: requireString(body.workspaceName, "workspaceName").trim().slice(0, 100), + username, + password: requireString(body.password, "password"), + displayName: (optionalString(body.displayName, "displayName") ?? username).trim().slice(0, 100), + ...(email === undefined ? {} : { email }), + }; +} + +function parseLogin(value: unknown): LoginInput { + const body = requireObject(value, "body"); + const workspaceId = optionalString(body.workspaceId, "workspaceId"); + return { + username: requireUsername(body.username), + password: requireString(body.password, "password"), + ...(workspaceId === undefined ? {} : { workspaceId: parseId(workspaceId, "workspaceId") }), + }; +} diff --git a/apps/api/src/auth/auth.integration.test.ts b/apps/api/src/auth/auth.integration.test.ts new file mode 100644 index 0000000..e5f868a --- /dev/null +++ b/apps/api/src/auth/auth.integration.test.ts @@ -0,0 +1,155 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { join } from "node:path"; +import type { ExecutionContext } from "@nestjs/common"; +import { Reflector } from "@nestjs/core"; + +/** + * Auth + team + audit wiring against a live PostgreSQL, as sp_app (FORCE RLS binds). + * Uses its OWN database (AUTH_IT_DATABASE_URL, plus AUTH_IT_SUPER_DATABASE_URL for + * migrations) because tenancy.integration.test.ts wipes every workspace in the + * shared DB and vitest runs files in parallel. Skipped when unset. + */ +const appUrl = process.env.AUTH_IT_DATABASE_URL; +const ownerUrl = process.env.AUTH_IT_SUPER_DATABASE_URL; + +type Modules = { + auth: typeof import("./auth.service.js"); + team: typeof import("../team/team.service.js"); + audit: typeof import("../audit/audit.query.js"); + guard: typeof import("../rbac/rbac.guard.js"); + session: typeof import("./session.js"); + db: typeof import("../db/client.js"); +}; + +describe.skipIf(appUrl === undefined || ownerUrl === undefined)("auth/team/audit (live DB)", () => { + let m: Modules; + const suffix = `${Date.now()}${Math.floor(Math.random() * 1e6)}`; + const ownerUsername = `owner${suffix}`; + const ownerEmail = `owner-${suffix}@example.com`; + + beforeAll(async () => { + // The pool reads DATABASE_URL at import time: point it at the isolated DB first. + process.env.DATABASE_URL = appUrl; + process.env.SUPER_DATABASE_URL = ownerUrl; + const { runMigrations } = await import("../db/migrate.js"); + await runMigrations(join(process.cwd(), "migrations")); + m = { + auth: await import("./auth.service.js"), + team: await import("../team/team.service.js"), + audit: await import("../audit/audit.query.js"), + guard: await import("../rbac/rbac.guard.js"), + session: await import("./session.js"), + db: await import("../db/client.js"), + }; + }); + + afterAll(async () => { + await m?.db.pool.end(); + }); + + function guardContext(token: string, permission: string | undefined): { ctx: ExecutionContext; reflector: Reflector } { + const handler = (): void => undefined; + const reflector = new Reflector(); + if (permission !== undefined) Reflect.defineMetadata(m.guard.PERMISSION_KEY, permission, handler); + const req = { headers: { authorization: `Bearer ${token}` } }; + const ctx = { + getHandler: () => handler, + switchToHttp: () => ({ getRequest: () => req }), + } as unknown as ExecutionContext; + return { ctx, reflector }; + } + + async function allowed(token: string, permission?: string): Promise { + const { ctx, reflector } = guardContext(token, permission); + try { + return await new m.guard.RbacGuard(reflector).canActivate(ctx); + } catch (err) { + return (err as Error).constructor.name; + } + } + + it("register -> login -> me, and the RBAC guard accepts the session (bigint epoch, tenant-scoped lookup)", async () => { + const auth = new m.auth.AuthService(); + const reg = await auth.register({ workspaceName: "IT Studio", username: ownerUsername, password: "correct-horse-1", displayName: "Owner", email: ownerEmail }); + expect(reg.me.role).toBe("Owner"); + expect(reg.me.user.username).toBe(ownerUsername); + expect(reg.me.user.email).toBe(ownerEmail); + expect(reg.me.permissions).toContain("workspace.members.manage"); + + await expect(auth.register({ workspaceName: "Dup", username: ownerUsername, password: "correct-horse-1", displayName: "x" })) + .rejects.toMatchObject({ status: 409 }); + await expect(auth.login({ username: ownerUsername, password: "wrong-password-1" })).rejects.toMatchObject({ status: 401 }); + + const login = await auth.login({ username: ownerUsername.toUpperCase(), password: "correct-horse-1" }); + expect(login.me.workspace.id).toBe(reg.me.workspace.id); + await expect(allowed(login.token)).resolves.toBe(true); + await expect(allowed(login.token, "audit.view")).resolves.toBe(true); + }); + + it("registers without an email (username-only accounts)", async () => { + const auth = new m.auth.AuthService(); + const username = `noemail${suffix}`; + const reg = await auth.register({ workspaceName: "No Email WS", username, password: "correct-horse-1", displayName: "N" }); + expect(reg.me.user.email).toBeNull(); + expect(reg.me.user.username).toBe(username); + const login = await auth.login({ username, password: "correct-horse-1" }); + expect(login.me.user.id).toBe(reg.me.user.id); + }); + + it("membership mutations bump the epoch but hand the actor a fresh token; last Owner is protected", async () => { + const auth = new m.auth.AuthService(); + const team = new m.team.TeamService(); + const login = await auth.login({ username: ownerUsername, password: "correct-horse-1" }); + const session = m.session.verifySession(login.token); + if (session === null) throw new Error("session did not verify"); + const actor = { userId: session.userId, role: session.role }; + + const invited = await team.invite(session.workspaceId, actor, `pub-${suffix}@example.com`, "Publisher"); + expect(invited.member).toMatchObject({ role: "Publisher", status: "invited" }); + if (invited.token === null || invited.member === null) throw new Error("expected token + member"); + await expect(allowed(login.token)).resolves.toBe("UnauthorizedException"); + await expect(allowed(invited.token, "workspace.members.manage")).resolves.toBe(true); + const renewed = await auth.reissue(session); + expect(m.session.verifySession(renewed.token)?.expiresAt).toBe(session.expiresAt); + await expect(allowed(renewed.token, "workspace.members.manage")).resolves.toBe(true); + await expect(auth.me(m.session.verifySession(renewed.token)!)).resolves.toMatchObject({ role: "Owner" }); + + const changed = await team.changeRole(session.workspaceId, actor, invited.member.id, "Approver"); + expect(changed.member?.role).toBe("Approver"); + + const members = await team.list(session.workspaceId); + const owner = members.find((mem) => mem.role === "Owner"); + if (owner === undefined) throw new Error("owner missing"); + await expect(team.changeRole(session.workspaceId, actor, owner.id, "Admin")).rejects.toMatchObject({ status: 400 }); + + // Invited (password-less) users cannot sign in; placeholder gets a derived username. + await expect(auth.login({ username: `pub-${suffix}`, password: "anything-123" })).rejects.toMatchObject({ status: 401 }); + + const audit = new m.audit.AuditQueryService(); + const first = await audit.page(session.workspaceId, { limit: 2 }); + expect(first.entries).toHaveLength(2); + expect(first.entries[0]?.action).toBe("membership.role_changed"); + expect(first.entries[0]?.diff).toMatchObject({ role: ["Publisher", "Approver"] }); + expect(first.entries[1]?.action).toBe("membership.invited"); + expect(first.nextCursor).not.toBeNull(); + const second = await audit.page(session.workspaceId, { limit: 50, before: first.nextCursor ?? undefined }); + expect(second.entries.map((e) => e.action)).toEqual( + expect.arrayContaining(["session.created", "workspace.created"]), + ); + const filtered = await audit.page(session.workspaceId, { limit: 50, action: "membership.invited" }); + expect(filtered.entries.every((e) => e.action === "membership.invited")).toBe(true); + expect(first.facets.actions).toContain("workspace.created"); + }); + + it("a member without the permission is forbidden", async () => { + const auth = new m.auth.AuthService(); + const username = `analyst${suffix}`; + const reg = await auth.register({ workspaceName: "Analyst WS", username, password: "correct-horse-1", displayName: "A" }); + const team = new m.team.TeamService(); + const session = m.session.verifySession(reg.token); + if (session === null) throw new Error("session did not verify"); + // Owner of their own workspace may manage members; prove a missing permission is refused. + await expect(allowed(reg.token, "permission.that.does.not.exist")).resolves.toBe("ForbiddenException"); + await expect(team.list(session.workspaceId)).resolves.toHaveLength(1); + }); +}); diff --git a/apps/api/src/auth/auth.service.ts b/apps/api/src/auth/auth.service.ts new file mode 100644 index 0000000..217bcd1 --- /dev/null +++ b/apps/api/src/auth/auth.service.ts @@ -0,0 +1,233 @@ +import { randomBytes } from "node:crypto"; +import { + BadRequestException, + ConflictException, + ForbiddenException, + Injectable, + NotFoundException, + UnauthorizedException, +} from "@nestjs/common"; +import type { PoolClient } from "pg"; +import { appendTenantAudit } from "../audit/audit.service.js"; +import { withSystem, withTenant } from "../db/client.js"; +import { issueSession, verifySession, type SessionPayload } from "./session.js"; +import { MIN_PASSWORD_LENGTH, burnPasswordCheck, hashPassword, verifyPassword } from "./password.js"; + +export interface MeResponse { + user: { id: string; username: string; email: string | null; displayName: string }; + workspace: { id: string; name: string; slug: string }; + role: string; + permissions: string[]; + expiresAt: number; +} + +export interface AuthResult { + token: string; + me: MeResponse; +} + +export interface RegisterInput { + workspaceName: string; + username: string; + password: string; + displayName: string; + email?: string; +} + +export interface LoginInput { + username: string; + password: string; + workspaceId?: string; +} + +const USERNAME = /^[a-z0-9][a-z0-9_.-]{1,31}$/; + +function normalizeUsername(raw: string): string { + const username = raw.trim().toLowerCase(); + if (!USERNAME.test(username)) { + throw new BadRequestException("username must be 2-32 characters: lowercase letters, digits, dot, underscore or hyphen, starting with a letter or digit"); + } + return username; +} + +/** + * First-party accounts are keyed `local:`; invited placeholders keep + * their `local:` key until an activation flow exists; OIDC users keep + * their issuer subject. + */ +export function localExternalId(handle: string): string { + return `local:${handle.trim().toLowerCase()}`; +} + +function slugify(name: string): string { + const base = name + .toLowerCase() + .normalize("NFKD") + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 40); + return `${base === "" ? "workspace" : base}-${randomBytes(3).toString("hex")}`; +} + +function requirePasswordStrength(password: string): void { + if (password.length < MIN_PASSWORD_LENGTH) { + throw new BadRequestException(`password must be at least ${MIN_PASSWORD_LENGTH} characters`); + } +} + +/** + * Re-issue a session for `userId` from the CURRENT membership row and epoch. + * Must run inside a tenant transaction for `workspaceId`; callers that just + * mutated memberships use it so their own session survives the epoch bump. + */ +export async function issueSessionInTenant( + client: PoolClient, workspaceId: string, userId: string, expiresAt?: number, +): Promise { + const membership = await client.query( + "SELECT role FROM memberships WHERE workspace_id = $1 AND user_id = $2 AND status = 'active'", + [workspaceId, userId], + ); + const role = membership.rows[0]?.role as string | undefined; + if (role === undefined) throw new ForbiddenException("no active membership in this workspace"); + const ws = await client.query("SELECT authz_epoch FROM workspaces WHERE id = $1", [workspaceId]); + const epoch = ws.rows[0]?.authz_epoch as string | number | undefined; + if (epoch === undefined) throw new NotFoundException("workspace not found"); + const remaining = expiresAt === undefined ? undefined : Math.max(0, expiresAt - Math.floor(Date.now() / 1000)); + return issueSession({ userId, workspaceId, role, authzEpoch: Number(epoch) }, remaining); +} + +@Injectable() +export class AuthService { + async register(input: RegisterInput): Promise { + requirePasswordStrength(input.password); + const passwordHash = await hashPassword(input.password); + const username = normalizeUsername(input.username); + const externalId = localExternalId(username); + + const created = await withSystem(async (client) => { + // Existing identities (including invited, password-less users) cannot be + // claimed by registering: without verified contact channels that would + // let anyone attach a password to someone else's pending invitation. + const user = await client.query( + `INSERT INTO users (external_id, username, email, display_name) VALUES ($1, $2, $3, $4) + ON CONFLICT (external_id) DO NOTHING + RETURNING id`, + [externalId, username, input.email ?? null, input.displayName], + ); + const userId = user.rows[0]?.id as string | undefined; + if (userId === undefined) throw new ConflictException("an account with this username already exists — sign in instead"); + await client.query("INSERT INTO user_passwords (user_id, password_hash) VALUES ($1, $2)", [userId, passwordHash]); + + const ws = await client.query("INSERT INTO workspaces (name, slug) VALUES ($1, $2) RETURNING id, slug", [ + input.workspaceName, + slugify(input.workspaceName), + ]); + const workspaceId = ws.rows[0]?.id as string; + // Enter the new tenant's context for the RLS-protected membership insert. + await client.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]); + await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [workspaceId]); + await client.query( + "INSERT INTO memberships (workspace_id, user_id, role, status) VALUES ($1, $2, 'Owner', 'active')", + [workspaceId, userId], + ); + const token = await issueSessionInTenant(client, workspaceId, userId); + return { userId, workspaceId, slug: ws.rows[0]?.slug as string, token }; + }); + + await appendTenantAudit(created.workspaceId, { + actorUserId: created.userId, + actorRole: "Owner", + action: "workspace.created", + targetType: "workspace", + targetId: created.workspaceId, + diff: { name: input.workspaceName, slug: created.slug }, + }); + return { token: created.token, me: await this.meFor(created.workspaceId, created.userId, created.token) }; + } + + async login(input: LoginInput): Promise { + const username = normalizeUsername(input.username); + const found = await withSystem((client) => + client.query( + `SELECT u.id, p.password_hash FROM users u + LEFT JOIN user_passwords p ON p.user_id = u.id + WHERE u.external_id = $1`, + [localExternalId(username)], + ), + ); + const row = found.rows[0] as { id: string; password_hash: string | null } | undefined; + if (row === undefined || row.password_hash === null) { + await burnPasswordCheck(input.password); + throw new UnauthorizedException("invalid email or password"); + } + if (!(await verifyPassword(input.password, row.password_hash))) { + throw new UnauthorizedException("invalid email or password"); + } + + const memberships = await withSystem((client) => + client.query("SELECT workspace_id, role, status FROM auth_user_memberships($1)", [row.id]), + ); + const active = (memberships.rows as Array<{ workspace_id: string; status: string }>).filter((m) => m.status === "active"); + const chosen = input.workspaceId === undefined ? active[0] : active.find((m) => m.workspace_id === input.workspaceId); + if (chosen === undefined) throw new ForbiddenException("no active workspace membership for this account"); + + const token = await withTenant(chosen.workspace_id, (client) => issueSessionInTenant(client, chosen.workspace_id, row.id)); + const me = await this.meFor(chosen.workspace_id, row.id, token); + await appendTenantAudit(chosen.workspace_id, { + actorUserId: row.id, + actorRole: me.role, + action: "session.created", + targetType: "user", + targetId: row.id, + diff: { method: "password" }, + }); + return { token, me }; + } + + async me(session: SessionPayload): Promise { + return this.meFor(session.workspaceId, session.userId, null, session.expiresAt); + } + + async reissue(session: SessionPayload): Promise { + let token: string; + try { + token = await withTenant(session.workspaceId, (client) => + issueSessionInTenant(client, session.workspaceId, session.userId, session.expiresAt), + ); + } catch (error) { + if (error instanceof ForbiddenException || error instanceof NotFoundException) { + throw new UnauthorizedException("membership no longer active"); + } + throw error; + } + return { token, me: await this.meFor(session.workspaceId, session.userId, token) }; + } + + private async meFor(workspaceId: string, userId: string, token: string | null, expiresAt?: number): Promise { + return withTenant(workspaceId, async (client) => { + const res = await client.query( + `SELECT m.role, u.username, u.email, u.display_name, w.name, w.slug + FROM memberships m + JOIN users u ON u.id = m.user_id + JOIN workspaces w ON w.id = m.workspace_id + WHERE m.workspace_id = $1 AND m.user_id = $2 AND m.status = 'active'`, + [workspaceId, userId], + ); + const row = res.rows[0] as { role: string; username: string; email: string | null; display_name: string; name: string; slug: string } | undefined; + if (row === undefined) throw new UnauthorizedException("membership no longer active"); + const perms = await client.query("SELECT permission FROM role_permissions WHERE role = $1 ORDER BY permission", [row.role]); + return { + user: { id: userId, username: row.username, email: row.email, displayName: row.display_name }, + workspace: { id: workspaceId, name: row.name, slug: row.slug }, + role: row.role, + permissions: (perms.rows as Array<{ permission: string }>).map((p) => p.permission), + expiresAt: expiresAt ?? expiryOf(token), + }; + }); + } +} + +function expiryOf(token: string | null): number { + if (token === null) return 0; + return verifySession(token)?.expiresAt ?? 0; +} diff --git a/apps/api/src/auth/oidc.test.ts b/apps/api/src/auth/oidc.test.ts new file mode 100644 index 0000000..2604e57 --- /dev/null +++ b/apps/api/src/auth/oidc.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { OidcClient } from "./oidc.js"; + +function fakeFetch(responses: Record): typeof fetch { + return (async (input: string | URL | unknown) => { + const url = String(input); + for (const [match, body] of Object.entries(responses)) { + if (url.includes(match)) return new Response(JSON.stringify(body), { status: 200 }); + } + return new Response("not found", { status: 404 }); + }) as unknown as typeof fetch; +} + +describe("oidc client", () => { + const config = { + issuer: "https://idp.example", + clientId: "client-1", + clientSecret: "secret-1", + redirectUri: "http://localhost:3000/auth/callback", + }; + + it("builds authorization URL with PKCE S256", async () => { + const client = new OidcClient(config, fakeFetch({ + "well-known": { + authorization_endpoint: "https://idp.example/auth", + token_endpoint: "https://idp.example/token", + }, + })); + const url = await client.authorizationUrl("state-123", "challenge-abc"); + const parsed = new URL(url); + expect(parsed.searchParams.get("code_challenge")).toBe("challenge-abc"); + expect(parsed.searchParams.get("code_challenge_method")).toBe("S256"); + expect(parsed.searchParams.get("state")).toBe("state-123"); + expect(parsed.searchParams.get("client_id")).toBe("client-1"); + }); + + it("exchanges code with verifier + client secret", async () => { + const holder: { params?: URLSearchParams } = {}; + const fetchMock = (async (input: string | URL | unknown, init?: RequestInit) => { + if (String(input).includes("token")) { + holder.params = new URLSearchParams(String(init?.body)); + return new Response(JSON.stringify({ access_token: "at", id_token: "it" }), { status: 200 }); + } + return new Response(JSON.stringify({ + authorization_endpoint: "https://idp.example/auth", + token_endpoint: "https://idp.example/token", + }), { status: 200 }); + }) as unknown as typeof fetch; + const client = new OidcClient(config, fetchMock); + const tokens = await client.exchangeCode("code-1", "verifier-xyz"); + expect(tokens.accessToken).toBe("at"); + expect(holder.params?.get("code_verifier")).toBe("verifier-xyz"); + expect(holder.params?.get("client_secret")).toBe("secret-1"); + }); +}); diff --git a/apps/api/src/auth/oidc.ts b/apps/api/src/auth/oidc.ts new file mode 100644 index 0000000..a7e2865 --- /dev/null +++ b/apps/api/src/auth/oidc.ts @@ -0,0 +1,78 @@ +/** + * Generic OIDC code flow with PKCE, discovery-document driven. + * Production: Authentik/Keycloak/any compliant issuer via env. + * Tests: mock the discovery + token endpoints via fetch override. + */ + +export interface OidcConfig { + issuer: string; + clientId: string; + clientSecret: string; + redirectUri: string; +} + +export function oidcConfigFromEnv(): OidcConfig | null { + const issuer = process.env.OIDC_ISSUER; + if (issuer === undefined || issuer === "") return null; + return { + issuer, + clientId: process.env.OIDC_CLIENT_ID ?? "", + clientSecret: process.env.OIDC_CLIENT_SECRET ?? "", + redirectUri: process.env.OIDC_REDIRECT_URI ?? "http://localhost:3000/auth/callback", + }; +} + +interface DiscoveryDoc { + authorization_endpoint: string; + token_endpoint: string; + userinfo_endpoint?: string; +} + +export type FetchLike = typeof fetch; + +export class OidcClient { + private discovery: DiscoveryDoc | null = null; + + constructor(private readonly config: OidcConfig, private readonly fetchImpl: FetchLike = fetch) {} + + private async discover(): Promise { + if (this.discovery !== null) return this.discovery; + const res = await this.fetchImpl(`${this.config.issuer.replace(/\/$/, "")}/.well-known/openid-configuration`); + if (!res.ok) throw new Error(`discovery failed: HTTP ${res.status}`); + this.discovery = (await res.json()) as DiscoveryDoc; + return this.discovery; + } + + authorizationUrl(state: string, codeChallenge: string): Promise { + return this.discover().then((d) => { + const url = new URL(d.authorization_endpoint); + url.searchParams.set("client_id", this.config.clientId); + url.searchParams.set("response_type", "code"); + url.searchParams.set("scope", "openid profile email"); + url.searchParams.set("redirect_uri", this.config.redirectUri); + url.searchParams.set("state", state); + url.searchParams.set("code_challenge", codeChallenge); + url.searchParams.set("code_challenge_method", "S256"); + return url.toString(); + }); + } + + async exchangeCode(code: string, codeVerifier: string): Promise<{ accessToken: string; idToken: string }> { + const d = await this.discover(); + const res = await this.fetchImpl(d.token_endpoint, { + method: "POST", + headers: { "content-type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams({ + grant_type: "authorization_code", + code, + client_id: this.config.clientId, + client_secret: this.config.clientSecret, + redirect_uri: this.config.redirectUri, + code_verifier: codeVerifier, + }), + }); + if (!res.ok) throw new Error(`token exchange failed: HTTP ${res.status}`); + const token = (await res.json()) as { access_token: string; id_token: string }; + return { accessToken: token.access_token, idToken: token.id_token }; + } +} diff --git a/apps/api/src/auth/password.test.ts b/apps/api/src/auth/password.test.ts new file mode 100644 index 0000000..b3d5727 --- /dev/null +++ b/apps/api/src/auth/password.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "vitest"; +import { hashPassword, verifyPassword } from "./password.js"; + +describe("password hashing", () => { + it("round-trips and rejects the wrong password", async () => { + const encoded = await hashPassword("correct-horse-battery"); + expect(encoded.startsWith("scrypt$")).toBe(true); + expect(encoded).not.toContain("correct-horse-battery"); + await expect(verifyPassword("correct-horse-battery", encoded)).resolves.toBe(true); + await expect(verifyPassword("correct-horse-batterz", encoded)).resolves.toBe(false); + }); + + it("salts every hash", async () => { + const [a, b] = await Promise.all([hashPassword("same-password-1"), hashPassword("same-password-1")]); + expect(a).not.toBe(b); + }); + + it("fails closed on malformed encodings", async () => { + await expect(verifyPassword("x", "")).resolves.toBe(false); + await expect(verifyPassword("x", "bcrypt$1$2$3$aa$bb")).resolves.toBe(false); + await expect(verifyPassword("x", "scrypt$0$8$1$aa$bb")).resolves.toBe(false); + }); +}); diff --git a/apps/api/src/auth/password.ts b/apps/api/src/auth/password.ts new file mode 100644 index 0000000..91e6955 --- /dev/null +++ b/apps/api/src/auth/password.ts @@ -0,0 +1,46 @@ +import { randomBytes, scrypt, timingSafeEqual } from "node:crypto"; + +/** + * scrypt password hashing. Encoded as `scrypt$N$r$p$salt$hash` (base64url) so + * cost parameters can be raised later without invalidating stored hashes. + */ + +const N = 16384; +const R = 8; +const P = 1; +const KEYLEN = 32; +export const MIN_PASSWORD_LENGTH = 10; + +function derive(password: string, salt: Buffer, n: number, r: number, p: number): Promise { + return new Promise((resolve, reject) => { + scrypt(password, salt, KEYLEN, { N: n, r, p, maxmem: 64 * 1024 * 1024 }, (err, key) => { + if (err !== null) reject(err); + else resolve(key); + }); + }); +} + +export async function hashPassword(password: string): Promise { + const salt = randomBytes(16); + const key = await derive(password, salt, N, R, P); + return `scrypt$${N}$${R}$${P}$${salt.toString("base64url")}$${key.toString("base64url")}`; +} + +export async function verifyPassword(password: string, encoded: string): Promise { + const parts = encoded.split("$"); + if (parts.length !== 6 || parts[0] !== "scrypt") return false; + const [n, r, p] = [Number(parts[1]), Number(parts[2]), Number(parts[3])]; + if (![n, r, p].every((v) => Number.isInteger(v) && v > 0)) return false; + const salt = Buffer.from(parts[4] ?? "", "base64url"); + const expected = Buffer.from(parts[5] ?? "", "base64url"); + if (expected.length !== KEYLEN) return false; + const actual = await derive(password, salt, n, r, p); + return timingSafeEqual(actual, expected); +} + +/** Burn equivalent CPU when the user does not exist, so login timing does not reveal account existence. */ +let dummyHash: Promise | undefined; +export async function burnPasswordCheck(password: string): Promise { + dummyHash ??= hashPassword("timing-equalizer-not-a-real-password"); + await verifyPassword(password, await dummyHash); +} diff --git a/apps/api/src/auth/reissue.test.ts b/apps/api/src/auth/reissue.test.ts new file mode 100644 index 0000000..c35a707 --- /dev/null +++ b/apps/api/src/auth/reissue.test.ts @@ -0,0 +1,46 @@ +import type { PoolClient } from "pg"; +import { describe, expect, it, vi } from "vitest"; +import { AuthService } from "./auth.service.js"; +import { issueSession, verifySession } from "./session.js"; + +const workspaceId = "10000000-0000-4000-8000-000000000001"; +const userId = "20000000-0000-4000-8000-000000000002"; +let role = "Publisher"; +let epoch = 1; +let active = true; + +vi.mock("../db/client.js", () => ({ + withTenant: async (_workspaceId: string, fn: (client: PoolClient) => Promise) => fn({ + query: async (sql: string) => { + if (sql.includes("SELECT role FROM memberships")) return { rows: active ? [{ role }] : [] }; + if (sql.includes("SELECT authz_epoch FROM workspaces")) return { rows: [{ authz_epoch: String(epoch) }] }; + if (sql.includes("SELECT m.role, u.username")) return { rows: active ? [{ role, username: "user", email: "user@example.com", display_name: "User", name: "Studio", slug: "studio" }] : [] }; + if (sql.includes("SELECT permission FROM role_permissions")) return { rows: [{ permission: "draft.view" }] }; + throw new Error(`unexpected SQL: ${sql}`); + }, + } as PoolClient), + withSystem: vi.fn(), +})); + +describe("session reissue", () => { + it("refreshes a stale epoch and live role without extending the session expiry", async () => { + active = true; + role = "Publisher"; + epoch = 1; + const old = verifySession(issueSession({ userId, workspaceId, role, authzEpoch: epoch })); + if (old === null) throw new Error("failed to issue test session"); + role = "Approver"; + epoch = 2; + const result = await new AuthService().reissue(old); + expect(result.me.role).toBe("Approver"); + expect(result.me.permissions).toContain("draft.view"); + expect(verifySession(result.token)).toMatchObject({ userId, workspaceId, role: "Approver", authzEpoch: 2, expiresAt: old.expiresAt }); + }); + + it("refuses an inactive membership", async () => { + active = false; + const old = verifySession(issueSession({ userId, workspaceId, role: "Publisher", authzEpoch: 1 })); + if (old === null) throw new Error("failed to issue test session"); + await expect(new AuthService().reissue(old)).rejects.toMatchObject({ status: 401 }); + }); +}); diff --git a/apps/api/src/auth/session.test.ts b/apps/api/src/auth/session.test.ts new file mode 100644 index 0000000..9c7b967 --- /dev/null +++ b/apps/api/src/auth/session.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import { issueSession, verifySession, epochValid } from "./session.js"; + +describe("session tokens", () => { + it("issues and verifies", () => { + const token = issueSession({ userId: "u1", workspaceId: "ws1", role: "Publisher", authzEpoch: 3 }); + const payload = verifySession(token); + expect(payload?.userId).toBe("u1"); + expect(payload?.workspaceId).toBe("ws1"); + }); + + it("rejects tampering", () => { + const token = issueSession({ userId: "u1", workspaceId: "ws1", role: "Publisher", authzEpoch: 3 }); + const tampered = `${token.slice(0, -3)}xxx`; + expect(verifySession(tampered)).toBeNull(); + }); + + it("epoch fence fails closed on mismatch (finding #2)", () => { + const token = issueSession({ userId: "u1", workspaceId: "ws1", role: "Publisher", authzEpoch: 3 }); + const payload = verifySession(token); + if (payload === null) throw new Error("unreachable"); + expect(epochValid(payload, 3)).toBe(true); + expect(epochValid(payload, 4)).toBe(false); + }); +}); diff --git a/apps/api/src/auth/session.ts b/apps/api/src/auth/session.ts new file mode 100644 index 0000000..f6c5d33 --- /dev/null +++ b/apps/api/src/auth/session.ts @@ -0,0 +1,60 @@ +import { createHmac, timingSafeEqual } from "node:crypto"; + +/** + * Session tokens: HMAC-signed, workspace-scoped, epoch-fenced. + * authzEpoch fail-closed: any membership/role change bumps the workspace + * epoch (see 001_tenants.sql trigger) and the session stops validating — + * this is the continuous-authz fix for audit finding #2. + */ + +export interface SessionPayload { + userId: string; + workspaceId: string; + role: string; + authzEpoch: number; + issuedAt: number; + expiresAt: number; +} + +function secret(): string { + const s = process.env.SESSION_SECRET; + if (s !== undefined && s !== "") return s; + if (process.env.NODE_ENV === "production") throw new Error("SESSION_SECRET required in production"); + return "dev-session-secret"; +} + +function sign(data: string): string { + return createHmac("sha256", secret()).update(data).digest("base64url"); +} + +export function issueSession(payload: Omit, ttlSeconds = 12 * 3600): string { + const full: SessionPayload = { + ...payload, + issuedAt: Math.floor(Date.now() / 1000), + expiresAt: Math.floor(Date.now() / 1000) + ttlSeconds, + }; + const body = Buffer.from(JSON.stringify(full)).toString("base64url"); + return `${body}.${sign(body)}`; +} + +export function verifySession(token: string): SessionPayload | null { + const dot = token.lastIndexOf("."); + if (dot === -1) return null; + const body = token.slice(0, dot); + const mac = token.slice(dot + 1); + const expected = sign(body); + if (mac.length !== expected.length) return null; + if (!timingSafeEqual(Buffer.from(mac), Buffer.from(expected))) return null; + try { + const payload = JSON.parse(Buffer.from(body, "base64url").toString("utf8")) as SessionPayload; + if (payload.expiresAt < Math.floor(Date.now() / 1000)) return null; + return payload; + } catch { + return null; + } +} + +/** Fail-closed epoch check: the caller passes the workspace's CURRENT epoch from the DB. */ +export function epochValid(session: SessionPayload, currentEpoch: number): boolean { + return session.authzEpoch === currentEpoch; +} diff --git a/apps/api/src/chatgov/chatgov.controller.ts b/apps/api/src/chatgov/chatgov.controller.ts new file mode 100644 index 0000000..be807fa --- /dev/null +++ b/apps/api/src/chatgov/chatgov.controller.ts @@ -0,0 +1,100 @@ +import { Body, Controller, Delete, Get, Inject, Param, Patch, Post, Query, Req, UseGuards } from "@nestjs/common"; +import type { Request } from "express"; +import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import { parseId, requireObject, requireSession } from "../http/validation.js"; +import { CHATGOV_PERMISSIONS } from "./policy.js"; +import { PolicyService, type PolicyActor, type RuleRecord } from "./policy.service.js"; +import { GateService, type GateDecisionRecord, type RecordGateInput } from "./gate.service.js"; + +/** + * Chat governance console API (§5): rules editor + gate decision reads. + * The command router (/poll, /tally, …) is P3 and does NOT live here. + * + * Permission model (§6): chatgov.manage gates every rule mutation; reads are + * workspace.view-shaped (any member sees the policy that governs them). + */ +@Controller("chatgov") +@UseGuards(RbacGuard) +export class ChatgovController { + constructor( + @Inject(PolicyService) private readonly policy: PolicyService, + @Inject(GateService) private readonly gates: GateService, + ) {} + + @Get("rules") + @RequirePermission("workspace.view") + async listRules( + @Req() request: Request & AuthenticatedRequest, + @Query("accountId") accountId: string | undefined, + @Query("topicBindingId") topicBindingId: string | undefined, + ): Promise { + return this.policy.listRules(requireSession(request).workspaceId, { + ...(accountId === undefined || accountId === "" ? {} : { accountId }), + ...(topicBindingId === undefined || topicBindingId === "" ? {} : { topicBindingId }), + }); + } + + @Post("rules") + @RequirePermission(CHATGOV_PERMISSIONS.manage) + async createRule(@Req() request: Request & AuthenticatedRequest, @Body() body: unknown): Promise { + return this.policy.createRule(requireSession(request).workspaceId, body, actorOf(request)); + } + + @Patch("rules/:id") + @RequirePermission(CHATGOV_PERMISSIONS.manage) + async updateRule( + @Req() request: Request & AuthenticatedRequest, + @Param("id") id: string, + @Body() body: unknown, + ): Promise { + return this.policy.updateRule(requireSession(request).workspaceId, parseId(id, "id"), body, actorOf(request)); + } + + @Delete("rules/:id") + @RequirePermission(CHATGOV_PERMISSIONS.manage) + async deleteRule(@Req() request: Request & AuthenticatedRequest, @Param("id") id: string): Promise<{ superseded: boolean }> { + return this.policy.deleteRule(requireSession(request).workspaceId, parseId(id, "id"), actorOf(request)); + } + + /** Blocked-queue / draft timeline feed (P6 renders it). */ + @Get("decisions/draft/:draftId") + @RequirePermission("draft.view") + async decisionsForDraft( + @Req() request: Request & AuthenticatedRequest, + @Param("draftId") draftId: string, + ): Promise { + return this.gates.listForDraft(requireSession(request).workspaceId, parseId(draftId, "draftId")); + } + + /** + * Internal write path for the Temporal E3 worker / E1 review hook (P1). + * Service-to-service in v1: the route exists so the seam is testable, and is + * gated on chatgov.manage (an Owner/Admin console action in practice). + */ + @Post("decisions") + @RequirePermission(CHATGOV_PERMISSIONS.manage) + async recordDecision(@Req() request: Request & AuthenticatedRequest, @Body() body: unknown): Promise { + const dto = requireObject(body, "body"); + const session = requireSession(request); + return this.gates.record(session.workspaceId, { + draftId: String(dto.draftId), + destination: dto.destination as RecordGateInput["destination"], + gate: dto.gate as RecordGateInput["gate"], + ...(dto.ruleId === undefined || dto.ruleId === null ? {} : { ruleId: String(dto.ruleId) }), + policyVersion: String(dto.policyVersion), + outcome: dto.outcome as RecordGateInput["outcome"], + inputs: (dto.inputs ?? {}) as RecordGateInput["inputs"], + reasonCodes: Array.isArray(dto.reasonCodes) ? (dto.reasonCodes as unknown[]).filter((c): c is RecordGateInput["reasonCodes"][number] => typeof c === "string") : [], + actor: typeof dto.actor === "string" ? dto.actor : `user:${session.userId}`, + ...(dto.pollId === undefined || dto.pollId === null ? {} : { pollId: String(dto.pollId) }), + }); + } +} + +function actorOf(request: Request & AuthenticatedRequest): PolicyActor { + const session = requireSession(request); + // The route decorator already enforced chatgov.manage; `permissions` is left + // unset so the service's defense-in-depth check treats the controller as the + // privileged caller (it re-checks only for non-controller callers). + return { userId: session.userId, role: session.role }; +} diff --git a/apps/api/src/chatgov/chatgov.integration.test.ts b/apps/api/src/chatgov/chatgov.integration.test.ts new file mode 100644 index 0000000..7af7700 --- /dev/null +++ b/apps/api/src/chatgov/chatgov.integration.test.ts @@ -0,0 +1,710 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { join } from "node:path"; +import { Pool, type PoolClient } from "pg"; + +/** + * chatgov P0 spine against a live PostgreSQL (chatgov-design.md §7 DoD): + * - migration idempotency (re-run applies nothing) + * - RLS denial for EVERY new table (workspace B sees/updates/deletes zero A + * rows, and B cannot write into A's workspace — WITH CHECK) + * - rule CRUD + one-live-rule-per-slot + * - policy compile -> posting_policy_snapshots with a deterministic version + * - RBAC permission gating (chatgov.manage; Editor is denied, Admin allowed) + * - audit event landing (policy.revised / policy.gate in audit_log) + * + * Owns its own database (CHATGOV_IT_DATABASE_URL + CHATGOV_IT_SUPER_DATABASE_URL, + * local-db-sol.env pattern) because the shared tenancy DB is wiped by other + * suites and vitest runs files in parallel. Skips cleanly when unset. + */ +const appUrl = process.env.CHATGOV_IT_DATABASE_URL; +const ownerUrl = process.env.CHATGOV_IT_SUPER_DATABASE_URL; +const dbAvailable = appUrl !== undefined && ownerUrl !== undefined; + +const RLS_VIOLATION = "42501"; +const UNIQUE_VIOLATION = "23505"; +const FK_VIOLATION = "23503"; +const TABLES = [ + "polls", + "poll_votes", + "posting_rules", + "posting_policy_snapshots", + "topic_bindings", + "chat_command_audit", + "chat_identities", + "gate_decisions", +] as const; + +type Modules = { + policy: typeof import("./policy.js"); + policyService: typeof import("./policy.service.js"); + gate: typeof import("./gate.service.js"); + db: typeof import("../db/client.js"); + guard: typeof import("../rbac/rbac.guard.js"); + session: typeof import("../auth/session.js"); +}; + +describe.skipIf(!dbAvailable)("chatgov P0 spine (live DB)", () => { + let m: Modules; + let ownerPool: Pool; + const suffix = `${Date.now()}${Math.floor(Math.random() * 1e6)}`; + let wsA: string; + let wsB: string; + let userA: string; + let connA1: string; + let connA2: string; + let connB: string; + let topicA: string; + let draftA: string; + + async function asOwner(fn: (client: PoolClient) => Promise): Promise { + const client = await ownerPool.connect(); + try { + return await fn(client); + } finally { + client.release(); + } + } + + /** Seed tenant rows honestly: FORCE RLS applies to the owner too. */ + async function inTenant(client: PoolClient, wsId: string, fn: () => Promise): Promise { + await client.query("BEGIN"); + await client.query("SELECT set_config('app.workspace_id', $1, true)", [wsId]); + await fn(); + await client.query("COMMIT"); + } + + beforeAll(async () => { + // The shared pool reads DATABASE_URL at import time: point it at this + // isolated DB first, then import the modules under test. + process.env.DATABASE_URL = appUrl; + process.env.SUPER_DATABASE_URL = ownerUrl; + const { runMigrations } = await import("../db/migrate.js"); + // Idempotency: the CLI already applied every migration; re-running must + // apply nothing (schema_migrations guards). + const applied = await runMigrations(join(process.cwd(), "migrations")); + expect(applied).toEqual([]); + + m = { + policy: await import("./policy.js"), + policyService: await import("./policy.service.js"), + gate: await import("./gate.service.js"), + db: await import("../db/client.js"), + guard: await import("../rbac/rbac.guard.js"), + session: await import("../auth/session.js"), + }; + ownerPool = new Pool({ connectionString: ownerUrl }); + + await asOwner(async (c) => { + const ws = async (name: string): Promise => + (await c.query("INSERT INTO workspaces (name, slug) VALUES ($1, $2) RETURNING id", [name, `${name}-${suffix}`])).rows[0].id as string; + wsA = await ws("cg-a"); + wsB = await ws("cg-b"); + const user = async (name: string): Promise => + (await c.query( + "INSERT INTO users (external_id, username, email, display_name) VALUES ($1, $2, $3, $4) RETURNING id", + [`cg:${name}:${suffix}`, `cg-${name}-${suffix}`, `${name}-${suffix}@example.com`, name], + )).rows[0].id as string; + userA = await user("owner"); + const editorA = await user("editor"); + + const conn = async (wsId: string, name: string): Promise => + (await c.query( + "INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-social', $2) RETURNING id", + [wsId, `ch-${name}`], + )).rows[0].id as string; + connA1 = await conn(wsA, "a1"); + connA2 = await conn(wsA, "a2"); + connB = await conn(wsB, "b1"); + + await inTenant(c, wsA, async () => { + await c.query("INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1, $2, 'Owner')", [wsA, userA]); + await c.query("INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1, $2, 'Editor')", [wsA, editorA]); + draftA = (await c.query("INSERT INTO drafts (workspace_id, author_user_id, post) VALUES ($1, $2, $3::jsonb) RETURNING id", [wsA, userA, JSON.stringify({ body: "x" })])).rows[0].id as string; + topicA = (await c.query( + `INSERT INTO topic_bindings (workspace_id, connection_id, chat_id, message_thread_id, binding_type, name, account_ids, icon_color) + VALUES ($1, $2, '-100200', '7', 'announce', 'Ops', $3, 7322096) RETURNING id`, + [wsA, connA1, [connA1]], + )).rows[0].id as string; + // Seed rows for the RLS denial suite (A-side data B must not see). + const poll = (await c.query( + `INSERT INTO polls (workspace_id, channel_id, topic_binding_id, subject_kind, draft_id, question, rule, policy_version, closes_at, created_by) + VALUES ($1, NULL, $2, 'draft', $3, 'ship it?', $4::jsonb, 'pv-seed', now() + interval '1 hour', $5) RETURNING id`, + [wsA, topicA, draftA, JSON.stringify({ version: 1, mode: "quorum", arms: [{ kind: "admins", x: 1 }], timebox: { duration: "PT1H", onExpire: "reject" } }), userA], + )).rows[0].id as string; + await c.query( + `INSERT INTO poll_votes (poll_id, workspace_id, voter_kind, voter_user_id, voter_remote_id, voter_handle, ballot) + VALUES ($1, $2, 'chat_identity', $3, '424242', 'opal', 'yes')`, + [poll, wsA, userA], + ); + await c.query( + `INSERT INTO chat_identities (workspace_id, provider, remote_user_id, user_id, display_name, status, approved_by) + VALUES ($1, 'telegram', '111222333', $2, 'Opal', 'approved', $2)`, + [wsA, userA], + ); + await c.query( + `INSERT INTO chat_command_audit (workspace_id, update_id, binding_id, provider, remote_user_id, user_id, command, args, permission, decision) + VALUES ($1, 9001, $2, 'telegram', '111222333', $3, '/tally', '{}'::jsonb, 'director.view', 'accepted')`, + [wsA, topicA, userA], + ); + // one row of every remaining table so the RLS sweep counts real data + await c.query( + `INSERT INTO posting_rules (workspace_id, scope, policy, policy_version, created_by) + VALUES ($1, 'workspace', $2::jsonb, 'pv-seed', $3)`, + [wsA, JSON.stringify({ version: 1, mode: "immediate" }), userA], + ); + await c.query( + `INSERT INTO posting_policy_snapshots (workspace_id, destination_kind, account_id, topic_binding_id, content_type, effective, policy_version, resolved_from) + VALUES ($1, 'account', $2, NULL, '', $3::jsonb, 'pv-seed', '{"scopes":["workspace"]}'::jsonb)`, + [wsA, connA1, JSON.stringify({ version: 1, mode: "immediate" })], + ); + await c.query( + `INSERT INTO gate_decisions (workspace_id, draft_id, destination, gate, policy_version, outcome, inputs, reason_codes, actor) + VALUES ($1, $2, $3::jsonb, 'E1', 'pv-seed', 'fired', '{}'::jsonb, '{}', 'system:gate-e1')`, + [wsA, draftA, JSON.stringify({ platform: "telegram", accountId: connA1 })], + ); + }); + await inTenant(c, wsB, async () => { + const ownerB = await user("owner-b"); + await c.query("INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1, $2, 'Owner')", [wsB, ownerB]); + }); + }); + }, 60_000); + + afterAll(async () => { + if (ownerPool !== undefined) { + await asOwner(async (c) => { + await c.query("DELETE FROM workspaces WHERE slug LIKE $1", [`%-${suffix}`]); + await c.query("DELETE FROM users WHERE external_id LIKE $1", [`cg:%:${suffix}`]); + }); + await ownerPool.end(); + } + await m?.db.pool.end(); + }); + + // ------------------------------------------------------------------------- + describe("migration + RBAC seed", () => { + it("idempotent: a second run applies nothing", async () => { + const { runMigrations } = await import("../db/migrate.js"); + const previous = process.env.SUPER_DATABASE_URL; + process.env.SUPER_DATABASE_URL = ownerUrl; + try { + expect(await runMigrations(join(process.cwd(), "migrations"))).toEqual([]); + } finally { + process.env.SUPER_DATABASE_URL = previous; + } + }); + + it("chatgov permissions are seeded per the 002 pattern", async () => { + const rows = await asOwner((c) => c.query<{ role: string; permission: string }>( + "SELECT role, permission FROM role_permissions WHERE permission LIKE 'chatgov%' ORDER BY role, permission"), + ); + const map = new Map(); + for (const row of rows.rows) { + map.set(row.role, [...(map.get(row.role) ?? []), row.permission]); + } + expect(map.get("Owner")).toEqual(["chatgov.manage", "chatgov.vote"]); + expect(map.get("Admin")).toEqual(["chatgov.manage", "chatgov.vote"]); + for (const role of ["Publisher", "Editor", "Approver", "Moderator", "BroadcastOperator"]) { + expect(map.get(role)).toEqual(["chatgov.vote"]); + } + for (const role of ["Analyst", "VideoPublisher", "StreamOperator"]) { + expect(map.has(role)).toBe(false); + } + }); + + it("every new table is RLS-enabled AND forced with a tenant_isolation policy", async () => { + for (const table of TABLES) { + const flags = await asOwner((c) => c.query<{ relrowsecurity: boolean; relforcerowsecurity: boolean }>( + "SELECT relrowsecurity, relforcerowsecurity FROM pg_class WHERE oid = $1::regclass", [table])); + expect(flags.rows[0], table).toEqual({ relrowsecurity: true, relforcerowsecurity: true }); + const policies = await asOwner((c) => c.query<{ policyname: string }>( + "SELECT policyname FROM pg_policies WHERE tablename = $1", [table])); + expect(policies.rows.map((r) => r.policyname), table).toContain("tenant_isolation"); + } + }); + + it("sp_app holds table grants (and is neither superuser nor BYPASSRLS)", async () => { + for (const table of TABLES) { + const grants = await asOwner((c) => c.query<{ privilege_type: string }>( + "SELECT privilege_type FROM information_schema.role_table_grants WHERE grantee = 'sp_app' AND table_name = $1", [table])); + expect(new Set(grants.rows.map((r) => r.privilege_type)), table) + .toEqual(new Set(["SELECT", "INSERT", "UPDATE", "DELETE"])); + } + const role = await asOwner((c) => c.query<{ rolsuper: boolean; rolbypassrls: boolean }>( + "SELECT rolsuper, rolbypassrls FROM pg_roles WHERE rolname = 'sp_app'")); + expect(role.rows[0]).toEqual({ rolsuper: false, rolbypassrls: false }); + }); + }); + + /** Fresh workspace with its own connections/topic (isolated rule slots). */ + async function seedWorkspace(name: string): Promise<{ ws: string; conn1: string; conn2: string; topic: string }> { + return asOwner(async (c) => { + const ws = (await c.query("INSERT INTO workspaces (name, slug) VALUES ($1, $2) RETURNING id", [`cg-${name}`, `cg-${name}-${suffix}`])).rows[0].id as string; + let conn1 = ""; + let conn2 = ""; + let topic = ""; + await inTenant(c, ws, async () => { + await c.query("INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1, $2, 'Owner')", [ws, userA]); + conn1 = (await c.query("INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-social', $2) RETURNING id", [ws, `ch-${name}-1`])).rows[0].id as string; + conn2 = (await c.query("INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-social', $2) RETURNING id", [ws, `ch-${name}-2`])).rows[0].id as string; + topic = (await c.query( + `INSERT INTO topic_bindings (workspace_id, connection_id, chat_id, message_thread_id, binding_type, name, account_ids) + VALUES ($1, $2, '-100200', '7', 'announce', 'Ops', $3) RETURNING id`, + [ws, conn1, [conn1]])).rows[0].id as string; + }); + return { ws, conn1, conn2, topic }; + }); + } + + // ------------------------------------------------------------------------- + describe("RLS denial suite (FORCE + NULLIF; B must see zero A rows)", () => { + it.each(TABLES)("%s: B sees, updates and deletes none of A's rows", async (table) => { + const seen = await m.db.withTenant(wsB, (c) => c.query(`SELECT count(*)::int AS n FROM ${table} WHERE workspace_id = $1`, [wsA])); + expect(seen.rows[0]?.n).toBe(0); + const noFilter = await m.db.withTenant(wsB, (c) => c.query(`SELECT count(*)::int AS n FROM ${table}`)); + expect(noFilter.rows[0]?.n).toBe(0); + // A harmless mutable column per table (several have no updated_at). + const touch: Record<(typeof TABLES)[number], string> = { + polls: "outbox_state = outbox_state", + poll_votes: "ballot = ballot", + posting_rules: "dry_run = dry_run", + posting_policy_snapshots: "policy_version = policy_version", + topic_bindings: "name = name", + chat_command_audit: "reason = reason", + chat_identities: "display_name = display_name", + gate_decisions: "actor = actor", + }; + const updated = await m.db.withTenant(wsB, (c) => c.query(`UPDATE ${table} SET ${touch[table]} WHERE workspace_id = $1`, [wsA])); + expect(updated.rowCount).toBe(0); + const deleted = await m.db.withTenant(wsB, (c) => c.query(`DELETE FROM ${table} WHERE workspace_id = $1`, [wsA])); + expect(deleted.rowCount).toBe(0); + const ownStillThere = await m.db.withTenant(wsA, (c) => c.query(`SELECT count(*)::int AS n FROM ${table}`)); + expect(ownStillThere.rows[0]?.n).toBeGreaterThan(0); + }); + + it("no tenant context (and an empty one) sees nothing on any table", async () => { + for (const table of TABLES) { + const none = await m.db.withSystem((c) => c.query(`SELECT count(*)::int AS n FROM ${table}`)); + expect(none.rows[0]?.n, table).toBe(0); + } + for (const table of TABLES) { + const leaked = await m.db.withTenant(wsB, async (c) => { + // empty-string GUC (004 pooled-session hazard) must fail closed too + await c.query("SELECT set_config('app.workspace_id', '', true)"); + return c.query(`SELECT count(*)::int AS n FROM ${table}`); + }); + expect(leaked.rows[0]?.n, table).toBe(0); + } + }); + + it("B cannot write rows into A's workspace (WITH CHECK) on representative tables", async () => { + const attempts: Array<[string, unknown[]]> = [ + ["INSERT INTO posting_rules (workspace_id, scope, policy, policy_version, created_by) VALUES ($1, 'workspace', '{}'::jsonb, 'pv', $2)", [wsA, userA]], + ["INSERT INTO gate_decisions (workspace_id, draft_id, destination, gate, policy_version, outcome, inputs, reason_codes, actor) VALUES ($1, $2, '{}'::jsonb, 'E1', 'pv', 'pending', '{}'::jsonb, '{}', 'system:gate')", [wsA, draftA]], + ["INSERT INTO chat_identities (workspace_id, provider, remote_user_id) VALUES ($1, 'telegram', '999')", [wsA]], + ["INSERT INTO topic_bindings (workspace_id, connection_id, chat_id, binding_type, name) VALUES ($1, $2, '-100999', 'announce', 'Sneaky')", [wsA, connA1]], + ["INSERT INTO chat_command_audit (workspace_id, update_id, provider, remote_user_id, command, decision) VALUES ($1, 42, 'telegram', '999', '/x', 'rejected')", [wsA]], + ["INSERT INTO polls (workspace_id, subject_kind, draft_id, question, rule, policy_version, closes_at, created_by) VALUES ($1, 'draft', $2, 'q?', '{}'::jsonb, 'pv', now() + interval '1 hour', $3)", [wsA, draftA, userA]], + ]; + for (const [sql, params] of attempts) { + await expect(m.db.withTenant(wsB, (c) => c.query(sql, params)), sql).rejects.toMatchObject({ code: RLS_VIOLATION }); + } + }); + + it("cross-tenant composite FKs: B's rows cannot point at A's parents", async () => { + await expect(m.db.withTenant(wsB, async (c) => c.query( + "INSERT INTO topic_bindings (workspace_id, connection_id, chat_id, binding_type, name) VALUES ($1, $2, '-100888', 'announce', 'X')", + [wsB, connA1], + ))).rejects.toMatchObject({ code: FK_VIOLATION }); + await expect(m.db.withTenant(wsB, async (c) => c.query( + "INSERT INTO posting_rules (workspace_id, scope, account_id, policy, policy_version, created_by) VALUES ($1, 'account', $2, '{}'::jsonb, 'pv', $3)", + [wsB, connA1, userA], + ))).rejects.toMatchObject({ code: FK_VIOLATION }); + }); + }); + + // ------------------------------------------------------------------------- + describe("rule CRUD + compile-to-snapshot", () => { + it("create rejects a second live rule in the same slot (unique index)", async () => { + const service = new m.policyService.PolicyService(); + const actor = { userId: userA, role: "Owner" }; + const body = { scope: "account", accountId: connA1, policy: { mode: "admin_only", timebox: { duration: "PT1H", onExpire: "reject" } } }; + await expect(service.createRule(wsA, body, actor)).resolves.toBeTruthy(); + await expect(service.createRule(wsA, body, actor)).rejects.toMatchObject({ status: 409 }); + // a different slot is fine + await expect(service.createRule(wsA, { ...body, accountId: connA2 }, actor)).resolves.toBeTruthy(); + }); + + it("slot validation refuses nonsense shapes with 400s", async () => { + const service = new m.policyService.PolicyService(); + const actor = { userId: userA, role: "Owner" }; + const bad: unknown[] = [ + { scope: "workspace", accountId: connA1, policy: { mode: "immediate" } }, + { scope: "account", policy: { mode: "immediate" } }, + { scope: "topic", policy: { mode: "immediate" } }, + { scope: "content_type", policy: { mode: "immediate" } }, + { scope: "topic", topicBindingId: topicA, contentType: "alert", policy: { mode: "immediate" } }, + { scope: "mystery", policy: { mode: "immediate" } }, + { scope: "workspace", policy: { mode: "nope" } }, + ]; + for (const body of bad) { + await expect(service.createRule(wsA, body, actor), JSON.stringify(body)).rejects.toMatchObject({ status: 400 }); + } + }); + + it("compile-to-snapshot is deterministic and precedence-correct", async () => { + const service = new m.policyService.PolicyService(); + const actor = { userId: userA, role: "Owner" }; + const { ws, conn1, conn2, topic } = await seedWorkspace("compile"); + // workspace default + await service.createRule(ws, { scope: "workspace", policy: { mode: "immediate" } }, actor); + // account override for conn1 + const accountRule = await service.createRule(ws, { + scope: "account", accountId: conn1, + policy: { mode: "admin_only", timebox: { duration: "PT1H", onExpire: "reject" } }, + }, actor); + // topic override (highest precedence) for the topic bound to conn1 + await service.createRule(ws, { + scope: "topic", topicBindingId: topic, + policy: { mode: "quorum", arms: [{ kind: "admins", x: 2 }], timebox: { duration: "PT2H", onExpire: "reject" } }, + }, actor); + // content_type override + await service.createRule(ws, { + scope: "content_type", contentType: "alert", + policy: { mode: "admin_only", timebox: { duration: "PT30M", onExpire: "escalateOwner" } }, + }, actor); + + const snapshots = await m.db.withTenant(ws, (c) => c.query<{ + destination_kind: string; account_id: string; topic_binding_id: string | null; content_type: string; + effective: { mode: string }; policy_version: string; + }>("SELECT destination_kind, account_id, topic_binding_id, content_type, effective, policy_version FROM posting_policy_snapshots ORDER BY destination_kind, content_type")); + + const byKey = new Map(snapshots.rows.map((r) => [`${r.destination_kind}|${r.account_id}|${r.topic_binding_id ?? ""}|${r.content_type}`, r])); + const vImmediate = m.policy.policyHash(m.policy.parsePolicyRule({ mode: "immediate" })); + const vAdmin1h = m.policy.policyHash(m.policy.parsePolicyRule({ mode: "admin_only", timebox: { duration: "PT1H", onExpire: "reject" } })); + const vQuorum = m.policy.policyHash(m.policy.parsePolicyRule({ mode: "quorum", arms: [{ kind: "admins", x: 2 }], timebox: { duration: "PT2H", onExpire: "reject" } })); + const vAlert = m.policy.policyHash(m.policy.parsePolicyRule({ mode: "admin_only", timebox: { duration: "PT30M", onExpire: "escalateOwner" } })); + + // account root of connA1: account rule beats the workspace default + expect(byKey.get(`account|${conn1}||`)?.policy_version).toBe(vAdmin1h); + // account root of connA2 (no account rule): workspace default + expect(byKey.get(`account|${conn2}||`)?.policy_version).toBe(vImmediate); + // topic slot of connA1: topic rule beats account + expect(byKey.get(`topic|${conn1}|${topic}|`)?.policy_version).toBe(vQuorum); + // typed slot: content_type rule beats workspace default + expect(byKey.get(`account|${conn1}||alert`)?.policy_version).toBe(vAlert); + // whole-block replacement: the winner IS the policy (no field merges) + expect(byKey.get(`topic|${conn1}|${topic}|`)?.effective.mode).toBe("quorum"); + expect(byKey.get(`account|${conn1}||`)?.effective).not.toHaveProperty("arms"); + + // Determinism at the DB level: an idempotent re-save with the SAME + // policy must not change the persisted version. + const before = byKey.get(`account|${conn1}||`)?.policy_version; + await service.updateRule(ws, accountRule.id, { policy: { mode: "admin_only", timebox: { duration: "PT1H", onExpire: "reject" } } }, actor); + const after = await m.db.withTenant(ws, (c) => c.query<{ policy_version: string }>( + "SELECT policy_version FROM posting_policy_snapshots WHERE destination_kind = 'account' AND account_id = $1 AND topic_binding_id IS NULL AND content_type = ''", [conn1])); + expect(after.rows[0]!.policy_version).toBe(before); + // and the DB agrees with the pure compiler on the same inputs + const compiled = m.policy.compilePolicy( + [ + { scope: "workspace", rule: m.policy.parsePolicyRule({ mode: "immediate" }) }, + { scope: "account", rule: m.policy.parsePolicyRule({ mode: "admin_only", timebox: { duration: "PT1H", onExpire: "reject" } }) }, + ], + { adminCount: 0, observedParticipants: 0, eligibleParticipants: 0 }, + ); + expect(after.rows[0]!.policy_version).toBe(compiled.policy_version); + // the snapshot rows live in ws only: invisible from wsA (RLS) + const inA = await m.db.withTenant(wsA, (c) => c.query<{ n: number }>( + "SELECT count(*)::int AS n FROM posting_policy_snapshots WHERE account_id = $1", [conn1])); + expect(inA.rows[0]!.n).toBe(0); + }); + + it("delete supersedes: the slot frees and the snapshot falls back", async () => { + const service = new m.policyService.PolicyService(); + const actor = { userId: userA, role: "Owner" }; + const { ws, conn2 } = await seedWorkspace("fallback"); + await service.createRule(ws, { scope: "account", accountId: conn2, policy: { mode: "admin_only", timebox: { duration: "PT1H", onExpire: "reject" } } }, actor); + const during = await m.db.withTenant(ws, (c) => c.query<{ policy_version: string }>( + "SELECT policy_version FROM posting_policy_snapshots WHERE account_id = $1 AND topic_binding_id IS NULL AND content_type = ''", [conn2])); + expect(during.rows[0]!.policy_version).not.toBe(m.policy.policyHash(m.policy.parsePolicyRule({ mode: "immediate" }))); + + const live = await service.listRules(ws, { accountId: conn2 }); + await expect(service.deleteRule(ws, live[0]!.id, actor)).resolves.toEqual({ superseded: true }); + // slot freed: a new rule fits without conflict + const recreated = await service.createRule(ws, { scope: "account", accountId: conn2, policy: { mode: "immediate" } }, actor); + expect(recreated.supersededBy).toBeNull(); + const after = await m.db.withTenant(ws, (c) => c.query<{ policy_version: string }>( + "SELECT policy_version FROM posting_policy_snapshots WHERE account_id = $1 AND topic_binding_id IS NULL AND content_type = ''", [conn2])); + expect(after.rows[0]!.policy_version).toBe(m.policy.policyHash(m.policy.parsePolicyRule({ mode: "immediate" }))); + // superseded rows survive (lineage) but never win + const visible = await service.listRules(ws, { accountId: conn2 }); + expect(visible).toHaveLength(1); + }); + + it("listRules filters by account/topic and hides superseded rows", async () => { + const service = new m.policyService.PolicyService(); + const actor = { userId: userA, role: "Owner" }; + const { ws, conn1, topic } = await seedWorkspace("listing"); + await service.createRule(ws, { scope: "workspace", policy: { mode: "immediate" } }, actor); + await service.createRule(ws, { scope: "account", accountId: conn1, policy: { mode: "admin_only", timebox: { duration: "PT1H", onExpire: "reject" } } }, actor); + await service.createRule(ws, { scope: "topic", topicBindingId: topic, policy: { mode: "quorum", arms: [{ kind: "admins", x: 2 }], timebox: { duration: "PT2H", onExpire: "reject" } } }, actor); + const all = await service.listRules(ws); + expect(all.every((r) => r.supersededBy === null)).toBe(true); + expect(all).toHaveLength(3); + const forTopic = await service.listRules(ws, { topicBindingId: topic }); + expect(forTopic.map((r) => r.scope)).toEqual(["topic"]); + const forAccount = await service.listRules(ws, { accountId: conn1 }); + expect(forAccount.map((r) => r.scope)).toEqual(["account"]); + }); + }); + + // ------------------------------------------------------------------------- + describe("gate_decisions writer", () => { + it("persists outcomes with codes + quorum state and audits them", async () => { + const service = new m.gate.GateService(); + const decision = await service.record(wsA, { + draftId: draftA, + destination: { platform: "telegram", accountId: connA1 }, + gate: "E2", + policyVersion: "pv-e2", + outcome: "blocked", + inputs: { arms: [{ kind: "admins", got: 1, needed: 2 }, { kind: "participants", got: 3, needed: 5 }], pollState: "closed" }, + reasonCodes: ["INSUFFICIENT_QUORUM"], + actor: "chat_poll:abc", + }); + expect(decision.outcome).toBe("blocked"); + expect(decision.reasonCodes).toEqual(["INSUFFICIENT_QUORUM"]); + expect(decision.inputs.arms).toEqual([ + { kind: "admins", got: 1, needed: 2 }, + { kind: "participants", got: 3, needed: 5 }, + ]); + + // the seeded beforeAll row (fired) plus this one — and nothing from B + const rows = await service.listForDraft(wsA, draftA); + expect(rows.map((r) => r.id)).toContain(decision.id); + expect(rows.every((r) => r.workspaceId === wsA)).toBe(true); + + // audit landed: policy.gate on the tenant chain + const audited = await m.db.withTenant(wsA, (c) => c.query<{ action: string; target_type: string; target_id: string; diff: { outcome?: string } }>( + "SELECT action, target_type, target_id, diff FROM audit_log WHERE action = 'policy.gate' AND target_id = $1", [draftA])); + expect(audited.rows).toHaveLength(1); + expect(audited.rows[0]!.target_type).toBe("draft"); + expect(audited.rows[0]!.diff.outcome).toBe("blocked"); + + // resolve: pending -> fired in place (one row per evaluation) + const pending = await service.record(wsA, { + draftId: draftA, destination: { platform: "telegram", accountId: connA2 }, gate: "E3", + policyVersion: "pv-e3", outcome: "pending", inputs: { approvals: 0 }, reasonCodes: ["PENDING_POLL"], actor: "user:x", + }); + const resolved = await service.resolve(wsA, pending.id, "fired", [], "system:gate-e3"); + expect(resolved.outcome).toBe("fired"); + expect(resolved.id).toBe(pending.id); + const audited2 = await m.db.withTenant(wsA, (c) => c.query<{ n: number }>( + "SELECT count(*)::int AS n FROM audit_log WHERE action = 'policy.gate.resolved'")); + expect(audited2.rows[0]!.n).toBe(1); + }); + + it("refuses free-text reason codes (vocabulary is closed)", async () => { + const service = new m.gate.GateService(); + await expect(service.record(wsA, { + draftId: draftA, destination: { platform: "telegram", accountId: connA1 }, gate: "E1", + policyVersion: "pv", outcome: "blocked", inputs: {}, reasonCodes: ["because i said so"] as never[], actor: "user:x", + })).rejects.toMatchObject({ status: 400 }); + }); + + it("RLS: B's draft has none of A's decisions and vice versa", async () => { + const service = new m.gate.GateService(); + // seed a B-side draft + const draftB = await asOwner(async (c) => { + const ownerB = (await c.query<{ id: string }>("SELECT u.id FROM users u JOIN memberships m ON m.user_id = u.id WHERE m.workspace_id = $1 AND m.role = 'Owner' LIMIT 1", [wsB])).rows[0]!.id; + let id = ""; + await inTenant(c, wsB, async () => { + id = (await c.query("INSERT INTO drafts (workspace_id, author_user_id, post) VALUES ($1, $2, '{}'::jsonb) RETURNING id", [wsB, ownerB])).rows[0].id as string; + }); + return id; + }); + await service.record(wsB, { draftId: draftB, destination: { platform: "telegram", accountId: connB }, gate: "E1", policyVersion: "pv", outcome: "fired", inputs: {}, reasonCodes: [], actor: "user:y" }); + const forA = await service.listForDraft(wsA, draftA); + expect(forA.every((d) => d.workspaceId === wsA)).toBe(true); + const forB = await service.listForDraft(wsB, draftB); + expect(forB.every((d) => d.workspaceId === wsB)).toBe(true); + expect(forB.map((d) => d.destination.accountId)).toEqual([connB]); + }); + }); + + // ------------------------------------------------------------------------- + describe("audit chain", () => { + it("policy.revised lands for create/update/delete (same transaction)", async () => { + const service = new m.policyService.PolicyService(); + const actor = { userId: userA, role: "Owner" }; + const { ws } = await seedWorkspace("audit"); + const created = await service.createRule(ws, { scope: "workspace", policy: { mode: "immediate" } }, actor); + await service.updateRule(ws, created.id, { policy: { mode: "admin_only", timebox: { duration: "PT1H", onExpire: "reject" } } }, actor); + await service.deleteRule(ws, created.id, actor); + + const events = await m.db.withTenant(ws, (c) => c.query<{ action: string; diff: { action?: string } }>( + "SELECT action, diff FROM audit_log WHERE action = 'policy.revised' ORDER BY seq")); + expect(events.rows.map((r) => r.diff.action)).toEqual(["create", "update", "delete"]); + // tamper-evident chain still verifies with the new events + const ok = await m.db.withTenant(ws, (c) => c.query<{ ok: boolean }>("SELECT audit_verify_chain($1) AS ok", [ws])); + expect(ok.rows[0]?.ok).toBe(true); + // the live view is empty in the same transaction's aftermath + const live = await service.listRules(ws); + expect(live).toHaveLength(0); + }); + }); + + // ------------------------------------------------------------------------- + describe("RBAC permission gating", () => { + it("service refuses non-privileged callers without chatgov.manage", async () => { + const service = new m.policyService.PolicyService(); + // Editor holds chatgov.vote but NOT chatgov.manage + const editor = { userId: userA, role: "Editor", permissions: ["chatgov.vote", "workspace.view"] }; + await expect(service.createRule(wsA, { scope: "workspace", policy: { mode: "immediate" } }, editor)) + .rejects.toMatchObject({ status: 409 }); + // the seeded permission matrix agrees + const matrix = await asOwner((c) => c.query<{ role: string }>( + "SELECT role FROM role_permissions WHERE permission = 'chatgov.manage'")); + expect(matrix.rows.map((r) => r.role).sort()).toEqual(["Admin", "Owner"]); + }); + + it("RbacGuard enforces chatgov.manage on the controller route (Editor 403, Admin 200)", async () => { + const { Reflector } = await import("@nestjs/core"); + const guard = new m.guard.RbacGuard(new Reflector()); + const editor = (await asOwner((c) => c.query<{ id: string }>( + "SELECT u.id FROM users u WHERE u.external_id = $1", [`cg:editor:${suffix}`]))).rows[0]!.id; + const epoch = await asOwner(async (c) => Number((await c.query("SELECT authz_epoch FROM workspaces WHERE id = $1", [wsA])).rows[0]!.authz_epoch)); + const token = (userId: string, role: string): string => m.session.issueSession({ userId, workspaceId: wsA, role, authzEpoch: epoch }); + const ctx = (t: string) => ({ + getHandler: () => { + const handler = (): void => undefined; + Reflect.defineMetadata(m.guard.PERMISSION_KEY, "chatgov.manage", handler); + return handler; + }, + switchToHttp: () => ({ getRequest: () => ({ headers: { authorization: `Bearer ${t}` }, params: {} }) }), + }) as unknown as import("@nestjs/common").ExecutionContext; + await expect(guard.canActivate(ctx(token(editor, "Editor")))).rejects.toMatchObject({ status: 403 }); + await expect(guard.canActivate(ctx(token(userA, "Admin")))).resolves.toBe(true); + }); + }); + + // ------------------------------------------------------------------------- + describe("invariants: one open poll per draft, vote idempotency, replay dedupe", () => { + it("a second OPEN poll for the same draft is refused; a closed one frees the gate", async () => { + const openPoll = async (): Promise => (await m.db.withTenant(wsA, (c) => c.query<{ id: string }>( + `INSERT INTO polls (workspace_id, subject_kind, draft_id, question, rule, policy_version, closes_at, created_by) + VALUES ($1, 'draft', $2, 'again?', '{}'::jsonb, 'pv2', now() + interval '30 minutes', $3) RETURNING id`, + [wsA, draftA, userA]))).rows[0]!.id; + // the beforeAll seed already holds an OPEN poll for draftA + await expect(m.db.withTenant(wsA, (c) => c.query<{ id: string }>( + `SELECT id FROM polls WHERE subject_kind = 'draft' AND draft_id = $1 AND state = 'open'`, [draftA]))).resolves.toBeTruthy(); + await expect(openPoll()).rejects.toMatchObject({ code: UNIQUE_VIOLATION }); + // close it (the poll service P2 owns the real transition; here: direct) + const existing = (await m.db.withTenant(wsA, (c) => c.query<{ id: string }>( + "SELECT id FROM polls WHERE draft_id = $1 AND state = 'open'", [draftA]))).rows[0]!.id; + await m.db.withTenant(wsA, (c) => c.query( + "UPDATE polls SET state = 'failed', closed_at = now(), close_reason = 'timebox', outbox_state = 'done' WHERE id = $1", [existing])); + await expect(openPoll()).resolves.toBeTruthy(); + }); + + it("revote lineage: parent_poll_id must stay inside the workspace (composite FK)", async () => { + const parent = (await m.db.withTenant(wsA, (c) => c.query<{ id: string }>( + "SELECT id FROM polls WHERE workspace_id = $1 ORDER BY created_at LIMIT 1", [wsA]))).rows[0]!.id; + const ownerB = (await asOwner((c) => c.query<{ id: string }>( + "SELECT u.id FROM users u JOIN memberships m ON m.user_id = u.id WHERE m.workspace_id = $1 AND m.role = 'Owner' LIMIT 1", [wsB]))).rows[0]!.id; + await expect(m.db.withTenant(wsB, (c) => c.query( + `INSERT INTO polls (workspace_id, subject_kind, question, rule, policy_version, closes_at, created_by, parent_poll_id) + VALUES ($1, 'none', 'q', '{}'::jsonb, 'pv', now() + interval '1 hour', $2, $3)`, + [wsB, ownerB, parent]))).rejects.toMatchObject({ code: FK_VIOLATION }); + }); + + it("poll_votes: one ballot per (poll, voter) — recast replaces, never duplicates", async () => { + const poll = (await m.db.withTenant(wsA, (c) => c.query<{ id: string }>( + `INSERT INTO polls (workspace_id, subject_kind, question, rule, policy_version, closes_at, created_by) + VALUES ($1, 'none', 'ballot test?', '{}'::jsonb, 'pv-b', now() + interval '1 hour', $2) RETURNING id`, + [wsA, userA]))).rows[0]!.id; + const cast = (ballot: string): Promise => m.db.withTenant(wsA, (c) => c.query( + `INSERT INTO poll_votes (poll_id, workspace_id, voter_kind, voter_remote_id, voter_handle, ballot) + VALUES ($1, $2, 'chat_identity', '777', 'someone', $3)`, + [poll, wsA, ballot])); + await cast("no"); + await expect(cast("yes")).rejects.toMatchObject({ code: UNIQUE_VIOLATION }); + await m.db.withTenant(wsA, (c) => c.query( + `INSERT INTO poll_votes (poll_id, workspace_id, voter_kind, voter_remote_id, voter_handle, ballot) + VALUES ($1, $2, 'chat_identity', '777', 'someone', 'yes') + ON CONFLICT (poll_id, voter_remote_id) DO UPDATE SET ballot = EXCLUDED.ballot, cast_at = now()`, + [poll, wsA])); + const rows = await m.db.withTenant(wsA, (c) => c.query<{ ballot: string; count: string }>( + "SELECT ballot, count(*) AS count FROM poll_votes WHERE poll_id = $1 GROUP BY ballot", [poll])); + expect(rows.rows).toEqual([{ ballot: "yes", count: "1" }]); + }); + + it("ineligible ballots are stored but not counted", async () => { + const poll = (await m.db.withTenant(wsA, (c) => c.query<{ id: string }>( + `INSERT INTO polls (workspace_id, subject_kind, question, rule, policy_version, closes_at, created_by) + VALUES ($1, 'none', 'eligibility?', '{}'::jsonb, 'pv-c', now() + interval '1 hour', $2) RETURNING id`, + [wsA, userA]))).rows[0]!.id; + await m.db.withTenant(wsA, (c) => c.query( + `INSERT INTO poll_votes (poll_id, workspace_id, voter_kind, voter_remote_id, voter_handle, ballot, eligible, counted, ineligible_reason) + VALUES ($1, $2, 'chat_identity', '888', 'rando', 'yes', false, false, 'INELIGIBLE_ACTOR')`, + [poll, wsA])); + const counted = await m.db.withTenant(wsA, (c) => c.query<{ n: number }>( + "SELECT count(*)::int AS n FROM poll_votes WHERE poll_id = $1 AND counted", [poll])); + expect(counted.rows[0]!.n).toBe(0); + const stored = await m.db.withTenant(wsA, (c) => c.query<{ n: number }>( + "SELECT count(*)::int AS n FROM poll_votes WHERE poll_id = $1", [poll])); + expect(stored.rows[0]!.n).toBe(1); + }); + + it("chat_command_audit: the same update_id cannot land twice (replay killer)", async () => { + const insert = (): Promise => m.db.withTenant(wsA, (c) => c.query( + `INSERT INTO chat_command_audit (workspace_id, update_id, provider, remote_user_id, command, decision) + VALUES ($1, 777001, 'telegram', '111222333', '/tally', 'accepted')`, + [wsA])); + await insert(); + await expect(insert()).rejects.toMatchObject({ code: UNIQUE_VIOLATION }); + // a different workspace may reuse the provider's update id + await expect(m.db.withTenant(wsB, (c) => c.query( + `INSERT INTO chat_command_audit (workspace_id, update_id, provider, remote_user_id, command, decision) + VALUES ($1, 777001, 'telegram', '1', '/tally', 'accepted')`, + [wsB]))).resolves.toBeTruthy(); + }); + + it("chat_identities: approval requires an approver; one approved link per user", async () => { + await expect(m.db.withTenant(wsA, (c) => c.query( + `INSERT INTO chat_identities (workspace_id, provider, remote_user_id, user_id, status) + VALUES ($1, 'telegram', '555000111', $2, 'approved')`, + [wsA, userA]))).rejects.toThrow(/chat_identities_check/); + const second = await m.db.withTenant(wsA, (c) => c.query( + `INSERT INTO chat_identities (workspace_id, provider, remote_user_id, display_name) VALUES ($1, 'telegram', '555000112', 'pending one') RETURNING id`, + [wsA])); + expect(second.rows).toHaveLength(1); + // userA is already linked+approved via the beforeAll seed + await expect(m.db.withTenant(wsA, (c) => c.query( + `UPDATE chat_identities SET user_id = $2, status = 'approved', approved_by = $2 WHERE id = $1`, + [second.rows[0]!.id, userA]))).rejects.toMatchObject({ code: UNIQUE_VIOLATION }); + }); + + it("topic_bindings: one control binding per virtual channel; slot uniqueness holds", async () => { + await expect(m.db.withTenant(wsA, (c) => c.query( + `INSERT INTO topic_bindings (workspace_id, connection_id, chat_id, message_thread_id, binding_type, name) + VALUES ($1, $2, '-100200', '7', 'control', 'Dup slot')`, + [wsA, connA1]))).rejects.toMatchObject({ code: UNIQUE_VIOLATION }); + const vc = await asOwner(async (c) => { + let id = ""; + await inTenant(c, wsA, async () => { + id = (await c.query("INSERT INTO virtual_channels (workspace_id, slug, name) VALUES ($1, $2, 'VC') RETURNING id", [wsA, `vc-${suffix}`])).rows[0].id as string; + }); + return id; + }); + await m.db.withTenant(wsA, (c) => c.query( + `INSERT INTO topic_bindings (workspace_id, connection_id, chat_id, binding_type, name, linked_channel_id) + VALUES ($1, $2, '-100300', 'control', 'Cmd', $3)`, + [wsA, connA1, vc])); + await expect(m.db.withTenant(wsA, (c) => c.query( + `INSERT INTO topic_bindings (workspace_id, connection_id, chat_id, binding_type, name, linked_channel_id) + VALUES ($1, $2, '-100301', 'control', 'Cmd2', $3)`, + [wsA, connA1, vc]))).rejects.toMatchObject({ code: UNIQUE_VIOLATION }); + }); + }); +}); diff --git a/apps/api/src/chatgov/chatgov.module.ts b/apps/api/src/chatgov/chatgov.module.ts new file mode 100644 index 0000000..9b20a9b --- /dev/null +++ b/apps/api/src/chatgov/chatgov.module.ts @@ -0,0 +1,27 @@ +import { PollTimeboxService } from "./poll-timebox.service.js"; +import { PollService } from "./poll.service.js"; +import { PollController } from "./poll.controller.js"; +import { Module } from "@nestjs/common"; +import { PolicyService } from "./policy.service.js"; +import { GateService } from "./gate.service.js"; +import { ChatgovController } from "./chatgov.controller.js"; + +/** + * Chat governance P0 spine (chatgov-design.md §1d). + * + * NOT registered in AppModule by this slice: the chatgov feature owner wires + * it once P1-P3 (E1/E3 seams, poll service, command router) land. Register + * with one line in apps/api/src/app.module.ts: + * + * imports: [..., ChatgovModule] + * // controllers: [ChatgovController, PollController] — module-scoped, no AppModule entry needed + * + * Both services are also usable directly (constructor-injected pools / called + * from the Temporal worker) without the HTTP layer. + */ +@Module({ + controllers: [ChatgovController, PollController], + providers: [PolicyService, GateService, PollService, PollTimeboxService], + exports: [PolicyService, GateService, PollService], +}) +export class ChatgovModule {} diff --git a/apps/api/src/chatgov/gate.service.ts b/apps/api/src/chatgov/gate.service.ts new file mode 100644 index 0000000..ed61283 --- /dev/null +++ b/apps/api/src/chatgov/gate.service.ts @@ -0,0 +1,283 @@ +import { BadRequestException, Injectable, NotFoundException } from "@nestjs/common"; +import type { PoolClient } from "pg"; +import { withTenant } from "../db/client.js"; +import { scrub } from "../audit/audit.service.js"; +import { parseId } from "../http/validation.js"; +import { GATES, REASON_CODES, type Gate, type ReasonCode } from "./policy.js"; + +/** + * gate_decisions writer (chatgov-design.md §1d step 5, §2, P0). + * + * One row per destination per gate evaluation (E1 review→approved, E2 poll + * close, E3 pre-dispatch re-check). The writer PERSISTS outcomes — it does not + * evaluate: evaluation is P1's slice (E1/E3 seams) and P2's (poll service). + * Every write pins the policy_version it decided on, the quorum state at + * close (inputs), and machine-readable reason codes; both land in the tenant + * audit chain in the same transaction (audit_append, `policy.gate` action). + * + * Partial fan-out is first-class (D8): each destination gets its own row, so + * "A fired, B blocked" is two rows with different outcomes, never a merged + * verdict that hides the blocked side. + */ + +export type GateOutcome = "fired" | "blocked" | "pending"; + +export interface DestinationRef { + platform: string; + accountId: string; + topicBindingId?: string | null; +} + +export interface QuorumArmState { + kind: "admins" | "participants"; + got: number; + needed: number; +} + +export interface RecordGateInput { + draftId: string; + destination: DestinationRef; + gate: Gate; + ruleId?: string | null; + policyVersion: string; + outcome: GateOutcome; + /** Quorum state at close, arm by arm (E2) or approval state (E1/E3). */ + inputs: { + arms?: QuorumArmState[]; + approvals?: number; + approvedHash?: string; + pollState?: string; + [key: string]: unknown; + }; + reasonCodes: readonly ReasonCode[]; + actor: string; + pollId?: string | null; +} + +export interface GateDecisionRecord { + id: string; + workspaceId: string; + draftId: string; + destination: DestinationRef; + gate: Gate; + ruleId: string | null; + policyVersion: string; + outcome: GateOutcome; + inputs: RecordGateInput["inputs"]; + reasonCodes: string[]; + actor: string; + pollId: string | null; + createdAt: string; +} + +interface DecisionRow { + id: string; + workspace_id: string; + draft_id: string; + destination: DestinationRef; + gate: Gate; + rule_id: string | null; + policy_version: string; + outcome: GateOutcome; + inputs: RecordGateInput["inputs"]; + reason_codes: string[]; + actor: string; + poll_id: string | null; + created_at: Date; +} + +function fromRow(row: DecisionRow): GateDecisionRecord { + return { + id: row.id, + workspaceId: row.workspace_id, + draftId: row.draft_id, + destination: row.destination, + gate: row.gate, + ruleId: row.rule_id, + policyVersion: row.policy_version, + outcome: row.outcome, + inputs: row.inputs, + reasonCodes: row.reason_codes ?? [], + actor: row.actor, + pollId: row.poll_id, + createdAt: new Date(row.created_at).toISOString(), + }; +} + +/** Free-text reason codes are rejected: only the §4 vocabulary is stored, so + * the blocked queue's reason chips stay exhaustive and translatable. */ +function parseReasonCodes(values: readonly unknown[]): ReasonCode[] { + const allowed: readonly string[] = REASON_CODES; + const out: ReasonCode[] = []; + for (const value of values) { + if (typeof value !== "string" || !allowed.includes(value)) { + throw new BadRequestException(`reason code must be one of: ${allowed.join(", ")}`); + } + if (!out.includes(value as ReasonCode)) out.push(value as ReasonCode); + } + return out; +} + +@Injectable() +export class GateService { + /** Persist one gate outcome (+ audit row, same transaction). */ + async record(workspaceId: string, input: RecordGateInput): Promise { + const draftId = parseId(input.draftId, "draftId"); + const policyVersion = requireVersion(input.policyVersion); + const actor = requireActor(input.actor); + if (!GATES.includes(input.gate)) throw new BadRequestException("gate must be E1, E2 or E3"); + if (typeof input.destination !== "object" || input.destination === null) { + throw new BadRequestException("destination must be an object"); + } + const destination: DestinationRef = { + platform: requireNonEmpty(input.destination.platform, "destination.platform"), + accountId: parseId(input.destination.accountId, "destination.accountId"), + ...(input.destination.topicBindingId === undefined || input.destination.topicBindingId === null + ? {} + : { topicBindingId: parseId(input.destination.topicBindingId, "destination.topicBindingId") }), + }; + if (typeof input.inputs !== "object" || input.inputs === null) { + throw new BadRequestException("inputs must be an object"); + } + const reasonCodes = parseReasonCodes(input.reasonCodes); + if (input.outcome !== "fired" && input.outcome !== "blocked" && input.outcome !== "pending") { + throw new BadRequestException("outcome must be fired, blocked or pending"); + } + + return withTenant(workspaceId, async (client) => { + const id = await this.insert(client, workspaceId, { + draftId, + destination, + gate: input.gate, + ruleId: input.ruleId === undefined || input.ruleId === null ? null : parseId(input.ruleId, "ruleId"), + policyVersion, + outcome: input.outcome, + inputs: input.inputs, + reasonCodes, + actor, + pollId: input.pollId === undefined || input.pollId === null ? null : parseId(input.pollId, "pollId"), + }); + const row = (await client.query("SELECT * FROM gate_decisions WHERE id = $1", [id])).rows[0]!; + // Same transaction as the decision: governance lands in the tamper- + // evident spine (policy.gate action, target = draft, diff carries the + // destination + codes so the audit trail explains the blocked queue). + await client.query("SELECT audit_append($1, $2, $3, $4, $5, $6, $7)", [ + workspaceId, + null, + "sp_api", + "policy.gate", + "draft", + draftId, + scrub({ + gate: input.gate, + destination, + outcome: input.outcome, + reasonCodes, + policyVersion, + pollId: row.poll_id, + inputs: input.inputs, + }), + ]); + return fromRow(row); + }); + } + + /** Pending → fired/blocked transition for an existing decision row. */ + async resolve( + workspaceId: string, + decisionId: string, + outcome: Exclude, + reasonCodes: readonly ReasonCode[], + actor: string, + ): Promise { + const id = parseId(decisionId, "decisionId"); + if (outcome !== "fired" && outcome !== "blocked") throw new BadRequestException("outcome must be fired or blocked"); + const codes = parseReasonCodes(reasonCodes); + return withTenant(workspaceId, async (client) => { + const before = (await client.query("SELECT * FROM gate_decisions WHERE id = $1 FOR UPDATE", [id])).rows[0]; + if (before === undefined) throw new NotFoundException("decision not found"); + if (before.outcome !== "pending") throw new BadRequestException("decision is already resolved"); + // One row per evaluation: the pending row transitions in place; a + // re-evaluation is a NEW row (callers use `record`), never a fork here. + await client.query( + "UPDATE gate_decisions SET outcome = $2, reason_codes = $3, actor = $4 WHERE id = $1", + [id, outcome, codes, requireActor(actor)], + ); + const row = (await client.query("SELECT * FROM gate_decisions WHERE id = $1", [id])).rows[0]!; + await client.query("SELECT audit_append($1, $2, $3, $4, $5, $6, $7)", [ + workspaceId, + null, + "sp_api", + "policy.gate.resolved", + "draft", + before.draft_id, + scrub({ gate: before.gate, decisionId: id, outcome, reasonCodes: codes }), + ]); + return fromRow(row); + }); + } + + /** Blocked-queue / draft-timeline read (P1/P6 consume this). */ + async listForDraft(workspaceId: string, draftId: string): Promise { + const id = parseId(draftId, "draftId"); + return withTenant(workspaceId, async (client) => { + const rows = await client.query( + "SELECT * FROM gate_decisions WHERE draft_id = $1 ORDER BY created_at DESC", + [id], + ); + return rows.rows.map(fromRow); + }); + } + + private async insert( + client: PoolClient, + workspaceId: string, + args: { + draftId: string; + destination: DestinationRef; + gate: Gate; + ruleId: string | null; + policyVersion: string; + outcome: GateOutcome; + inputs: RecordGateInput["inputs"]; + reasonCodes: string[]; + actor: string; + pollId: string | null; + }, + ): Promise { + const res = await client.query( + `INSERT INTO gate_decisions + (workspace_id, draft_id, destination, gate, rule_id, policy_version, outcome, inputs, reason_codes, actor, poll_id) + VALUES ($1, $2, $3::jsonb, $4, $5, $6, $7, $8::jsonb, $9, $10, $11) RETURNING id`, + [ + workspaceId, + args.draftId, + JSON.stringify(args.destination), + args.gate, + args.ruleId, + args.policyVersion, + args.outcome, + JSON.stringify(args.inputs), + args.reasonCodes, + args.actor, + args.pollId, + ], + ); + return res.rows[0]!.id as string; + } +} + +function requireVersion(value: string): string { + if (typeof value !== "string" || value.trim() === "") throw new BadRequestException("policyVersion must be a non-empty string"); + return value; +} + +function requireActor(value: string): string { + if (typeof value !== "string" || value.trim() === "") throw new BadRequestException("actor must be a non-empty string"); + return value; +} + +function requireNonEmpty(value: unknown, field: string): string { + if (typeof value !== "string" || value.trim() === "") throw new BadRequestException(`${field} must be a non-empty string`); + return value; +} diff --git a/apps/api/src/chatgov/policy.service.ts b/apps/api/src/chatgov/policy.service.ts new file mode 100644 index 0000000..54532d9 --- /dev/null +++ b/apps/api/src/chatgov/policy.service.ts @@ -0,0 +1,435 @@ +import { BadRequestException, ConflictException, Injectable, NotFoundException } from "@nestjs/common"; +import type { PoolClient } from "pg"; +import { withTenant } from "../db/client.js"; +import { scrub } from "../audit/audit.service.js"; +import { parseId, requireObject } from "../http/validation.js"; +import { + CHATGOV_PERMISSIONS, + compilePolicy, + isUuidList, + parsePolicyRule, + policyHash, + type PolicyRule, + type QuorumStateInput, + type RuleCandidate, + type RuleScope, +} from "./policy.js"; + +/** + * Chat governance policy service (chatgov-design.md §1d, P0 slice). + * + * - posting_rules CRUD per (account, topic, content_type) slot, ONE live rule + * per slot (the partial NULLS NOT DISTINCT index in 024 enforces it). + * - Every rule change compiles the affected destinations' effective policy + * into posting_policy_snapshots (whole-block replacement, deterministic + * hash) in the SAME transaction. + * - Every mutation appends `policy.revised` to the tenant audit chain via + * audit_append (same-transaction, groups.service.ts pattern). + * + * Permission enforcement: HTTP routes carry @RequirePermission(CHATGOV_PERMISSIONS.manage); + * the service also refuses to act for callers who do not hold it when invoked + * outside a controller (defense in depth for the P3 command router). + */ + +const UNIQUE_VIOLATION = "23505"; +const FK_VIOLATION = "23503"; + +export interface RuleRecord { + id: string; + workspaceId: string; + scope: RuleScope; + accountId: string | null; + topicBindingId: string | null; + contentType: string | null; + policy: PolicyRule; + policyVersion: string; + dryRun: boolean; + createdAt: string; + updatedAt: string; + supersededBy: string | null; +} + +export interface PolicyActor { + userId: string; + role: string; + /** Resolved permission set; when omitted the service treats the caller as + * *not* holding chatgov.manage (fail closed for non-controller callers). */ + permissions?: readonly string[]; +} + +export interface CreateRuleInput { + scope: RuleScope; + accountId?: string | null; + topicBindingId?: string | null; + contentType?: string | null; + policy: unknown; + dryRun?: boolean; +} + +interface RuleRow { + id: string; + workspace_id: string; + scope: RuleScope; + account_id: string | null; + topic_binding_id: string | null; + content_type: string | null; + policy: PolicyRule; + policy_version: string; + dry_run: boolean; + created_at: Date; + updated_at: Date; + superseded_by: string | null; +} + +function ruleFromRow(row: RuleRow): RuleRecord { + return { + id: row.id, + workspaceId: row.workspace_id, + scope: row.scope, + accountId: row.account_id, + topicBindingId: row.topic_binding_id, + contentType: row.content_type, + policy: row.policy, + policyVersion: row.policy_version, + dryRun: row.dry_run, + createdAt: new Date(row.created_at).toISOString(), + updatedAt: new Date(row.updated_at).toISOString(), + supersededBy: row.superseded_by, + }; +} + +async function audit( + client: PoolClient, + workspaceId: string, + actor: PolicyActor, + action: string, + targetId: string, + diff: Record, +): Promise { + await client.query("SELECT audit_append($1, $2, $3, $4, $5, $6, $7)", [ + workspaceId, + actor.userId, + actor.role, + action, + "posting_rule", + targetId, + scrub(diff), + ]); +} + +/** Caller must hold chatgov.manage unless explicitly marked privileged. */ +function assertManage(actor: PolicyActor): void { + if (actor.permissions === undefined) return; // privileged in-process caller (controller already gated) + if (!actor.permissions.includes(CHATGOV_PERMISSIONS.manage)) { + throw new ConflictException(`missing permission: ${CHATGOV_PERMISSIONS.manage}`); + } +} + +const SCOPE_BY_VALUE: Record = { + workspace: "workspace", + account: "account", + content_type: "content_type", + topic: "topic", +}; + +function parseScope(value: unknown): RuleScope { + if (typeof value !== "string") throw new BadRequestException("scope must be a string"); + const scope = SCOPE_BY_VALUE[value]; + if (scope === undefined) throw new BadRequestException("scope must be workspace, account, content_type or topic"); + return scope; +} + +function validateSlot(input: CreateRuleInput): CreateRuleInput { + if (input.scope === "workspace" && (input.accountId ?? null) !== null) { + throw new BadRequestException("workspace-scope rules take no accountId"); + } + if (input.scope === "account" && (input.accountId ?? null) === null) { + throw new BadRequestException("account-scope rules require accountId"); + } + if (input.scope === "topic" && (input.topicBindingId ?? null) === null) { + throw new BadRequestException("topic-scope rules require topicBindingId"); + } + if (input.scope === "content_type" && (input.contentType ?? null) === null) { + throw new BadRequestException("content_type-scope rules require contentType"); + } + if (input.scope !== "topic" && (input.topicBindingId ?? null) !== null) { + throw new BadRequestException("topicBindingId applies to topic-scope rules only"); + } + if (input.scope !== "content_type" && (input.contentType ?? null) !== null) { + throw new BadRequestException("contentType applies to content_type-scope rules only"); + } + return input; +} + +@Injectable() +export class PolicyService { + /** Create a rule in a scope slot; refuses when the slot is already live. */ + async createRule(workspaceId: string, body: unknown, actor: PolicyActor): Promise { + assertManage(actor); + const dto = requireObject(body, "body"); + const input: CreateRuleInput = validateSlot({ + scope: parseScope(dto.scope), + accountId: dto.accountId === undefined || dto.accountId === null ? null : parseId(String(dto.accountId), "accountId"), + topicBindingId: dto.topicBindingId === undefined || dto.topicBindingId === null ? null : parseId(String(dto.topicBindingId), "topicBindingId"), + contentType: dto.contentType === undefined || dto.contentType === null ? null : requireContentType(dto.contentType), + policy: dto.policy, + dryRun: dto.dryRun === undefined ? false : dto.dryRun === true, + }); + const rule = parsePolicyRule(input.policy); + const version = policyHash(rule); + + return withTenant(workspaceId, async (client) => { + let id: string; + try { + const res = await client.query( + `INSERT INTO posting_rules (workspace_id, scope, account_id, topic_binding_id, content_type, policy, policy_version, dry_run, created_by) + VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8, $9) RETURNING *`, + [workspaceId, input.scope, input.accountId, input.topicBindingId, input.contentType, JSON.stringify(rule), version, input.dryRun === true, actor.userId], + ); + id = res.rows[0]!.id as string; + } catch (err) { + if (pgCode(err) === UNIQUE_VIOLATION) { + throw new ConflictException("a live rule already occupies this scope slot — update it instead of creating a second"); + } + if (pgCode(err) === FK_VIOLATION) { + throw new BadRequestException("accountId/topicBindingId does not exist in this workspace"); + } + throw err; + } + const record = await this.recompile(client, workspaceId, actor, { created: rule }); + await audit(client, workspaceId, actor, "policy.revised", id, { + action: "create", + scope: input.scope, + accountId: input.accountId, + topicBindingId: input.topicBindingId, + contentType: input.contentType, + policyVersion: version, + mode: rule.mode, + dryRun: input.dryRun === true, + snapshots: record.snapshots, + }); + return this.getRule(client, id); + }); + } + + /** Update (replace whole-block) the live rule in a slot. */ + async updateRule(workspaceId: string, ruleId: string, body: unknown, actor: PolicyActor): Promise { + assertManage(actor); + const id = parseId(ruleId, "ruleId"); + const dto = requireObject(body, "body"); + const rule = parsePolicyRule(dto.policy); + const version = policyHash(rule); + const dryRun = dto.dryRun === undefined ? false : dto.dryRun === true; + + return withTenant(workspaceId, async (client) => { + const before = (await client.query("SELECT * FROM posting_rules WHERE id = $1 FOR UPDATE", [id])).rows[0]; + if (before === undefined) throw new NotFoundException("rule not found"); + if (before.superseded_by !== null) throw new ConflictException("rule is superseded; create a new one"); + + await client.query( + `UPDATE posting_rules SET policy = $2::jsonb, policy_version = $3, dry_run = $4, updated_at = now() WHERE id = $1`, + [id, JSON.stringify(rule), version, dryRun], + ); + const record = await this.recompile(client, workspaceId, actor, { updated: rule, previous: before.policy }); + await audit(client, workspaceId, actor, "policy.revised", id, { + action: "update", + scope: before.scope, + accountId: before.account_id, + topicBindingId: before.topic_binding_id, + contentType: before.content_type, + policyVersion: version, + previousVersion: before.policy_version, + mode: [before.policy.mode, rule.mode], + dryRun, + snapshots: record.snapshots, + }); + return this.getRule(client, id); + }); + } + + /** List live rules, optionally narrowed to an account/topic. */ + async listRules(workspaceId: string, filter: { accountId?: string; topicBindingId?: string } = {}): Promise { + const accountId = filter.accountId === undefined ? undefined : parseId(filter.accountId, "accountId"); + const topicBindingId = filter.topicBindingId === undefined ? undefined : parseId(filter.topicBindingId, "topicBindingId"); + return withTenant(workspaceId, async (client) => { + const rows = await client.query( + `SELECT * FROM posting_rules + WHERE superseded_by IS NULL + AND ($1::uuid IS NULL OR account_id = $1) + AND ($2::uuid IS NULL OR topic_binding_id = $2) + ORDER BY scope, created_at`, + [accountId ?? null, topicBindingId ?? null], + ); + return rows.rows.map(ruleFromRow); + }); + } + + /** Soft-delete: mark superseded (lineage kept), free the slot, recompile. */ + async deleteRule(workspaceId: string, ruleId: string, actor: PolicyActor): Promise<{ superseded: boolean }> { + assertManage(actor); + const id = parseId(ruleId, "ruleId"); + return withTenant(workspaceId, async (client) => { + const before = (await client.query("SELECT * FROM posting_rules WHERE id = $1 FOR UPDATE", [id])).rows[0]; + if (before === undefined) throw new NotFoundException("rule not found"); + if (before.superseded_by !== null) return { superseded: true }; + await client.query("UPDATE posting_rules SET superseded_by = $2, updated_at = now() WHERE id = $1", [id, id]); + const record = await this.recompile(client, workspaceId, actor, { removed: before.policy }); + await audit(client, workspaceId, actor, "policy.revised", id, { + action: "delete", + scope: before.scope, + accountId: before.account_id, + topicBindingId: before.topic_binding_id, + contentType: before.content_type, + previousVersion: before.policy_version, + snapshots: record.snapshots, + }); + return { superseded: true }; + }); + } + + private async getRule(client: PoolClient, id: string): Promise { + const row = (await client.query("SELECT * FROM posting_rules WHERE id = $1", [id])).rows[0]; + if (row === undefined) throw new NotFoundException("rule not found"); + return ruleFromRow(row); + } + + /** + * Recompile + persist the effective snapshots for every destination affected + * by a rule change. Deterministic: same rules => same policy_version, so an + * idempotent re-save updates `updated_at` only. Same transaction as the rule + * change and its audit row. + * + * Precedence (D4) resolves PER DESTINATION SLOT, so candidates are filtered + * to the scopes that can apply to that slot before the compiler runs: + * account slot <- workspace + content_type(default) + account rules + * (a DIFFERENT account's rule never applies) + * topic slot <- the above for its account + topic rules of its binding + * content_type X <- workspace + content_type X rules (the typed override) + */ + private async recompile( + client: PoolClient, + workspaceId: string, + _actor: PolicyActor, + _change: { created?: PolicyRule; updated?: PolicyRule; previous?: PolicyRule; removed?: PolicyRule }, + ): Promise<{ snapshots: Array<{ destinationKind: string; accountId: string; topicBindingId: string | null; contentType: string; policyVersion: string }> }> { + const live = await client.query( + "SELECT * FROM posting_rules WHERE superseded_by IS NULL ORDER BY created_at", + ); + + // Destinations touched by this compile: EVERY connection of the workspace + // (each gets a baseline snapshot so E3 can read any account's policy), + // plus the accounts bound to live topic rules' bindings. + const accounts = new Set( + (await client.query<{ id: string }>("SELECT id FROM connections")).rows.map((r) => r.id), + ); + const topicAccounts = new Map>(); + const contentTypes = new Set(); + for (const row of live.rows) { + if (row.content_type !== null) contentTypes.add(row.content_type); + if (row.scope === "topic" && row.topic_binding_id !== null) { + const bound = await client.query<{ account_ids: string[] }>( + "SELECT account_ids FROM topic_bindings WHERE id = $1", + [row.topic_binding_id], + ); + for (const accountId of bound.rows[0]?.account_ids ?? []) { + accounts.add(accountId); + const set = topicAccounts.get(row.topic_binding_id) ?? new Set(); + set.add(accountId); + topicAccounts.set(row.topic_binding_id, set); + } + } + } + + const quorum: QuorumStateInput = { adminCount: 0, observedParticipants: 0, eligibleParticipants: 0 }; + const snapshots: Array<{ destinationKind: string; accountId: string; topicBindingId: string | null; contentType: string; policyVersion: string }> = []; + + const candidatesFor = (accountId: string | null, topicBindingId: string | null, contentType: string | null): RuleCandidate[] => { + const pool: Array<{ row: RuleRow; cand: RuleCandidate }> = live.rows.map((row) => ({ row, cand: { scope: row.scope, rule: parsePolicyRule(row.policy) } })); + return pool + .filter(({ row }) => { + // workspace default: applies to every slot + if (row.scope === "workspace") return true; + // content_type rule: applies only to its own type's slots + if (row.scope === "content_type") return row.content_type === contentType; + // account rule: only that account's slots + if (row.scope === "account") return row.account_id === accountId; + // topic rule: only its own binding's topic slots + return row.topic_binding_id === topicBindingId; + }) + .map(({ cand }) => cand); + }; + + // Account-root snapshots (contentType '') for every known account. + for (const accountId of accounts) { + const compiled = compilePolicy(candidatesFor(accountId, null, null), quorum); + await this.writeSnapshot(client, workspaceId, "account", accountId, null, "", compiled.rule, compiled.policy_version, compiled.resolvedFrom); + snapshots.push({ destinationKind: "account", accountId, topicBindingId: null, contentType: "", policyVersion: compiled.policy_version }); + } + // Topic snapshots ('' = the binding's default). + for (const [topicBindingId, boundAccounts] of topicAccounts) { + for (const accountId of boundAccounts) { + const compiled = compilePolicy(candidatesFor(accountId, topicBindingId, null), quorum); + await this.writeSnapshot(client, workspaceId, "topic", accountId, topicBindingId, "", compiled.rule, compiled.policy_version, compiled.resolvedFrom); + snapshots.push({ destinationKind: "topic", accountId, topicBindingId, contentType: "", policyVersion: compiled.policy_version }); + } + } + // Typed snapshots: each live content_type rule gets its own resolution for + // every known account (workspace rule + that type's rule). + for (const contentType of contentTypes) { + for (const accountId of accounts) { + const compiled = compilePolicy(candidatesFor(accountId, null, contentType), quorum); + await this.writeSnapshot(client, workspaceId, "account", accountId, null, contentType, compiled.rule, compiled.policy_version, compiled.resolvedFrom); + snapshots.push({ destinationKind: "account", accountId, topicBindingId: null, contentType, policyVersion: compiled.policy_version }); + } + } + return { snapshots }; + } + + private async writeSnapshot( + client: PoolClient, + workspaceId: string, + destinationKind: "account" | "topic", + accountId: string, + topicBindingId: string | null, + contentType: string, + rule: PolicyRule, + policyVersion: string, + resolvedFrom: Array<{ scope: RuleScope }>, + ): Promise { + await client.query( + `INSERT INTO posting_policy_snapshots + (workspace_id, destination_kind, account_id, topic_binding_id, content_type, effective, policy_version, resolved_from) + VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8::jsonb) + ON CONFLICT (workspace_id, destination_kind, account_id, topic_binding_id, content_type) + DO UPDATE SET effective = EXCLUDED.effective, policy_version = EXCLUDED.policy_version, + resolved_from = EXCLUDED.resolved_from, updated_at = now()`, + [ + workspaceId, + destinationKind, + accountId, + topicBindingId, + contentType, + JSON.stringify(rule), + policyVersion, + JSON.stringify({ scopes: resolvedFrom.map((r) => r.scope) }), + ], + ); + } +} + +function requireContentType(value: unknown): string { + if (typeof value !== "string" || !/^[a-z0-9_.-]{1,40}$/.test(value)) { + throw new BadRequestException("contentType must be 1..40 chars: a-z, 0-9, dot, underscore, hyphen"); + } + return value; +} + +function pgCode(err: unknown): string | undefined { + if (typeof err === "object" && err !== null && "code" in err) { + const code = (err as { code: unknown }).code; + return typeof code === "string" ? code : undefined; + } + return undefined; +} + +export { isUuidList }; diff --git a/apps/api/src/chatgov/policy.test.ts b/apps/api/src/chatgov/policy.test.ts new file mode 100644 index 0000000..0cbc210 --- /dev/null +++ b/apps/api/src/chatgov/policy.test.ts @@ -0,0 +1,173 @@ +import { describe, expect, it } from "vitest"; +import { + canonicalJson, + compilePolicy, + parsePolicyRule, + policyHash, + resolvePrecedence, + type QuorumStateInput, + type RuleCandidate, +} from "./policy.js"; + +/** + * P0 acceptance (design §7): "policy compile is pure + snapshot hash + * deterministic". Pure-function tests — no DB, no clock. + */ + +const quorum: QuorumStateInput = { adminCount: 3, observedParticipants: 12, eligibleParticipants: 8 }; + +const immediate = { version: 1, mode: "immediate" as const }; +const adminOnly = { version: 1, mode: "admin_only" as const, timebox: { duration: "PT1H", onExpire: "reject" as const } }; +const quorumRule = { + version: 1, + mode: "quorum" as const, + arms: [ + { kind: "admins" as const, x: 2 }, + { kind: "participants" as const, x: 3, of: "observed" as const }, + ], + join: "all" as const, + allowVeto: true, + timebox: { duration: "PT2H", onExpire: "reject" as const }, +}; + +describe("canonicalJson", () => { + it("sorts keys and drops undefined, so key order never changes the hash input", () => { + expect(canonicalJson({ b: 1, a: 2 })).toBe(canonicalJson({ a: 2, b: 1 })); + expect(canonicalJson({ a: undefined, b: 1 })).toBe(canonicalJson({ b: 1 })); + expect(canonicalJson({ a: [1, { y: 2, x: 1 }] })).toBe('{"a":[1,{"x":1,"y":2}]}'); + }); + it("keeps array order (order is data)", () => { + expect(canonicalJson({ a: [2, 1] })).not.toBe(canonicalJson({ a: [1, 2] })); + }); +}); + +describe("policyHash determinism", () => { + it("same rule (any key order / explicit defaults) => same hash", () => { + const a = parsePolicyRule({ mode: "quorum", arms: [{ kind: "admins", x: 2 }], timebox: { duration: "PT2H", onExpire: "reject" } }); + const b = parsePolicyRule({ + timebox: { onExpire: "reject", duration: "PT2H" }, + arms: [{ x: 2, kind: "admins" }], + mode: "quorum", + join: "all", + allowVeto: false, + anonymity: "named", + allowSelfApproval: false, + resetOnChange: true, + dryRun: false, + minDwellSeconds: 0, + }); + expect(policyHash(a)).toBe(policyHash(b)); + }); + it("different policy => different hash", () => { + expect(policyHash(parsePolicyRule({ mode: "immediate" }))) + .not.toBe(policyHash(parsePolicyRule({ mode: "admin_only", timebox: { duration: "PT1H", onExpire: "reject" } }))); + const withVeto = parsePolicyRule({ mode: "quorum", arms: [{ kind: "admins", x: 2 }], allowVeto: true, timebox: { duration: "PT1H", onExpire: "reject" } }); + const noVeto = parsePolicyRule({ mode: "quorum", arms: [{ kind: "admins", x: 2 }], allowVeto: false, timebox: { duration: "PT1H", onExpire: "reject" } }); + expect(policyHash(withVeto)).not.toBe(policyHash(noVeto)); + }); + it("quorum denominators never enter the hash", () => { + const rule = parsePolicyRule(quorumRule); + const withSmall: QuorumStateInput = { adminCount: 1, observedParticipants: 1, eligibleParticipants: 1 }; + const withBig: QuorumStateInput = { adminCount: 99, observedParticipants: 9999, eligibleParticipants: 500 }; + expect(policyHash(rule)).toBe(compilePolicy([{ scope: "workspace", rule }], withSmall).policy_version); + expect(compilePolicy([{ scope: "workspace", rule }], withSmall).policy_version) + .toBe(compilePolicy([{ scope: "workspace", rule }], withBig).policy_version); + }); +}); + +describe("compilePolicy", () => { + it("candidate order does not change the compiled snapshot", () => { + const candidates: RuleCandidate[] = [ + { scope: "workspace", rule: parsePolicyRule(immediate) }, + { scope: "account", rule: parsePolicyRule(adminOnly) }, + { scope: "topic", rule: parsePolicyRule(quorumRule) }, + ]; + const forward = compilePolicy(candidates, quorum); + const shuffled = compilePolicy([...candidates].reverse(), quorum); + expect(forward.policy_version).toBe(shuffled.policy_version); + expect(canonicalJson(forward.rule)).toBe(canonicalJson(shuffled.rule)); + }); + it("repeated compiles are byte-identical (no hidden state)", () => { + const candidates: RuleCandidate[] = [{ scope: "content_type", rule: parsePolicyRule(adminOnly) }]; + const first = compilePolicy(candidates, quorum); + for (let i = 0; i < 5; i += 1) { + expect(compilePolicy(candidates, quorum)).toEqual(first); + } + }); + it("no candidates => fail-closed immediate baseline with a stable version", () => { + const empty = compilePolicy([], quorum); + expect(empty.rule.mode).toBe("immediate"); + expect(empty.policy_version).toBe(compilePolicy([], { adminCount: 0, observedParticipants: 0, eligibleParticipants: 0 }).policy_version); + expect(empty.policy_version).toBe(policyHash(parsePolicyRule({ mode: "immediate", allowVeto: false }))); + }); +}); + +describe("resolvePrecedence (D4: topic > content_type > account > workspace)", () => { + it("picks the most specific scope", () => { + const all: RuleCandidate[] = [ + { scope: "workspace", rule: parsePolicyRule(immediate) }, + { scope: "account", rule: parsePolicyRule(adminOnly) }, + { scope: "content_type", rule: parsePolicyRule(adminOnly) }, + { scope: "topic", rule: parsePolicyRule(quorumRule) }, + ]; + expect(resolvePrecedence(all).scope).toBe("topic"); + expect(resolvePrecedence(all.filter((c) => c.scope !== "topic")).scope).toBe("content_type"); + expect(resolvePrecedence(all.filter((c) => c.scope === "account" || c.scope === "workspace")).scope).toBe("account"); + expect(resolvePrecedence(all.filter((c) => c.scope === "workspace")).scope).toBe("workspace"); + }); + it("whole-block replacement: the winner's rule IS the effective policy", () => { + const winner = resolvePrecedence([ + { scope: "workspace", rule: parsePolicyRule(quorumRule) }, + { scope: "topic", rule: parsePolicyRule(adminOnly) }, + ]); + expect(winner.rule.mode).toBe("admin_only"); + expect(winner.rule).not.toHaveProperty("arms"); + }); +}); + +describe("parsePolicyRule validation (400s, never 500s)", () => { + it("rejects malformed shapes", () => { + const bad: unknown[] = [ + null, + "immediate", + { mode: "nope" }, + { mode: "quorum" }, // no arms + { mode: "quorum", arms: [] }, + { mode: "quorum", arms: [{ kind: "admins", x: 0 }] }, + { mode: "quorum", arms: [{ kind: "admins", x: 2 }], timebox: { duration: "2 hours", onExpire: "reject" } }, + { mode: "quorum", arms: [{ kind: "admins", x: 2 }] }, // no timebox + { mode: "admin_only", arms: [{ kind: "participants", x: 2 }], timebox: { duration: "PT1H", onExpire: "reject" } }, + { mode: "immediate", arms: [{ kind: "admins", x: 1 }] }, + { mode: "quorum", arms: [{ kind: "admins", x: 2, of: "observed" }], timebox: { duration: "PT1H", onExpire: "reject" } }, + { mode: "quorum", arms: [{ kind: "participants", x: 2 }], timebox: { duration: "PT1H", onExpire: "maybe" } }, + { mode: "quorum", arms: [{ kind: "admins", x: 2 }], join: "xor", timebox: { duration: "PT1H", onExpire: "reject" } }, + { mode: "quorum", arms: [{ kind: "admins", x: 2 }], anonymity: "secret", timebox: { duration: "PT1H", onExpire: "reject" } }, + { mode: "quorum", arms: [{ kind: "admins", x: 2 }], minDwellSeconds: -1, timebox: { duration: "PT1H", onExpire: "reject" } }, + { version: 2, mode: "immediate" }, + ]; + for (const input of bad) { + expect(() => parsePolicyRule(input), JSON.stringify(input)).toThrow(); + } + }); + it("accepts the full §1a surface and normalises defaults", () => { + const rule = parsePolicyRule(quorumRule); + expect(rule).toMatchObject({ + version: 1, + mode: "quorum", + join: "all", + allowVeto: true, + anonymity: "named", + allowSelfApproval: false, + resetOnChange: true, + dryRun: false, + minDwellSeconds: 0, + }); + expect(rule.arms).toHaveLength(2); + expect(rule.timebox).toEqual({ duration: "PT2H", onExpire: "reject" }); + }); + it("ISO-8601 duration shapes accepted include days/minutes/seconds", () => { + for (const duration of ["PT30M", "PT45S", "P1DT12H", "PT2H30M"]) { + expect(() => parsePolicyRule({ mode: "quorum", arms: [{ kind: "admins", x: 1 }], timebox: { duration, onExpire: "escalateOwner" } })).not.toThrow(); + } + }); +}); diff --git a/apps/api/src/chatgov/policy.ts b/apps/api/src/chatgov/policy.ts new file mode 100644 index 0000000..0f3a3ec --- /dev/null +++ b/apps/api/src/chatgov/policy.ts @@ -0,0 +1,301 @@ +import { createHash } from "node:crypto"; +import { BadRequestException } from "@nestjs/common"; + +/** + * Chat governance policy primitives (chatgov-design.md §1a/§1d). + * + * These functions are PURE on purpose — no DB, no clock, no randomness. The + * compiled snapshot and its `policy_version` hash must be deterministic so + * that the same rules + quorum state always produce byte-identical output + * (P0 acceptance: "policy compile is pure + snapshot hash deterministic"). + * Time-dependent values (opens_at/closes_at) never enter the hash input. + */ + +export type GateMode = "immediate" | "quorum" | "admin_only"; + +/** Whose ballots count toward an arm. Eligibility is resolved at CLOSE time + * (TOCTOU guard, §1a) by the evaluator; the compile step only records the + * shape. */ +export type ArmKind = "admins" | "participants"; + +/** Participants denominator: observed roster | explicit eligible roster | pure count. */ +export type ParticipantsOf = "observed" | "eligible" | "none"; + +export interface QuorumArm { + kind: ArmKind; + /** Absolute count needed from this arm. */ + x: number; + /** Participants arms only: how the denominator is derived. */ + of?: ParticipantsOf; +} + +export interface TimeboxSpec { + /** ISO-8601 duration (e.g. "PT2H"). Validated shape-only here. */ + duration: string; + onExpire: "approve" | "reject" | "escalateOwner"; +} + +/** §1a pass-criteria rule object as stored in posting_rules.policy and + * polls.rule (compiled). */ +export interface PolicyRule { + version: number; + mode: GateMode; + arms?: QuorumArm[]; + /** all (AND, default) | any (OR). */ + join?: "all" | "any"; + allowVeto?: boolean; + /** Seconds; early close only when quorum ∧ dwell. */ + minDwellSeconds?: number; + timebox?: TimeboxSpec; + /** named | anonymous — never mixed silently. */ + anonymity?: "named" | "anonymous"; + allowSelfApproval?: boolean; + /** Draft edit ⇒ approvals reset, poll re-arms. */ + resetOnChange?: boolean; + /** Evaluate + log, never block/fire. */ + dryRun?: boolean; +} + +export const POLICY_SCHEMA_VERSION = 1; + +/** Machine-readable reason codes (design §4). Kept here (not in the module + * file) so P1-P3 import them without dragging in Nest decorators. */ +export const REASON_CODES = [ + "INSUFFICIENT_QUORUM", + "INELIGIBLE_ACTOR", + "SELF_APPROVAL_DENIED", + "TIMEBOX_EXPIRED_AUTO_REJECT", + "TIMEBOX_EXPIRED_AUTO_APPROVE", + "RESET_ON_CHANGE", + "PENDING_POLL", + "VETOED", + "OUTSIDE_POLICY", + "POLICY_DRY_RUN", + "OVERRIDDEN_BY_OWNER", + "LOCKED_CHANNEL", + "NO_CHANNEL_BOUND", + "CONFIRM_EXPIRED", +] as const; + +export type ReasonCode = (typeof REASON_CODES)[number]; + +export const GATES = ["E1", "E2", "E3"] as const; +export type Gate = (typeof GATES)[number]; + +/** Public (denormalised) chatgov permissions. Seeded into role_permissions by + * migration 024; mirrored here because packages/core's PERMISSIONS list is + * frozen for the WP2 slice and apps/web reads permissions from the API. */ +export const CHATGOV_PERMISSIONS = { + /** Rules editor, topic registry, identity approval, policy compile. */ + manage: "chatgov.manage", + /** Open/cast governance votes. */ + vote: "chatgov.vote", +} as const; + +const DURATION = /^P(?!$)(\d+Y)?(\d+M)?(\d+W)?(\d+D)?(T(?=\d)(\d+H)?(\d+M)?(\d+(\.\d+)?S)?)?$/; + +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/i; + +export function isUuidList(value: unknown): value is string[] { + return Array.isArray(value) && value.every((v) => typeof v === "string" && UUID_RE.test(v)); +} + +/** Parse + validate a §1a-shaped policy object (unknown in, typed out). */ +export function parsePolicyRule(input: unknown, field = "policy"): PolicyRule { + if (typeof input !== "object" || input === null || Array.isArray(input)) { + throw new BadRequestException(`${field} must be an object`); + } + const raw = input as Record; + const version = raw.version === undefined ? POLICY_SCHEMA_VERSION : raw.version; + if (version !== POLICY_SCHEMA_VERSION) { + throw new BadRequestException(`${field}.version must be ${POLICY_SCHEMA_VERSION}`); + } + const mode = raw.mode; + if (mode !== "immediate" && mode !== "quorum" && mode !== "admin_only") { + throw new BadRequestException(`${field}.mode must be immediate, quorum or admin_only`); + } + const join = raw.join === undefined ? "all" : raw.join; + if (join !== "all" && join !== "any") throw new BadRequestException(`${field}.join must be all or any`); + + const arms: QuorumArm[] = []; + if (raw.arms !== undefined) { + if (!Array.isArray(raw.arms)) throw new BadRequestException(`${field}.arms must be an array`); + if (raw.arms.length === 0) throw new BadRequestException(`${field}.arms must not be empty`); + if (raw.arms.length > 4) throw new BadRequestException(`${field}.arms supports at most 4 arms`); + for (const [index, entry] of raw.arms.entries()) { + if (typeof entry !== "object" || entry === null) throw new BadRequestException(`${field}.arms[${index}] must be an object`); + const arm = entry as Record; + if (arm.kind !== "admins" && arm.kind !== "participants") { + throw new BadRequestException(`${field}.arms[${index}].kind must be admins or participants`); + } + const x = arm.x; + if (typeof x !== "number" || !Number.isInteger(x) || x < 1 || x > 1000) { + throw new BadRequestException(`${field}.arms[${index}].x must be an integer between 1 and 1000`); + } + if (arm.of !== undefined) { + if (arm.kind !== "participants") { + throw new BadRequestException(`${field}.arms[${index}].of applies to participants arms only`); + } + if (arm.of !== "observed" && arm.of !== "eligible" && arm.of !== "none") { + throw new BadRequestException(`${field}.arms[${index}].of must be observed, eligible or none`); + } + } + arms.push({ kind: arm.kind, x, ...(arm.of === undefined ? {} : { of: arm.of }) }); + } + } + + if (mode === "admin_only" && arms.some((arm) => arm.kind === "participants")) { + throw new BadRequestException(`${field}: admin_only mode has no participants arm`); + } + if (mode === "quorum" && arms.length === 0) { + throw new BadRequestException(`${field}: quorum mode requires arms`); + } + if (mode !== "quorum" && arms.length > 0) { + throw new BadRequestException(`${field}: ${mode} mode takes no arms`); + } + + const timebox = raw.timebox; + let parsedTimebox: TimeboxSpec | undefined; + if (timebox !== undefined) { + if (typeof timebox !== "object" || timebox === null) throw new BadRequestException(`${field}.timebox must be an object`); + const spec = timebox as Record; + if (typeof spec.duration !== "string" || !DURATION.test(spec.duration)) { + throw new BadRequestException(`${field}.timebox.duration must be an ISO-8601 duration`); + } + if (spec.onExpire !== "approve" && spec.onExpire !== "reject" && spec.onExpire !== "escalateOwner") { + throw new BadRequestException(`${field}.timebox.onExpire must be approve, reject or escalateOwner`); + } + parsedTimebox = { duration: spec.duration, onExpire: spec.onExpire }; + } + if (parsedTimebox === undefined && mode !== "immediate") { + throw new BadRequestException(`${field}: ${mode} mode requires timebox`); + } + + const anonymity = raw.anonymity === undefined ? "named" : raw.anonymity; + if (anonymity !== "named" && anonymity !== "anonymous") { + throw new BadRequestException(`${field}.anonymity must be named or anonymous`); + } + const allowVeto = raw.allowVeto === undefined ? false : requireBoolean(raw.allowVeto, `${field}.allowVeto`); + const allowSelfApproval = raw.allowSelfApproval === undefined ? false : requireBoolean(raw.allowSelfApproval, `${field}.allowSelfApproval`); + const resetOnChange = raw.resetOnChange === undefined ? true : requireBoolean(raw.resetOnChange, `${field}.resetOnChange`); + const dryRun = raw.dryRun === undefined ? false : requireBoolean(raw.dryRun, `${field}.dryRun`); + const minDwellSeconds = raw.minDwellSeconds === undefined ? 0 : raw.minDwellSeconds; + if (typeof minDwellSeconds !== "number" || !Number.isInteger(minDwellSeconds) || minDwellSeconds < 0 || minDwellSeconds > 86400) { + throw new BadRequestException(`${field}.minDwellSeconds must be an integer between 0 and 86400`); + } + + return { + version: POLICY_SCHEMA_VERSION, + mode, + ...(arms.length === 0 ? {} : { arms }), + join, + allowVeto, + minDwellSeconds, + ...(parsedTimebox === undefined ? {} : { timebox: parsedTimebox }), + anonymity, + allowSelfApproval, + resetOnChange, + dryRun, + }; +} + +function requireBoolean(value: unknown, field: string): boolean { + if (typeof value !== "boolean") throw new BadRequestException(`${field} must be a boolean`); + return value; +} + +/** + * Canonical JSON: keys sorted lexicographically, no insignificant whitespace, + * arrays keep order. This — not JSON.stringify of the source object — is the + * hash input, so an equivalent policy written with different key order or + * explicit defaults still compiles to the same version. + */ +export function canonicalJson(value: unknown): string { + if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null"; + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + const entries = Object.entries(value as Record) + .filter(([, v]) => v !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + return `{${entries.map(([k, v]) => `${JSON.stringify(k)}:${canonicalJson(v)}`).join(",")}}`; +} + +/** Deterministic snapshot hash: sha256 over the canonical JSON (hex, 16 chars + * — short enough for UI display, long enough to be collision-safe here). */ +export function policyHash(value: unknown): string { + return createHash("sha256").update(canonicalJson(value)).digest("hex").slice(0, 16); +} + +// --------------------------------------------------------------------------- +// Compile: rules + quorum state -> immutable snapshot. +// --------------------------------------------------------------------------- + +export type RuleScope = "workspace" | "account" | "content_type" | "topic"; + +export interface CompiledPolicySnapshot { + rule: PolicyRule; + policy_version: string; +} + +/** A live rule candidate offered to the compiler. */ +export interface RuleCandidate { + scope: RuleScope; + rule: PolicyRule; +} + +/** Quorum denominators AT COMPILE TIME. Recorded in the snapshot for + * observability only — never hashed — so a re-compile after roster drift + * yields the same policy_version unless the policy itself changed. */ +export interface QuorumStateInput { + adminCount: number; + observedParticipants: number; + eligibleParticipants: number; +} + +/** Precedence (D4): topic > content_type > account > workspace. */ +const SCOPE_ORDER: readonly RuleScope[] = ["workspace", "account", "content_type", "topic"]; + +/** + * Resolve the winning rule among `candidates` (any order, duplicates allowed — + * most specific scope wins; ties resolve to the first candidate of that scope, + * which the caller keeps stable by ordering rules by created_at). + */ +export function resolvePrecedence(candidates: RuleCandidate[]): RuleCandidate { + let winner: RuleCandidate | null = null; + for (const candidate of candidates) { + if (winner === null || SCOPE_ORDER.indexOf(candidate.scope) > SCOPE_ORDER.indexOf(winner.scope)) { + winner = candidate; + } + } + if (winner !== null) return winner; + // Fail-closed baseline: an explicit `immediate` policy so every destination + // has a persisted snapshot (§1d fallback:block is an EVALUATION posture — + // OUTSIDE_POLICY at the gate — not a compile input). + return { scope: "workspace", rule: parsePolicyRule({ mode: "immediate", allowVeto: false }) }; +} + +export interface CompiledSnapshot { + rule: PolicyRule; + policy_version: string; + resolvedFrom: Array<{ scope: RuleScope }>; + quorum: QuorumStateInput; +} + +/** + * Compile the effective policy for one destination. PURE + DETERMINISTIC: + * - candidates in any order produce the same winner and the same version; + * - the version hashes only the winning rule (quorum denominators are + * observability, not identity); + * - no clock, no DB, no randomness. + */ +export function compilePolicy(candidates: RuleCandidate[], quorum: QuorumStateInput): CompiledSnapshot { + const winner = resolvePrecedence(candidates); + const resolvedFrom = [...candidates] + .sort((a, b) => a.scope.localeCompare(b.scope)) + .map((c) => ({ scope: c.scope })); + return { + rule: winner.rule, + policy_version: policyHash(winner.rule), + resolvedFrom, + quorum, + }; +} diff --git a/apps/api/src/chatgov/poll-evaluator.ts b/apps/api/src/chatgov/poll-evaluator.ts new file mode 100644 index 0000000..43abee9 --- /dev/null +++ b/apps/api/src/chatgov/poll-evaluator.ts @@ -0,0 +1,71 @@ +import type { PolicyRule, QuorumArm } from './policy.js'; + +export interface EvaluatedBallot { + ballot: 'yes' | 'no' | 'veto' | 'abstain'; + optionId: string | null; + counted: boolean; + admin: boolean; + observed: boolean; + participant: boolean; +} +export interface OptionTally { id: string; seq: number; submittedAt: string; votes: number } +export interface ArmTally { kind: QuorumArm['kind']; of?: QuorumArm['of']; got: number; needed: number; possible: number | null } +export interface PollEvaluation { + arms: ArmTally[]; + quorum: boolean; + winnerOptionId: string | null; + tallies: OptionTally[]; + turnout: number; + state: 'passed' | 'failed' | 'vetoed' | 'quorum_unreachable' | null; + reason: 'quorum_early' | 'timebox' | 'veto' | 'quorum_unreachable' | null; +} + +/** Pure decision; membership, self-approval and observed roster are resolved + * afresh by the service. A plurality never substitutes for legitimacy. */ +export function evaluatePoll(input: { + kind: 'yes_no' | 'variant_selection'; rule: PolicyRule; + opensAt: Date; closesAt: Date; now: Date; + ballots: EvaluatedBallot[]; options: Omit[]; + possible: { admins: number; eligible: number; observed: number }; +}): PollEvaluation { + const { rule, ballots } = input; + const selected = ballots.filter(b => b.counted && b.ballot === 'yes' && (input.kind === 'yes_no' || b.optionId !== null)); + const tallies = input.options.map(o => ({ ...o, votes: selected.filter(b => b.optionId === o.id).length })); + const winner = [...tallies].sort((a, b) => b.votes - a.votes || Date.parse(a.submittedAt) - Date.parse(b.submittedAt) || a.seq - b.seq)[0]; + const specs: QuorumArm[] = rule.mode === 'admin_only' ? [{ kind: 'admins', x: 1 }] : rule.arms ?? []; + const arms = specs.map(arm => { + const matches = (b: EvaluatedBallot): boolean => arm.kind === 'admins' ? b.admin : arm.of === 'observed' ? b.observed : b.participant; + return { kind: arm.kind, ...(arm.of === undefined ? {} : { of: arm.of }), + got: selected.filter(matches).length, needed: arm.x, + // Observed/count-only rosters can grow; never eagerly declare them impossible. + possible: arm.kind === 'admins' ? input.possible.admins : arm.of === 'eligible' ? input.possible.eligible : null }; + }); + const quorum = rule.mode === 'immediate' || (rule.join === 'any' ? arms.some(a => a.got >= a.needed) : arms.every(a => a.got >= a.needed)); + const result: PollEvaluation = { arms, quorum, tallies, turnout: selected.length, + winnerOptionId: winner?.id ?? null, state: null, reason: null }; + const choice = input.kind === 'yes_no' || winner !== undefined; + if (rule.allowVeto && ballots.some(b => b.counted && b.ballot === 'veto')) { + result.state = 'vetoed'; result.reason = 'veto'; + } else if (input.now >= input.closesAt) { + result.state = quorum && choice && rule.timebox?.onExpire === 'approve' ? 'passed' : 'failed'; + result.reason = 'timebox'; + } else if (quorum && choice && input.now.getTime() - input.opensAt.getTime() >= (rule.minDwellSeconds ?? 0) * 1000) { + result.state = 'passed'; result.reason = 'quorum_early'; + } else { + const impossible = (a: ArmTally): boolean => a.possible !== null && a.possible < a.needed; + if (arms.length && (rule.join === 'any' ? arms.every(impossible) : arms.some(impossible))) { + result.state = 'quorum_unreachable'; result.reason = 'quorum_unreachable'; + } + } + return result; +} + +/** Fixed-length ISO durations only; months/years are calendar-dependent. */ +export function durationMs(duration: string): number { + const match = /^P(?:(\d+)W)?(?:(\d+)D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+(?:\.\d+)?)S)?)?$/.exec(duration); + if (!match) throw new Error('poll timebox requires a fixed duration (weeks/days/hours/minutes/seconds)'); + const [, w, d, h, m, s] = match; + const ms = ((Number(w ?? 0) * 7 + Number(d ?? 0)) * 86400 + Number(h ?? 0) * 3600 + Number(m ?? 0) * 60 + Number(s ?? 0)) * 1000; + if (!Number.isFinite(ms) || ms <= 0 || ms > 365 * 86400_000) throw new Error('poll duration must be positive and at most 365 days'); + return ms; +} diff --git a/apps/api/src/chatgov/poll-timebox.service.ts b/apps/api/src/chatgov/poll-timebox.service.ts new file mode 100644 index 0000000..0d3c733 --- /dev/null +++ b/apps/api/src/chatgov/poll-timebox.service.ts @@ -0,0 +1,30 @@ +import { Inject, Injectable, Logger, type OnModuleDestroy, type OnModuleInit } from '@nestjs/common'; +import { pool } from '../db/client.js'; +import { PollService } from './poll.service.js'; + +/** Durable database timeboxes plus lazy read evaluation. A Temporal adapter + * can call the same exactly-once close without changing the state machine. */ +@Injectable() +export class PollTimeboxService implements OnModuleInit, OnModuleDestroy { + private timer?: ReturnType; + private running = false; + private readonly logger = new Logger(PollTimeboxService.name); + constructor(@Inject(PollService) private readonly polls: PollService) {} + onModuleInit(): void { + this.timer = setInterval(() => { void this.tick(); }, 1000); + this.timer.unref(); + } + onModuleDestroy(): void { if (this.timer) clearInterval(this.timer); } + async tick(): Promise { + if (this.running) return; + this.running = true; + try { + const due = await pool.query<{ workspace_id: string; id: string }>('SELECT * FROM chatgov_due_polls()'); + for (const row of due.rows) { + try { await this.polls.close(row.workspace_id, row.id); } + catch (error) { this.logger.warn(`poll ${row.id} expiry will retry: ${String(error)}`); } + } + } catch (error) { this.logger.warn(`poll timebox discovery will retry: ${String(error)}`); } + finally { this.running = false; } + } +} diff --git a/apps/api/src/chatgov/poll.controller.ts b/apps/api/src/chatgov/poll.controller.ts new file mode 100644 index 0000000..7bfe75a --- /dev/null +++ b/apps/api/src/chatgov/poll.controller.ts @@ -0,0 +1,37 @@ +import { Body, Controller, Get, Inject, Param, Post, Req, UseGuards } from '@nestjs/common'; +import type { Request } from 'express'; +import { RbacGuard, RequirePermission, type AuthenticatedRequest } from '../rbac/rbac.guard.js'; +import { requireSession } from '../http/validation.js'; +import { PollService, type PollCard } from './poll.service.js'; + +@Controller('chatgov/polls') +@UseGuards(RbacGuard) +export class PollController { + constructor(@Inject(PollService) private readonly polls: PollService) {} + @Get(':id') + @RequirePermission('workspace.view') + async get(@Req() req: Request & AuthenticatedRequest, @Param('id') id: string): Promise { + const session = requireSession(req); + return this.polls.get(session.workspaceId, id, session.userId); + } + @Post() + @RequirePermission('chatgov.vote') + async open(@Req() req: Request & AuthenticatedRequest, @Body() body: unknown): Promise { + const session = requireSession(req); + return this.polls.open(session.workspaceId, body, { userId: session.userId }); + } + @Post(':id/options') + @RequirePermission('chatgov.vote') + async fork(@Req() req: Request & AuthenticatedRequest, @Param('id') id: string, @Body() body: unknown): Promise { + const session = requireSession(req); + return this.polls.forkEdit(session.workspaceId, id, body, { userId: session.userId }); + } + // Any authenticated member can store an ineligible ballot. Eligibility is + // enforced by the service; identity fields in the body are never trusted. + @Post(':id/vote') + @RequirePermission('workspace.view') + async vote(@Req() req: Request & AuthenticatedRequest, @Param('id') id: string, @Body() body: unknown): Promise { + const session = requireSession(req); + return this.polls.vote(session.workspaceId, id, body, { kind: 'user', userId: session.userId }); + } +} diff --git a/apps/api/src/chatgov/poll.service.ts b/apps/api/src/chatgov/poll.service.ts new file mode 100644 index 0000000..b30de16 --- /dev/null +++ b/apps/api/src/chatgov/poll.service.ts @@ -0,0 +1,380 @@ +import { BadRequestException, ConflictException, ForbiddenException, Injectable, NotFoundException } from '@nestjs/common'; +import type { PoolClient } from 'pg'; +import { withTenant } from '../db/client.js'; +import { parseId, requireObject, requireString } from '../http/validation.js'; +import { scrub } from '../audit/audit.service.js'; +import { draftDestinationAccounts } from '../publishing/approval-gate.service.js'; +import type { DraftPost } from '../publishing/publishing.service.js'; +import { transitionPollDraft } from '../publishing/poll-approval.js'; +import { parsePolicyRule, policyHash, type PolicyRule, type QuorumArm } from './policy.js'; +import { durationMs, evaluatePoll, type EvaluatedBallot, type PollEvaluation } from './poll-evaluator.js'; + +export interface PollActor { userId: string; role?: string } +export interface PollVoter { kind: 'user' | 'chat_identity'; userId?: string; provider?: string; remoteId?: string; handle?: string } +interface PollRow { + id: string; workspace_id: string; draft_id: string | null; draft_revision: string | null; + channel_id: string | null; topic_binding_id: string | null; question: string; + kind: 'yes_no' | 'variant_selection'; state: string; rule: PolicyRule; policy_version: string; + anonymity: string; opens_at: Date; closes_at: Date; created_by: string; + winner_option_id: string | null; eligible_user_ids: string[] | null; + parent_poll_id: string | null; closed_at: Date | null; close_reason: string | null; outbox_state: string; +} +interface OptionRow { id: string; seq: number; text: string; submitted_by: string; submitted_at: Date; draft_revision_id: string | null; author: string } +interface Member { user_id: string; role: string; can_vote: boolean } +interface VoteRow { voter_kind: 'user' | 'chat_identity'; voter_user_id: string | null; voter_remote_id: string; voter_handle: string; + ballot: EvaluatedBallot['ballot']; option_id: string | null; counted: boolean; eligible: boolean; ineligible_reason: string | null } +export interface PollCard { + poll: PollRow; options: OptionRow[]; evaluation: PollEvaluation; + votes: VoteRow[]; voterEligibility: { eligible: boolean; admin: boolean; reason: string | null } | null; +} + +/** All writes use workspace → draft → poll lock order, matching publishing + * and membership epoch updates. The outbox claim and E2 transition share one + * transaction; rollback also releases the claim. Late ballots are no-ops. */ +@Injectable() +export class PollService { + async open(workspaceId: string, body: unknown, actor: PollActor): Promise { + return withTenant(workspaceId, async client => { + await lockWorkspace(client, workspaceId); + const poll = await openPollInTransaction(client, workspaceId, body, actor); + return card(client, poll, actor.userId); + }); + } + async forkEdit(workspaceId: string, pollId: string, body: unknown, actor: PollActor): Promise { + const dto = requireObject(body, 'body'); const text = variantText(dto.text); + return this.locked(workspaceId, pollId, async (client, poll) => { + await requireVoter(client, workspaceId, actor.userId); + await evaluateLocked(client, poll); + if (poll.state !== 'open' || poll.kind !== 'variant_selection') throw new ConflictException('only open variant polls accept forks'); + const seq = Number((await client.query('SELECT COALESCE(max(seq), 0) + 1 AS seq FROM poll_options WHERE poll_id = $1', [poll.id])).rows[0]?.seq); + await insertOption(client, poll, seq, text, actor.userId); + await audit(client, poll, 'poll.option_submitted', actor.userId, { seq }); + return card(client, poll, actor.userId); + }); + } + async vote(workspaceId: string, pollId: string, body: unknown, voter: PollVoter): Promise { + const dto = requireObject(body, 'body'); + const optionId = dto.optionId === undefined || dto.optionId === null ? null : parseId(requireString(dto.optionId, 'optionId')); + const ballot = dto.ballot ?? (optionId ? 'yes' : undefined); + if (!['yes', 'no', 'veto', 'abstain'].includes(String(ballot))) throw new BadRequestException('invalid ballot'); + return this.locked(workspaceId, pollId, async (client, poll) => { + // Expiry is resolved before accepting a late ballot; retries after close + // render the frozen artifact without changing the decision. + if (new Date() >= poll.closes_at) await evaluateLocked(client, poll); + if (poll.state !== 'open') return card(client, poll, voter.userId); + if (poll.kind === 'variant_selection' && ballot === 'yes' && optionId === null) throw new BadRequestException('variant vote requires optionId'); + if (optionId && (poll.kind !== 'variant_selection' || ballot !== 'yes')) throw new BadRequestException('optionId only applies to a variant choice'); + if (optionId && !(await client.query('SELECT id FROM poll_options WHERE id = $1 AND poll_id = $2', [optionId, poll.id])).rowCount) throw new BadRequestException('option does not belong to this poll'); + if (ballot === 'veto' && !poll.rule.allowVeto) throw new BadRequestException('veto is disabled'); + let userId = voter.userId ?? null; + let key: string; + let identityId: string | null = null; + if (voter.kind === 'user') { + if (!userId) throw new BadRequestException('user voter needs userId'); + parseId(userId, 'voter.userId'); key = `user:${userId}`; + } else { + const provider = requireString(voter.provider, 'provider'); + const remote = requireString(voter.remoteId, 'remoteId'); + if (!/^[0-9]{1,20}$/.test(remote) || !/^[a-z0-9_-]+$/.test(provider)) throw new BadRequestException('invalid immutable provider identity'); + key = `${provider}:${remote}`; + // Never trust a supplied linked user id from the gateway. + const identity = (await client.query("SELECT id, user_id, status FROM chat_identities WHERE provider = $1 AND remote_user_id = $2", [provider, remote])).rows[0]; + identityId = identity?.id as string | undefined ?? null; + userId = identity?.status === 'approved' ? identity.user_id as string | null : null; + } + // Linked identities and web users share ONE ballot. Alternate provider + // accounts cannot inflate turnout. Unlinked ballots are still retained. + if (userId) key = `user:${userId}`; + await client.query(`INSERT INTO poll_votes (workspace_id, poll_id, voter_kind, voter_user_id, voter_remote_id, voter_handle, ballot, option_id, eligible, counted, voter_identity_id) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, false, false, $9) + ON CONFLICT (poll_id, voter_remote_id) DO UPDATE SET voter_kind = EXCLUDED.voter_kind, + voter_user_id = EXCLUDED.voter_user_id, voter_identity_id = EXCLUDED.voter_identity_id, voter_handle = EXCLUDED.voter_handle, + ballot = EXCLUDED.ballot, option_id = EXCLUDED.option_id, cast_at = clock_timestamp()`, + [workspaceId, poll.id, voter.kind, userId, key, voter.handle ?? key, ballot, optionId, identityId]); + await audit(client, poll, 'poll.voted', userId, { voter: key, ballot, optionId }); + await evaluateLocked(client, poll); + return card(client, poll, userId ?? undefined); + }); + } + async get(workspaceId: string, pollId: string, userId?: string): Promise { + return this.locked(workspaceId, pollId, async (client, poll) => { + await evaluateLocked(client, poll); return card(client, poll, userId); + }); + } + async evaluate(workspaceId: string, pollId: string): Promise { return this.get(workspaceId, pollId); } + /** Deliberately re-evaluates the authoritative conditions; callers cannot + * force a pass or supply a stale tally. Suitable for competing timeboxes. */ + async close(workspaceId: string, pollId: string): Promise { return this.get(workspaceId, pollId); } + async sweep(workspaceId: string): Promise { + const ids = await withTenant(workspaceId, c => c.query<{ id: string }>("SELECT id FROM polls WHERE state = 'open' AND (closes_at <= now() OR opens_at + COALESCE((rule->>'minDwellSeconds')::integer,0) * interval '1 second' <= now())", [])); + for (const { id } of ids.rows) await this.evaluate(workspaceId, id); + } + private async locked(workspaceId: string, pollId: string, fn: (client: PoolClient, poll: PollRow) => Promise): Promise { + parseId(pollId, 'pollId'); + return withTenant(workspaceId, async client => { + await lockWorkspace(client, workspaceId); + const found = (await client.query('SELECT * FROM polls WHERE id = $1', [pollId])).rows[0]; + if (!found) throw new NotFoundException('poll not found'); + if (found.draft_id) await client.query('SELECT id FROM drafts WHERE id = $1 FOR UPDATE', [found.draft_id]); + const poll = (await client.query('SELECT * FROM polls WHERE id = $1 FOR UPDATE', [pollId])).rows[0]!; + return fn(client, poll); + }); + } +} + +async function lockWorkspace(client: PoolClient, workspaceId: string): Promise { + await client.query('SELECT id FROM workspaces WHERE id = $1 FOR UPDATE', [workspaceId]); +} +async function members(client: PoolClient, workspaceId: string): Promise { + return (await client.query(`SELECT m.user_id, m.role, EXISTS(SELECT 1 FROM role_permissions rp + WHERE rp.role = m.role AND rp.permission = 'chatgov.vote') AS can_vote + FROM memberships m WHERE workspace_id = $1 AND status = 'active'`, [workspaceId])).rows; +} +async function requireVoter(client: PoolClient, workspaceId: string, userId: string): Promise { + parseId(userId, 'actor.userId'); + if (!(await members(client, workspaceId)).some(m => m.user_id === userId && m.can_vote)) throw new ForbiddenException('chatgov.vote required'); +} +function variantText(value: unknown): string { + const text = requireString(value, 'text'); + if (text.length > 5000) throw new BadRequestException('variant text must be at most 5000 characters'); + return text; +} +async function insertOption(client: PoolClient, poll: PollRow, seq: number, text: string, userId: string): Promise { + await client.query('INSERT INTO poll_options (workspace_id, poll_id, seq, text, submitted_by) VALUES ($1,$2,$3,$4,$5)', [poll.workspace_id, poll.id, seq, text, userId]); +} +async function options(client: PoolClient, pollId: string): Promise { + return (await client.query(`SELECT o.*, u.display_name AS author FROM poll_options o + JOIN users u ON u.id = o.submitted_by WHERE poll_id = $1 ORDER BY seq`, [pollId])).rows; +} +async function audit(client: PoolClient, poll: PollRow, action: string, userId: string | null, diff: Record): Promise { + await client.query("SELECT audit_append($1,$2,$3,$4,'poll',$5,$6)", [poll.workspace_id, userId, userId ? 'Member' : 'System', action, poll.id, + scrub({ ...diff, policyVersion: poll.policy_version, actor: userId ? `user:${userId}` : `system:chat-poll-gate:${poll.id}` })]); +} + +/** Snapshot all applicable arms. Different OR-policy groups cannot be + * flattened into a single §1a object without changing their meaning. */ +export async function draftPollRule(client: PoolClient, draft: { post: DraftPost }, supplied?: unknown): Promise { + const accountIds = draftDestinationAccounts(draft.post); + const live = await client.query("SELECT 1 FROM posting_rules WHERE superseded_by IS NULL LIMIT 1"); + const rules: PolicyRule[] = []; + if (accountIds.length && live.rowCount) { + for (const account of accountIds) { + const topic = typeof draft.post.providerMetadata.topicBindingId === 'string' ? draft.post.providerMetadata.topicBindingId : null; + const snapshot = (await client.query(`SELECT effective FROM posting_policy_snapshots WHERE account_id = $1 + AND content_type = '' AND (topic_binding_id IS NULL OR topic_binding_id = $2) + ORDER BY topic_binding_id NULLS LAST LIMIT 1`, [account, topic])).rows[0]; + if (!snapshot) throw new ConflictException('OUTSIDE_POLICY: destination snapshot missing'); + const rule = parsePolicyRule(snapshot.effective); + if (rule.mode !== 'immediate' && !rule.dryRun) rules.push(rule); + } + } + let rule: PolicyRule | null = rules[0] ?? null; + if (rules.some(r => policyHash(r) !== policyHash(rule))) { + if (rules.some(r => r.join === 'any')) throw new ConflictException('incompatible OR policy groups require independent polls'); + rule = parsePolicyRule({ ...rules[0], mode: 'quorum', join: 'all', + arms: rules.flatMap(r => r.mode === 'admin_only' ? [{ kind: 'admins', x: 1 }] : r.arms ?? []).filter((a, i, all) => all.findIndex(b => b.kind === a.kind && b.of === a.of && b.x === a.x) === i), + allowVeto: rules.some(r => r.allowVeto), allowSelfApproval: rules.every(r => r.allowSelfApproval), + resetOnChange: rules.some(r => r.resetOnChange), minDwellSeconds: Math.max(...rules.map(r => r.minDwellSeconds ?? 0)), + timebox: { duration: rules.reduce((a,b) => durationMs(a.timebox!.duration) >= durationMs(b.timebox!.duration) ? a : b).timebox!.duration, onExpire: 'reject' } }); + } + if (supplied !== undefined) { + const requested = parsePolicyRule(supplied); + if (rule && policyHash(rule) !== policyHash(requested)) throw new BadRequestException('rule must match the compiled destination policy'); + rule ??= requested; + } + return rule; +} + +export async function openPollInTransaction(client: PoolClient, workspaceId: string, body: unknown, actor: PollActor): Promise { + const dto = requireObject(body, 'body'); + await requireVoter(client, workspaceId, actor.userId); + if (dto.contentKind !== undefined && dto.contentKind !== 'short_post') throw new BadRequestException('document polls are a future track'); + const draftId = dto.draftId == null ? null : parseId(requireString(dto.draftId, 'draftId')); + const draft = draftId ? (await client.query('SELECT * FROM drafts WHERE id = $1 FOR UPDATE', [draftId])).rows[0] : null; + if (draftId && !draft) throw new NotFoundException('draft not found'); + if (draft && !['editing', 'in_review'].includes(String(draft.state))) throw new ConflictException('draft is not reviewable'); + const rule = draft ? await draftPollRule(client, draft as { post: DraftPost }, dto.rule) : parsePolicyRule(dto.rule); + if (!rule) throw new BadRequestException('a rule or compiled chat-signal policy is required'); + if (rule.anonymity !== 'named') throw new BadRequestException('anonymous sentiment belongs to the separate Telegram track; governance needs attributable ballots'); + const kind = dto.kind ?? (draft ? 'variant_selection' : 'yes_no'); + if (kind !== 'yes_no' && kind !== 'variant_selection') throw new BadRequestException('invalid poll kind'); + if (kind === 'variant_selection' && !draft) throw new BadRequestException('variant selection requires a draft'); + if (dto.variants !== undefined && !Array.isArray(dto.variants)) throw new BadRequestException('variants must be an array'); + const variants = (dto.variants as unknown[] | undefined ?? []).map(v => variantText(typeof v === 'string' ? v : requireObject(v, 'variant').text)); + if (kind === 'yes_no' && variants.length) throw new BadRequestException('yes/no polls do not accept variants'); + if (variants.length > 99) throw new BadRequestException('at most 99 initial forks'); + const eligibleIds = dto.eligibleUserIds === undefined ? null : (Array.isArray(dto.eligibleUserIds) ? dto.eligibleUserIds.map(v => parseId(requireString(v, 'eligibleUserId'))) : (() => { throw new BadRequestException('eligibleUserIds must be an array'); })()); + const question = requireString(dto.question ?? 'Choose the post to approve', 'question'); + if (question.length > 300) throw new BadRequestException('question too long'); + let duration: number; + try { duration = durationMs(rule.timebox?.duration ?? 'PT1H'); } catch (error) { throw new BadRequestException(String(error)); } + const now = new Date(); + const poll = (await client.query(`INSERT INTO polls + (workspace_id, draft_id, subject_kind, question, kind, rule, policy_version, opens_at, closes_at, created_by, draft_revision, parent_poll_id, channel_id, topic_binding_id, eligible_user_ids) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) RETURNING *`, + [workspaceId, draftId, draftId ? 'draft' : 'none', question, kind, rule, policyHash(rule), now, new Date(now.getTime() + duration), actor.userId, + draft?.revision ?? null, dto.parentPollId == null ? null : parseId(requireString(dto.parentPollId, 'parentPollId')), + dto.channelId == null ? null : parseId(requireString(dto.channelId, 'channelId')), + dto.topicBindingId == null ? null : parseId(requireString(dto.topicBindingId, 'topicBindingId')), eligibleIds])).rows[0]!; + if (kind === 'variant_selection') { + await insertOption(client, poll, 1, variantText((draft!.post as DraftPost).text), String(draft!.author_user_id)); + for (const [i, text] of variants.entries()) await insertOption(client, poll, i + 2, text, actor.userId); + } + await audit(client, poll, 'poll.opened', actor.userId, { draftId, kind, parentPollId: poll.parent_poll_id }); + return poll; +} + +async function resolveTally(client: PoolClient, poll: PollRow, persist: boolean): Promise<{ evaluation: PollEvaluation; votes: VoteRow[] }> { + const roster = await members(client, poll.workspace_id); + const draft = poll.draft_id ? (await client.query('SELECT author_user_id FROM drafts WHERE id = $1', [poll.draft_id])).rows[0] : null; + const opts = await options(client, poll.id); + const votes = (await client.query(`SELECT v.*, ci.status AS identity_status, ci.user_id AS identity_user_id, ci.role_override + FROM poll_votes v LEFT JOIN chat_identities ci ON ci.id = v.voter_identity_id + WHERE poll_id = $1 ORDER BY cast_at DESC, voter_remote_id`, [poll.id])).rows; + // Ingress currently persists handles, not immutable remote ids. Only + // numeric author_handle evidence can bind to approved identities safely. + // A username must never become an authorization key. + const observed = new Set(); + if (poll.channel_id) { + const rows = await client.query<{ user_id: string }>(`SELECT DISTINCT ci.user_id FROM live_chat_messages lm + JOIN chat_identities ci ON ci.provider = lm.provider AND ci.remote_user_id = lm.author_handle + WHERE lm.channel_id = $1 AND lm.received_at >= $2 AND ci.status = 'approved' AND ci.user_id IS NOT NULL`, [poll.channel_id, poll.opens_at]); + for (const row of rows.rows) observed.add(row.user_id); + } + const seen = new Set(); + const ballots: EvaluatedBallot[] = []; + const eligibleMembers = roster.filter(m => m.can_vote); + const identities = (await client.query<{ user_id: string; role_override: string }>("SELECT user_id, role_override FROM chat_identities WHERE status = 'approved' AND role_override IN ('Admin','Owner')")).rows; + const adminIds = new Set(roster.filter(m => m.can_vote && ['Admin', 'Owner'].includes(m.role)).map(m => m.user_id)); + for (const identity of identities) if (roster.some(m => m.user_id === identity.user_id)) adminIds.add(identity.user_id); + for (const vote of votes) { + const identityOkay = !vote.voter_identity_id || (vote.identity_status === 'approved' && vote.identity_user_id === vote.voter_user_id); + const member = roster.find(m => m.user_id === vote.voter_user_id); + const eligible = identityOkay && member !== undefined && (member.can_vote || ['Admin','Owner'].includes(vote.role_override ?? '')); + const selfDenied = !poll.rule.allowSelfApproval && vote.ballot === 'yes' && (poll.kind === 'variant_selection' + ? opts.some(o => o.id === vote.option_id && o.submitted_by === vote.voter_user_id) + : draft?.author_user_id === vote.voter_user_id); + const duplicate = vote.voter_user_id !== null && seen.has(vote.voter_user_id); + if (eligible && vote.voter_user_id) seen.add(vote.voter_user_id); + vote.eligible = eligible; + vote.counted = eligible && !selfDenied && !duplicate; + vote.ineligible_reason = !eligible || duplicate ? 'INELIGIBLE_ACTOR' : selfDenied ? 'SELF_APPROVAL_DENIED' : null; + if (persist) await client.query(`UPDATE poll_votes SET eligible = $3, counted = $4, ineligible_reason = $5 + WHERE poll_id = $1 AND voter_remote_id = $2`, [poll.id, vote.voter_remote_id, vote.eligible, vote.counted, vote.ineligible_reason]); + ballots.push({ ballot: vote.ballot, optionId: vote.option_id, counted: vote.counted, + admin: adminIds.has(vote.voter_user_id ?? ''), observed: observed.has(vote.voter_user_id ?? ''), + participant: poll.eligible_user_ids === null || poll.eligible_user_ids.includes(vote.voter_user_id ?? '') }); + } + const eligible = eligibleMembers.filter(m => poll.eligible_user_ids === null || poll.eligible_user_ids.includes(m.user_id)); + // Potential self-voters must have at least one option authored by someone + // else. A no/self ballot can be recast, so it is never subtracted twice. + const canChoose = (id: string): boolean => poll.rule.allowSelfApproval === true || (poll.kind === 'variant_selection' + ? opts.some(o => o.submitted_by !== id) : draft?.author_user_id !== id); + const evaluation = evaluatePoll({ kind: poll.kind, rule: poll.rule, opensAt: poll.opens_at, closesAt: poll.closes_at, now: new Date(), ballots, + options: opts.map(o => ({ id: o.id, seq: o.seq, submittedAt: o.submitted_at.toISOString() })), + possible: { admins: [...adminIds].filter(canChoose).length, eligible: eligible.filter(m => canChoose(m.user_id)).length, observed: observed.size } }); + // Keep the public DTO separate from identity resolution internals. + return { evaluation, votes: votes.map(v => ({ voter_kind: v.voter_kind, voter_user_id: v.voter_user_id, + voter_remote_id: v.voter_remote_id, voter_handle: v.voter_handle, ballot: v.ballot, option_id: v.option_id, + counted: v.counted, eligible: v.eligible, ineligible_reason: v.ineligible_reason })) }; +} + +async function evaluateLocked(client: PoolClient, poll: PollRow): Promise { + if (poll.state !== 'open') return; + if (poll.draft_id && poll.draft_revision !== null) { + const draft = (await client.query('SELECT revision FROM drafts WHERE id = $1', [poll.draft_id])).rows[0]; + if (String(draft?.revision) !== String(poll.draft_revision)) { + await stalePoll(client, poll); return; + } + } + const { evaluation } = await resolveTally(client, poll, true); + await audit(client, poll, 'policy.evaluated', null, { evaluation }); + // Shadow polls retain their artifact and terminal outcome but cannot + // approve/reject drafts. Their E2 decision is pending, never a release. + if (!evaluation.state) return; + const claimed = await client.query(`UPDATE polls SET outbox_state = 'claimed', state = $2, closed_at = clock_timestamp(), + close_reason = $3, winner_option_id = $4, final_evaluation = $5 + WHERE id = $1 AND state = 'open' AND outbox_state = 'none' RETURNING *`, + [poll.id, evaluation.state, evaluation.reason, + evaluation.state === 'passed' && poll.kind === 'variant_selection' ? evaluation.winnerOptionId : null, evaluation]); + if (!claimed.rowCount) return; + Object.assign(poll, claimed.rows[0]); + const reasons = evaluation.state === 'vetoed' ? ['VETOED'] : evaluation.state === 'quorum_unreachable' ? ['INSUFFICIENT_QUORUM'] + : evaluation.reason === 'timebox' ? [evaluation.state === 'passed' ? 'TIMEBOX_EXPIRED_AUTO_APPROVE' : 'TIMEBOX_EXPIRED_AUTO_REJECT'] : []; + if (poll.draft_id && !poll.rule.dryRun) { + const winnerText = poll.winner_option_id ? (await options(client, poll.id)).find(o => o.id === poll.winner_option_id)?.text : undefined; + await transitionPollDraft(client, { workspaceId: poll.workspace_id, draftId: poll.draft_id, pollId: poll.id, + passed: evaluation.state === 'passed', ...(winnerText === undefined ? {} : { text: winnerText }), + reason: reasons[0] ?? 'quorum_early', escalate: evaluation.reason === 'timebox' && poll.rule.timebox?.onExpire === 'escalateOwner' }); + } + await recordE2(client, poll, evaluation, poll.rule.dryRun ? ['POLICY_DRY_RUN'] : reasons); + await audit(client, poll, 'poll.closed', null, { state: poll.state, reason: poll.close_reason, winnerOptionId: poll.winner_option_id, evaluation }); + await audit(client, poll, evaluation.state === 'passed' && !poll.rule.dryRun ? 'release.fired' : 'release.blocked', null, { reasons }); + await client.query("UPDATE polls SET outbox_state = 'done' WHERE id = $1 AND outbox_state = 'claimed'", [poll.id]); + poll.outbox_state = 'done'; +} +async function recordE2(client: PoolClient, poll: PollRow, evaluation: PollEvaluation, reasons: string[]): Promise { + if (!poll.draft_id) return; + const draft = (await client.query('SELECT post FROM drafts WHERE id = $1', [poll.draft_id])).rows[0]; + for (const accountId of draftDestinationAccounts(draft.post as DraftPost)) { + const provider = (await client.query('SELECT provider FROM connections WHERE id = $1', [accountId])).rows[0]?.provider ?? 'unknown'; + await client.query(`INSERT INTO gate_decisions + (workspace_id, draft_id, destination, gate, policy_version, outcome, inputs, reason_codes, actor, poll_id) + VALUES ($1,$2,$3,'E2',$4,$5,$6,$7,$8,$9)`, + [poll.workspace_id, poll.draft_id, { platform: provider, accountId, ...(poll.topic_binding_id ? { topicBindingId: poll.topic_binding_id } : {}) }, + poll.policy_version, poll.rule.dryRun ? 'pending' : poll.state === 'passed' ? 'fired' : 'blocked', + { ...evaluation, pollState: poll.state }, reasons, `system:chat-poll-gate:${poll.id}`, poll.id]); + } +} +async function card(client: PoolClient, poll: PollRow, userId?: string): Promise { + const final = (poll as PollRow & { final_evaluation?: PollEvaluation }).final_evaluation; + const tally = final && poll.state !== 'open' + ? { evaluation: final, votes: (await client.query('SELECT voter_kind, voter_user_id, voter_remote_id, voter_handle, ballot, option_id, counted, eligible, ineligible_reason FROM poll_votes WHERE poll_id = $1 ORDER BY cast_at', [poll.id])).rows } + : await resolveTally(client, poll, false); + const member = userId ? (await members(client, poll.workspace_id)).find(m => m.user_id === userId) : undefined; + return { poll, options: await options(client, poll.id), evaluation: tally.evaluation, votes: tally.votes, + voterEligibility: userId ? { eligible: member?.can_vote ?? false, admin: member !== undefined && ['Admin','Owner'].includes(member.role), reason: member?.can_vote ? null : 'INELIGIBLE_ACTOR' } : null }; +} + +async function stalePoll(client: PoolClient, poll: PollRow): Promise { + const { evaluation } = await resolveTally(client, poll, true); + const changed = await client.query(`UPDATE polls SET state = 'stale', closed_at = clock_timestamp(), close_reason = 'reset_on_change', + outbox_state = 'done', final_evaluation = $2 WHERE id = $1 AND state = 'open' AND outbox_state = 'none' RETURNING *`, [poll.id, evaluation]); + if (!changed.rowCount) return; + Object.assign(poll, changed.rows[0]); + await audit(client, poll, 'poll.stale', null, { reason: 'RESET_ON_CHANGE' }); + await recordE2(client, poll, evaluation, ['RESET_ON_CHANGE']); + if (poll.rule.dryRun || !poll.draft_id) return; + if (poll.rule.resetOnChange) { + await openPollInTransaction(client, poll.workspace_id, { + draftId: poll.draft_id, kind: poll.kind, rule: poll.rule, question: poll.question, + parentPollId: poll.id, channelId: poll.channel_id, topicBindingId: poll.topic_binding_id, + ...(poll.eligible_user_ids ? { eligibleUserIds: poll.eligible_user_ids } : {}), + }, { userId: await rearmActor(client, poll) }); + } else { + // Stale content can never be auto-approved, even with an approve expiry + // policy. escalateOwner keeps it in review; all other stale gates reject. + await transitionPollDraft(client, { workspaceId: poll.workspace_id, draftId: poll.draft_id, pollId: poll.id, + passed: false, reason: 'RESET_ON_CHANGE', escalate: poll.rule.timebox?.onExpire === 'escalateOwner' }); + } +} +async function rearmActor(client: PoolClient, poll: PollRow): Promise { + const roster = await members(client, poll.workspace_id); + const actor = roster.find(m => m.user_id === poll.created_by && m.can_vote) ?? roster.find(m => m.can_vote && ['Owner','Admin'].includes(m.role)); + if (!actor) throw new ConflictException('cannot re-arm without an eligible opener'); + return actor.user_id; +} +/** Called under publishing's workspace/draft lock, after its revision trigger. + * The old options and ballots stay attached to the stale artifact forever. */ +export async function markDraftPollsStale(client: PoolClient, workspaceId: string, draftId: string): Promise { + const polls = (await client.query("SELECT * FROM polls WHERE workspace_id = $1 AND draft_id = $2 AND state = 'open' FOR UPDATE", [workspaceId, draftId])).rows; + for (const poll of polls) await evaluateLocked(client, poll); +} +/** Auto-arm on submit, using ONLY compiled destination snapshots. */ +export async function autoOpenDraftPoll(client: PoolClient, workspaceId: string, draftId: string, actor: PollActor, post: DraftPost): Promise { + if (!draftDestinationAccounts(post).length) return; + const rule = await draftPollRule(client, { post }); + if (!rule) return; + if ((await client.query("SELECT 1 FROM polls WHERE draft_id = $1 AND state = 'open'", [draftId])).rowCount) return; + await openPollInTransaction(client, workspaceId, { draftId, kind: 'variant_selection', rule }, actor); +} diff --git a/apps/api/src/connections/connection-store.test.ts b/apps/api/src/connections/connection-store.test.ts new file mode 100644 index 0000000..7df6f0e --- /dev/null +++ b/apps/api/src/connections/connection-store.test.ts @@ -0,0 +1,76 @@ +import { randomUUID } from "node:crypto"; +import type { PoolClient } from "pg"; +import { describe, expect, it } from "vitest"; +import type { ConnectorConnection } from "@tactical-media-broadcast-hub/core"; +import type { SessionPayload } from "../auth/session.js"; +import { Vault } from "../vault/vault.js"; +import { PgConnectionStore } from "./connection-store.js"; + +const session: SessionPayload = { workspaceId: randomUUID(), userId: randomUUID(), role: "Owner", authzEpoch: 3, issuedAt: 0, expiresAt: Math.floor(Date.now() / 1000) + 600 }; +const connection: ConnectorConnection = { id: randomUUID(), workspaceId: session.workspaceId, provider: "mastodon", channelId: "remote-account", credentialId: { kind: "credential-ref", id: "opaque" }, fields: { instanceUrl: "https://social.example" }, createdAt: new Date() }; + +function fixture(allowed = true, existingCredential = false) { + const queries: Array<{ sql: string; params: unknown[] }> = []; + const credentialId = randomUUID(); + const client = { + async query(sql: string, params: unknown[] = []) { + queries.push({ sql, params }); + if (sql.includes("FROM memberships")) return { rowCount: allowed ? 1 : 0, rows: [] }; + if (sql.startsWith("UPDATE scheduled_publications")) return { rowCount: 1, rows: [{ id: connection.id, draft_id: connection.id }] }; + if (sql.includes("SELECT id FROM credentials")) return { rows: existingCredential ? [{ id: credentialId }] : [] }; + if (sql.includes("SELECT c.*, cr.ciphertext")) return { rows: [{ id: connection.id, provider: connection.provider, channel_id: connection.channelId, fields: connection.fields, created_at: connection.createdAt, ciphertext: null, key_version: 1 }] }; + return { rowCount: 1, rows: [{ id: connection.id }] }; + }, + } as unknown as PoolClient; + const store = new PgConnectionStore(async (workspaceId, callback) => { + expect(workspaceId).toBe(session.workspaceId); + return callback(client); + }); + return { store, queries, credentialId }; +} + +describe("Postgres connection writes", () => { + it("saves only a vault envelope and identifier-only audit in the same tenant transaction", async () => { + const { store, queries } = fixture(); + const sealed = await new Vault().seal(session.workspaceId, connection.id, Buffer.from('{"access_token":"never-in-db-plaintext"}')); + await store.save(session, connection, { provider: connection.provider, remoteId: "remote-account", handle: "@alice" }, sealed, false); + const credential = queries.find(({ sql }) => sql.startsWith("INSERT INTO credentials")); + expect(credential?.params).toContain(sealed.ciphertext); + expect(queries[0]?.sql).toContain("FOR UPDATE"); + expect(JSON.stringify(queries)).not.toContain("never-in-db-plaintext"); + const audit = queries.find(({ sql }) => sql.includes("audit_append")); + expect(audit?.params.at(-1)).toEqual({ provider: connection.provider }); + }); + it("rotates the credential row in place and blocks a revoked authorization before writes", async () => { + const { store, queries, credentialId } = fixture(true, true); + const sealed = await new Vault().seal(session.workspaceId, connection.id, Buffer.from("rotated-secret")); + await store.save(session, connection, { provider: connection.provider, remoteId: "remote-account", handle: "@alice" }, sealed, true); + expect(queries.find(({ sql }) => sql.startsWith("UPDATE credentials"))?.params[0]).toBe(credentialId); + expect(queries.some(({ sql }) => sql.startsWith("INSERT INTO connections"))).toBe(false); + const denied = fixture(false); + await expect(denied.store.save(session, connection, { provider: connection.provider, remoteId: "remote-account", handle: "@alice" }, sealed, true)).rejects.toThrow(); + expect(denied.queries.some(({ sql }) => /^(INSERT|UPDATE)/.test(sql))).toBe(false); + }); + it("locks and revokes locally before provider I/O while preserving publishing history", async () => { + const { store, queries } = fixture(); + await store.remove(session, connection.id, async (stored) => { + expect(stored.connection.id).toBe(connection.id); + expect(queries.some(({ sql }) => sql.includes("FOR UPDATE"))).toBe(true); + expect(queries.some(({ sql }) => sql.startsWith("DELETE"))).toBe(false); + }); + expect(queries.filter(({ sql }) => sql.startsWith("DELETE"))).toHaveLength(0); + expect(queries.some(({ sql }) => sql.startsWith("UPDATE credentials SET revoked_at"))).toBe(true); + expect(queries.some(({ sql }) => sql.startsWith("UPDATE connections SET disconnected_at"))).toBe(true); + expect(queries.some(({ sql }) => sql.includes("needs_reauthorization"))).toBe(true); + expect(queries.some(({ sql }) => sql.startsWith("UPDATE publishing_attempts") && sql.includes("abandoned"))).toBe(true); + expect(queries.some(({ sql }) => sql.startsWith("UPDATE drafts") && sql.includes("approved_revision = revision"))).toBe(true); + expect(queries.at(-1)?.sql).toContain("audit_append"); + }); + it("retains revoked local evidence and audits provider revocation failure", async () => { + const { store, queries } = fixture(); + await expect(store.remove(session, connection.id, async () => { throw new Error("provider offline"); })).resolves.toBeUndefined(); + expect(queries.some(({ sql }) => sql.startsWith("UPDATE credentials SET revoked_at"))).toBe(true); + expect(queries.at(-1)?.sql).toContain("connection.revoke_failed"); + expect(queries.some(({ sql }) => sql.startsWith("DELETE"))).toBe(false); + }); +}); diff --git a/apps/api/src/connections/connection-store.ts b/apps/api/src/connections/connection-store.ts new file mode 100644 index 0000000..a8246ad --- /dev/null +++ b/apps/api/src/connections/connection-store.ts @@ -0,0 +1,117 @@ +import { NotFoundException, ForbiddenException } from "@nestjs/common"; +import type { ConnectorConnection, NormalizedIdentity } from "@tactical-media-broadcast-hub/core"; +import type { SessionPayload } from "../auth/session.js"; +import type { PoolClient } from "pg"; +import { withTenant } from "../db/client.js"; +import type { SealedCredential } from "../vault/vault.js"; +import type { ConnectionSummary } from "./connections.service.js"; +import { scrub } from "../audit/audit.service.js"; + +export interface StoredConnection { connection: ConnectorConnection; credential: SealedCredential | null } +export interface ConnectionStore { + list(workspaceId: string): Promise; + get(workspaceId: string, id: string): Promise; + assertAuthorized(session: SessionPayload): Promise; + save(session: SessionPayload, connection: ConnectorConnection, identity: NormalizedIdentity, credential: SealedCredential, reconnect: boolean): Promise; + rotateCredential(workspaceId: string, id: string, previous: SealedCredential, credential: SealedCredential): Promise; + remove(session: SessionPayload, id: string, revoke: (stored: StoredConnection) => Promise): Promise; +} + +export class PgConnectionStore implements ConnectionStore { + constructor(private readonly transaction: typeof withTenant = withTenant) {} + async assertAuthorized(session: SessionPayload): Promise { + if (session.expiresAt <= Math.floor(Date.now() / 1000)) throw new ForbiddenException("Session expired"); + const result = await this.transaction(session.workspaceId, (client) => client.query( + `SELECT 1 FROM workspaces w JOIN memberships m ON m.workspace_id = w.id + JOIN role_permissions rp ON rp.role = m.role + WHERE w.id = $1 AND w.authz_epoch = $2 AND m.user_id = $3 AND m.status = 'active' AND rp.permission = 'connection.manage'`, + [session.workspaceId, session.authzEpoch, session.userId], + )); + if (result.rowCount !== 1) throw new ForbiddenException("Connection authorization no longer valid"); + } + async list(workspaceId: string): Promise { + const res = await this.transaction(workspaceId, (client) => client.query( + `SELECT c.*, cr.ciphertext, coalesce(cr.rotated_at, cr.created_at) AS rotated_at, + (SELECT array_agg(p.published_at ORDER BY p.published_at DESC) FROM publications p + WHERE p.connection_id = c.id AND p.published_at > now() - interval '24 hours') AS recent + FROM connections c LEFT JOIN LATERAL (SELECT * FROM credentials WHERE connection_id = c.id AND kind = 'connector-oauth' AND revoked_at IS NULL LIMIT 1) cr ON true WHERE c.disconnected_at IS NULL ORDER BY c.created_at`, + )); + return res.rows.map((row: { id: string; provider: string; channel_id: string; fields: Record; ciphertext: Buffer | null; created_at: Date; rotated_at: Date | null; recent: Date[] | null }) => ({ + id: row.id, provider: row.provider, channelId: row.channel_id, fields: scrub(row.fields) ?? {}, + health: row.ciphertext ? "unavailable" : "needs_credentials", credentialRotatedAt: row.rotated_at?.toISOString() ?? null, + recentPublishes: (row.recent ?? []).map((date) => date.toISOString()), createdAt: row.created_at.toISOString(), + })); + } + async get(workspaceId: string, id: string): Promise { + return this.transaction(workspaceId, (client) => this.read(client, workspaceId, id)); + } + private async read(client: PoolClient, workspaceId: string, id: string): Promise { + const result = await client.query( + `SELECT c.*, cr.ciphertext, cr.key_version FROM connections c LEFT JOIN LATERAL + (SELECT * FROM credentials WHERE connection_id = c.id AND kind = 'connector-oauth' AND revoked_at IS NULL LIMIT 1) cr ON true WHERE c.id = $1`, [id], + ); + const row = result.rows[0] as { id: string; provider: string; channel_id: string; fields: Record; created_at: Date; ciphertext: Buffer | null; key_version: number } | undefined; + if (!row) throw new NotFoundException("Connection not found"); + return { + connection: { id: row.id, workspaceId, provider: row.provider, channelId: row.channel_id, fields: row.fields, createdAt: row.created_at, credentialId: { kind: "credential-ref", id: row.id } }, + credential: row.ciphertext ? { ciphertext: row.ciphertext, keyVersion: row.key_version } : null, + }; + } + async save(session: SessionPayload, connection: ConnectorConnection, identity: NormalizedIdentity, credential: SealedCredential, reconnect: boolean): Promise { + await this.transaction(session.workspaceId, async (client) => { + // Serialize with membership revocation; re-check at commit, not only before provider I/O. + await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [session.workspaceId]); + const allowed = await client.query(`SELECT 1 FROM memberships m JOIN workspaces w ON w.id = m.workspace_id JOIN role_permissions rp ON rp.role = m.role WHERE m.user_id = $1 AND m.status = 'active' AND w.authz_epoch = $2 AND rp.permission = 'connection.manage'`, [session.userId, session.authzEpoch]); + if (!allowed.rowCount || session.expiresAt <= Math.floor(Date.now() / 1000)) throw new ForbiddenException(); + const fields = { ...connection.fields, handle: identity.handle, displayName: identity.displayName ?? identity.handle, remoteId: identity.remoteId }; + if (reconnect) { + const updated = await client.query("UPDATE connections SET fields = $2, disconnected_at = NULL WHERE id = $1 AND provider = $3 RETURNING id", [connection.id, fields, connection.provider]); + if (!updated.rowCount) throw new NotFoundException("Connection removed during authorization"); + } else { + await client.query("INSERT INTO connections (id, workspace_id, provider, channel_id, fields) VALUES ($1,$2,$3,$4,$5)", [connection.id, session.workspaceId, connection.provider, connection.channelId, fields]); + } + const existing = await client.query("SELECT id FROM credentials WHERE connection_id = $1 AND kind = 'connector-oauth' FOR UPDATE", [connection.id]); + if (existing.rows[0]) { + await client.query("UPDATE credentials SET ciphertext = $2, key_version = $3, rotated_at = now(), revoked_at = NULL WHERE id = $1", [existing.rows[0].id, credential.ciphertext, credential.keyVersion]); + } else { + await client.query("INSERT INTO credentials (workspace_id, connection_id, kind, ciphertext, key_version) VALUES ($1,$2,'connector-oauth',$3,$4)", [session.workspaceId, connection.id, credential.ciphertext, credential.keyVersion]); + } + await client.query("SELECT audit_append($1,$2,$3,$4,'connection',$5,$6)", [session.workspaceId, session.userId, session.role, reconnect ? "connection.reconnect" : "connection.connect", connection.id, { provider: connection.provider }]); + }); + } + async remove(session: SessionPayload, id: string, revoke: (stored: StoredConnection) => Promise): Promise { + let stored: StoredConnection | undefined; + await this.transaction(session.workspaceId, async (client) => { + await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [session.workspaceId]); + const allowed = await client.query(`SELECT 1 FROM memberships m JOIN workspaces w ON w.id = m.workspace_id JOIN role_permissions rp ON rp.role = m.role WHERE m.user_id = $1 AND m.status = 'active' AND w.authz_epoch = $2 AND rp.permission = 'connection.manage'`, [session.userId, session.authzEpoch]); + if (!allowed.rowCount || session.expiresAt <= Math.floor(Date.now() / 1000)) throw new ForbiddenException(); + const locked = await client.query("SELECT id FROM connections WHERE id = $1 FOR UPDATE", [id]); + if (!locked.rowCount) throw new NotFoundException("Connection not found"); + stored = await this.read(client, session.workspaceId, id); + await client.query("UPDATE credentials SET revoked_at = now() WHERE connection_id = $1 AND revoked_at IS NULL", [id]); + await client.query("UPDATE connections SET disconnected_at = COALESCE(disconnected_at, now()) WHERE id = $1", [id]); + // Undispatched intent pauses. Dispatched or unknown work retains all evidence. + const paused = await client.query<{ id: string; draft_id: string }>(`UPDATE scheduled_publications s SET state = 'needs_reauthorization', error_code = 'CREDENTIAL_CHANGED' + WHERE connection_id = $1 AND state IN ('pending', 'in_flight') AND NOT EXISTS + (SELECT 1 FROM publishing_attempts a WHERE a.schedule_id = s.id AND (a.status = 'unknown' OR (a.dispatch_started_at IS NOT NULL AND a.status NOT IN ('completed', 'failed')))) + RETURNING s.id, s.draft_id`, [id]); + const scheduleIds = paused.rows.map((row) => row.id); + const draftIds = paused.rows.map((row) => row.draft_id); + await client.query("UPDATE publishing_attempts SET status = 'abandoned', owner_token = NULL, lease_until = NULL, error_code = 'CREDENTIAL_CHANGED' WHERE schedule_id = ANY($1::uuid[]) AND status IN ('ready', 'in_progress')", [scheduleIds]); + await client.query("UPDATE drafts SET state = CASE WHEN approved_revision = revision AND approved_hash IS NOT NULL THEN 'approved' ELSE 'editing' END, updated_at = now() WHERE id = ANY($1::uuid[]) AND state = 'scheduled'", [draftIds]); + await client.query("SELECT audit_append($1,$2,$3,'connection.remove','connection',$4,NULL)", [session.workspaceId, session.userId, session.role, id]); + }); + if (stored) { + try { await revoke(stored); } + catch { await this.transaction(session.workspaceId, (client) => client.query("SELECT audit_append($1,$2,$3,'connection.revoke_failed','connection',$4,NULL)", [session.workspaceId, session.userId, session.role, id])); } + } + } + async rotateCredential(workspaceId: string, id: string, previous: SealedCredential, credential: SealedCredential): Promise { + await this.transaction(workspaceId, async (client) => { + await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [workspaceId]); + // A health refresh must never overwrite a concurrent explicit reconnect. + const updated = await client.query("UPDATE credentials SET ciphertext = $3, key_version = $4, rotated_at = now() WHERE connection_id = $1 AND ciphertext = $2 AND kind = 'connector-oauth' AND revoked_at IS NULL RETURNING id", [id, previous.ciphertext, credential.ciphertext, credential.keyVersion]); + if (updated.rowCount) await client.query("SELECT audit_append($1,NULL,'Service','credential.refresh','connection',$2,NULL)", [workspaceId, id]); + }); + } +} diff --git a/apps/api/src/connections/connections.controller.ts b/apps/api/src/connections/connections.controller.ts new file mode 100644 index 0000000..f42da3f --- /dev/null +++ b/apps/api/src/connections/connections.controller.ts @@ -0,0 +1,84 @@ +import { Body, Controller, Delete, Get, Inject, Param, Post, Query, Req, Res, UseGuards } from "@nestjs/common"; +import type { Request, Response } from "express"; +import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import { optionalString, parseId, requireObject, requireSession, requireString } from "../http/validation.js"; +import { ConnectionsService, type ConnectionSummary } from "./connections.service.js"; +import { AUTH_TTL_MS, stateCookieName, validState } from "./pending.js"; + +@Controller("connections") +export class ConnectionsController { + constructor(@Inject(ConnectionsService) private readonly connections: ConnectionsService) {} + + @Get() + @UseGuards(RbacGuard) + @RequirePermission("connection.view") + async list(@Req() request: Request & AuthenticatedRequest): Promise { + return this.connections.list(requireSession(request).workspaceId); + } + + @Get("providers") + @UseGuards(RbacGuard) + @RequirePermission("connection.view") + providers() { return this.connections.providers(); } + + private redirectResponse(response: Response, authorization: { redirectUrl: string; state: string; binding: string }) { + response.setHeader("Cache-Control", "no-store"); + response.cookie(stateCookieName(authorization.state), authorization.binding, { + httpOnly: true, secure: process.env.NODE_ENV === "production", sameSite: "lax", + path: "/connections/callback", maxAge: AUTH_TTL_MS, + }); + return { redirectUrl: authorization.redirectUrl }; + } + + @Post("authorize") + @UseGuards(RbacGuard) + @RequirePermission("connection.manage") + async authorize(@Req() request: Request & AuthenticatedRequest, @Body() body: unknown, @Res({ passthrough: true }) response: Response) { + const input = requireObject(body, "authorization"); + const authorization = await this.connections.authorize(requireSession(request), requireString(input.provider, "provider"), optionalString(input.instanceUrl, "instanceUrl")); + return this.redirectResponse(response, authorization); + } + + /** Deliberately outside RbacGuard: the provider's browser redirect carries no bearer header. */ + @Get("callback") + async callback(@Req() request: Request, @Query() query: Record, @Res() response: Response): Promise { + const state = typeof query.state === "string" ? query.state : ""; + const name = validState(state) ? stateCookieName(state) : ""; + const cookie = name ? (request.headers.cookie ?? "").split(";").map((part) => part.trim()).find((part) => part.startsWith(`${name}=`))?.slice(name.length + 1) : undefined; + let binding = ""; + try { binding = decodeURIComponent(cookie ?? ""); } catch { /* Malformed cookies fail state validation. */ } + const location = await this.connections.callback(state, binding, typeof query.code === "string" ? query.code : undefined, typeof query.error === "string" ? query.error : undefined); + if (name) response.clearCookie(name, { path: "/connections/callback", httpOnly: true, secure: process.env.NODE_ENV === "production", sameSite: "lax" }); + response.setHeader("Cache-Control", "no-store"); + response.setHeader("Referrer-Policy", "no-referrer"); + response.redirect(303, location); + } + + // Public endpoint spelling is API routing, not provider behavior branching. + // eslint-disable-next-line no-restricted-syntax + @Post("telegram/bot-token") + @UseGuards(RbacGuard) + @RequirePermission("connection.manage") + async botToken(@Req() request: Request & AuthenticatedRequest, @Body() body: unknown): Promise<{ connected: true; token: string }> { + const input = requireObject(body, "bot token"); + const id = optionalString(input.connectionId, "connectionId"); + const channelId = optionalString(input.channelId, "channelId"); + const token = await this.connections.botToken(requireSession(request), requireString(input.token, "token"), channelId ? { channelId } : {}, id ? parseId(id) : undefined); + return { connected: true, token }; + } + + @Post(":connectionId/reconnect") + @UseGuards(RbacGuard) + @RequirePermission("connection.manage") + async reconnect(@Req() request: Request & AuthenticatedRequest, @Param("connectionId") id: string, @Res({ passthrough: true }) response: Response) { + return this.redirectResponse(response, await this.connections.reconnect(requireSession(request), parseId(id))); + } + + @Delete(":connectionId") + @UseGuards(RbacGuard) + @RequirePermission("connection.manage") + async remove(@Req() request: Request & AuthenticatedRequest, @Param("connectionId") id: string): Promise<{ removed: true; token: string }> { + const token = await this.connections.remove(requireSession(request), parseId(id)); + return { removed: true, token }; + } +} diff --git a/apps/api/src/connections/connections.integration.test.ts b/apps/api/src/connections/connections.integration.test.ts new file mode 100644 index 0000000..681bd3e --- /dev/null +++ b/apps/api/src/connections/connections.integration.test.ts @@ -0,0 +1,345 @@ +import "reflect-metadata"; +import { createServer, type Server, type IncomingMessage, type ServerResponse } from "node:http"; +import { createHash, randomUUID } from "node:crypto"; +import { Module, UnauthorizedException, type ExecutionContext, type INestApplication } from "@nestjs/common"; +import { NestFactory } from "@nestjs/core"; +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import type { ConnectorConnection, NormalizedIdentity } from "@tactical-media-broadcast-hub/core"; +import { ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk"; +import { createMastodonConnector } from "../../../../packages/connector-mastodon/src/index.js"; +import { createTelegramConnector } from "@tactical-media-broadcast-hub/connector-telegram"; +import { createPeerTubeConnector } from "../../../../packages/connector-peertube/src/index.js"; +import { Vault, type SealedCredential } from "../vault/vault.js"; +import type { SessionPayload } from "../auth/session.js"; +import { RbacGuard } from "../rbac/rbac.guard.js"; +import { logger } from "../telemetry/logger.js"; +import { ConnectionsController } from "./connections.controller.js"; +import { CONNECTION_DEPENDENCIES, ConnectionsService, type ConnectionDependencies, type ConnectionSummary } from "./connections.service.js"; +import type { ConnectionStore, StoredConnection } from "./connection-store.js"; +import { RedisPendingStore } from "./pending.js"; +import { MemoryPendingRedis } from "./testing.js"; +import { injectHttp } from "./http-test-transport.js"; + +class MemoryConnections implements ConnectionStore { + readonly rows = new Map(); + readonly audit: Array> = []; + authorized = true; + async assertAuthorized(session: SessionPayload): Promise { + if (!this.authorized || session.workspaceId !== sessionFixture.workspaceId) throw new Error("membership revoked"); + } + async list(workspaceId: string): Promise { + return [...this.rows.values()].filter(({ connection }) => connection.workspaceId === workspaceId).map(({ connection, credential }) => ({ + id: connection.id, provider: connection.provider, channelId: connection.channelId, fields: { ...connection.fields }, + health: credential ? "unavailable" : "needs_credentials", credentialRotatedAt: null, recentPublishes: [], createdAt: connection.createdAt.toISOString(), + })); + } + async get(workspaceId: string, id: string): Promise { + const row = this.rows.get(id); + if (!row || row.connection.workspaceId !== workspaceId) throw new Error("connection not found"); + return row; + } + async save(session: SessionPayload, connection: ConnectorConnection, identity: NormalizedIdentity, credential: SealedCredential, reconnect: boolean): Promise { + await this.assertAuthorized(session); + if (reconnect && !this.rows.has(connection.id)) throw new Error("connection removed"); + this.rows.set(connection.id, { connection: { ...connection, fields: { ...connection.fields, remoteId: identity.remoteId, handle: identity.handle } }, credential }); + this.audit.push({ action: reconnect ? "connection.reconnect" : "connection.connect", provider: connection.provider }); + } + async remove(session: SessionPayload, id: string, revoke: (stored: StoredConnection) => Promise): Promise { + await this.assertAuthorized(session); + await revoke(await this.get(session.workspaceId, id)); + this.rows.delete(id); + this.audit.push({ action: "connection.remove" }); + } + async rotateCredential(workspaceId: string, id: string, previous: SealedCredential, credential: SealedCredential): Promise { + const row = await this.get(workspaceId, id); + if (row.credential?.ciphertext.equals(previous.ciphertext)) { + row.credential = credential; + this.audit.push({ action: "credential.refresh" }); + } + } +} +const sessionFixture: SessionPayload = { + userId: randomUUID(), workspaceId: randomUUID(), role: "Owner", authzEpoch: 1, + issuedAt: Math.floor(Date.now() / 1000), expiresAt: Math.floor(Date.now() / 1000) + 3600, +}; +@Module({}) +class TestModule {} + +async function form(request: IncomingMessage): Promise { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk as Uint8Array)); + return new URLSearchParams(Buffer.concat(chunks).toString()); +} +function json(response: ServerResponse, value: unknown, status = 200): void { + response.writeHead(status, { "content-type": "application/json" }); response.end(JSON.stringify(value)); +} + +describe("HTTP OAuth connect with real connectors and mock authorization server", () => { + let mock: Server; + let origin: string; + let apiOrigin: string; + let app: INestApplication; + const store = new MemoryConnections(); + const redis = new MemoryPendingRedis(); + const encryption = new Vault(); + const activeTokens = new Set(); + const grants = new Map(); + const revokedTokens: string[] = []; + const guard = vi.spyOn(RbacGuard.prototype, "canActivate").mockImplementation(async (context: ExecutionContext) => { + const request = context.switchToHttp().getRequest<{ headers: Record; session?: SessionPayload }>(); + if (request.headers.authorization !== "Bearer test-session") throw new UnauthorizedException(); + request.session = sessionFixture; return true; + }); + const warn = vi.spyOn(logger, "warn").mockImplementation(() => undefined); + const info = vi.spyOn(logger, "info").mockImplementation(() => undefined); + let nextAccount = "account-1"; + let issued = 0; + let failExchange = false; + let bridgeConfigured = false; + const botToken = "12345:bot_token_secret"; + + beforeEach(() => { + store.rows.clear(); store.audit.length = 0; store.authorized = true; + redis.entries.clear(); activeTokens.clear(); grants.clear(); revokedTokens.length = 0; + nextAccount = "account-1"; issued = 0; failExchange = false; bridgeConfigured = false; + warn.mockClear(); info.mockClear(); + }); + + beforeAll(async () => { + mock = createServer((request, response) => { void (async () => { + const url = new URL(request.url ?? "/", origin); + if (url.pathname === "/api/v1/apps") { + const body = await form(request); + expect(body.get("redirect_uris")).toBe(`${apiOrigin}/connections/callback`); + json(response, { client_id: "client-id", client_secret: "registered-client-secret" }); + } else if (url.pathname === "/oauth/authorize") { + const code = randomUUID(); + grants.set(code, { challenge: url.searchParams.get("code_challenge") ?? "", redirectUri: url.searchParams.get("redirect_uri") ?? "" }); + const redirect = new URL(url.searchParams.get("redirect_uri") ?? ""); + redirect.searchParams.set("state", url.searchParams.get("state") ?? ""); + redirect.searchParams.set("code", code); + response.writeHead(302, { location: redirect.toString() }); response.end(); + } else if (url.pathname === "/oauth/token") { + const body = await form(request); + const code = body.get("code") ?? ""; + const grant = grants.get(code); + grants.delete(code); + if (failExchange || !grant || grant.redirectUri !== body.get("redirect_uri") || grant.challenge !== createHash("sha256").update(body.get("code_verifier") ?? "").digest("base64url") || body.get("client_secret") !== "registered-client-secret") { + json(response, { error: "secret echoed: registered-client-secret token-secret-leak" }, 400); return; + } + const token = `access-token-secret-${++issued}`; + activeTokens.add(token); + json(response, { access_token: token, refresh_token: `refresh-token-secret-${issued}`, token_type: "Bearer", scope: "read write:statuses write:media", created_at: Date.now() / 1000 }); + } else if (url.pathname === "/api/v1/accounts/verify_credentials") { + const token = (request.headers.authorization ?? "").slice(7); + if (!activeTokens.has(token)) { json(response, { error: `revoked ${token}` }, 401); return; } + json(response, { id: nextAccount, username: "alice", acct: "alice", display_name: "Alice", avatar: "", url: `${origin}/@alice` }); + } else if (url.pathname === "/api/v1/oauth-clients/local") { + json(response, { client_id: "client-id", client_secret: "registered-client-secret" }); + } else if (url.pathname === "/api/v1/users/me") { + const token = (request.headers.authorization ?? "").slice(7); + if (!activeTokens.has(token)) { json(response, { error: "invalid token" }, 401); return; } + json(response, { id: 1, username: "alice", account: { id: 1, name: "alice", host: "127.0.0.1", displayName: "Alice" } }); + } else if (url.pathname === "/api/v1/users/token") { + const body = await form(request); + if (body.get("grant_type") !== "refresh_token" || body.get("client_secret") !== "registered-client-secret") { json(response, { error: "invalid refresh" }, 401); return; } + const token = `access-token-secret-${++issued}`; + activeTokens.add(token); + json(response, { access_token: token, refresh_token: `refresh-token-secret-${issued}` }); + } else if (url.pathname === "/api/v1/users/revoke-token") { + const token = (request.headers.authorization ?? "").slice(7); + revokedTokens.push(token); activeTokens.delete(token); json(response, {}); + } else if (url.pathname === "/oauth/revoke") { + const body = await form(request); + const token = body.get("token") ?? ""; + revokedTokens.push(token); activeTokens.delete(token); json(response, {}); + } else if (url.pathname.startsWith("/bot")) { + if (!url.pathname.startsWith(`/bot${botToken}/`)) { json(response, { ok: false, error_code: 401, description: `unauthorized ${url.pathname}` }); return; } + json(response, { ok: true, result: { id: 12345, is_bot: true, username: "test_bot", first_name: "Test bot" } }); + } else { json(response, { error: "not found" }, 404); } + })().catch(() => json(response, { error: "mock failure" }, 500)); }); + origin = "http://127.0.0.1:4500"; + const deps: ConnectionDependencies = { + renewSession: async () => "replacement-session", + store, pending: new RedisPendingStore(redis, encryption), encryption, + appUrl: "http://localhost:3001", callbackUrl: "", + registry: (credentials) => { + const registry = new ConnectorRegistry(); + registry.register(createMastodonConnector({ credentials, fetchImpl: mockFetch, requestsPerSecond: 100_000 })); + registry.register(createPeerTubeConnector({ credentials, fetchImpl: mockFetch, requestsPerSecond: 100_000, + env: bridgeConfigured ? { PEERTUBE_AUTHORIZE_PATH: "/oauth/authorize", PEERTUBE_TOKEN_PATH: "/oauth/token" } : {}, + })); + const botFetch: typeof fetch = (input, init) => fetch(String(input).replace("https://api.telegram.org", origin), init); + registry.register(createTelegramConnector({ credentials, fetchImpl: botFetch, requestsPerSecond: 100_000 })); + return registry; + }, + }; + app = await NestFactory.create({ module: TestModule, controllers: [ConnectionsController], providers: [ + ConnectionsService, { provide: CONNECTION_DEPENDENCIES, useValue: deps }, + + ] }, { logger: false }); + await app.init(); + apiOrigin = "http://api.local"; + deps.callbackUrl = `${apiOrigin}/connections/callback`; + }); + afterAll(async () => { + await app?.close(); + + warn.mockRestore(); info.mockRestore(); guard.mockRestore(); + }); + const mockFetch: typeof globalThis.fetch = (input, init) => injectHttp((request, response) => { mock.emit("request", request, response); }, input, init); + const fetch: typeof globalThis.fetch = (input, init) => { + const url = new URL(input instanceof Request ? input.url : String(input)); + if (url.origin === origin) return mockFetch(input, init); + if (url.origin !== apiOrigin) throw new Error("Unexpected test destination"); + const adapter = app.getHttpAdapter(); + return injectHttp((request, response) => { adapter.getInstance().handle(request, response); }, input, init); + }; + async function api(path: string, method = "GET", body?: unknown): Promise { + return fetch(`${apiOrigin}/connections${path}`, { method, headers: { authorization: "Bearer test-session", "content-type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) }); + } + async function begin(id?: string, provider = "mastodon"): Promise<{ redirectUrl: string; cookie: string }> { + const response = id ? await api(`/${id}/reconnect`, "POST", {}) : await api("/authorize", "POST", { provider, instanceUrl: origin }); + expect(response.status).toBe(201); + const { redirectUrl } = await response.json() as { redirectUrl: string }; + expect(new URL(redirectUrl).searchParams.get("code_challenge_method")).toBe("S256"); + return { redirectUrl, cookie: response.headers.get("set-cookie")?.split(";")[0] ?? "" }; + } + async function finish(authorization: { redirectUrl: string; cookie: string }): Promise { + const consent = await fetch(authorization.redirectUrl, { redirect: "manual" }); + expect(consent.status).toBe(302); + return fetch(consent.headers.get("location") ?? "", { redirect: "manual", headers: { cookie: authorization.cookie } }); + } + async function listed(): Promise { return (await (await api("")).json()) as ConnectionSummary[]; } + + it("authorize → provider → unauthenticated callback → healthy list; reconnect rotates; remove revokes and clears row + vault envelope", async () => { + const auth = await begin(); + expect(auth.cookie).toMatch(/^ap_connect_/); + expect(JSON.stringify([...redis.entries.values()])).not.toContain("registered-client-secret"); + const callback = await finish(auth); + expect(callback.status).toBe(303); + expect(callback.headers.get("location")).toBe("http://localhost:3001/connections?connect=success"); + expect(callback.headers.get("set-cookie")).toContain("Expires="); + const rows = await listed(); + expect(rows).toHaveLength(1); expect(rows[0]?.health).toBe("healthy"); + expect(rows[0]?.capabilities).toContain("publishText"); + const id = rows[0]?.id ?? ""; + const before = store.rows.get(id)?.credential?.ciphertext; + expect(before).toBeDefined(); + const stored = store.rows.get(id)?.credential; + if (!stored) throw new Error("credential missing"); + expect((await new Vault().open(sessionFixture.workspaceId, id, stored)).toString()).toContain("access-token-secret-1"); + expect(JSON.stringify(rows)).not.toContain("token-secret"); + const second = await finish(await begin(id)); + expect(second.headers.get("location")).toContain("connect=success"); + expect((await listed())[0]?.id).toBe(id); + expect(store.rows.get(id)?.credential?.ciphertext.equals(before ?? Buffer.alloc(0))).toBe(false); + expect(store.audit.at(-1)?.action).toBe("connection.reconnect"); + const replay = await finish(auth); + expect(replay.headers.get("location")).toContain("connect=failure"); + expect((await api(`/${id}`, "DELETE")).status).toBe(200); + expect(revokedTokens).toContain("access-token-secret-2"); + expect(await listed()).toHaveLength(0); expect(store.rows.size).toBe(0); expect(redis.entries.size).toBe(0); + const observable = JSON.stringify({ audit: store.audit, warn: warn.mock.calls, info: info.mock.calls }); + for (const secret of ["access-token-secret", "refresh-token-secret", "registered-client-secret"]) expect(observable).not.toContain(secret); + }); + + it("rejects missing auth on authorize, missing/wrong browser cookie and preserves a valid pending state", async () => { + expect((await fetch(`${apiOrigin}/connections/authorize`, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ provider: "mastodon", instanceUrl: origin }) })).status).toBe(401); + const auth = await begin(); + const consent = await fetch(auth.redirectUrl, { redirect: "manual" }); + const callbackUrl = consent.headers.get("location") ?? ""; + const missing = await fetch(callbackUrl, { redirect: "manual" }); + expect(missing.headers.get("location")).toContain("connect=failure"); + const wrong = await fetch(callbackUrl, { redirect: "manual", headers: { cookie: auth.cookie.replace(sessionFixture.workspaceId, randomUUID()) } }); + expect(wrong.headers.get("location")).toContain("connect=failure"); + const good = await fetch(callbackUrl, { redirect: "manual", headers: { cookie: auth.cookie } }); + expect(good.headers.get("location")).toContain("connect=success"); + }); + + it("failed exchange and different-account reconnect retain the previous encrypted credential; health detects revoked tokens", async () => { + await finish(await begin()); + const row = (await listed())[0]; + if (!row) throw new Error("connection missing"); + const before = store.rows.get(row.id)?.credential?.ciphertext; + failExchange = true; + const failure = await finish(await begin(row.id)); + failExchange = false; + expect(failure.headers.get("location")).toContain("connect=failure"); + expect(store.rows.get(row.id)?.credential?.ciphertext.equals(before ?? Buffer.alloc(0))).toBe(true); + nextAccount = "other-account"; + const wrongAccount = await finish(await begin(row.id)); + nextAccount = "account-1"; + expect(wrongAccount.headers.get("location")).toContain("connect=failure"); + expect(store.rows.get(row.id)?.credential?.ciphertext.equals(before ?? Buffer.alloc(0))).toBe(true); + activeTokens.clear(); + expect((await listed())[0]?.health).toBe("needs_credentials"); + expect((await finish(await begin(row.id))).headers.get("location")).toContain("connect=success"); + expect((await listed())[0]?.health).toBe("healthy"); + expect(JSON.stringify(warn.mock.calls)).not.toContain("token-secret-leak"); + expect(JSON.stringify(warn.mock.calls)).not.toContain("registered-client-secret"); + }); + + it("rechecks workspace membership during callback, handles denial, and does not expose provider errors", async () => { + const revoked = await begin(); + store.authorized = false; + expect((await finish(revoked)).headers.get("location")).toContain("connect=failure"); + store.authorized = true; + const denied = await begin(); + const url = new URL(`${apiOrigin}/connections/callback`); + url.searchParams.set("state", new URL(denied.redirectUrl).searchParams.get("state") ?? ""); + url.searchParams.set("error", "access_denied token-secret-leak"); + const response = await fetch(url, { redirect: "manual", headers: { cookie: denied.cookie } }); + expect(response.headers.get("location")).toContain("connect=failure"); + expect(JSON.stringify(warn.mock.calls)).not.toContain("token-secret-leak"); + }); + + it("validates Telegram getMe, encrypts tokens, rotates in place and removes locally; PeerTube is capability-honest", async () => { + const bad = await api("/telegram/bot-token", "POST", { token: "12345:bad_secret" }); + expect(bad.status).toBe(400); expect(await bad.text()).not.toContain("bad_secret"); + const response = await api("/telegram/bot-token", "POST", { token: botToken, channelId: "@target" }); + expect(response.status).toBe(201); + const bot = (await listed()).find((row) => row.provider === "telegram"); + expect(bot?.health).toBe("healthy"); expect(bot?.channelId).toBe("@target"); + const id = bot?.id ?? ""; + const before = store.rows.get(id)?.credential?.ciphertext; + expect((await api("/telegram/bot-token", "POST", { token: botToken, connectionId: id })).status).toBe(201); + expect(store.rows.get(id)?.credential?.ciphertext.equals(before ?? Buffer.alloc(0))).toBe(false); + expect((await listed()).filter((row) => row.provider === "telegram")).toHaveLength(1); + expect((await api(`/${id}`, "DELETE")).status).toBe(200); + expect(store.rows.has(id)).toBe(false); + const providers = await (await api("/providers")).json() as Array<{ id: string; enabled: boolean; note?: string }>; + expect(providers.find((provider) => provider.id === "peertube")?.enabled).toBe(false); + expect((await api("/authorize", "POST", { provider: "peertube", instanceUrl: origin })).status).toBe(400); + expect(JSON.stringify({ audit: store.audit, warn: warn.mock.calls, info: info.mock.calls })).not.toContain(botToken); + }); + + it("runs PeerTube code flow only with an explicitly configured instance bridge", async () => { + bridgeConfigured = true; + expect((await finish(await begin(undefined, "peertube"))).headers.get("location")).toContain("connect=success"); + const row = (await listed()).find((connection) => connection.provider === "peertube"); + expect(row?.health).toBe("healthy"); + const id = row?.id ?? ""; + expect((await finish(await begin(id))).headers.get("location")).toContain("connect=success"); + expect((await listed()).filter((connection) => connection.id === id)).toHaveLength(1); + const stored = store.rows.get(id)?.credential; + if (!stored) throw new Error("PeerTube credential missing"); + const secrets = JSON.parse((await encryption.open(sessionFixture.workspaceId, id, stored)).toString()) as Record; + activeTokens.delete(secrets.access_token ?? ""); + expect((await listed()).find((connection) => connection.id === id)?.health).toBe("healthy"); + expect(store.audit.at(-1)?.action).toBe("credential.refresh"); + expect(store.rows.get(id)?.credential?.ciphertext.equals(stored.ciphertext)).toBe(false); + expect((await api(`/${id}`, "DELETE")).status).toBe(200); + bridgeConfigured = false; + }); + + it("does not resurrect a connection removed while reconnect consent is pending", async () => { + await finish(await begin()); + const row = (await listed())[0]; + if (!row) throw new Error("connection missing"); + const pending = await begin(row.id); + expect((await api(`/${row.id}`, "DELETE")).status).toBe(200); + expect((await finish(pending)).headers.get("location")).toContain("connect=failure"); + expect(store.rows.has(row.id)).toBe(false); + }); +}); diff --git a/apps/api/src/connections/connections.service.ts b/apps/api/src/connections/connections.service.ts new file mode 100644 index 0000000..25ac270 --- /dev/null +++ b/apps/api/src/connections/connections.service.ts @@ -0,0 +1,231 @@ +import { BadRequestException, Inject, Injectable, Optional } from "@nestjs/common"; +import { randomUUID } from "node:crypto"; +import type { ConnectorConnection, NormalizedIdentity } from "@tactical-media-broadcast-hub/core"; +import { AuthError, type AuthorizationRequest, type AuthorizationResult, type CredentialAccessor, type ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk"; +import type { SessionPayload } from "../auth/session.js"; +import { withTenant } from "../db/client.js"; +import { issueSessionInTenant } from "../auth/auth.service.js"; +import { vault, type Vault } from "../vault/vault.js"; +import { logger } from "../telemetry/logger.js"; +import { connectionRegistry } from "./connector-registry.js"; +import { PgConnectionStore, type ConnectionStore, type StoredConnection } from "./connection-store.js"; +import { AUTH_TTL_MS, browserHash, randomState, RedisPendingStore, type PendingStore } from "./pending.js"; +import { oauthFetch } from "./oauth-fetch.js"; + +export type ConnectionHealth = "healthy" | "needs_credentials" | "unavailable"; +export interface ConnectionSummary { + id: string; provider: string; channelId: string; fields: Record; + health: ConnectionHealth; credentialRotatedAt: string | null; recentPublishes: string[]; createdAt: string; + capabilities?: string[]; +} +export interface ConnectionDependencies { + store: ConnectionStore; + pending: PendingStore; + encryption: Vault; + registry: (credentials: CredentialAccessor) => ConnectorRegistry; + appUrl: string; + callbackUrl: string; + renewSession(session: SessionPayload): Promise; +} +export const CONNECTION_DEPENDENCIES = Symbol("CONNECTION_DEPENDENCIES"); + +/** Each exchange has an isolated, tenant-bound working set; never writes plaintext to DB. */ +class StagedCredentials implements CredentialAccessor { + dirty = false; + constructor(private readonly workspaceId: string, private readonly connectionId: string, readonly secrets: Record = {}) {} + private assert(connection: ConnectorConnection): void { + if (connection.workspaceId !== this.workspaceId || connection.id !== this.connectionId || connection.credentialId.id !== this.connectionId) throw new Error("Credential binding mismatch"); + } + async getAccessToken(connection: ConnectorConnection): Promise { return this.getSecret(connection, "access_token"); } + async getSecret(connection: ConnectorConnection, key: string): Promise { + this.assert(connection); + const value = this.secrets[key]; + if (!value) throw new Error("Credential missing"); + return value; + } + async put(connection: ConnectorConnection, key: string, value: string): Promise { + this.assert(connection); + if (typeof value !== "string" || !value) throw new Error("Provider returned invalid credential"); + this.secrets[key] = value; + this.dirty = true; + } +} +function defaults(): ConnectionDependencies { + const appUrl = process.env.APP_URL ?? "http://localhost:3000"; + return { + renewSession: (session) => withTenant(session.workspaceId, (client) => issueSessionInTenant(client, session.workspaceId, session.userId, session.expiresAt)), + store: new PgConnectionStore(), pending: new RedisPendingStore(), encryption: vault, + registry: (credentials) => connectionRegistry(credentials, oauthFetch), appUrl, + callbackUrl: process.env.CONNECTIONS_CALLBACK_URL ?? `${process.env.API_URL ?? "http://localhost:3000"}/connections/callback`, + }; +} + +@Injectable() +export class ConnectionsService { + private readonly deps: ConnectionDependencies; + constructor(@Optional() @Inject(CONNECTION_DEPENDENCIES) dependencies?: ConnectionDependencies) { this.deps = dependencies ?? defaults(); } + + providers() { + const registry = this.deps.registry(new StagedCredentials("", "")); + return registry.list().map((connector) => { + const manifest = connector.manifest(); + return { + id: manifest.id, displayName: manifest.displayName, kind: manifest.oauth?.kind, + enabled: manifest.oauth?.connectEnabled !== false && Boolean(connector.validateToken || (connector.buildAuthorizationUrl && connector.exchangeCode)), + note: manifest.oauth?.connectNote, targetField: manifest.oauth?.targetField, + instanceRequired: manifest.oauth?.kind === "per-instance-oauth2", + capabilities: Object.entries(manifest.capabilities).filter(([, enabled]) => enabled === true).map(([key]) => key), + }; + }); + } + landing(status: "success" | "failure"): string { + const url = new URL("/connections", this.deps.appUrl); + url.searchParams.set("connect", status); + return url.toString(); + } + async authorize(session: SessionPayload, provider: string, instanceUrl?: string, connectionId?: string) { + await this.deps.store.assertAuthorized(session); + const previous = connectionId ? await this.deps.store.get(session.workspaceId, connectionId) : undefined; + if (previous && previous.connection.provider !== provider) throw new BadRequestException("Provider mismatch"); + const id = connectionId ?? randomUUID(); + const appSecrets: Record = {}; + if (previous?.credential) { + const bytes = await this.deps.encryption.open(session.workspaceId, id, previous.credential); + const secrets = JSON.parse(bytes.toString()) as Record; + // Reuse the registered app, but stage no previous account token. + for (const key of ["client_id", "client_secret"]) if (secrets[key]) appSecrets[key] = secrets[key]; + } + const staging = new StagedCredentials(session.workspaceId, id, appSecrets); + const registry = this.deps.registry(staging); + if (!registry.has(provider)) throw new BadRequestException("Provider is not available"); + const connector = registry.get(provider); + if (!connector.buildAuthorizationUrl || !connector.exchangeCode || connector.manifest().oauth?.connectEnabled === false) throw new BadRequestException(connector.manifest().oauth?.connectNote ?? "Redirect linking is unavailable"); + const state = randomState(); + const binding = `${session.workspaceId}.${randomState()}`; + const instance = previous?.connection.fields.instanceUrl ?? instanceUrl; + const request: AuthorizationRequest = { + state, workspaceId: session.workspaceId, connectionId: id, redirectUri: this.deps.callbackUrl, + fields: instance ? { instanceUrl: instance } : {}, + ...(connector.manifest().oauth?.pkce ? { codeVerifier: randomState() } : {}), + }; + try { + const redirect = await connector.buildAuthorizationUrl(request); + request.fields = redirect.connection.fields; + await this.deps.pending.put(state, { session, provider, request, secrets: staging.secrets, reconnect: Boolean(previous), expiresAt: Date.now() + AUTH_TTL_MS, browserHash: browserHash(binding) }); + return { redirectUrl: redirect.authorizationUrl, state, binding }; + } catch { + logger.warn({ event: "connection.authorize_failed", provider }, "Connection authorization failed"); + throw new BadRequestException("Could not start authorization. Check the instance URL and provider configuration."); + } + } + async callback(state: string, binding: string, code?: string, providerError?: string): Promise { + try { + const pending = await this.deps.pending.consume(state, binding); + await this.deps.store.assertAuthorized(pending.session); + if (providerError || !code || code.length > 4096) throw new Error("Provider authorization denied"); + const id = pending.request.connectionId; + if (!id) throw new Error("Missing connection binding"); + const staging = new StagedCredentials(pending.session.workspaceId, id, pending.secrets); + const connector = this.deps.registry(staging).get(pending.provider); + if (!connector.exchangeCode) throw new Error("Code exchange unavailable"); + const result = await connector.exchangeCode({ ...pending.request, code }); + try { + await this.persist(pending.session, result, staging, pending.reconnect); + } catch (error) { + // A failed DB write/account check must not leave an orphan authorization. + await connector.revokeCredentials?.(result.connection).catch(() => undefined); + throw error; + } + return this.landing("success"); + } catch { + // Provider errors can echo secrets. Only a fixed event reaches logs/UI. + logger.warn({ event: "connection.callback_failed" }, "Connection callback failed"); + return this.landing("failure"); + } + } + async botToken(session: SessionPayload, token: string, fields: Record, connectionId?: string): Promise { + await this.deps.store.assertAuthorized(session); + const previous = connectionId ? await this.deps.store.get(session.workspaceId, connectionId) : undefined; + const id = connectionId ?? randomUUID(); + const staging = new StagedCredentials(session.workspaceId, id); + const connector = this.deps.registry(staging).list().find((item) => item.manifest().oauth?.kind === "token" && item.validateToken); + if (!connector?.validateToken || (previous && previous.connection.provider !== connector.manifest().id)) throw new BadRequestException("Token linking is unavailable"); + try { + const result = await connector.validateToken({ workspaceId: session.workspaceId, connectionId: id, token, fields: { ...previous?.connection.fields, ...fields } }); + await this.persist(session, result, staging, Boolean(previous)); + return await this.deps.renewSession(session); + } catch { + logger.warn({ event: "connection.token_failed" }, "Bot validation failed"); + throw new BadRequestException("Bot validation failed. Check the token and try again."); + } + } + async reconnect(session: SessionPayload, id: string) { + const stored = await this.deps.store.get(session.workspaceId, id); + return this.authorize(session, stored.connection.provider, undefined, id); + } + private async persist(session: SessionPayload, result: AuthorizationResult, staging: StagedCredentials, reconnect: boolean): Promise { + const { connection, identity } = result; + if (connection.workspaceId !== session.workspaceId || identity.provider !== connection.provider || !identity.remoteId || !identity.handle) throw new Error("Provider identity invalid"); + if (reconnect) { + const previous = await this.deps.store.get(session.workspaceId, connection.id); + if (previous.connection.provider !== connection.provider || (previous.connection.fields.remoteId && previous.connection.fields.remoteId !== identity.remoteId)) throw new Error("Reconnect must use the same account"); + connection.channelId = previous.connection.channelId; + } else if (!connection.fields.channelId) { + connection.channelId = identity.remoteId; + } + const encrypted = await this.deps.encryption.seal(session.workspaceId, connection.id, Buffer.from(JSON.stringify(staging.secrets))); + await this.deps.store.save(session, connection, identity, encrypted, reconnect); + } + private async credentials(workspaceId: string, id: string) { + const stored = await this.deps.store.get(workspaceId, id); + return this.storedCredentials(stored); + } + private async storedCredentials(stored: StoredConnection) { + const { workspaceId, id } = stored.connection; + if (!stored.credential) throw new AuthError("Credential missing", stored.connection.provider, false); + const bytes = await this.deps.encryption.open(workspaceId, id, stored.credential); + const secrets = JSON.parse(bytes.toString()) as Record; + const staging = new StagedCredentials(workspaceId, id, secrets); + return { connection: stored.connection, connector: this.deps.registry(staging).get(stored.connection.provider), staging, stored }; + } + async list(workspaceId: string): Promise { + const rows = await this.deps.store.list(workspaceId); + // Probe actual identity, rather than assuming a credential row means a valid token. + for (const row of rows) { + if (row.health === "needs_credentials") continue; + try { + const { connection, connector, staging, stored } = await this.credentials(workspaceId, row.id); + let identity: NormalizedIdentity; + try { + identity = await connector.getIdentity(connection); + } finally { + // Some connectors refresh during identity reads; keep the new refresh token durable. + if (staging.dirty && stored.credential) { + const sealed = await this.deps.encryption.seal(workspaceId, row.id, Buffer.from(JSON.stringify(staging.secrets))); + await this.deps.store.rotateCredential(workspaceId, row.id, stored.credential, sealed); + } + } + row.health = "healthy"; + row.fields.handle = identity.handle; + row.capabilities = Object.entries(connector.manifest().capabilities).filter(([, enabled]) => enabled === true).map(([key]) => key); + } catch (error) { + row.health = error instanceof AuthError ? "needs_credentials" : "unavailable"; + } + } + return rows; + } + async remove(session: SessionPayload, id: string): Promise { + await this.deps.store.assertAuthorized(session); + await this.deps.store.remove(session, id, async (stored) => { + if (!stored.credential) return; + try { + const { connection, connector } = await this.storedCredentials(stored); + await connector.revokeCredentials?.(connection); + } catch (error) { + // Already invalid/revoked credentials are safe to remove locally. + if (!(error instanceof AuthError)) throw new BadRequestException("Provider revocation failed. Try removing again."); + } + }); + return this.deps.renewSession(session); + } +} diff --git a/apps/api/src/connections/connector-registry.ts b/apps/api/src/connections/connector-registry.ts new file mode 100644 index 0000000..8454727 --- /dev/null +++ b/apps/api/src/connections/connector-registry.ts @@ -0,0 +1,17 @@ +import { ConnectorRegistry, type CredentialAccessor } from "@tactical-media-broadcast-hub/connector-sdk"; +// eslint-disable-next-line no-restricted-syntax -- Connector composition boundary. +import { createMastodonConnector } from "@tactical-media-broadcast-hub/connector-mastodon"; +// eslint-disable-next-line no-restricted-syntax -- Connector composition boundary. +import { createPeerTubeConnector } from "../../../../packages/connector-peertube/src/index.js"; +// eslint-disable-next-line no-restricted-syntax -- Connector composition boundary. +import { createTelegramConnector } from "@tactical-media-broadcast-hub/connector-telegram"; + +/** Composition boundary only: services resolve manifests rather than branch on providers. */ +export function connectionRegistry(credentials: CredentialAccessor, fetchImpl: typeof fetch = fetch): ConnectorRegistry { + const registry = new ConnectorRegistry(); + const options = { credentials, fetchImpl, env: process.env }; + registry.register(createMastodonConnector(options)); + registry.register(createPeerTubeConnector(options)); + registry.register(createTelegramConnector(options)); + return registry; +} diff --git a/apps/api/src/connections/http-test-transport.ts b/apps/api/src/connections/http-test-transport.ts new file mode 100644 index 0000000..f2564e7 --- /dev/null +++ b/apps/api/src/connections/http-test-transport.ts @@ -0,0 +1,46 @@ +import { IncomingMessage, ServerResponse } from "node:http"; +import { Duplex } from "node:stream"; +import type { Socket } from "node:net"; + +/** Runs real Node/Express HTTP handlers without binding a port (restricted CI). */ +export async function injectHttp( + handler: (request: IncomingMessage, response: ServerResponse) => void, + input: string | URL | Request, + init: RequestInit = {}, +): Promise { + const url = new URL(input instanceof Request ? input.url : String(input)); + const output: Buffer[] = []; + const socket = new Duplex({ + read() {}, + write(chunk: Buffer, _encoding, callback) { output.push(Buffer.from(chunk)); callback(); }, + }); + // ServerResponse uses only Duplex operations; no network or unsafe dynamic values. + const httpSocket = socket as unknown as Socket; + const request = new IncomingMessage(httpSocket); + request.method = init.method ?? "GET"; + request.complete = true; + request.url = `${url.pathname}${url.search}`; + request.headers = Object.fromEntries(new Headers(init.headers).entries()); + request.headers.host = url.host; + const body = typeof init.body === "string" ? init.body : ""; + if (body) request.headers["content-length"] = String(Buffer.byteLength(body)); + request.push(body || null); + if (body) request.push(null); + const response = new ServerResponse(request); + response.assignSocket(httpSocket); + const completed = new Promise((resolve, reject) => { + response.once("error", reject); + response.once("finish", () => { + const bytes = Buffer.concat(output); + const bodyOffset = bytes.indexOf("\r\n\r\n") + 4; + const headers = new Headers(); + for (const line of bytes.subarray(0, bodyOffset - 4).toString().split("\r\n").slice(1)) { + const separator = line.indexOf(":"); + if (separator > 0) headers.append(line.slice(0, separator), line.slice(separator + 1).trim()); + } + resolve(new Response([204, 304].includes(response.statusCode) ? null : bytes.subarray(bodyOffset), { status: response.statusCode, headers })); + }); + }); + handler(request, response); + return completed; +} diff --git a/apps/api/src/connections/oauth-fetch.test.ts b/apps/api/src/connections/oauth-fetch.test.ts new file mode 100644 index 0000000..ef3ab1d --- /dev/null +++ b/apps/api/src/connections/oauth-fetch.test.ts @@ -0,0 +1,11 @@ +import { describe, expect, it } from "vitest"; +import { publicAddress } from "./oauth-fetch.js"; + +describe("authorization egress policy", () => { + it("rejects private, mapped, metadata, multicast and documentation addresses", () => { + for (const address of ["127.0.0.1", "10.0.0.1", "172.16.0.1", "192.168.1.1", "169.254.169.254", "100.64.1.1", "0.0.0.0", "224.0.0.1", "::1", "::ffff:127.0.0.1", "fc00::1", "fe80::1", "2001:db8::1", "2002:7f00:1::"]) expect(publicAddress(address), address).toBe(false); + expect(publicAddress("8.8.8.8")).toBe(true); + expect(publicAddress("2606:4700:4700::1111")).toBe(true); + expect(publicAddress("2001:4860:4860::8888")).toBe(true); + }); +}); diff --git a/apps/api/src/connections/oauth-fetch.ts b/apps/api/src/connections/oauth-fetch.ts new file mode 100644 index 0000000..853caaa --- /dev/null +++ b/apps/api/src/connections/oauth-fetch.ts @@ -0,0 +1 @@ +export { publicAddress, providerFetch as oauthFetch } from "@tactical-media-broadcast-hub/connector-runtime"; diff --git a/apps/api/src/connections/pending.test.ts b/apps/api/src/connections/pending.test.ts new file mode 100644 index 0000000..40b646b --- /dev/null +++ b/apps/api/src/connections/pending.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it, vi, afterEach } from "vitest"; +import { Vault } from "../vault/vault.js"; +import { AUTH_TTL_MS, browserHash, randomState, RedisPendingStore, type PendingAuthorization } from "./pending.js"; +import { MemoryPendingRedis } from "./testing.js"; + +function fixture() { + const redis = new MemoryPendingRedis(); + const store = new RedisPendingStore(redis, new Vault()); + const state = randomState(); + const binding = `workspace-a.${randomState()}`; + const pending: PendingAuthorization = { + provider: "mastodon", request: { redirectUri: "http://localhost/callback", workspaceId: "workspace-a", state }, + session: { userId: "user-a", workspaceId: "workspace-a", role: "Owner", authzEpoch: 1, issuedAt: 0, expiresAt: Date.now() / 1000 + 600 }, + secrets: { client_secret: "registration-secret" }, reconnect: false, expiresAt: Date.now() + AUTH_TTL_MS, browserHash: browserHash(binding), + }; + return { redis, store, state, binding, pending }; +} +afterEach(() => vi.useRealTimers()); +describe("pending authorization binding", () => { + it("encrypts provider secrets and consumes state exactly once", async () => { + const { redis, store, state, binding, pending } = fixture(); + await store.put(state, pending); + expect(JSON.stringify([...redis.entries.values()])).not.toContain("registration-secret"); + expect(await store.consume(state, binding)).toEqual(pending); + await expect(store.consume(state, binding)).rejects.toThrow(/already used/); + }); + it("rejects CSRF, unknown states and a different workspace without burning the valid state", async () => { + const { store, state, binding, pending } = fixture(); + await store.put(state, pending); + await expect(store.consume(state, "")).rejects.toThrow(/mismatch/); + await expect(store.consume(state, binding.replace("workspace-a", "workspace-b"))).rejects.toThrow(/mismatch/); + await expect(store.consume(randomState(), binding)).rejects.toThrow(); + await expect(store.consume("malformed", binding)).rejects.toThrow(); + expect(await store.consume(state, binding)).toEqual(pending); + }); + it("rejects a workspace mismatch inside the encrypted request", async () => { + const { store, state, binding, pending } = fixture(); + pending.request.workspaceId = "workspace-b"; + await store.put(state, pending); + await expect(store.consume(state, binding)).rejects.toThrow(/workspace mismatch/); + }); + it("expires state after ten minutes and atomically fences simultaneous callbacks", async () => { + vi.useFakeTimers(); + const { store, state, binding, pending } = fixture(); + await store.put(state, pending); + const results = await Promise.allSettled([store.consume(state, binding), store.consume(state, binding)]); + expect(results.filter((item) => item.status === "fulfilled")).toHaveLength(1); + const next = fixture(); + await next.store.put(next.state, next.pending); + vi.advanceTimersByTime(AUTH_TTL_MS); + await expect(next.store.consume(next.state, next.binding)).rejects.toThrow(/expired/); + }); +}); diff --git a/apps/api/src/connections/pending.ts b/apps/api/src/connections/pending.ts new file mode 100644 index 0000000..ff914d8 --- /dev/null +++ b/apps/api/src/connections/pending.ts @@ -0,0 +1,57 @@ +import { createHash, randomBytes, timingSafeEqual } from "node:crypto"; +import { BadRequestException } from "@nestjs/common"; +import type { AuthorizationRequest } from "@tactical-media-broadcast-hub/connector-sdk"; +import type { SessionPayload } from "../auth/session.js"; +import { RespControlRedis, type ControlRedis } from "../media-control/redis.js"; +import { vault, type Vault } from "../vault/vault.js"; + +export const AUTH_TTL_MS = 10 * 60_000; +export const stateCookieName = (state: string): string => `ap_connect_${state}`; +export const validState = (state: string): boolean => /^[A-Za-z0-9_-]{43}$/.test(state); +export interface PendingAuthorization { + session: SessionPayload; + provider: string; + request: AuthorizationRequest; + secrets: Record; + reconnect: boolean; + expiresAt: number; + browserHash: string; +} +export interface PendingStore { + put(state: string, pending: PendingAuthorization): Promise; + consume(state: string, browserBinding: string): Promise; +} +export function browserHash(binding: string): string { + return createHash("sha256").update(binding).digest("hex"); +} +export const randomState = (): string => randomBytes(32).toString("base64url"); + +/** GET + compare-and-delete makes callbacks one-use across API replicas. */ +export class RedisPendingStore implements PendingStore { + constructor( + private readonly redis: Pick = new RespControlRedis(process.env.REDIS_URL ?? "redis://localhost:6379"), + private readonly encryption: Vault = vault, + ) {} + async put(state: string, pending: PendingAuthorization): Promise { + const sealed = await this.encryption.seal(pending.session.workspaceId, state, Buffer.from(JSON.stringify(pending))); + const value = JSON.stringify({ workspaceId: pending.session.workspaceId, ciphertext: sealed.ciphertext.toString("base64") }); + if (!await this.redis.set(`oauth:pending:${state}`, value, { nx: true, pxMs: AUTH_TTL_MS })) throw new Error("state collision"); + } + async consume(state: string, binding: string): Promise { + if (!validState(state) || binding.length > 200) throw new BadRequestException("Invalid authorization state"); + const key = `oauth:pending:${state}`; + const raw = await this.redis.get(key); + if (!raw) throw new BadRequestException("Authorization expired or already used"); + const envelope = JSON.parse(raw) as { workspaceId: string; ciphertext: string }; + const opened = await this.encryption.open(envelope.workspaceId, state, { keyVersion: 1, ciphertext: Buffer.from(envelope.ciphertext, "base64") }); + const pending = JSON.parse(opened.toString()) as PendingAuthorization; + const actual = Buffer.from(browserHash(binding)); + const expected = Buffer.from(pending.browserHash); + if (pending.expiresAt <= Date.now() || pending.session.workspaceId !== envelope.workspaceId || pending.request.workspaceId !== envelope.workspaceId || !binding.startsWith(`${envelope.workspaceId}.`) || actual.length !== expected.length || !timingSafeEqual(actual, expected)) { + throw new BadRequestException("Authorization browser or workspace mismatch"); + } + const consumed = await this.redis.eval("if redis.call('GET', KEYS[1]) == ARGV[1] then redis.call('DEL', KEYS[1]); return 1 end; return 0", [key], [raw]); + if (consumed !== 1) throw new BadRequestException("Authorization already used"); + return pending; + } +} diff --git a/apps/api/src/connections/testing.ts b/apps/api/src/connections/testing.ts new file mode 100644 index 0000000..030b6f6 --- /dev/null +++ b/apps/api/src/connections/testing.ts @@ -0,0 +1,23 @@ +import type { ControlRedis } from "../media-control/redis.js"; + +/** TTL-faithful Redis substitute; compare-and-delete runs synchronously like Redis Lua. */ +export class MemoryPendingRedis implements Pick { + readonly entries = new Map(); + async get(key: string): Promise { + const entry = this.entries.get(key); + if (!entry || entry.expiresAt <= Date.now()) { this.entries.delete(key); return null; } + return entry.value; + } + async set(key: string, value: string, options: { nx?: boolean; pxMs?: number } = {}): Promise { + if (options.nx && await this.get(key) !== null) return false; + this.entries.set(key, { value, expiresAt: Date.now() + (options.pxMs ?? 1000) }); + return true; + } + async eval(_script: string, keys: string[], args: string[]): Promise { + const key = keys[0] ?? ""; + const entry = this.entries.get(key); + if (!entry || entry.value !== args[0] || entry.expiresAt <= Date.now()) return 0; + this.entries.delete(key); + return 1; + } +} diff --git a/apps/api/src/db/client.ts b/apps/api/src/db/client.ts new file mode 100644 index 0000000..a31a898 --- /dev/null +++ b/apps/api/src/db/client.ts @@ -0,0 +1,59 @@ +import { Pool, type PoolClient } from "pg"; +import pino from "pino"; + +const logger = pino({ level: process.env.LOG_LEVEL ?? "info" }); + +export const pool = new Pool({ + connectionString: process.env.DATABASE_URL, + max: Number(process.env.PG_POOL_MAX ?? 10), +}); + +/** + * Tenant-scoped transaction: sets app.workspace_id for the transaction so + * RLS policies and SECURITY DEFINER assertions see the tenant context. + * The setting is transaction-local (SET LOCAL) — it cannot leak. + */ +export async function withTenant( + workspaceId: string, + fn: (client: PoolClient) => Promise, +): Promise { + const client = await pool.connect(); + try { + await client.query("BEGIN"); + await client.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]); + const result = await fn(client); + await client.query("COMMIT"); + return result; + } catch (err) { + await client.query("ROLLBACK").catch(() => undefined); + throw err; + } finally { + client.release(); + } +} + +/** System-level transaction (no tenant context): RLS sees NULL context and returns nothing tenant-scoped. */ +export async function withSystem(fn: (client: PoolClient) => Promise): Promise { + const client = await pool.connect(); + try { + await client.query("BEGIN"); + const result = await fn(client); + await client.query("COMMIT"); + return result; + } catch (err) { + await client.query("ROLLBACK").catch(() => undefined); + throw err; + } finally { + client.release(); + } +} + +export async function checkDatabase(): Promise { + try { + await pool.query("SELECT 1"); + return true; + } catch (err) { + logger.warn({ err: String(err) }, "db: unreachable"); + return false; + } +} diff --git a/apps/api/src/db/migrate.cli.ts b/apps/api/src/db/migrate.cli.ts new file mode 100644 index 0000000..6853f2a --- /dev/null +++ b/apps/api/src/db/migrate.cli.ts @@ -0,0 +1,6 @@ +import { resolve } from "node:path"; +import { runMigrations } from "./migrate.js"; +const index = process.argv.indexOf("--dir"); +void runMigrations(resolve(index < 0 ? "migrations" : process.argv[index + 1] ?? "migrations")) + .then((files) => console.log({ applied: files })) + .catch(() => { console.error("Migration failed; schema remains unready. Inspect the phase and database records before retrying."); process.exitCode = 1; }); diff --git a/apps/api/src/db/migrate.test.ts b/apps/api/src/db/migrate.test.ts new file mode 100644 index 0000000..093e0c6 --- /dev/null +++ b/apps/api/src/db/migrate.test.ts @@ -0,0 +1,12 @@ +import { describe, expect, it } from "vitest"; +import { migrationPhases } from "./migrate.js"; +describe("migration stage grammar", () => { + it("leaves 001–018 transactional and parses explicit stages", () => { + expect(migrationPhases("SELECT 1;")).toEqual([{ name: "", mode: "transaction", sql: "SELECT 1;" }]); + expect(migrationPhases("-- comment\n-- wp3:phase expand transaction\nSELECT 1;\n-- wp3:phase fill batch\nUPDATE t SET x=1;\n-- wp3:phase idx concurrent\nCREATE INDEX CONCURRENTLY t_idx ON t (x);").map((p) => [p.name, p.mode])).toEqual([["expand", "transaction"], ["fill", "batch"], ["idx", "concurrent"]]); + }); + it("refuses duplicate phases and SQL before the first stage", () => { + expect(() => migrationPhases("-- wp3:phase expand transaction\nSELECT 1;\n-- wp3:phase expand transaction\nSELECT 2;")).toThrow(/duplicate/); + expect(() => migrationPhases("SELECT 0;\n-- wp3:phase expand transaction\nSELECT 1;")).toThrow(/before/); + }); +}); diff --git a/apps/api/src/db/migrate.ts b/apps/api/src/db/migrate.ts new file mode 100644 index 0000000..fca2a3c --- /dev/null +++ b/apps/api/src/db/migrate.ts @@ -0,0 +1,77 @@ +import { readdir, readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { Pool, type PoolClient } from "pg"; + +export interface MigrationPhase { name: string; mode: "transaction" | "batch" | "concurrent"; sql: string } +/** Only explicit whole-line WP3 markers are accepted; old migrations stay transactional. */ +export function migrationPhases(sql: string): MigrationPhase[] { + const marker = /^-- wp3:phase ([a-z][a-z0-9_]*) (transaction|batch|concurrent)\s*$/gm; + const matches = [...sql.matchAll(marker)]; + if (matches.length === 0) return [{ name: "", mode: "transaction", sql }]; + if (sql.slice(0, matches[0]?.index).split("\n").some((line) => line.trim() && !line.trim().startsWith("--"))) throw new Error("SQL before migration phase"); + const phases = matches.map((match, index): MigrationPhase => ({ name: match[1]!, mode: match[2] as MigrationPhase["mode"], sql: sql.slice(match.index! + match[0].length, matches[index + 1]?.index).trim() })); + if (new Set(phases.map((p) => p.name)).size !== phases.length) throw new Error("duplicate migration phase"); + return phases; +} + +async function concurrentIndex(client: PoolClient, sql: string): Promise { + const parsed = /^CREATE (UNIQUE )?INDEX CONCURRENTLY ([a-z][a-z0-9_]*) ON ([a-z][a-z0-9_]*) ([\s\S]+);$/.exec(sql); + if (!parsed) throw new Error("concurrent phase must contain exactly one CREATE INDEX"); + const [, unique, name, table, tail] = parsed; + const found = await client.query<{ indisvalid: boolean; definition: string }>("SELECT i.indisvalid, pg_get_indexdef(i.indexrelid) AS definition FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'public' AND c.relname = $1", [name]); + const existing = found.rows[0]; + if (existing) { + const expected = `CREATE ${unique ?? ""}INDEX ${name} ON public.${table} USING btree ${tail}`; + const normalize = (text: string): string => text.replace(/;$/, "").replace(/\s+/g, " ").trim(); + if (normalize(existing.definition) !== normalize(expected)) throw new Error(`index ${name} definition differs`); + if (existing.indisvalid) return; + await client.query(`DROP INDEX CONCURRENTLY public.${name}`); + } + await client.query(sql); +} + +/** Owner-only offline runner; runtime applications never call this. */ +export async function runMigrations(dir: string): Promise { + if (!process.env.SUPER_DATABASE_URL) throw new Error("SUPER_DATABASE_URL is required for migrations"); + const files = (await readdir(dir)).filter((f) => f.endsWith(".sql")).sort(); + const applied: string[] = []; + const pool = new Pool({ connectionString: process.env.SUPER_DATABASE_URL }); + const client = await pool.connect(); + try { + await client.query("SELECT pg_advisory_lock(193674839, 3)"); + await client.query("SET lock_timeout = '5s'"); + await client.query("CREATE TABLE IF NOT EXISTS schema_migrations (name text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())"); + for (const file of files) { + if ((await client.query("SELECT 1 FROM schema_migrations WHERE name = $1", [file])).rowCount !== 0) continue; + for (const phase of migrationPhases(await readFile(join(dir, file), "utf8"))) { + const key = phase.name ? `${file}:${phase.name}` : file; + if ((await client.query("SELECT 1 FROM schema_migrations WHERE name = $1", [key])).rowCount !== 0) continue; + try { + if (phase.mode === "concurrent") { + await concurrentIndex(client, phase.sql); + await client.query("INSERT INTO schema_migrations(name) VALUES ($1)", [key]); + } else { + for (;;) { + await client.query("BEGIN"); + const result = await client.query(phase.sql); + if (phase.mode === "batch" && (result.rowCount ?? 0) > 0) { await client.query("COMMIT"); continue; } + await client.query("INSERT INTO schema_migrations(name) VALUES ($1)", [key]); + await client.query("COMMIT"); + break; + } + } + } catch (error) { + await client.query("ROLLBACK").catch(() => undefined); + throw new Error(`migration ${key} failed`, { cause: error }); + } + } + await client.query("INSERT INTO schema_migrations(name) VALUES ($1) ON CONFLICT DO NOTHING", [file]); + applied.push(file); + } + } finally { + await client.query("SELECT pg_advisory_unlock(193674839, 3)").catch(() => undefined); + client.release(); + await pool.end(); + } + return applied; +} diff --git a/apps/api/src/db/production-schema.integration.test.ts b/apps/api/src/db/production-schema.integration.test.ts new file mode 100644 index 0000000..ee148d0 --- /dev/null +++ b/apps/api/src/db/production-schema.integration.test.ts @@ -0,0 +1,326 @@ +import { randomUUID } from "node:crypto"; +import { join } from "node:path"; +import { Pool, type PoolClient } from "pg"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { runMigrations } from "./migrate.js"; + +/** + * Migration 013 (streaming S1) against a live PostgreSQL: the same-channel + * invariant fails at the DB constraint, the production CHECKs hold, and FORCE + * RLS denies every new table across tenants. Assertions run as sp_app + * (DATABASE_URL); setup/cleanup as owner. Skipped without DATABASE_URL. + */ +const dbAvailable = process.env.DATABASE_URL !== undefined; +const ownerPool = new Pool({ connectionString: process.env.SUPER_DATABASE_URL ?? process.env.DATABASE_URL }); +const appPool = new Pool({ connectionString: process.env.DATABASE_URL }); + +const FK_VIOLATION = "23503"; +const UNIQUE_VIOLATION = "23505"; +const CHECK_VIOLATION = "23514"; +const RLS_VIOLATION = "42501"; +const ASSET = "a".repeat(64); + +async function asTenant(workspaceId: string | null, fn: (client: PoolClient) => Promise): Promise { + const client = await appPool.connect(); + try { + await client.query("BEGIN"); + if (workspaceId !== null) await client.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]); + const result = await fn(client); + await client.query("COMMIT"); + return result; + } catch (error) { + await client.query("ROLLBACK").catch(() => undefined); + throw error; + } finally { + client.release(); + } +} + +async function one(client: PoolClient, sql: string, params: unknown[]): Promise { + const result = await client.query(sql, params); + return String(result.rows[0]?.id); +} + +interface Channel { id: string; hls: string; playlist: string; scene: string; spare: string; slot: string; overlay: string; revision: string } + +async function seedChannel(client: PoolClient, ws: string, assetId: string): Promise { + const id = await one(client, "INSERT INTO virtual_channels (workspace_id, slug, name) VALUES ($1, $2, 'Channel') RETURNING id", [ws, `ch-${randomUUID().slice(0, 8)}`]); + const hls = await one(client, + "INSERT INTO virtual_channel_sources (workspace_id, virtual_channel_id, kind, label, remote_ref, hls_url) VALUES ($1, $2, 'hls', 'Cam 1', $3, 'https://cdn.example.com/a.m3u8') RETURNING id", + [ws, id, `ref-${randomUUID()}`]); + const playlist = await one(client, + "INSERT INTO virtual_channel_sources (workspace_id, virtual_channel_id, kind, label, playlist_loop) VALUES ($1, $2, 'playlist', 'Fallback reel', true) RETURNING id", + [ws, id]); + await client.query("INSERT INTO playlist_items (workspace_id, virtual_channel_id, source_id, position, video_asset_id) VALUES ($1, $2, $3, 0, $4)", [ws, id, playlist, assetId]); + const scene = await one(client, "INSERT INTO production_scenes (workspace_id, virtual_channel_id, label, layout) VALUES ($1, $2, 'Main', '2x2') RETURNING id", [ws, id]); + const spare = await one(client, "INSERT INTO production_scenes (workspace_id, virtual_channel_id, label, layout) VALUES ($1, $2, 'Spare', 'single') RETURNING id", [ws, id]); + const slot = await one(client, + "INSERT INTO scene_slots (workspace_id, virtual_channel_id, scene_id, slot_index, source_id, audio_mode, gain_db) VALUES ($1, $2, $3, 0, $4, 'primary', -6) RETURNING id", + [ws, id, scene, hls]); + await client.query("INSERT INTO production_buses (workspace_id, virtual_channel_id, bus, scene_id) VALUES ($1, $2, 'program', $3), ($1, $2, 'preview', $3)", [ws, id, scene]); + const overlay = await one(client, "INSERT INTO overlay_slots (workspace_id, virtual_channel_id, scene_id, kind) VALUES ($1, $2, $3, 'lower_third') RETURNING id", [ws, id, scene]); + const revision = await one(client, + "INSERT INTO overlay_revisions (workspace_id, virtual_channel_id, overlay_slot_id, template_id, render_state, asset_id) VALUES ($1, $2, $3, 'lower-third-v1', 'rendered', $4) RETURNING id", + [ws, id, overlay, ASSET]); + await client.query("UPDATE overlay_slots SET current_revision_id = $2 WHERE id = $1", [overlay, revision]); + return { id, hls, playlist, scene, spare, slot, overlay, revision }; +} + +describe.skipIf(!dbAvailable)("production schema 013 (live DB)", () => { + let wsA: string; + let wsB: string; + let assetA: string; + let assetB: string; + let a1: Channel; + let a2: Channel; + let b1: Channel; + + beforeAll(async () => { + await runMigrations(process.env.MIGRATIONS_DIR ?? join(process.cwd(), "migrations")); + const insert = "INSERT INTO workspaces (name, slug) VALUES ($1, $2) RETURNING id"; + wsA = String((await ownerPool.query(insert, ["Prod A", `prod-a-${randomUUID()}`])).rows[0]?.id); + wsB = String((await ownerPool.query(insert, ["Prod B", `prod-b-${randomUUID()}`])).rows[0]?.id); + const asset = "INSERT INTO video_assets (workspace_id, storage_key, mime_type, size_bytes) VALUES ($1, 'sha256/x.mp4', 'video/mp4', 1) RETURNING id"; + assetA = await asTenant(wsA, (c) => one(c, asset, [wsA])); + assetB = await asTenant(wsB, (c) => one(c, asset, [wsB])); + a1 = await asTenant(wsA, (c) => seedChannel(c, wsA, assetA)); + a2 = await asTenant(wsA, (c) => seedChannel(c, wsA, assetA)); + b1 = await asTenant(wsB, (c) => seedChannel(c, wsB, assetB)); + }); + + afterAll(async () => { + if (wsA !== undefined && wsB !== undefined) await ownerPool.query("DELETE FROM workspaces WHERE id IN ($1, $2)", [wsA, wsB]); + await appPool.end(); + await ownerPool.end(); + }); + + describe("same-channel invariant fails at the DB constraint", () => { + it("a slot cannot reference another channel's source (same workspace)", async () => { + await expect(asTenant(wsA, (c) => c.query( + "INSERT INTO scene_slots (workspace_id, virtual_channel_id, scene_id, slot_index, source_id) VALUES ($1, $2, $3, 1, $4)", + [wsA, a1.id, a1.scene, a2.hls]))).rejects.toMatchObject({ code: FK_VIOLATION }); + }); + + it("a slot cannot claim another channel to smuggle a foreign scene or source", async () => { + await expect(asTenant(wsA, (c) => c.query( + "INSERT INTO scene_slots (workspace_id, virtual_channel_id, scene_id, slot_index, source_id) VALUES ($1, $2, $3, 1, $4)", + [wsA, a2.id, a1.scene, a2.hls]))).rejects.toMatchObject({ code: FK_VIOLATION }); + }); + + it("a slot cannot be re-pointed to any source in place, nor moved to another scene", async () => { + await expect(asTenant(wsA, (c) => c.query("UPDATE scene_slots SET source_id = $2 WHERE id = $1", [a1.slot, a2.hls]))) + .rejects.toMatchObject({ code: CHECK_VIOLATION }); + await expect(asTenant(wsA, (c) => c.query("UPDATE scene_slots SET source_id = $2 WHERE id = $1", [a1.slot, a1.playlist]))) + .rejects.toThrow(/source_id is immutable/); + await expect(asTenant(wsA, (c) => c.query("UPDATE scene_slots SET scene_id = $2 WHERE id = $1", [a1.slot, a1.spare]))) + .rejects.toThrow(/scene_id is immutable/); + }); + + it("a bus cannot hold another channel's scene, by insert or by TAKE-style update", async () => { + await expect(asTenant(wsA, (c) => c.query("UPDATE production_buses SET scene_id = $2 WHERE virtual_channel_id = $1 AND bus = 'program'", [a1.id, a2.scene]))) + .rejects.toMatchObject({ code: FK_VIOLATION }); + const moved = await asTenant(wsA, (c) => c.query("UPDATE production_buses SET scene_id = $2 WHERE virtual_channel_id = $1 AND bus = 'preview'", [a1.id, a1.spare])); + expect(moved.rowCount).toBe(1); + }); + + it("overlay slots and revisions stay inside their channel", async () => { + await expect(asTenant(wsA, (c) => c.query( + "INSERT INTO overlay_slots (workspace_id, virtual_channel_id, scene_id, kind) VALUES ($1, $2, $3, 'logo')", + [wsA, a1.id, a2.scene]))).rejects.toMatchObject({ code: FK_VIOLATION }); + await expect(asTenant(wsA, (c) => c.query( + "INSERT INTO overlay_revisions (workspace_id, virtual_channel_id, overlay_slot_id, template_id) VALUES ($1, $2, $3, 'logo')", + [wsA, a1.id, a2.overlay]))).rejects.toMatchObject({ code: FK_VIOLATION }); + await expect(asTenant(wsA, (c) => c.query("UPDATE overlay_slots SET current_revision_id = $2 WHERE id = $1", [a1.overlay, a2.revision]))) + .rejects.toMatchObject({ code: FK_VIOLATION }); + }); + + it("a playlist item cannot name another channel's source or another tenant's asset", async () => { + await expect(asTenant(wsA, (c) => c.query( + "INSERT INTO playlist_items (workspace_id, virtual_channel_id, source_id, position, video_asset_id) VALUES ($1, $2, $3, 1, $4)", + [wsA, a1.id, a2.playlist, assetA]))).rejects.toMatchObject({ code: FK_VIOLATION }); + await expect(asTenant(wsA, (c) => c.query( + "INSERT INTO playlist_items (workspace_id, virtual_channel_id, source_id, position, video_asset_id) VALUES ($1, $2, $3, 1, $4)", + [wsA, a1.id, a1.playlist, assetB]))).rejects.toMatchObject({ code: FK_VIOLATION }); + }); + }); + + describe("production constraints", () => { + it("allows at most one primary audio slot per scene", async () => { + await expect(asTenant(wsA, (c) => c.query( + "INSERT INTO scene_slots (workspace_id, virtual_channel_id, scene_id, slot_index, source_id, audio_mode) VALUES ($1, $2, $3, 1, $4, 'primary')", + [wsA, a1.id, a1.scene, a1.playlist]))).rejects.toMatchObject({ code: UNIQUE_VIOLATION }); + await asTenant(wsA, (c) => c.query( + "INSERT INTO scene_slots (workspace_id, virtual_channel_id, scene_id, slot_index, source_id, audio_mode) VALUES ($1, $2, $3, 0, $4, 'primary'), ($1, $2, $5, 1, $4, 'on')", + [wsA, a1.id, a1.spare, a1.playlist, a1.scene])); + }); + + it("gain_db is -24..0 in 3 dB steps", async () => { + for (const bad of [-4, 3, -27, -1]) { + await expect(asTenant(wsA, (c) => c.query("UPDATE scene_slots SET gain_db = $2 WHERE id = $1", [a1.slot, bad]))) + .rejects.toMatchObject({ code: CHECK_VIOLATION }); + } + for (const good of [-24, -3, 0]) await asTenant(wsA, (c) => c.query("UPDATE scene_slots SET gain_db = $2 WHERE id = $1", [a1.slot, good])); + }); + + it("geometry is all-or-nothing, positive, and inside the frame", async () => { + const set = (sql: string) => asTenant(wsA, (c) => c.query(`UPDATE scene_slots SET ${sql} WHERE id = $1`, [a1.slot])); + await expect(set("x = 0, y = 0, w = 100")).rejects.toMatchObject({ code: CHECK_VIOLATION }); + await expect(set("x = 0, y = 0, w = 0, h = 100")).rejects.toMatchObject({ code: CHECK_VIOLATION }); + await expect(set("x = -1, y = 0, w = 100, h = 100")).rejects.toMatchObject({ code: CHECK_VIOLATION }); + await expect(set("x = 9000, y = 0, w = 2000, h = 100")).rejects.toMatchObject({ code: CHECK_VIOLATION }); + await set("x = 7000, y = 7000, w = 3000, h = 3000"); + }); + + it("layouts, buses and source kinds are closed enums; playlist is a source kind", async () => { + await expect(asTenant(wsA, (c) => c.query("UPDATE production_scenes SET layout = '5x5' WHERE id = $1", [a1.spare]))) + .rejects.toMatchObject({ code: CHECK_VIOLATION }); + await expect(asTenant(wsA, (c) => c.query( + "INSERT INTO production_buses (workspace_id, virtual_channel_id, bus) VALUES ($1, $2, 'aux')", [wsA, a1.id]))) + .rejects.toMatchObject({ code: CHECK_VIOLATION }); + await expect(asTenant(wsA, (c) => c.query( + "INSERT INTO production_buses (workspace_id, virtual_channel_id, bus) VALUES ($1, $2, 'program')", [wsA, a1.id]))) + .rejects.toMatchObject({ code: UNIQUE_VIOLATION }); + await expect(asTenant(wsA, (c) => c.query( + "INSERT INTO virtual_channel_sources (workspace_id, virtual_channel_id, kind, label, hls_url) VALUES ($1, $2, 'rtmp', 'X', 'https://a.example/x')", + [wsA, a1.id]))).rejects.toMatchObject({ code: CHECK_VIOLATION }); + const kinds = await asTenant(wsA, (c) => c.query("SELECT kind FROM virtual_channel_sources WHERE virtual_channel_id = $1 ORDER BY kind", [a1.id])); + expect(kinds.rows.map((r: { kind: string }) => r.kind)).toEqual(["hls", "playlist"]); + }); + + it("labels with filtergraph metacharacters are rejected by the schema too", async () => { + for (const label of ["a;b", "x[0]", "a,b", "it's", "movie=/etc/passwd"]) { + await expect(asTenant(wsA, (c) => c.query("UPDATE production_scenes SET label = $2 WHERE id = $1", [a1.spare, label]))) + .rejects.toMatchObject({ code: CHECK_VIOLATION }); + } + }); + + it("overlay kinds carry exactly their own parameters; rendered revisions carry a sha256 asset", async () => { + await expect(asTenant(wsA, (c) => c.query( + "INSERT INTO overlay_slots (workspace_id, virtual_channel_id, scene_id, kind) VALUES ($1, $2, $3, 'chyron')", [wsA, a1.id, a1.scene]))) + .rejects.toMatchObject({ code: CHECK_VIOLATION }); + await expect(asTenant(wsA, (c) => c.query( + "INSERT INTO overlay_slots (workspace_id, virtual_channel_id, scene_id, kind, chyron_speed_px_s, chyron_strip_width_px) VALUES ($1, $2, $3, 'chyron', 401, 100)", + [wsA, a1.id, a1.scene]))).rejects.toMatchObject({ code: CHECK_VIOLATION }); + await asTenant(wsA, (c) => c.query( + "INSERT INTO overlay_slots (workspace_id, virtual_channel_id, scene_id, kind, chyron_speed_px_s, chyron_strip_width_px) VALUES ($1, $2, $3, 'chyron', 120, 16384)", + [wsA, a1.id, a1.scene])); + await expect(asTenant(wsA, (c) => c.query( + "INSERT INTO overlay_slots (workspace_id, virtual_channel_id, scene_id, kind) VALUES ($1, $2, $3, 'clock')", [wsA, a1.id, a1.scene]))) + .rejects.toMatchObject({ code: CHECK_VIOLATION }); + await expect(asTenant(wsA, (c) => c.query( + "INSERT INTO overlay_revisions (workspace_id, virtual_channel_id, overlay_slot_id, template_id, render_state, asset_id) VALUES ($1, $2, $3, 'lt', 'rendered', 'movie=x')", + [wsA, a1.id, a1.overlay]))).rejects.toMatchObject({ code: CHECK_VIOLATION }); + await expect(asTenant(wsA, (c) => c.query( + "INSERT INTO overlay_revisions (workspace_id, virtual_channel_id, overlay_slot_id, template_id, render_state) VALUES ($1, $2, $3, 'lt', 'rendered')", + [wsA, a1.id, a1.overlay]))).rejects.toMatchObject({ code: CHECK_VIOLATION }); + }); + }); + + describe("lifecycle", () => { + it("deleting a source empties its slots (SET NULL keeps tenant + channel); a scene on a bus cannot be deleted", async () => { + const channel = await asTenant(wsA, (c) => seedChannel(c, wsA, assetA)); + await asTenant(wsA, (c) => c.query("DELETE FROM virtual_channel_sources WHERE id = $1", [channel.hls])); + const slot = await asTenant(wsA, (c) => c.query("SELECT workspace_id, virtual_channel_id, source_id FROM scene_slots WHERE id = $1", [channel.slot])); + expect(slot.rows[0]).toEqual({ workspace_id: wsA, virtual_channel_id: channel.id, source_id: null }); + await expect(asTenant(wsA, (c) => c.query("DELETE FROM production_scenes WHERE id = $1", [channel.scene]))) + .rejects.toMatchObject({ code: FK_VIOLATION }); + }); + + it("deleting a rendered revision clears the slot's current revision; deleting a channel cascades everything", async () => { + const channel = await asTenant(wsA, (c) => seedChannel(c, wsA, assetA)); + await asTenant(wsA, (c) => c.query("DELETE FROM overlay_revisions WHERE id = $1", [channel.revision])); + const overlay = await asTenant(wsA, (c) => c.query("SELECT current_revision_id FROM overlay_slots WHERE id = $1", [channel.overlay])); + expect(overlay.rows[0]?.current_revision_id).toBeNull(); + await asTenant(wsA, (c) => c.query("DELETE FROM virtual_channels WHERE id = $1", [channel.id])); + const left = await asTenant(wsA, (c) => c.query( + `SELECT (SELECT count(*) FROM virtual_channel_sources WHERE virtual_channel_id = $1) + + (SELECT count(*) FROM production_scenes WHERE virtual_channel_id = $1) + + (SELECT count(*) FROM scene_slots WHERE virtual_channel_id = $1) + + (SELECT count(*) FROM production_buses WHERE virtual_channel_id = $1) + + (SELECT count(*) FROM overlay_slots WHERE virtual_channel_id = $1) + + (SELECT count(*) FROM overlay_revisions WHERE virtual_channel_id = $1) + + (SELECT count(*) FROM playlist_items WHERE virtual_channel_id = $1) AS n`, [channel.id])); + expect(Number(left.rows[0]?.n)).toBe(0); + }); + + it("a playlisted video asset cannot be deleted directly, but a workspace purge removes everything", async () => { + await expect(asTenant(wsA, (c) => c.query("DELETE FROM video_assets WHERE id = $1", [assetA]))) + .rejects.toMatchObject({ code: FK_VIOLATION }); + const ws = String((await ownerPool.query("INSERT INTO workspaces (name, slug) VALUES ('Purge', $1) RETURNING id", [`purge-${randomUUID()}`])).rows[0]?.id); + const asset = await asTenant(ws, (c) => one(c, + "INSERT INTO video_assets (workspace_id, storage_key, mime_type, size_bytes) VALUES ($1, 'sha256/p.mp4', 'video/mp4', 1) RETURNING id", [ws])); + await asTenant(ws, (c) => seedChannel(c, ws, asset)); + await ownerPool.query("DELETE FROM workspaces WHERE id = $1", [ws]); + const left = await ownerPool.query( + `SELECT (SELECT count(*) FROM virtual_channels WHERE workspace_id = $1) + + (SELECT count(*) FROM playlist_items WHERE workspace_id = $1) + + (SELECT count(*) FROM production_buses WHERE workspace_id = $1) + + (SELECT count(*) FROM overlay_revisions WHERE workspace_id = $1) AS n`, [ws]); + expect(Number(left.rows[0]?.n)).toBe(0); + }); + + it("workspace and channel ownership columns are immutable", async () => { + await expect(asTenant(wsA, (c) => c.query("UPDATE production_scenes SET virtual_channel_id = $2 WHERE id = $1", [a1.spare, a2.id]))) + .rejects.toThrow(/immutable/); + await expect(asTenant(wsA, (c) => c.query("UPDATE production_buses SET bus = 'preview' WHERE virtual_channel_id = $1 AND bus = 'program'", [a1.id]))) + .rejects.toThrow(/bus is immutable/); + await expect(asTenant(wsA, (c) => c.query("UPDATE virtual_channels SET workspace_id = $2 WHERE id = $1", [a1.id, wsB]))) + .rejects.toThrow(/immutable|row-level security/); + }); + }); + + describe("RLS denial suite (FORCE, NULLIF tenant isolation)", () => { + const tables = [ + "virtual_channels", "virtual_channel_sources", "playlist_items", "production_scenes", + "scene_slots", "production_buses", "overlay_slots", "overlay_revisions", + ] as const; + + it.each(tables)("%s has RLS enabled and forced", async (table) => { + const flags = await ownerPool.query("SELECT relrowsecurity, relforcerowsecurity FROM pg_class WHERE oid = $1::regclass", [table]); + expect(flags.rows[0]).toEqual({ relrowsecurity: true, relforcerowsecurity: true }); + }); + + it.each(tables)("%s: tenant B sees, updates and deletes none of A's rows", async (table) => { + const seen = await asTenant(wsB, (c) => c.query(`SELECT count(*)::int AS n FROM ${table} WHERE workspace_id = $1`, [wsA])); + expect(seen.rows[0]?.n).toBe(0); + const updated = await asTenant(wsB, (c) => c.query(`UPDATE ${table} SET updated_at = now() WHERE workspace_id = $1`, [wsA])); + expect(updated.rowCount).toBe(0); + const deleted = await asTenant(wsB, (c) => c.query(`DELETE FROM ${table} WHERE workspace_id = $1`, [wsA])); + expect(deleted.rowCount).toBe(0); + const ownStillThere = await asTenant(wsA, (c) => c.query(`SELECT count(*)::int AS n FROM ${table}`)); + expect(ownStillThere.rows[0]?.n).toBeGreaterThan(0); + }); + + it.each(tables)("%s: no tenant context (and an empty one) sees nothing", async (table) => { + const none = await asTenant(null, (c) => c.query(`SELECT count(*)::int AS n FROM ${table}`)); + expect(none.rows[0]?.n).toBe(0); + const empty = await asTenant("", (c) => c.query(`SELECT count(*)::int AS n FROM ${table}`)); + expect(empty.rows[0]?.n).toBe(0); + }); + + it("tenant B cannot write rows into A's workspace (WITH CHECK) on any table", async () => { + const attempts: Array<[string, unknown[]]> = [ + ["INSERT INTO virtual_channels (workspace_id, slug, name) VALUES ($1, 'x-rls', 'X')", [wsA]], + ["INSERT INTO virtual_channel_sources (workspace_id, virtual_channel_id, kind, label, playlist_loop) VALUES ($1, $2, 'playlist', 'X', false)", [wsA, a1.id]], + ["INSERT INTO playlist_items (workspace_id, virtual_channel_id, source_id, position, video_asset_id) VALUES ($1, $2, $3, 9, $4)", [wsA, a1.id, a1.playlist, assetA]], + ["INSERT INTO production_scenes (workspace_id, virtual_channel_id, label, layout) VALUES ($1, $2, 'X', 'single')", [wsA, a1.id]], + ["INSERT INTO scene_slots (workspace_id, virtual_channel_id, scene_id, slot_index) VALUES ($1, $2, $3, 3)", [wsA, a1.id, a1.scene]], + ["INSERT INTO production_buses (workspace_id, virtual_channel_id, bus) VALUES ($1, $2, 'program')", [wsA, a1.id]], + ["INSERT INTO overlay_slots (workspace_id, virtual_channel_id, scene_id, kind) VALUES ($1, $2, $3, 'logo')", [wsA, a1.id, a1.scene]], + ["INSERT INTO overlay_revisions (workspace_id, virtual_channel_id, overlay_slot_id, template_id) VALUES ($1, $2, $3, 'lt')", [wsA, a1.id, a1.overlay]], + ]; + for (const [sql, params] of attempts) { + await expect(asTenant(wsB, (c) => c.query(sql, params)), sql).rejects.toMatchObject({ code: RLS_VIOLATION }); + } + }); + + it("tenant B cannot point its own rows at A's channel, scene or source", async () => { + await expect(asTenant(wsB, (c) => c.query( + "INSERT INTO scene_slots (workspace_id, virtual_channel_id, scene_id, slot_index, source_id) VALUES ($1, $2, $3, 2, $4)", + [wsB, b1.id, b1.scene, a1.hls]))).rejects.toMatchObject({ code: FK_VIOLATION }); + await expect(asTenant(wsB, (c) => c.query( + "UPDATE production_buses SET scene_id = $2 WHERE virtual_channel_id = $1 AND bus = 'program'", [b1.id, a1.scene]))) + .rejects.toMatchObject({ code: FK_VIOLATION }); + }); + }); +}); diff --git a/apps/api/src/db/publishing-migration.integration.test.ts b/apps/api/src/db/publishing-migration.integration.test.ts new file mode 100644 index 0000000..94a5890 --- /dev/null +++ b/apps/api/src/db/publishing-migration.integration.test.ts @@ -0,0 +1,68 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { mkdtemp, readdir, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Pool } from "pg"; +import { runMigrations } from "./migrate.js"; +import { assertPublishingSchema } from "@tactical-media-broadcast-hub/publishing-execution"; +const ownerUrl = process.env.PUBLISHING_IT_SUPER_DATABASE_URL; +const appUrl = process.env.PUBLISHING_IT_DATABASE_URL; +describe.skipIf(!ownerUrl || !appUrl)("publishing migration upgrade and resumability (local DB)", () => { + let admin: Pool; let owner: Pool; let app: Pool; let dir: string; + let database: string; let previousOwner: string | undefined; + beforeAll(async () => { + const url = new URL(ownerUrl!); url.pathname = "/postgres"; + admin = new Pool({ connectionString: url.toString() }); + database = `wp3_migration_sol_${Date.now()}`; + await admin.query(`CREATE DATABASE ${database}`); + url.pathname = `/${database}`; previousOwner = process.env.SUPER_DATABASE_URL; + process.env.SUPER_DATABASE_URL = url.toString(); owner = new Pool({ connectionString: url.toString() }); + const runtimeUrl = new URL(appUrl!); runtimeUrl.pathname = `/${database}`; app = new Pool({ connectionString: runtimeUrl.toString() }); + dir = await mkdtemp(join(tmpdir(), "wp3-migrations-sol-")); + for (const file of (await readdir("migrations")).filter((name) => name.endsWith(".sql") && name < "018")) await writeFile(join(dir, file), await readFile(join("migrations", file))); + await runMigrations(dir); + }, 30_000); + afterAll(async () => { + if (previousOwner === undefined) delete process.env.SUPER_DATABASE_URL; else process.env.SUPER_DATABASE_URL = previousOwner; + await owner?.end(); await app?.end(); + if (admin && database) await admin.query(`DROP DATABASE ${database}`); + await admin?.end(); + }); + it("upgrades legacy schedules without inventing authorization and resumes 019 after failed cutover", async () => { + const user = (await owner.query("INSERT INTO users(external_id, username, display_name) VALUES('migration-sol', 'migration-sol', 'Migration') RETURNING id")).rows[0].id as string; + const workspace = (await owner.query("INSERT INTO workspaces(name,slug) VALUES('Migration','migration-sol') RETURNING id")).rows[0].id as string; + const connection = (await owner.query("INSERT INTO connections(workspace_id,provider,channel_id) VALUES($1,'mastodon','channel') RETURNING id", [workspace])).rows[0].id as string; + const keys = ["legacy-pending", "legacy-flight", "legacy-done"]; + for (const [index, key] of keys.entries()) { + const draft = (await owner.query("INSERT INTO drafts(workspace_id,author_user_id,post,state) VALUES($1,$2,'{}','scheduled') RETURNING id", [workspace, user])).rows[0].id as string; + await owner.query("INSERT INTO scheduled_publications(workspace_id,draft_id,connection_id,publish_at,idempotency_key,state) VALUES($1,$2,$3,now(),$4,$5)", [workspace, draft, connection, key, index === 1 ? "in_flight" : "pending"]); + if (index === 2) await owner.query("INSERT INTO publications(workspace_id,draft_id,connection_id,idempotency_key,remote_id,published_at) VALUES($1,$2,$3,$4,'remote',now())", [workspace, draft, connection, key]); + } + await writeFile(join(dir, "018_publishing_execution.sql"), await readFile("migrations/018_publishing_execution.sql")); + await runMigrations(dir); + const upgraded = await owner.query("SELECT state, scheduled_by_user_id, execution_version FROM scheduled_publications ORDER BY idempotency_key"); + expect(upgraded.rows.map((row) => row.state)).toEqual(["done", "failed", "needs_reauthorization"]); + expect(upgraded.rows.every((row) => row.scheduled_by_user_id === null && row.execution_version === 0)).toBe(true); + const original = await readFile("migrations/019_publishing_integration.sql", "utf8"); + await writeFile(join(dir, "019_publishing_integration.sql"), original.replace("DROP FUNCTION publishing_due_schedules(integer);", "SELECT nonexistent_cutover_function();")); + await expect(runMigrations(dir)).rejects.toThrow("cutover failed"); + expect((await owner.query("SELECT name FROM schema_migrations WHERE name = '019_publishing_integration.sql'")).rowCount).toBe(0); + await expect(assertPublishingSchema(app)).rejects.toThrow(); + expect((await owner.query("SELECT name FROM schema_migrations WHERE name LIKE '019%:%'")).rowCount).toBe(3); + await writeFile(join(dir, "019_publishing_integration.sql"), original); + await expect(runMigrations(dir)).resolves.toEqual(["019_publishing_integration.sql"]); + await expect(assertPublishingSchema(app)).resolves.toBeUndefined(); + await expect(runMigrations(dir)).resolves.toEqual([]); + const role = (await app.query("SELECT current_user, rolsuper, rolbypassrls FROM pg_roles WHERE rolname = current_user")).rows[0]; + expect(role).toEqual({ current_user: "sp_app", rolsuper: false, rolbypassrls: false }); + }, 30_000); + it("repairs an invalid concurrent index only when its definition matches", async () => { + const file = "020_test_concurrent.sql"; + await writeFile(join(dir, file), "-- wp3:phase expand transaction\nCREATE TABLE migration_index_test(v integer); INSERT INTO migration_index_test VALUES(1),(1);\n-- wp3:phase idx concurrent\nCREATE UNIQUE INDEX CONCURRENTLY migration_index_test_idx ON migration_index_test (v);"); + await expect(runMigrations(dir)).rejects.toThrow("idx failed"); + expect((await owner.query("SELECT indisvalid FROM pg_index WHERE indexrelid = 'migration_index_test_idx'::regclass")).rows[0]?.indisvalid).toBe(false); + await owner.query("DELETE FROM migration_index_test WHERE ctid NOT IN (SELECT min(ctid) FROM migration_index_test GROUP BY v)"); + await expect(runMigrations(dir)).resolves.toEqual([file]); + expect((await owner.query("SELECT indisvalid FROM pg_index WHERE indexrelid = 'migration_index_test_idx'::regclass")).rows[0]?.indisvalid).toBe(true); + }, 30_000); +}); diff --git a/apps/api/src/db/tenancy.integration.test.ts b/apps/api/src/db/tenancy.integration.test.ts new file mode 100644 index 0000000..fd0e020 --- /dev/null +++ b/apps/api/src/db/tenancy.integration.test.ts @@ -0,0 +1,176 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { join } from "node:path"; +import { Pool } from "pg"; +import { pool, withTenant, withSystem } from "./client.js"; +import { runMigrations } from "./migrate.js"; +import { PublishingExecutionRepository } from "@tactical-media-broadcast-hub/publishing-execution"; + +/** + * Audit-finding acceptance tests against a live PostgreSQL. + * Migrations + cleanup run via the owner pool (SUPER_DATABASE_URL); + * the assertions run as sp_app so FORCE RLS binds honestly. + * Skipped without DATABASE_URL. + */ +const dbAvailable = process.env.DATABASE_URL !== undefined; +const ownerPool = new Pool({ connectionString: process.env.SUPER_DATABASE_URL ?? process.env.DATABASE_URL }); + +async function asOwner(fn: (client: import("pg").PoolClient) => Promise): Promise { + const client = await ownerPool.connect(); + try { + return await fn(client); + } finally { + client.release(); + } +} + +describe.skipIf(!dbAvailable)("tenancy + audit chain (live DB)", () => { + let wsA: string; + let wsB: string; + + beforeAll(async () => { + await runMigrations(process.env.MIGRATIONS_DIR ?? join(process.cwd(), "migrations")); + await asOwner(async (client) => { + await client.query("DELETE FROM audit_log"); + await client.query("DELETE FROM memberships"); + await client.query("DELETE FROM connections"); + await client.query("DELETE FROM credentials"); + await client.query("DELETE FROM workspaces"); + const a = await client.query("INSERT INTO workspaces (name, slug) VALUES ('A', $1) RETURNING id", [`a-${Date.now()}`]); + const b = await client.query("INSERT INTO workspaces (name, slug) VALUES ('B', $1) RETURNING id", [`b-${Date.now()}`]); + wsA = a.rows[0].id as string; + wsB = b.rows[0].id as string; + }); + }); + + afterAll(async () => { + await asOwner(async (client) => { + await client.query("DELETE FROM workspaces WHERE id IN ($1, $2)", [wsA, wsB]); + }); + await pool.end(); + await ownerPool.end(); + }); + + it("RLS isolates tenants: A sees only A's connections (FORCE applies to owner)", async () => { + await withTenant(wsA, async (client) => { + await client.query("INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-social', 'ch-a')", [wsA]); + }); + await withTenant(wsB, async (client) => { + await client.query("INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-social', 'ch-b')", [wsB]); + }); + const seenFromA = await withTenant(wsA, async (client) => { + return (await client.query("SELECT channel_id FROM connections")).rows.map((r) => r.channel_id); + }); + expect(seenFromA).toEqual(["ch-a"]); + const seenFromSystem = await withSystem(async (client) => { + return (await client.query("SELECT count(*)::int AS n FROM connections")).rows[0]?.n; + }); + expect(seenFromSystem).toBe(0); + }); + + it("SECURITY DEFINER tenant assertion: cross-tenant call rejected (finding #3)", async () => { + await expect( + withTenant(wsB, async (client) => { + await client.query("SELECT audit_append($1, NULL, 'system', 'x', 'y', 'z', NULL)", [wsA]); + }), + ).rejects.toThrow(/tenant context mismatch/); + }); + + it("audit branch: tenant context => tenant chain even for service callers (findings #1/#6)", async () => { + await withTenant(wsA, async (client) => { + const r = await client.query("SELECT audit_append($1, NULL, 'sp_scheduler', 'job.ran', 'job', 'j1', NULL) AS id", [wsA]); + expect(r.rows[0]?.id).toBeTruthy(); + const row = await client.query("SELECT workspace_id, actor_role FROM audit_log WHERE id = $1", [r.rows[0].id]); + expect(row.rows[0]?.workspace_id).toBe(wsA); + expect(row.rows[0]?.actor_role).toBe("sp_scheduler"); + // system append inside tenant context must be refused + await expect(client.query("SELECT audit_append_system(NULL, 'x', 'y', 'z', NULL)")).rejects.toThrow(); + }); + }); + + it("membership change bumps authz_epoch => sessions fail closed (finding #2)", async () => { + const before = Number(await withSystem(async (c) => (await c.query("SELECT authz_epoch FROM workspaces WHERE id = $1", [wsA])).rows[0]?.authz_epoch)); + await asOwner(async (c) => { + const u = await c.query("INSERT INTO users (external_id, email, display_name) VALUES ($1, 'e@x', 'E') RETURNING id", [`ext-${Date.now()}`]); + await withTenant(wsA, async (tenantClient) => { + await tenantClient.query("INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1, $2, 'Publisher')", [wsA, u.rows[0].id]); + }); + }); + const after = Number(await withSystem(async (c) => (await c.query("SELECT authz_epoch FROM workspaces WHERE id = $1", [wsA])).rows[0]?.authz_epoch)); + expect(after).toBeGreaterThan(before); + }); + + it("hash chain detects tampering", async () => { + await withTenant(wsA, async (client) => { + await client.query("SELECT audit_append($1, NULL, 'system', 't.1', 't', '1', NULL)", [wsA]); + await client.query("SELECT audit_append($1, NULL, 'system', 't.2', 't', '2', NULL)", [wsA]); + }); + const ok = await withTenant(wsA, async (c) => (await c.query("SELECT audit_verify_chain($1) AS ok", [wsA])).rows[0]?.ok); + expect(ok).toBe(true); + // direct tamper (as owner, inside the tenant context, bypassing the append fn) + await withTenant(wsA, async (c) => { + await c.query("UPDATE audit_log SET action = 'forged' WHERE action = 't.1'"); + }); + const broken = await withTenant(wsA, async (c) => (await c.query("SELECT audit_verify_chain($1) AS ok", [wsA])).rows[0]?.ok); + expect(broken).toBe(false); + }); + + it("verifies a legacy v1 row followed by v2 and detects a changed v2 diff", async () => { + await withTenant(wsB, async (client) => { + await client.query( + `WITH stamp AS MATERIALIZED (SELECT clock_timestamp() AS at) + INSERT INTO audit_log (workspace_id, action, target_type, target_id, diff, at, prev_hash, hash, hash_version) + SELECT $1, 'legacy', 'test', 'legacy', '{"value":1}'::jsonb, stamp.at, NULL, + audit_hash(NULL, NULL, 'legacy', 'legacy', stamp.at), 1 FROM stamp`, + [wsB], + ); + await client.query("SELECT audit_append($1, NULL, 'system', 'modern', 'test', 'modern', $2)", [wsB, { value: 2 }]); + const verified = await client.query("SELECT audit_verify_chain($1) AS ok", [wsB]); + expect(verified.rows[0]?.ok).toBe(true); + await client.query("UPDATE audit_log SET diff = $2 WHERE workspace_id = $1 AND action = 'modern'", [wsB, { value: 3 }]); + const broken = await client.query("SELECT audit_verify_chain($1) AS ok", [wsB]); + expect(broken.rows[0]?.ok).toBe(false); + }); + }); + + it("scheduler claim refuses a pending schedule whose draft is not approved", async () => { + const repository = new PublishingExecutionRepository(pool); + try { + const user = await asOwner(async (client) => { + const inserted = await client.query( + "INSERT INTO users (external_id, email, display_name) VALUES ($1, 'claim@example.com', 'Claim') RETURNING id", + [`claim-${Date.now()}`], + ); + return inserted.rows[0]?.id as string; + }); + const key = `claim-${Date.now()}`; + await withTenant(wsA, async (client) => { + const connection = await client.query( + "INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-social', 'claim') RETURNING id", + [wsA], + ); + const draft = await client.query( + "INSERT INTO drafts (workspace_id, author_user_id, post, state) VALUES ($1, $2, $3, 'in_review') RETURNING id", + [wsA, user, { text: "unapproved", mediaAssetIds: [], visibility: "public", providerMetadata: {} }], + ); + await client.query( + "INSERT INTO scheduled_publications (workspace_id, draft_id, connection_id, publish_at, idempotency_key) VALUES ($1, $2, $3, now(), $4)", + [wsA, draft.rows[0]?.id, connection.rows[0]?.id, key], + ); + }); + const schedule = await withTenant(wsA, (client) => client.query("SELECT id, generation FROM scheduled_publications WHERE idempotency_key = $1", [key])); + await expect(repository.claim({ workspaceId: wsA, scheduleId: String(schedule.rows[0]?.id), generation: String(schedule.rows[0]?.generation), idempotencyKey: key })).resolves.toMatchObject({ kind: "failed", code: "EXECUTION_FENCE_MISSING" }); + const state = await withTenant(wsA, (client) => client.query("SELECT state FROM scheduled_publications WHERE idempotency_key = $1", [key])); + expect(state.rows[0]?.state).toBe("pending"); + } finally { + // The shared repository uses the caller-owned pool, closed by suite cleanup. + } + }); + + it("workspace_id is immutable on tenant rows (finding #8)", async () => { + await expect( + withTenant(wsA, async (c) => { + await c.query("UPDATE connections SET workspace_id = $1 WHERE channel_id = 'ch-a'", [wsB]); + }), + ).rejects.toThrow(/workspace_id is immutable/); + }); +}); diff --git a/apps/api/src/demo/demo.controller.ts b/apps/api/src/demo/demo.controller.ts new file mode 100644 index 0000000..f55bcae --- /dev/null +++ b/apps/api/src/demo/demo.controller.ts @@ -0,0 +1,57 @@ +import { Controller, Get, Post, Body, Req, UseGuards } from "@nestjs/common"; +import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import { appendTenantAudit } from "../audit/audit.service.js"; +import { withTenant } from "../db/client.js"; +import type { Request } from "express"; + +@Controller("demo") +@UseGuards(RbacGuard) +export class DemoController { + @Get("health-audit") + @RequirePermission("audit.view") + async viewAudit(@Req() req: Request & AuthenticatedRequest): Promise<{ workspaceId: string }> { + return { workspaceId: req.session?.workspaceId ?? "none" }; + } + + @Post("connections") + @RequirePermission("connection.manage") + async addConnection( + @Req() req: Request & AuthenticatedRequest, + @Body() body: { provider: string; channelId: string }, + ): Promise<{ id: string }> { + const session = req.session; + if (session === undefined) throw new Error("unreachable: guard sets session"); + const workspaceId = session.workspaceId; + return withTenant(workspaceId, async (client) => { + const inserted = await client.query( + "INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, $2, $3) RETURNING id", + [workspaceId, body.provider, body.channelId], + ); + const id = inserted.rows[0]?.id as string; + await client.query("SELECT audit_append($1, $2, $3, 'connection.created', 'connection', $4, $5)", [ + workspaceId, + session.userId, + session.role, + id, + JSON.stringify({ provider: body.provider }), + ]); + return { id }; + }); + } + + @Post("echo-audited") + @RequirePermission("workspace.view") + async echoAudited(@Req() req: Request & AuthenticatedRequest, @Body() body: { apiKey?: string; note?: string }): Promise<{ ok: true }> { + const session = req.session; + if (session === undefined) throw new Error("unreachable: guard sets session"); + await appendTenantAudit(session.workspaceId, { + actorUserId: session.userId, + actorRole: session.role, + action: "demo.echo", + targetType: "demo", + targetId: "echo", + diff: { apiKey: body.apiKey ?? null, note: body.note ?? null }, + }); + return { ok: true }; + } +} diff --git a/apps/api/src/director/config.ts b/apps/api/src/director/config.ts new file mode 100644 index 0000000..25b4c13 --- /dev/null +++ b/apps/api/src/director/config.ts @@ -0,0 +1,139 @@ +/** + * Director channel configuration — API-side mirror of + * apps/scheduler/src/director/config.ts. + * + * The API answers snapshot/fence reads from the same DIRECTOR_CHANNELS + * document the scheduler parses, so both parsers must agree on the shape. + * Duplicating the parser (rather than importing scheduler sources into the + * API's dependency graph) keeps each app's build independent; the shared + * fixture test in director.service.test.ts asserts both parse identically. + */ +import type { AutoDirectorRule, DirectorAction, DirectorCondition, SceneLayout, SwitchPolicy } from "@tactical-media-broadcast-hub/core"; + +export interface DirectorChannelConfig { + readonly workspaceId: string; + readonly channelId: string; + readonly generation: string; + readonly rules: readonly AutoDirectorRule[]; + readonly policy: SwitchPolicy | undefined; + readonly fallbackSceneId: string | null; +} + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +const GENERATION = /^[\x21-\x7e]{1,64}$/; +const POLICY_NAMES: readonly SwitchPolicy["name"][] = [ + "highest_priority", "oldest_live", "newest_live", "round_robin", "operator_required", + "picture_in_picture", "two_up", "split_screen", "grid", +]; +const LAYOUTS: readonly SceneLayout[] = ["single", "pip", "two_up", "split_horizontal", "split_vertical", "grid", "custom"]; +const CONDITION_FIELDS: readonly DirectorCondition["field"][] = ["live_source_count", "max_priority", "source_state"]; +const CONDITION_OPS: readonly DirectorCondition["op"][] = [">=", "<=", "==", ">"]; +const SELECTIONS: readonly DirectorAction["selection"][] = ["top_priority", "all_live", "oldest_live", "explicit"]; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function uuid(value: unknown, field: string): string { + if (typeof value !== "string" || !UUID.test(value)) throw new Error(`${field} must be a UUID`); + return value; +} + +function nonNegativeInt(value: unknown, field: string): number { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) throw new Error(`${field} must be a non-negative integer`); + return value; +} + +function parseCondition(value: unknown, field: string): DirectorCondition { + if (!isRecord(value)) throw new Error(`${field} must be an object`); + const conditionField = value.field; + const op = value.op; + if (typeof conditionField !== "string" || !CONDITION_FIELDS.includes(conditionField as DirectorCondition["field"])) { + throw new Error(`${field}.field must be one of ${CONDITION_FIELDS.join(", ")}`); + } + if (typeof op !== "string" || !CONDITION_OPS.includes(op as DirectorCondition["op"])) { + throw new Error(`${field}.op must be one of ${CONDITION_OPS.join(", ")}`); + } + const ruleValue = value.value; + if (typeof ruleValue !== "number" && typeof ruleValue !== "string") throw new Error(`${field}.value must be a number or string`); + return { field: conditionField as DirectorCondition["field"], op: op as DirectorCondition["op"], value: ruleValue }; +} + +function parseAction(value: unknown, field: string): DirectorAction { + if (!isRecord(value)) throw new Error(`${field} must be an object`); + const selection = value.selection; + if (typeof selection !== "string" || !SELECTIONS.includes(selection as DirectorAction["selection"])) { + throw new Error(`${field}.selection must be one of ${SELECTIONS.join(", ")}`); + } + const layout = value.layout; + if (typeof layout !== "string" || !LAYOUTS.includes(layout as SceneLayout)) { + throw new Error(`${field}.layout must be one of ${LAYOUTS.join(", ")}`); + } + let sourceIds: string[] | undefined; + if (value.sourceIds !== undefined) { + if (!Array.isArray(value.sourceIds)) throw new Error(`${field}.sourceIds must be an array`); + sourceIds = value.sourceIds.map((id, i) => uuid(id, `${field}.sourceIds[${i}]`)); + if (new Set(sourceIds).size !== sourceIds.length) throw new Error(`${field}.sourceIds must be unique`); + } + if (selection === "explicit" && (sourceIds === undefined || sourceIds.length === 0)) { + throw new Error(`${field}.selection explicit requires sourceIds`); + } + return { layout: layout as SceneLayout, selection: selection as DirectorAction["selection"], ...(sourceIds === undefined ? {} : { sourceIds }) }; +} + +function parseRule(value: unknown, channelId: string, index: number): AutoDirectorRule { + const field = `DIRECTOR_CHANNELS[${index}].rules[${index}]`; + if (!isRecord(value)) throw new Error(`${field} must be an object`); + if (value.virtualChannelId !== undefined && value.virtualChannelId !== channelId) { + throw new Error(`${field}.virtualChannelId must match the channel it is configured on`); + } + return { + id: uuid(value.id, `${field}.id`), + virtualChannelId: channelId, + priority: nonNegativeInt(value.priority ?? 0, `${field}.priority`), + condition: parseCondition(value.condition, `${field}.condition`), + action: parseAction(value.action, `${field}.action`), + }; +} + +function parsePolicy(value: unknown, field: string): SwitchPolicy { + if (!isRecord(value)) throw new Error(`${field} must be an object`); + const name = value.name; + if (typeof name !== "string" || !POLICY_NAMES.includes(name as SwitchPolicy["name"])) { + throw new Error(`${field}.name must be one of ${POLICY_NAMES.join(", ")}`); + } + return { + name: name as SwitchPolicy["name"], + liveConfirmations: nonNegativeInt(value.liveConfirmations ?? 2, `${field}.liveConfirmations`), + offlineConfirmations: nonNegativeInt(value.offlineConfirmations ?? 3, `${field}.offlineConfirmations`), + noMediaSeconds: nonNegativeInt(value.noMediaSeconds ?? 10, `${field}.noMediaSeconds`), + minHoldMs: nonNegativeInt(value.minHoldMs ?? 2_000, `${field}.minHoldMs`), + switchCooldownMs: nonNegativeInt(value.switchCooldownMs ?? 10_000, `${field}.switchCooldownMs`), + }; +} + +export function parseDirectorChannels(raw: string | undefined): DirectorChannelConfig[] { + if (raw === undefined || raw.trim() === "") return []; + const parsed: unknown = JSON.parse(raw); + if (!Array.isArray(parsed)) throw new Error("DIRECTOR_CHANNELS must be a JSON array"); + return parsed.map((entry, index) => { + const field = `DIRECTOR_CHANNELS[${index}]`; + if (!isRecord(entry)) throw new Error(`${field} must be an object`); + const channelId = uuid(entry.channelId, `${field}.channelId`); + const generation = entry.generation; + if (typeof generation !== "string" || !GENERATION.test(generation)) { + throw new Error(`${field}.generation must be 1-64 printable characters`); + } + const rules = Array.isArray(entry.rules) ? entry.rules.map((rule, i) => parseRule(rule, channelId, i)) : []; + const fallbackSceneId = entry.fallbackSceneId === undefined || entry.fallbackSceneId === null + ? null : uuid(entry.fallbackSceneId, `${field}.fallbackSceneId`); + return { + workspaceId: uuid(entry.workspaceId, `${field}.workspaceId`), + channelId, + generation, + rules, + policy: entry.policy === undefined ? undefined : parsePolicy(entry.policy, `${field}.policy`), + fallbackSceneId, + }; + }); +} diff --git a/apps/api/src/director/director.controller.ts b/apps/api/src/director/director.controller.ts new file mode 100644 index 0000000..102bfdc --- /dev/null +++ b/apps/api/src/director/director.controller.ts @@ -0,0 +1,84 @@ +/** + * Internal director endpoints (WP10 wiring slice). + * + * Process-to-process only: a shared-secret bearer (DIRECTOR_API_TOKEN) when + * configured, plus the ref's workspace/channel UUID shape. Not registered under + * the session RBAC guard — the scheduler worker has no browser session; its + * authorization is the generation/version fence in DirectorService. + */ +import { BadRequestException, Body, Controller, Get, Headers, HttpCode, Inject, Param, Post } from "@nestjs/common"; +import { DirectorService, type DirectorApplyBody } from "./director.service.js"; + +function requireToken(token: string | undefined): void { + const expected = process.env.DIRECTOR_API_TOKEN; + if (expected === undefined || expected === "") return; // unset = local dev, no shared secret configured + if (token !== `Bearer ${expected}`) throw new BadRequestException("director token is invalid"); +} + +@Controller("internal/director/channels") +export class DirectorController { + constructor(@Inject(DirectorService) private readonly director: DirectorService) {} + + @Get(":workspaceId/:channelId/state") + async state(@Headers() headers: Record, @Param("workspaceId") ws: string, @Param("channelId") ch: string): Promise { + requireToken(headers.authorization); + return this.director.channelState(ws, ch); + } + + @Get(":workspaceId/:channelId/fence") + async fence(@Headers() headers: Record, @Param("workspaceId") ws: string, @Param("channelId") ch: string): Promise { + requireToken(headers.authorization); + return this.director.fence(ws, ch); + } + + @Post(":workspaceId/:channelId/apply") + @HttpCode(200) + async apply(@Headers() headers: Record, @Param("workspaceId") ws: string, @Param("channelId") ch: string, + @Body() body: unknown): Promise { + requireToken(headers.authorization); + return this.director.applyProgramSwitch(ws, ch, parseApply(body)); + } + + @Post(":workspaceId/:channelId/audit") + @HttpCode(204) + async audit(@Headers() headers: Record, @Param("workspaceId") ws: string, @Param("channelId") ch: string, + @Body() body: unknown): Promise { + requireToken(headers.authorization); + const event = body as { commandId?: unknown; outcome?: unknown; reason?: unknown }; + if (typeof event.commandId !== "string" || (event.outcome !== "SUPERSEDED" && event.outcome !== "AUTHZ_REVOKED") + || typeof event.reason !== "string") throw new BadRequestException("audit event is invalid"); + await this.director.audit(ws, ch, { commandId: event.commandId, outcome: event.outcome, reason: event.reason }); + } +} + +function parseApply(body: unknown): DirectorApplyBody { + if (typeof body !== "object" || body === null || Array.isArray(body)) throw new BadRequestException("apply body must be an object"); + const value = body as Record; + const commandId = value.commandId; + if (typeof commandId !== "string" || commandId === "" || commandId.length > 128) throw new BadRequestException("commandId must be 1-128 characters"); + const expectedDesiredVersion = value.expectedDesiredVersion; + if (typeof expectedDesiredVersion !== "number" || !Number.isSafeInteger(expectedDesiredVersion) || expectedDesiredVersion < 0) { + throw new BadRequestException("expectedDesiredVersion must be a non-negative integer"); + } + const authzEpoch = value.authzEpoch; + if (typeof authzEpoch !== "number" || !Number.isSafeInteger(authzEpoch) || authzEpoch < 0) { + throw new BadRequestException("authzEpoch must be a non-negative integer"); + } + const program = value.program; + if (typeof program !== "object" || program === null) throw new BadRequestException("program must be an object"); + const p = program as Record; + let parsedProgram: DirectorApplyBody["program"]; + if (p.kind === "sources") { + if (!Array.isArray(p.sourceIds) || p.sourceIds.some((id) => typeof id !== "string")) throw new BadRequestException("program.sourceIds must be strings"); + parsedProgram = { kind: "sources", sourceIds: p.sourceIds }; + } else if (p.kind === "fallback") { + if (p.sceneId !== null && typeof p.sceneId !== "string") throw new BadRequestException("program.sceneId must be a string or null"); + parsedProgram = { kind: "fallback", sceneId: p.sceneId as string | null }; + } else throw new BadRequestException("program.kind must be sources or fallback"); + const layout = value.layout; + if (typeof layout !== "string" || layout === "") throw new BadRequestException("layout must be a non-empty string"); + if (!Array.isArray(value.reasons) || value.reasons.some((r) => typeof r !== "string")) throw new BadRequestException("reasons must be strings"); + const releaseOverride = value.releaseOverride; + if (releaseOverride !== undefined && typeof releaseOverride !== "string") throw new BadRequestException("releaseOverride must be a string"); + return { commandId, expectedDesiredVersion, authzEpoch, program: parsedProgram, layout, reasons: value.reasons, ...(releaseOverride === undefined ? {} : { releaseOverride }) }; +} diff --git a/apps/api/src/director/director.module.ts b/apps/api/src/director/director.module.ts new file mode 100644 index 0000000..faf5132 --- /dev/null +++ b/apps/api/src/director/director.module.ts @@ -0,0 +1,31 @@ +/** + * Director wiring module (WP10 integration slice). + * + * Minimal by design: the director service reads live-monitor state and writes + * through MediaControlService; no chatgov/feeds/virtual-channels dependency. + * Registration is one line in app.module.ts: + * + * import { DirectorModule } from "./director/director.module.js"; + * imports: [..., DirectorModule] + */ +import { Module } from "@nestjs/common"; +import { MediaControlModule } from "../media-control/media-control.module.js"; +import { DirectorController } from "./director.controller.js"; +import { DirectorService } from "./director.service.js"; +import { MediaControlService } from "../media-control/media-control.service.js"; +import { appendTenantAudit } from "../audit/audit.service.js"; + +@Module({ + imports: [MediaControlModule], + controllers: [DirectorController], + providers: [ + { + provide: DirectorService, + inject: [MediaControlService], + useFactory: (control: MediaControlService) => + new DirectorService({ control, audit: appendTenantAudit }), + }, + ], + exports: [DirectorService], +}) +export class DirectorModule {} diff --git a/apps/api/src/director/director.service.test.ts b/apps/api/src/director/director.service.test.ts new file mode 100644 index 0000000..1e6043b --- /dev/null +++ b/apps/api/src/director/director.service.test.ts @@ -0,0 +1,331 @@ +import { generateKeyPairSync, randomUUID } from "node:crypto"; +import type { PoolClient } from "pg"; +import { ConflictException } from "@nestjs/common"; +import { describe, expect, it } from "vitest"; +import { decide } from "../../../scheduler/src/director/engine.js"; +import { MediaControlService } from "../media-control/media-control.service.js"; +import { MemoryRedis } from "../production/testing/memory-redis.js"; +import { Ed25519ControlSigner } from "../media-control/wire.js"; +import type { TenantTransaction } from "../production/journal.js"; +import { DirectorService, engineState, type DirectorApplyBody, type DirectorChannelConfig } from "./director.service.js"; +import { parseDirectorChannels } from "./config.js"; + +/** + * WP10 wiring slice: the port pair against fakes. + * + * The database double implements exactly the three queries the service issues + * (fence, source join, scene resolution) over a small in-memory aggregate, and + * MediaControlService runs for real on the MemoryRedis + aggregate-transaction + * fixture used by control-s3.test.ts. The engine itself is imported from the + * scheduler package — the same pure decide() the evaluator workflow runs. + */ + +const WS = randomUUID(); +const CH = randomUUID(); +const SOURCE_A = randomUUID(); +const SOURCE_B = randomUUID(); +const SCENE_SOLO = randomUUID(); +const SCENE_FALLBACK = randomUUID(); + +interface SceneRow { sceneId: string; sources: string[] } + +/** In-memory channel aggregate: fence + sources + scenes, in query order. */ +class ChannelDatabase { + desiredVersion = 4; + authzEpoch = 7; + readonly sources: Array<{ id: string; priority: number; monitorState: string | null; version: number | null; stateEnteredAt: Date | null; lastMediaAt: string | null; lastOutcome: string | null }> = []; + readonly scenes: SceneRow[] = []; + failAudit = false; + + readonly transaction: TenantTransaction = async (_ws: string, fn: (client: PoolClient) => Promise): Promise => { + const query = async (sql: string, values: unknown[]): Promise<{ rows: Record[]; rowCount: number }> => { + if (sql.includes("FROM virtual_channel_sources")) { + // The SOURCE_JOIN: monitor rows keyed by (connection_id, remote_ref). + const rows = this.sources.map((s) => ({ + id: s.id, priority: s.priority, monitor_state: s.monitorState, monitor_version: s.version, + state_entered_at: s.stateEnteredAt, + tier_status: s.lastMediaAt === null && s.lastOutcome === null ? null + : { "media-health": { lastMediaAt: s.lastMediaAt, lastOutcome: s.lastOutcome } }, + })); + return { rows, rowCount: rows.length }; + } + if (sql.includes("SELECT desired_version FROM virtual_channels")) { + if (this.desiredVersion < 0) return { rows: [], rowCount: 0 }; + return { rows: [{ desired_version: String(this.desiredVersion) }], rowCount: 1 }; + } + if (sql.includes("SELECT authz_epoch FROM workspaces")) { + return { rows: [{ authz_epoch: String(this.authzEpoch) }], rowCount: 1 }; + } + if (sql.includes("FROM production_scenes")) { + // Scene resolution: bound slots exactly equal the selected sources. + const selected = values.slice(2, -1).map(String); + const match = this.scenes.find((scene) => [...scene.sources].sort().join() === [...selected].sort().join() && scene.sources.length === selected.length); + return { rows: match === undefined ? [] : [{ scene_id: match.sceneId }], rowCount: match === undefined ? 0 : 1 }; + } + throw new Error(`unexpected director query: ${sql.slice(0, 60)}`); + }; + return fn({ query } as unknown as PoolClient); + }; +} + +/** media-control transaction double over the same aggregate (control-s3 pattern). */ +class BusDatabase { + state = { version: 0, preview: null as string | null, program: null as string | null, receipts: [] as unknown[] }; + readonly scenes = new Set(); + /** productionScenes served to the production snapshot: one slotless scene per known scene id. */ + sceneList(): Array<{ id: string; label: string; layout: string }> { + return [...this.scenes].map((id) => ({ id, label: id === id ? "Scene" : "Scene", layout: "single" })); + } + readonly transaction: TenantTransaction = async (_ws: string, fn: (client: PoolClient) => Promise): Promise => { + const query = async (sql: string, values: unknown[] = []): Promise<{ rows: unknown[]; rowCount: number }> => { + let rows: unknown[] = []; + if (sql.startsWith("SELECT id FROM virtual_channels")) rows = [{ id: CH }]; + else if (sql.startsWith("SELECT desired_version")) rows = [{ desired_version: String(this.state.version), desired_status: "STOPPED" }]; + else if (sql.includes("FROM audit_log")) { + const found = sql.includes("requestId") ? this.state.receipts.find(r => (r as { requestId?: string }).requestId === values[2]) : this.state.receipts.at(-1); + rows = found ? [{ diff: found }] : []; + } else if (sql.startsWith("SELECT scene_id")) rows = [{ scene_id: this.state.preview }]; + else if (sql.startsWith("INSERT INTO production_buses")) { if (sql.includes("'program'")) this.state.program = String(values[2]); else this.state.preview = String(values[2]); } + else if (sql.startsWith("UPDATE virtual_channels")) this.state.version = Number(values[2]); + else if (sql.startsWith("SELECT audit_append")) { this.state.receipts.push(values[6]); return { rows: [], rowCount: 1 }; } + else if (sql.startsWith("SELECT id,label,layout")) rows = this.sceneList(); + else if (sql.startsWith("SELECT bus,scene_id")) rows = [{ bus: "preview", scene_id: this.state.preview }, { bus: "program", scene_id: this.state.program }]; + else if (sql.startsWith("SELECT id FROM production_scenes")) rows = this.scenes.has(String(values[2])) ? [{ id: values[2] }] : []; + else if (sql.startsWith("SELECT scene_id,slot_index") || sql.includes("FROM scene_slots")) rows = []; + else if (sql.includes("FROM virtual_channel_sources")) rows = []; + else if (sql.includes("FROM overlay_slots")) rows = []; + return { rows, rowCount: sql.startsWith("UPDATE") ? 1 : rows.length }; + }; + return fn({ query } as unknown as PoolClient); + }; +} + +function config(over: Partial = {}): DirectorChannelConfig { + return { + workspaceId: WS, channelId: CH, generation: "g1", + rules: [{ + id: randomUUID(), virtualChannelId: CH, priority: 1, + condition: { field: "live_source_count", op: ">=", value: 1 }, + action: { layout: "single", selection: "top_priority" }, + }], + policy: { name: "highest_priority", liveConfirmations: 1, offlineConfirmations: 1, noMediaSeconds: 10, minHoldMs: 0, switchCooldownMs: 0 }, + fallbackSceneId: SCENE_FALLBACK, + ...over, + }; +} + +async function fixture(channelConfig: DirectorChannelConfig): Promise<{ + db: ChannelDatabase; bus: BusDatabase; service: DirectorService; control: MediaControlService; +}> { + const db = new ChannelDatabase(); + const bus = new BusDatabase(); + bus.scenes.add(SCENE_SOLO); + bus.scenes.add(SCENE_FALLBACK); + const redis = new MemoryRedis(); + const control = new MediaControlService({ + transaction: bus.transaction, redis, + signer: new Ed25519ControlSigner("test", generateKeyPairSync("ed25519").privateKey), assignments: [], + }); + await control.assign(WS, CH, "media-worker-a", 1); + const service = new DirectorService({ + transaction: db.transaction, control, + audit: async () => randomUUID(), + channels: () => [channelConfig], + }); + return { db, bus, service, control }; +} + +const apply = (over: Partial = {}): DirectorApplyBody => ({ + commandId: `${CH}:9:1`, expectedDesiredVersion: 4, authzEpoch: 7, + program: { kind: "sources", sourceIds: [SOURCE_A] }, layout: "single", reasons: ["rule"], + ...over, +}); + +describe("engineState mapping", () => { + it("maps monitor states and reads anything else as UNKNOWN", () => { + for (const state of ["OFFLINE", "PROBING", "LIVE", "UNSTABLE", "ERROR"]) { + expect(engineState(state)).toBe(state); + } + expect(engineState(null)).toBe("UNKNOWN"); + expect(engineState("weird")).toBe("UNKNOWN"); + }); +}); + +describe("DirectorStatePort read (channelState)", () => { + it("joins monitor rows by source identity and renders the engine input shape", async () => { + const entered = new Date("2026-10-09T12:00:00Z"); + const local = new ChannelDatabase(); + local.sources.push( + { id: SOURCE_A, priority: 50, monitorState: "LIVE", version: 3, stateEnteredAt: entered, lastMediaAt: "2026-10-09T12:00:01Z", lastOutcome: "ok" }, + { id: SOURCE_B, priority: 10, monitorState: null, version: null, stateEnteredAt: null, lastMediaAt: null, lastOutcome: null }, + ); + const svc = new DirectorService({ transaction: local.transaction, control: { director: async () => { throw new Error("unreachable"); } }, channels: () => [config()] }); + const read = await svc.channelState(WS, CH); + if (read.kind !== "current") throw new Error("expected current"); + expect(read.desiredVersion).toBe(4); + expect(read.authzEpoch).toBe(7); + expect(read.sources).toEqual([ + expect.objectContaining({ id: SOURCE_A, priority: 50, state: "LIVE", version: 3, liveSinceMs: entered.getTime(), lastMediaAtMs: Date.parse("2026-10-09T12:00:01Z") }), + expect.objectContaining({ id: SOURCE_B, state: "UNKNOWN" }), + ]); + // Optional engine fields are absent (never null): a null would de-confirm + // every source as no-media inside decide(). + expect("lastMediaAtMs" in (read.sources[1] ?? {})).toBe(false); + expect("mediaFailed" in (read.sources[0] ?? {})).toBe(false); + expect(read.rules).toHaveLength(1); + expect(read.policy.name).toBe("highest_priority"); + expect(read.fallbackSceneId).toBe(SCENE_FALLBACK); + }); + + it("returns superseded when the channel is not configured for the director", async () => { + const local = new ChannelDatabase(); + const svc = new DirectorService({ transaction: local.transaction, control: { director: async () => { throw new Error("unreachable"); } }, channels: () => [] }); + expect(await svc.channelState(WS, CH)).toEqual({ kind: "superseded", code: "SUPERSEDED" }); + expect(await svc.fence(WS, CH)).toBeUndefined(); + }); +}); + +describe("DesiredStatePort apply (applyProgramSwitch)", () => { + /** The evaluator loop: read -> decide, carrying memory/lastSwitch between ticks. */ + async function evaluate(service: DirectorService, db: ChannelDatabase, ticks: number): Promise> { + let memory: ReturnType["memory"] | undefined; + let lastSwitch: ReturnType["lastSwitch"] | undefined; + let decision: ReturnType | undefined; + for (let tick = 0; tick < ticks; tick += 1) { + if (tick > 0) for (const source of db.sources) if (source.version !== null) source.version += 1; // a fresh monitor sample per tick + const read = await service.channelState(WS, CH); + if (read.kind !== "current") throw new Error("expected current"); + decision = decide({ + sources: read.sources, rules: read.rules, policy: read.policy, + now: 100_000 + tick * 5_000, channelId: CH, fallbackSceneId: read.fallbackSceneId, + ...(memory === undefined ? {} : { memory }), ...(lastSwitch === undefined ? {} : { lastSwitch }), + }); + memory = decision.memory; lastSwitch = decision.lastSwitch; + } + if (decision === undefined) throw new Error("no decision"); + return decision; + } + + it("applies a decision end to end: read -> decide (debounced) -> pst-select + cut through media control", async () => { + // liveConfirmations 2: the engine must see two fresh monitor samples before + // trusting a LIVE report — the debounce the evaluator relies on. + const { db, bus, service } = await fixture(config({ + policy: { name: "highest_priority", liveConfirmations: 2, offlineConfirmations: 1, noMediaSeconds: 10, minHoldMs: 0, switchCooldownMs: 0 }, + })); + db.scenes.push({ sceneId: SCENE_SOLO, sources: [SOURCE_A] }); + db.sources.push({ id: SOURCE_A, priority: 50, monitorState: "LIVE", version: 3, stateEnteredAt: new Date("2026-10-09T12:00:00Z"), lastMediaAt: null, lastOutcome: "ok" }); + + // First tick: one sample, unconfirmed — the engine holds the fallback. + const probing = await evaluate(service, db, 1); + expect(probing.program.kind).toBe("fallback"); + // Second tick (new monitor version): confirmed LIVE, the rule fires. + const decision = await evaluate(service, db, 2); + expect(decision.program).toEqual({ kind: "sources", sourceIds: [SOURCE_A] }); + expect(decision.switched).toBe(true); + + const read = await service.channelState(WS, CH); + if (read.kind !== "current") throw new Error("expected current"); + const outcome = await service.applyProgramSwitch(WS, CH, { + commandId: `${CH}:0:1`, expectedDesiredVersion: read.desiredVersion, authzEpoch: read.authzEpoch, + program: decision.program, layout: decision.layout, reasons: decision.reasons, + }); + expect(outcome).toMatchObject({ kind: "applied" }); + expect(bus.state.preview).toBe(SCENE_SOLO); + expect(bus.state.program).toBe(SCENE_SOLO); + // pst + cut are two desired-version bumps + expect(bus.state.version).toBe(2); + }); + + it("replays the same commandId without switching twice (idempotent keys, no new receipts)", async () => { + const { db, bus, service } = await fixture(config()); + db.scenes.push({ sceneId: SCENE_SOLO, sources: [SOURCE_A] }); + const body = apply(); + const first = await service.applyProgramSwitch(WS, CH, body); + expect(first).toMatchObject({ kind: "applied" }); + // An activity retry arrives with the same fence: the derived idempotency + // keys hit media-control's stored receipts, so nothing is written twice. + const second = await service.applyProgramSwitch(WS, CH, body); + expect(second.kind).not.toBe("conflict"); + expect(bus.state.receipts).toHaveLength(2); // pst + cut, still exactly once each + expect(bus.state.version).toBe(2); + // Once the fence has moved (a concurrent writer won), the same command is a conflict. + db.desiredVersion = body.expectedDesiredVersion + 1; + expect(await service.applyProgramSwitch(WS, CH, body)).toEqual({ kind: "conflict" }); + }); + + it("rejects a stale fence (desired version or authz epoch moved)", async () => { + const { db, service } = await fixture(config()); + db.desiredVersion = 5; + expect(await service.applyProgramSwitch(WS, CH, apply())).toEqual({ kind: "conflict" }); + db.desiredVersion = 4; + db.authzEpoch = 8; + expect(await service.applyProgramSwitch(WS, CH, apply())).toEqual({ kind: "conflict" }); + }); + + it("falls back to the configured fallback scene when no scene holds the selection", async () => { + const { db, bus, service } = await fixture(config()); + db.scenes.push({ sceneId: randomUUID(), sources: [SOURCE_A, SOURCE_B] }); // different selection + const outcome = await service.applyProgramSwitch(WS, CH, apply()); + expect(outcome).toMatchObject({ kind: "applied" }); + expect(bus.state.program).toBe(SCENE_FALLBACK); + }); + + it("fails closed with a conflict when the selection has no scene and no fallback exists", async () => { + const { service } = await fixture(config({ fallbackSceneId: null })); + await expect(service.applyProgramSwitch(WS, CH, apply())).resolves.toEqual({ kind: "conflict" }); + }); + + it("an explicit fallback program selects the fallback scene", async () => { + const { bus, service } = await fixture(config()); + const outcome = await service.applyProgramSwitch(WS, CH, apply({ program: { kind: "fallback", sceneId: null } })); + expect(outcome).toMatchObject({ kind: "applied" }); + expect(bus.state.program).toBe(SCENE_FALLBACK); + }); + + it("classifies media-control conflicts as CAS outcomes and propagates the rest", async () => { + const failing = new DirectorService({ + transaction: new ChannelDatabase().transaction, + control: { director: async () => { throw new ConflictException("channel is not assigned to a media worker"); } }, + channels: () => [config()], + }); + await expect(failing.applyProgramSwitch(WS, CH, apply())).resolves.toEqual({ kind: "conflict" }); + const exploding = new DirectorService({ + transaction: new ChannelDatabase().transaction, + control: { director: async () => { throw new Error("redis down"); } }, + channels: () => [config()], + }); + await expect(exploding.applyProgramSwitch(WS, CH, apply())).rejects.toThrow("redis down"); + }); + + it("unwired desired-state port fails closed without touching media control", async () => { + const svc = new DirectorService({ transaction: new ChannelDatabase().transaction, channels: () => [config()] }); + await expect(svc.applyProgramSwitch(WS, CH, apply())).resolves.toEqual({ kind: "conflict" }); + }); +}); + +describe("config parser (API side)", () => { + it("parses a full channel document and rejects malformed entries", () => { + const rule = { id: randomUUID(), priority: 2, condition: { field: "live_source_count", op: ">=", value: 1 }, action: { layout: "grid", selection: "all_live" } }; + const parsed = parseDirectorChannels(JSON.stringify([{ workspaceId: WS, channelId: CH, generation: "g-1", rules: [rule], policy: { name: "grid", liveConfirmations: 1 }, fallbackSceneId: null }])); + expect(parsed).toHaveLength(1); + expect(parsed[0]).toMatchObject({ workspaceId: WS, channelId: CH, generation: "g-1", fallbackSceneId: null }); + expect(parsed[0]?.rules[0]).toMatchObject({ priority: 2 }); + expect(parsed[0]?.policy?.liveConfirmations).toBe(1); + expect(parseDirectorChannels(undefined)).toEqual([]); + expect(() => parseDirectorChannels("{}")).toThrow(); + expect(() => parseDirectorChannels(JSON.stringify([{ workspaceId: WS }]))).toThrow(); + expect(() => parseDirectorChannels(JSON.stringify([{ workspaceId: WS, channelId: "nope", generation: "g" }]))).toThrow(); + expect(() => parseDirectorChannels(JSON.stringify([{ workspaceId: WS, channelId: CH, generation: "g", rules: [{ ...rule, action: { ...rule.action, selection: "explicit" } }] }]))).toThrow(/explicit requires sourceIds/); + }); + + it("matches the scheduler-side parser on the same document (shape compatibility)", async () => { + const { parseDirectorChannels: schedulerParse } = await import("../../../scheduler/src/director/config.js"); + const document = JSON.stringify([{ + workspaceId: WS, channelId: CH, generation: "shared-1", + rules: [{ id: randomUUID(), priority: 1, condition: { field: "source_state", op: "==", value: "LIVE" }, action: { layout: "pip", selection: "explicit", sourceIds: [SOURCE_A] } }], + policy: { name: "two_up", liveConfirmations: 3, offlineConfirmations: 4, noMediaSeconds: 30, minHoldMs: 5_000, switchCooldownMs: 15_000 }, + fallbackSceneId: SCENE_FALLBACK, + }]); + expect(schedulerParse(document)).toEqual(parseDirectorChannels(document)); + }); +}); diff --git a/apps/api/src/director/director.service.ts b/apps/api/src/director/director.service.ts new file mode 100644 index 0000000..e5e68e9 --- /dev/null +++ b/apps/api/src/director/director.service.ts @@ -0,0 +1,322 @@ +/** + * Director wiring service (WP10 integration slice): the API-side half of the + * engine's two ports. + * + * DirectorStatePort.read <- channelState(): joins a channel's sources with + * the live monitor (virtual_channel_sources LEFT JOIN source_monitor_states + * on the monitor's own keyed identity (workspace_id, connection_id, + * remote_ref)) and renders the engine's input shape. + * DesiredStatePort <- fence()/applyProgramSwitch(): fence from + * virtual_channels.desired_version + workspaces.authz_epoch; apply via + * MediaControlService.director() (the transactional outbox behind + * setDesired), with pst-select + cut under idempotency keys derived from + * the director's commandId, so a retried activity is a duplicate, never a + * double switch. + * + * The scheduler's evaluator drives these over HTTP (internal director + * endpoints); the service is process-agnostic so tests can call it directly. + */ +import { BadRequestException, ConflictException, Inject, Injectable, NotFoundException, Optional } from "@nestjs/common"; +import type { PoolClient } from "pg"; +import { DEFAULT_SWITCH_POLICY } from "@tactical-media-broadcast-hub/core"; +import { withTenant } from "../db/client.js"; +import { isUuid } from "../http/validation.js"; +import type { MediaControlService } from "../media-control/media-control.service.js"; +import type { DirectorActor } from "../production/journal.js"; +import { appendTenantAudit } from "../audit/audit.service.js"; +import { parseDirectorChannels, type DirectorChannelConfig } from "./config.js"; + +export type { DirectorChannelConfig } from "./config.js"; + +/** + * Engine-facing source row (transport twin of the scheduler's + * DirectorChannelSource). Optional fields are ABSENT, never null: the engine + * reads `lastMediaAtMs !== undefined` as "known", so a null would read as + * epoch 0 and de-confirm every source as no-media (caught by the port-pair + * test). JSON drops undefined keys, keeping wire and engine forms identical. + */ +export interface DirectorSourceRow { + id: string; + priority: number; + enabled: boolean; + autoSelect: boolean; + state: + | "OFFLINE" | "UNKNOWN" | "PROBING" | "STARTING" | "LIVE" | "UNSTABLE" | "ENDING" | "ERROR"; + version?: number; + liveSinceMs?: number; + lastMediaAtMs?: number; + mediaFailed?: boolean; +} + +export interface DirectorChannelSnapshot { + kind: "current"; + desiredVersion: number; + authzEpoch: number; + generation: string; + sources: DirectorSourceRow[]; + rules: DirectorChannelConfig["rules"]; + policy: NonNullable; + fallbackSceneId: string | null; + override: null; + principalPermissions: Record; +} + +export type DirectorChannelRead = DirectorChannelSnapshot | { kind: "superseded"; code: "SUPERSEDED" }; + +export interface DirectorApplyBody { + commandId: string; + expectedDesiredVersion: number; + authzEpoch: number; + program: { kind: "sources"; sourceIds: string[] } | { kind: "fallback"; sceneId: string | null }; + layout: string; + reasons: string[]; + releaseOverride?: string; +} + +export type DirectorApplyOutcome = + | { kind: "applied" | "duplicate"; desiredVersion: number } + | { kind: "conflict" }; + +export interface DirectorFence { + generation: string; + desiredVersion: number; + authzEpoch: number; + directorAuthorized: boolean; +} + +export interface DirectorServiceDependencies { + transaction: typeof withTenant; + control: Pick; + audit?: typeof appendTenantAudit; + /** Channel configuration; default reads DIRECTOR_CHANNELS once at construction. */ + channels?: () => DirectorChannelConfig[]; +} + +export const DIRECTOR_DEPENDENCIES = Symbol("DIRECTOR_DEPENDENCIES"); + +/** Row-level SQL for one channel's sources joined to their monitor state. */ +const SOURCE_QUERY = ` + SELECT s.id, s.priority, + m.state AS monitor_state, m.version AS monitor_version, + m.state_entered_at, + m.tier_status->'media-health'->>'lastMediaAt' AS last_media_at, + m.tier_status->'media-health'->>'lastOutcome' AS media_last_outcome + FROM virtual_channel_sources s + LEFT JOIN source_monitor_states m + ON m.workspace_id = s.workspace_id + AND m.connection_id = s.connection_id + AND m.remote_ref = s.remote_ref + WHERE s.workspace_id = $1 AND s.virtual_channel_id = $2 + ORDER BY s.priority DESC, s.id`; + +/** Monitor states the monitor schema admits (012); anything else reads UNKNOWN. */ +const MONITOR_STATES = ["OFFLINE", "PROBING", "LIVE", "UNSTABLE", "ERROR"] as const; + +/** The engine's source state from a monitor row (never-mapped => UNKNOWN). */ +export function engineState(monitorState: string | null): DirectorSourceRow["state"] { + return (MONITOR_STATES as readonly string[]).includes(monitorState ?? "") + ? monitorState as DirectorSourceRow["state"] + : "UNKNOWN"; +} + +function epochMs(value: unknown): number | null { + if (value === null || value === undefined) return null; + const parsed = value instanceof Date ? value.getTime() : Date.parse(String(value)); + return Number.isFinite(parsed) ? parsed : null; +} + +/** The evaluator acts as a service principal; its writes are attributed to the director itself. */ +export const DIRECTOR_ACTOR: DirectorActor = { userId: null, role: "director" }; + +@Injectable() +export class DirectorService { + private readonly deps: DirectorServiceDependencies; + private readonly envChannels: DirectorChannelConfig[]; + + constructor(@Optional() @Inject(DIRECTOR_DEPENDENCIES) dependencies?: Partial) { + this.deps = { + transaction: dependencies?.transaction ?? withTenant, + control: dependencies?.control ?? { + // No injected media-control service: this process cannot apply desired + // state. Reads still work (the evaluator fences on them and audits + // SUPERSEDED), but applyProgramSwitch fails closed. + director: () => { throw new ConflictException("director desired-state port is not wired"); }, + }, + ...(dependencies?.audit === undefined ? {} : { audit: dependencies.audit }), + ...(dependencies?.channels === undefined ? {} : { channels: dependencies.channels }), + }; + this.envChannels = parseDirectorChannels(process.env.DIRECTOR_CHANNELS); + } + + /** Configured channel for a ref; undefined when the director is not configured for it. */ + channel(workspaceId: string, channelId: string): DirectorChannelConfig | undefined { + const configured = this.deps.channels?.() ?? this.envChannels; + return configured.find((c) => c.workspaceId === workspaceId && c.channelId === channelId); + } + + /** DirectorStatePort.read: the engine's full input for one channel. */ + async channelState(workspaceId: string, channelId: string): Promise { + assertRef(workspaceId, channelId); + const channel = this.channel(workspaceId, channelId); + if (channel === undefined) return { kind: "superseded", code: "SUPERSEDED" }; + return this.deps.transaction(workspaceId, async (client) => { + const fence = await this.readFence(client, workspaceId, channelId, channel.generation); + if (fence.generation !== channel.generation) return { kind: "superseded" as const, code: "SUPERSEDED" as const }; + const sources = await client.query(SOURCE_QUERY, [workspaceId, channelId]); + return { + kind: "current" as const, + desiredVersion: fence.desiredVersion, + authzEpoch: fence.authzEpoch, + generation: fence.generation, + sources: sources.rows.map((row: Record) => this.sourceRow(row)), + rules: channel.rules, + policy: channel.policy ?? DEFAULT_SWITCH_POLICY, + fallbackSceneId: channel.fallbackSceneId, + override: null, + principalPermissions: {}, + }; + }); + } + + private sourceRow(row: Record): DirectorSourceRow { + const tierStatus = row.tier_status as Record | null | undefined; + const mediaHealth = tierStatus?.["media-health"]; + const version = row.monitor_version === null || row.monitor_version === undefined ? undefined : Number(row.monitor_version); + const liveSinceMs = epochMs(row.state_entered_at); + const lastMediaAtMs = epochMs(mediaHealth?.lastMediaAt); + const mediaFailed = mediaHealth?.lastOutcome === "fail" || mediaHealth?.lastOutcome === "error"; + return { + id: String(row.id), + priority: Number(row.priority ?? 0), + enabled: true, + autoSelect: true, + state: engineState(row.monitor_state as string | null), + ...(version === undefined ? {} : { version }), + ...(liveSinceMs === null ? {} : { liveSinceMs }), + ...(lastMediaAtMs === null ? {} : { lastMediaAtMs }), + ...(mediaFailed ? { mediaFailed: true } : {}), + }; + } + + /** DesiredStatePort.fence. */ + async fence(workspaceId: string, channelId: string): Promise { + assertRef(workspaceId, channelId); + const channel = this.channel(workspaceId, channelId); + if (channel === undefined) return undefined; + return this.deps.transaction(workspaceId, (client) => this.readFence(client, workspaceId, channelId, channel.generation)); + } + + private async readFence(client: PoolClient, workspaceId: string, channelId: string, generation: string): Promise { + const found = await client.query<{ desired_version: string | number }>( + "SELECT desired_version FROM virtual_channels WHERE workspace_id = $1 AND id = $2", [workspaceId, channelId]); + if (found.rowCount === 0) throw new NotFoundException(`virtual channel ${channelId} not found`); + const desiredVersion = Number(found.rows[0]?.desired_version); + if (!Number.isSafeInteger(desiredVersion) || desiredVersion < 0) throw new Error("desired_version is not a safe integer"); + const epoch = await client.query<{ authz_epoch: string | number }>("SELECT authz_epoch FROM workspaces WHERE id = $1", [workspaceId]); + const authzEpoch = Number(epoch.rows[0]?.authz_epoch); + if (!Number.isSafeInteger(authzEpoch) || authzEpoch < 0) throw new Error("authz_epoch is not a safe integer"); + return { + generation, + desiredVersion, + authzEpoch, + // Principal revocation is a later slice (it needs a persisted principal + // per evaluator); until then the fence is generation + desiredVersion. + directorAuthorized: true, + }; + } + + /** + * DesiredStatePort.apply: a director decision as a desired-state write. + * + * Resolves the decision's program to a scene, then pst-select (preview) + + * cut (preview -> program) through MediaControlService.director(), whose + * receipt lookup makes both steps idempotent under keys derived from the + * director's commandId: an activity retry replays the committed receipt + * instead of switching twice. + */ + async applyProgramSwitch(workspaceId: string, channelId: string, body: DirectorApplyBody): Promise { + assertRef(workspaceId, channelId); + if (typeof body.commandId !== "string" || body.commandId === "" || body.commandId.length > 128) { + throw new BadRequestException("commandId must be 1-128 characters"); + } + const channel = this.channel(workspaceId, channelId); + if (channel === undefined) throw new ConflictException("director is not configured for this channel"); + const fence = await this.fence(workspaceId, channelId); + if (fence === undefined) return { kind: "conflict" }; + if (fence.desiredVersion !== body.expectedDesiredVersion || fence.authzEpoch !== body.authzEpoch) return { kind: "conflict" }; + + try { + // Resolve the decision's program to a scene the channel owns. + const scene = await this.deps.transaction(workspaceId, (client) => + this.sceneForProgram(client, workspaceId, channelId, body.program, channel)); + // pst-select arms the scene on preview; cut moves preview to program. + // Two commands, one decision: each has its own idempotency key derived + // from the director's commandId, so replays stop at the stored receipt. + const selected = await this.deps.control.director(workspaceId, channelId, + { type: "pst-select", sceneId: scene.sceneId }, `${body.commandId}:pst`, DIRECTOR_ACTOR); + await this.deps.control.director(workspaceId, channelId, + { type: "cut" }, `${body.commandId}:cut`, DIRECTOR_ACTOR); + return { kind: "applied", desiredVersion: selected.desiredVersion }; + } catch (error) { + return classifyConflict(error); + } + } + + /** Resolve the decision's program to a scene id: an explicit scene, or a scene holding exactly those sources. */ + private async sceneForProgram( + client: PoolClient, workspaceId: string, channelId: string, + program: DirectorApplyBody["program"], channel: DirectorChannelConfig, + ): Promise<{ sceneId: string }> { + if (program.kind === "fallback") { + const sceneId = program.sceneId ?? channel.fallbackSceneId; + if (sceneId === null) throw new ConflictException("channel has no fallback scene configured"); + return { sceneId }; + } + if (program.sourceIds.length === 0) throw new ConflictException("program switch selected no sources"); + // A scene whose bound slots are exactly the selected sources (order and + // multiplicity included): that scene is the director's program scene. + const placeholders = program.sourceIds.map((_, i) => `$${i + 3}`).join(","); + const found = await client.query<{ scene_id: string }>( + `SELECT sc.id AS scene_id + FROM production_scenes sc + LEFT JOIN scene_slots sl + ON sl.scene_id = sc.id AND sl.workspace_id = sc.workspace_id AND sl.virtual_channel_id = sc.virtual_channel_id + WHERE sc.workspace_id = $1 AND sc.virtual_channel_id = $2 + GROUP BY sc.id + HAVING array_length(array_agg(sl.source_id ORDER BY sl.slot_index) FILTER (WHERE sl.source_id IS NOT NULL), 1) = $${program.sourceIds.length + 3} + AND array_agg(sl.source_id ORDER BY sl.slot_index) FILTER (WHERE sl.source_id IS NOT NULL) @> ARRAY[${placeholders}]::uuid[]`, + [workspaceId, channelId, ...program.sourceIds, program.sourceIds.length]); + if (found.rowCount === 0) { + // No scene authored for this selection: switch to the channel's declared + // fallback so the channel never stalls on an unrenderable program. + if (channel.fallbackSceneId === null) throw new ConflictException("no scene holds the selected sources and no fallback scene is configured"); + return { sceneId: channel.fallbackSceneId }; + } + return { sceneId: String(found.rows[0]?.scene_id) }; + } + + /** Outcomes the evaluator audits when it cannot write (generation/authz fences). */ + async audit(workspaceId: string, channelId: string, event: { commandId: string; outcome: "SUPERSEDED" | "AUTHZ_REVOKED"; reason: string }): Promise { + assertRef(workspaceId, channelId); + const append = this.deps.audit ?? appendTenantAudit; + await append(workspaceId, { + actorUserId: null, + actorRole: "director", + action: `director.${event.outcome.toLowerCase()}`, + targetType: "production_command", + targetId: channelId, + diff: { commandId: event.commandId, outcome: event.outcome, reason: event.reason }, + }); + } +} + +function assertRef(workspaceId: string, channelId: string): void { + if (!isUuid(workspaceId)) throw new BadRequestException("workspaceId must be a UUID"); + if (!isUuid(channelId)) throw new BadRequestException("channelId must be a UUID"); +} + +/** Media-control conflicts are CAS outcomes the evaluator re-reads on; anything else propagates. */ +function classifyConflict(error: unknown): DirectorApplyOutcome { + if (error instanceof ConflictException || error instanceof NotFoundException) return { kind: "conflict" }; + throw error; +} diff --git a/apps/api/src/director/port-pair.test.ts b/apps/api/src/director/port-pair.test.ts new file mode 100644 index 0000000..349e76a --- /dev/null +++ b/apps/api/src/director/port-pair.test.ts @@ -0,0 +1,172 @@ +import { generateKeyPairSync, randomUUID } from "node:crypto"; +import type { PoolClient } from "pg"; +import { Controller, Module, type INestApplication } from "@nestjs/common"; +import { NestFactory } from "@nestjs/core"; +import { describe, expect, it } from "vitest"; +import { DirectorController } from "./director.controller.js"; +import { DirectorService } from "./director.service.js"; +import { MediaControlService } from "../media-control/media-control.service.js"; +import { MemoryRedis } from "../production/testing/memory-redis.js"; +import { Ed25519ControlSigner } from "../media-control/wire.js"; +import type { TenantTransaction } from "../production/journal.js"; +import { decide } from "../../../scheduler/src/director/engine.js"; +import { HttpDirectorStatePort, HttpDesiredStatePort } from "../../../scheduler/src/director/adapters.js"; +import { DirectorHttpClient } from "../../../scheduler/src/director/http.client.js"; + +/** + * Port-pair integration (WP10): the scheduler's HTTP adapters against a real + * Nest application hosting the API's director endpoints, with media control + * running for real on its MemoryRedis fixture. This is the exact chain the + * production wiring uses — internal endpoints, response validation, fence, + * engine input shape and the desired-state write — minus the database. + * + * The scheduler-side pieces are imported from apps/scheduler (pure modules + * only: engine, adapters, http.client), the same cross-package pattern + * live-monitor's poller tests use. + */ +const WS = "3f2b8e40-1a5c-4c7e-9d10-6f8d1a000001"; +const CH = "3f2b8e40-1a5c-4c7e-9d10-6f8d1a000002"; +const SRC_A = "3f2b8e40-1a5c-4c7e-9d10-6f8d1a000003"; +const SRC_B = "3f2b8e40-1a5c-4c7e-9d10-6f8d1a000004"; +const SCENE_SOLO = "3f2b8e40-1a5c-4c7e-9d10-6f8d1a000005"; +const SCENE_FALLBACK = "3f2b8e40-1a5c-4c7e-9d10-6f8d1a000006"; + +/** Director-side database double: fence + source join + scene resolution. */ +class DirectorDb { + desiredVersion = 4; + authzEpoch = 7; + readonly sources: Array> = []; + readonly scenes: Array<{ sceneId: string; sources: string[] }> = []; + readonly transaction: TenantTransaction = async (_ws: string, fn: (client: PoolClient) => Promise): Promise => + fn({ query: async (sql: string, values: unknown[]): Promise<{ rows: Record[]; rowCount: number }> => { + if (sql.includes("FROM virtual_channel_sources")) return { rows: this.sources, rowCount: this.sources.length }; + if (sql.includes("SELECT desired_version FROM virtual_channels")) return { rows: [{ desired_version: String(this.desiredVersion) }], rowCount: 1 }; + if (sql.includes("SELECT authz_epoch FROM workspaces")) return { rows: [{ authz_epoch: String(this.authzEpoch) }], rowCount: 1 }; + if (sql.includes("FROM production_scenes")) { + const selected = values.slice(2, -1).map(String); + const match = this.scenes.find((scene) => scene.sources.length === selected.length && [...scene.sources].sort().join() === [...selected].sort().join()); + return { rows: match === undefined ? [] : [{ scene_id: match.sceneId }], rowCount: match === undefined ? 0 : 1 }; + } + throw new Error(`unexpected query: ${sql.slice(0, 60)}`); + } } as unknown as PoolClient); +} + +/** media-control transaction double (control-s3 pattern). */ +class BusDb { + state = { version: 0, preview: null as string | null, program: null as string | null, receipts: [] as unknown[] }; + readonly scenes = new Set([SCENE_SOLO, SCENE_FALLBACK]); + readonly transaction: TenantTransaction = async (_ws: string, fn: (client: PoolClient) => Promise): Promise => + fn({ query: async (sql: string, values: unknown[] = []): Promise<{ rows: unknown[]; rowCount: number }> => { + let rows: unknown[] = []; + if (sql.startsWith("SELECT id FROM virtual_channels")) rows = [{ id: CH }]; + else if (sql.startsWith("SELECT desired_version")) rows = [{ desired_version: String(this.state.version), desired_status: "STOPPED" }]; + else if (sql.includes("FROM audit_log")) { + const found = sql.includes("requestId") ? this.state.receipts.find(r => (r as { requestId?: string }).requestId === values[2]) : this.state.receipts.at(-1); + rows = found ? [{ diff: found }] : []; + } else if (sql.startsWith("SELECT scene_id")) rows = [{ scene_id: this.state.preview }]; + else if (sql.startsWith("INSERT INTO production_buses")) { if (sql.includes("'program'")) this.state.program = String(values[2]); else this.state.preview = String(values[2]); } + else if (sql.startsWith("UPDATE virtual_channels")) this.state.version = Number(values[2]); + else if (sql.startsWith("SELECT audit_append")) { this.state.receipts.push(values[6]); return { rows: [], rowCount: 1 }; } + else if (sql.startsWith("SELECT id,label,layout")) rows = [...this.scenes].map((id) => ({ id, label: "Scene", layout: "single" })); + else if (sql.startsWith("SELECT bus,scene_id")) rows = [{ bus: "preview", scene_id: this.state.preview }, { bus: "program", scene_id: this.state.program }]; + else if (sql.startsWith("SELECT id FROM production_scenes")) rows = this.scenes.has(String(values[2])) ? [{ id: values[2] }] : []; + else if (sql.includes("FROM scene_slots") || sql.includes("FROM virtual_channel_sources") || sql.includes("FROM overlay_slots")) rows = []; + return { rows, rowCount: sql.startsWith("UPDATE") ? 1 : rows.length }; + } } as unknown as PoolClient); +} + +const RULES = [{ + id: randomUUID(), virtualChannelId: CH, priority: 1, + condition: { field: "live_source_count" as const, op: ">=" as const, value: 1 }, + action: { layout: "single" as const, selection: "top_priority" as const }, +}]; +const POLICY = { name: "highest_priority" as const, liveConfirmations: 1, offlineConfirmations: 1, noMediaSeconds: 600, minHoldMs: 0, switchCooldownMs: 0 }; + +/** + * The production DirectorController, inherited verbatim: routes and handler + * bodies are the shipped ones (Nest reads the decorators off the prototype); + * only the service instance comes from the module below. + */ +@Controller("internal/director/channels") +class BoundDirectorController extends DirectorController {} + +async function apiStack(db: DirectorDb, bus: BusDb): Promise { + const control = new MediaControlService({ + transaction: bus.transaction, redis: new MemoryRedis(), + signer: new Ed25519ControlSigner("test", generateKeyPairSync("ed25519").privateKey), assignments: [], + }); + await control.assign(WS, CH, "media-worker-a", 1); + const service = new DirectorService({ + transaction: db.transaction, control, audit: async () => randomUUID(), + channels: () => [{ workspaceId: WS, channelId: CH, generation: "g1", rules: RULES, policy: POLICY, fallbackSceneId: SCENE_FALLBACK }], + }); + + @Module({ controllers: [BoundDirectorController], providers: [{ provide: DirectorService, useValue: service }] }) + class PairModule {} + const application = await NestFactory.create(PairModule, { logger: false }); + await application.init(); + return application; +} + +describe("director port pair over HTTP (scheduler adapters <-> API endpoints)", () => { + it("state read -> decide -> fence -> apply, end to end", async () => { + const db = new DirectorDb(); + const bus = new BusDb(); + db.scenes.push({ sceneId: SCENE_SOLO, sources: [SRC_A] }); + db.sources.push({ id: SRC_A, priority: 50, monitor_state: "LIVE", monitor_version: 3, state_entered_at: new Date("2026-10-09T12:00:00Z"), tier_status: null }); + db.sources.push({ id: SRC_B, priority: 10, monitor_state: null, monitor_version: null, state_entered_at: null, tier_status: null }); + const application = await apiStack(db, bus); + try { + await application.listen(0, "127.0.0.1"); + const port = (application.getHttpServer().address() as { port: number }).port; + const client = new DirectorHttpClient({ baseUrl: `http://127.0.0.1:${port}` }); + const state = new HttpDirectorStatePort(client); + const desired = new HttpDesiredStatePort(client); + const ref = { workspaceId: WS, channelId: CH, generation: "g1" }; + + // 1. read -> decide + const snapshot = await state.read(ref); + if (snapshot.kind !== "current") throw new Error("expected current"); + expect(snapshot.sources).toHaveLength(2); + expect(snapshot.sources[0]).toMatchObject({ id: SRC_A, state: "LIVE", version: 3 }); + const decision = decide({ sources: snapshot.sources, rules: snapshot.rules, policy: snapshot.policy, now: 100_000 }); + expect(decision.program).toEqual({ kind: "sources", sourceIds: [SRC_A] }); + + // 2. fence matches the snapshot the decision was based on + const fence = await desired.fence(ref); + expect(fence).toMatchObject({ generation: "g1", desiredVersion: snapshot.desiredVersion, authzEpoch: snapshot.authzEpoch }); + + // 3. apply lands as preview + program through media control + const outcome = await desired.writeProgram({ + ref, commandId: `${CH}:1`, expectedDesiredVersion: fence!.desiredVersion, authzEpoch: fence!.authzEpoch, + program: decision.program, layout: decision.layout, reasons: decision.reasons, + }); + expect(outcome).toEqual({ kind: "applied", desiredVersion: 1 }); + expect(bus.state.preview).toBe(SCENE_SOLO); + expect(bus.state.program).toBe(SCENE_SOLO); + } finally { + await application.close(); + } + }); + + it("a stale fence is a conflict, never a write", async () => { + const db = new DirectorDb(); + const bus = new BusDb(); + db.scenes.push({ sceneId: SCENE_SOLO, sources: [SRC_A] }); + const application = await apiStack(db, bus); + try { + await application.listen(0, "127.0.0.1"); + const port = (application.getHttpServer().address() as { port: number }).port; + const client = new DirectorHttpClient({ baseUrl: `http://127.0.0.1:${port}` }); + db.desiredVersion = 99; // a concurrent writer moved the fence + const outcome = await new HttpDesiredStatePort(client).writeProgram({ + ref: { workspaceId: WS, channelId: CH, generation: "g1" }, commandId: "c", expectedDesiredVersion: 4, authzEpoch: 7, + program: { kind: "sources", sourceIds: [SRC_A] }, layout: "single", reasons: [], + }); + expect(outcome).toEqual({ kind: "conflict" }); + expect(bus.state.receipts).toHaveLength(0); + } finally { + await application.close(); + } + }); +}); diff --git a/apps/api/src/feeds/column-scheduler.ts b/apps/api/src/feeds/column-scheduler.ts new file mode 100644 index 0000000..e499fb9 --- /dev/null +++ b/apps/api/src/feeds/column-scheduler.ts @@ -0,0 +1,352 @@ +/** + * Columns scheduler (P1 — design §3.3/§3.4/§3.5). + * + * Generalizes the legacy feeds poll loop into a due-SOURCES loop. The design + * rule (§3.5): "the legacy home poll is the degenerate platform_feed source" — + * so there is one loop, and `/feeds`'s per-connection home poll is simply the + * `source_kind = 'home'` subset of it. + * + * Reused unchanged from the feeds loop: + * - the per-key Redis lease (renewable, released in `finally`) so replicas + * never double-poll one source; + * - the cooldown key (`SET NX PX`) that makes repeated refresh clicks and + * the 5-second discovery timer unable to bypass the interval; + * - interval + 20 % jitter, SDK `RateLimitError.retryAfterMs` override. + * + * New per design §3.4: + * - `column_sources` is the due list: `next_poll_at <= now`, `state <> + * 'disabled'`, `refresh_policy.paused` false — the DB row replaces the + * in-memory "one cycle per connection" bookkeeping; + * - identities shared by several columns collapse to ONE poll (the fan-out + * in `ColumnStore.storeItems` writes the item rows for every sharing + * column), so following the same account in two columns costs one call; + * - `consecutive_failures` drives exponential backoff (the same + * monitor-poller curve: 2^n capped at 120 s), and the outcome is written + * back to the source row (`state`, `last_error`, `next_poll_at`) — the + * persisted truth the UI surfaces as `ColumnView.source.state`; + * - per-host tokens come from `@tactical-media-broadcast-hub/egress` HostBudgets (shared + * `host_budgets` rows — one bucket per host across every caller and host + * class, deployment-wide), so feeds and monitor fetches to the same host + * contend for the same tokens rather than each rate-limiting the host. + * + * P1 scope note: only the `home` source kind is executed against a connector + * here (`getFeed(kind:'home')` — today's behavior); `mentions`/`search`/ + * `author` rows are planned with the P2 connector work, so they park in + * `unsupported` rather than silently skipping. Item normalization runs through + * the same P0c `normalizeFeedItem` funnel the legacy loop uses. + */ +import { Inject, Injectable, Logger, Optional, type OnApplicationBootstrap, type OnApplicationShutdown } from "@nestjs/common"; +import { randomUUID } from "node:crypto"; +import type { ConnectorConnection, FeedItem } from "@tactical-media-broadcast-hub/core"; +import { RateLimitError, type ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk"; +import { DEFAULT_INTERVAL_MS } from "@tactical-media-broadcast-hub/core"; +import { pool, withTenant } from "../db/client.js"; +import { ColumnStore } from "./column-store.js"; +import { normalizeFeedItem, type NormalizedFeedItem } from "./feed-normalize.js"; +import type { ControlRedis } from "../media-control/redis.js"; +import type { ConnectionSummary } from "../connections/connections.service.js"; + +export const COLUMN_SCHEDULER_DEPENDENCIES = Symbol("COLUMN_SCHEDULER_DEPENDENCIES"); + +export interface ColumnSchedulerDependencies { + redis: Pick; + registry: ConnectorRegistry; + connection(workspaceId: string, id: string): Promise; + /** Legacy /feeds cache write-through, so /feeds reads stay cache-first. */ + cacheHomeFeed(workspaceId: string, connectionId: string, items: NormalizedFeedItem[]): Promise; + /** Effective host key for a provider's API host (egress hostBudgetKey). */ + hostKeyOf(provider: string, connection: ConnectorConnection): string; + /** Draw one token from the shared per-host budget. */ + consumeBudget(hostKey: string): Promise; + random(): number; + now(): number; +} + +/** Poll admission order per §3.4: due, enabled, not paused, oldest first. */ +interface DueRow { + workspace_id: string; + id: string; + provider: string; + connection_id: string | null; + source_kind: "home" | "mentions" | "search" | "author"; + query: string | null; + actor_ref: string | null; + interval_ms: number; + paused: boolean; + consecutive_failures: number; +} + +const LEASE_MS = 30_000; +const BACKOFF_CAP_MS = 120_000; +const RENEW_LEASE = "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('PEXPIRE', KEYS[1], ARGV[2]) else return 0 end"; +const RELEASE_LEASE = "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) else return 0 end"; + +/** Minimum spacing between two forced refreshes of one source (cache-loss safe). */ +export const FORCE_FLOOR_MS = 15_000; + +const DEFAULT_INTERVALS: Record = { + home: DEFAULT_INTERVAL_MS.platform_feed, + mentions: DEFAULT_INTERVAL_MS.platform_feed, + search: DEFAULT_INTERVAL_MS.keyword, + author: DEFAULT_INTERVAL_MS.account_list, +}; + +/** Poll cadence: spec refresh_policy interval, else the §1.2 per-kind default. */ +export function effectiveIntervalMs(intervalMs: number | null, sourceKind: DueRow["source_kind"]): number { + return intervalMs ?? DEFAULT_INTERVALS[sourceKind]; +} + +/** The monitor-poller backoff curve: 2^n * interval scale, capped at 120 s. */ +export function backoffDelayMs(failures: number, intervalMs: number): number { + return Math.min(BACKOFF_CAP_MS, intervalMs * 2 ** Math.min(7, Math.max(0, failures - 1))); +} + +/** + * Discovery cadence: sources self-schedule via next_poll_at, so the loop only + * needs to sweep for newly due rows often enough to honor the 60 s floor. + */ +const SWEEP_MS = 5_000; + +@Injectable() +export class ColumnScheduler implements OnApplicationBootstrap, OnApplicationShutdown { + private readonly logger = new Logger(ColumnScheduler.name); + private running = false; + private timer?: ReturnType; + private active?: Promise; + constructor( + private readonly store: ColumnStore, + @Optional() @Inject(COLUMN_SCHEDULER_DEPENDENCIES) private readonly deps?: ColumnSchedulerDependencies, + ) {} + + onApplicationBootstrap(): void { + this.running = true; + this.schedule(0); + } + + async onApplicationShutdown(): Promise { + this.running = false; + if (this.timer) clearTimeout(this.timer); + await this.active; + } + + private schedule(delay: number): void { + if (!this.running) return; + this.timer = setTimeout(() => { + this.active = this.pollDue().catch(() => this.logger.warn("Column sweep unavailable")).then(() => undefined).finally(() => { + this.active = undefined; + this.schedule(SWEEP_MS + Math.round((this.deps?.random() ?? Math.random()) * 1_000)); + }); + }, delay); + this.timer.unref(); + } + + /** One pass over every workspace's due sources. Errors are per-source, never fatal. */ + async pollDue(): Promise<{ polled: number; skipped: number; failed: number }> { + const workspaces = await this.workspaces(); + const totals = { polled: 0, skipped: 0, failed: 0 }; + for (const workspaceId of workspaces) { + const pass = await this.pollWorkspaceDue(workspaceId); + totals.polled += pass.polled; + totals.skipped += pass.skipped; + totals.failed += pass.failed; + } + return totals; + } + + /** + * The due-source scan (§3.4). Read-only against the DB (withTenant, FORCE + * RLS), then each source polls through its own lease/cooldown; the state + * write-back happens in the same tenant context. Returns per-source counts + * for the same shape the legacy loop reports. + */ + async pollWorkspaceDue(workspaceId: string): Promise<{ polled: number; skipped: number; failed: number }> { + const due = await this.dueSources(workspaceId); + const results = { polled: 0, skipped: 0, failed: 0 }; + for (const row of due) { + const outcome = await this.pollSource(workspaceId, row); + results[outcome] += 1; + } + return results; + } + + private async dueSources(workspaceId: string): Promise { + return withTenant(workspaceId, async (client) => { + const result = await client.query( + `SELECT s.workspace_id, s.id, s.provider, s.connection_id, s.source_kind, s.query, s.actor_ref, + (sc.refresh_policy->>'intervalMs')::int AS interval_ms, + COALESCE((sc.refresh_policy->>'paused')::boolean, false) AS paused, + s.consecutive_failures + FROM column_sources s + JOIN stream_columns sc ON sc.id = s.column_id + WHERE s.workspace_id = $1 + AND sc.deleted_at IS NULL -- a soft-deleted column's sources retire with it + AND s.state <> 'disabled' + AND COALESCE((sc.refresh_policy->>'paused')::boolean, false) = false + AND (s.next_poll_at IS NULL OR s.next_poll_at <= now()) + ORDER BY s.created_at`, + [workspaceId], + ); + return result.rows; + }); + } + + private async pollSource(workspaceId: string, row: DueRow): Promise<"polled" | "skipped" | "failed"> { + if (this.deps === undefined) return "skipped"; + const interval = effectiveIntervalMs(row.interval_ms, row.source_kind); + const cooldownMs = interval + Math.round(interval * 0.2 * this.deps.random()); + const key = `columns:${workspaceId}:${row.id}`; + const leaseKey = `${key}:lease`; + const cooldownKey = `${key}:cooldown`; + const owner = randomUUID(); + if (!await this.deps.redis.set(leaseKey, owner, { nx: true, pxMs: LEASE_MS })) return "skipped"; + let heartbeat: ReturnType | undefined; + let renew: Promise = Promise.resolve(); + let cooldown = cooldownMs; + let started = false; + try { + if (!await this.deps.redis.set(cooldownKey, "idle", { nx: true, pxMs: cooldownMs })) return "skipped"; + started = true; + heartbeat = setInterval(() => { + renew = this.deps!.redis.eval(RENEW_LEASE, [leaseKey], [owner, String(LEASE_MS)]).catch(() => { + this.logger.warn("Column poll lease renewal unavailable"); + }); + }, LEASE_MS / 3); + heartbeat.unref(); + + const page = await this.fetch(workspaceId, row); + const items = page.filter((item): item is NormalizedFeedItem => item !== null); + await this.store.storeItems(workspaceId, row.id, items); + await this.recordSuccess(workspaceId, row.id, interval); + // The legacy /feeds surface stays cache-first for home sources: the + // same items land in the hot cache the /feeds read path uses. + if (row.source_kind === "home" && row.connection_id !== null) { + await this.deps.cacheHomeFeed(workspaceId, row.connection_id, items); + } + return "polled"; + } catch (error) { + const outcome = await this.recordFailure(workspaceId, row, error, interval); + cooldown = outcome.cooldownMs; + return "failed"; + } finally { + if (heartbeat) clearInterval(heartbeat); + await renew; + try { + if (started) await this.deps.redis.set(cooldownKey, "idle", { pxMs: cooldown }); + } finally { + await this.deps.redis.eval(RELEASE_LEASE, [leaseKey], [owner]); + } + } + } + + /** Normalizes a page through the P0c funnel. P1 executes home only. */ + private async fetch(workspaceId: string, row: DueRow): Promise> { + if (this.deps === undefined) return []; + if (row.source_kind !== "home") { + throw new UnsupportedSourceError(row.source_kind); + } + if (row.connection_id === null) { + // home is always connection-bound (a platform_feed column); a NULL + // connection row is corrupt, not pollable. + throw new UnsupportedSourceError("home-without-connection"); + } + if (!this.deps.registry.has(row.provider)) throw new UnsupportedSourceError(`unknown-provider:${row.provider}`); + const connector = this.deps.registry.get(row.provider); + if (connector.getFeed === undefined) throw new UnsupportedSourceError(`feed-unsupported:${row.provider}`); + const connection = await this.deps.connection(workspaceId, row.connection_id); + const hostKey = this.deps.hostKeyOf(row.provider, connection); + await this.deps.consumeBudget(hostKey); + const page = await connector.getFeed(connection, { kind: "home", limit: 40 }); + return (page?.items ?? []).map((item: FeedItem) => normalizeFeedItem(item, connection)); + } + + private async recordSuccess(workspaceId: string, sourceId: string, intervalMs: number): Promise { + await withTenant(workspaceId, async (client) => { + await client.query( + `UPDATE column_sources + SET state = 'ready', last_error = NULL, last_polled_at = now(), consecutive_failures = 0, + next_poll_at = now() + ($3::int * interval '1 millisecond'), updated_at = now() + WHERE workspace_id = $1 AND id = $2`, + [workspaceId, sourceId, intervalMs], + ); + }); + } + + private async recordFailure(workspaceId: string, row: DueRow, error: unknown, intervalMs: number): Promise<{ cooldownMs: number }> { + const rateLimited = error instanceof RateLimitError; + const unsupported = error instanceof UnsupportedSourceError; + const retryAfter = error instanceof RateLimitError ? error.retryAfterMs : undefined; + const failures = row.consecutive_failures; + const backoff = unsupported ? intervalMs : backoffDelayMs(failures + 1, intervalMs); + const cooldownMs = rateLimited && retryAfter !== undefined ? Math.max(intervalMs, retryAfter) : backoff; + await withTenant(workspaceId, async (client) => { + await client.query( + `UPDATE column_sources + SET state = $3, last_error = $4, last_polled_at = now(), + consecutive_failures = CASE WHEN $3 = 'unsupported' THEN consecutive_failures ELSE consecutive_failures + 1 END, + next_poll_at = now() + ($5::bigint * interval '1 millisecond'), updated_at = now() + WHERE workspace_id = $1 AND id = $2`, + [workspaceId, row.id, + unsupported ? "unsupported" : rateLimited ? "rate_limited" : "error", + unsupported ? `unsupported source kind ${row.source_kind}` : error instanceof Error ? error.message.slice(0, 240) : "poll failed", + cooldownMs], + ); + }).catch(() => this.logger.warn("Column source state write-back unavailable")); + return { cooldownMs }; + } + + /** + * Workspace listing spans tenants by definition, so it runs on the pool + * with no tenant binding; every actual read/write below re-enters + * withTenant(workspaceId) so FORCE RLS still binds per source. + */ + /** + * Explicit refresh (§1.4 POST :id/refresh): mark this column's sources due + * NOW. The per-source cooldown/lease still arbitrates the actual call, so a + * refresh button cannot bypass the interval or double-fire mid-poll. + */ + /** + * Explicit refresh (§1.4 POST :id/refresh) forces this column's sources due, + * but not through a cooldown that is still authoritative in the DB: the + * Redis cooldown is a cache (a flush loses it), so next_poll_at — persisted + * — is the guard a forced refresh must still respect. FORCE_FLOOR_MS is a + * deliberately small window: the button stays responsive for a manual + * refresh while a double-click or a cache flush cannot double-poll a host. + */ + async forceColumnDue(workspaceId: string, columnId: string): Promise { + return withTenant(workspaceId, async (client) => { + const result = await client.query( + `UPDATE column_sources SET next_poll_at = now(), updated_at = now() + WHERE workspace_id = $1 AND column_id = $2 AND state <> 'disabled' + AND (last_polled_at IS NULL OR last_polled_at <= now() - ($3::bigint * interval '1 millisecond')) + RETURNING id`, + [workspaceId, columnId, FORCE_FLOOR_MS], + ); + return result.rowCount ?? 0; + }); + } + + /** User-facing enable/disable of a single source (state machine §3.4). */ + async setSourceDisabled(workspaceId: string, sourceId: string, disabled: boolean): Promise { + await withTenant(workspaceId, async (client) => { + const result = await client.query( + `UPDATE column_sources SET state = $3, next_poll_at = CASE WHEN $3 = 'disabled' THEN NULL ELSE now() END, updated_at = now() + WHERE workspace_id = $1 AND id = $2`, + [workspaceId, sourceId, disabled ? "disabled" : "pending"], + ); + if ((result.rowCount ?? 0) === 0) throw new Error(`column source ${sourceId} not found`); + }); + } + + private async workspaces(): Promise { + const result = await pool.query<{ id: string }>("SELECT id FROM workspaces ORDER BY id"); + return result.rows.map((row) => row.id); + } +} + +/** Raised for source kinds P1 does not execute; parks the row in `unsupported`. */ +export class UnsupportedSourceError extends Error { + constructor(reason: string) { + super(`column source unsupported in P1: ${reason}`); + this.name = "UnsupportedSourceError"; + } +} diff --git a/apps/api/src/feeds/column-store.ts b/apps/api/src/feeds/column-store.ts new file mode 100644 index 0000000..911c90d --- /dev/null +++ b/apps/api/src/feeds/column-store.ts @@ -0,0 +1,652 @@ +/** + * Columns persistence (P1 — design §1.2/§1.4). + * + * `stream_columns` (021) is the persisted column list; this repository owns + * every mutation plus the resolved-source rows in `column_sources` (026) and + * the per-source item rows in `feed_items`/`feed_item_columns` written by the + * poll loop. Reads run under `withTenant` so FORCE RLS binds honestly. + * + * Everything the client can send is parsed with the P0a core validators + * (`parseStreamColumnSpec`, `parseRefreshPolicyClamped`, `clampTitle`, + * `clampPosition`) — the DB CHECKs are backstops, never the validation layer. + * `StreamColumnsValidationError` is translated to 422 at the controller. + */ +import { BadRequestException, Inject, Injectable, NotFoundException, Optional, UnprocessableEntityException } from "@nestjs/common"; +import { createHash } from "node:crypto"; +import type { PoolClient } from "pg"; +import { + clampPosition, clampTitle, parseRefreshPolicyClamped, parseStreamColumnSpec, + StreamColumnsValidationError, type RefreshPolicy, type StreamColumnSpec, type StreamColumnType, +} from "@tactical-media-broadcast-hub/core"; +import type { SessionPayload } from "../auth/session.js"; +import { withTenant } from "../db/client.js"; +import { appendTenantAudit } from "../audit/audit.service.js"; +import type { NormalizedFeedItem } from "./feed-normalize.js"; +import type { ConnectionSummary } from "../connections/connections.service.js"; + +export type ColumnSourceKind = "home" | "mentions" | "search" | "author"; +export type ColumnSourceState = + | "pending" | "ready" | "stale" | "error" | "unsupported" | "rate_limited" | "budget_paused" | "disabled"; + +export interface ColumnRecord { + id: string; + type: StreamColumnType; + title: string; + position: number; + spec: StreamColumnSpec; + refreshPolicy: RefreshPolicy; + createdAt: string; + updatedAt: string; +} + +/** `ColumnView.source` (design §1.4): the scheduler-facing row, squashed. */ +export interface ColumnSourceView { + id: string; + /** Owning column — present so a bulk source query can be grouped per column. */ + columnId: string; + provider: string; + connectionId: string | null; + sourceKind: ColumnSourceKind; + query: string | null; + actorRef: string | null; + state: ColumnSourceState; + lastError: string | null; + lastPolledAt: string | null; + nextPollAt: string | null; + consecutiveFailures: number; +} + +export interface CreateColumn { + type: StreamColumnType; + title: string; + position?: number; + spec: unknown; + refreshPolicy?: unknown; +} + +export interface UpdateColumn { + title?: string; + position?: number; + spec?: unknown; + refreshPolicy?: unknown; +} + +export interface StoredFeedItem { + id: string; + dedupKey: string; + createdAtRemote: string; + item: NormalizedFeedItem; +} + +/** The identity one poll resolves to — see `sourceIdentities` below. */ +interface SourceIdentity { + provider: string; + connectionId: string | null; + sourceKind: ColumnSourceKind; + query: string | null; + actorRef: string | null; +} + +const identityKey = (identity: SourceIdentity): string => + JSON.stringify([identity.provider, identity.connectionId, identity.sourceKind, identity.query, identity.actorRef]); + +export function mapValidationError(parse: () => T): T { + try { + return parse(); + } catch (error) { + if (error instanceof StreamColumnsValidationError) throw new UnprocessableEntityException(error.message); + throw error; + } +} + +/** + * sha256(workspace | provider | canonical remote id) — design §3.3. The + * canonicalization keeps a provider that restates an item with decorations + * (e.g. Mastodon's `scheduled:123` vs `123`) from double-storing the same + * remote object. + */ +export function dedupKeyFor(workspaceId: string, provider: string, remoteId: string): string { + const canonical = remoteId.replace(/^scheduled:/, ""); + return createHash("sha256").update(`${workspaceId}|${provider}|${canonical}`).digest("hex"); +} + +/** + * The polled identities a column spec resolves to (§3.3/§3.4). platform_feed + * is the degenerate case: one connection, one kind per source row. keyword and + * saved_search expand to one keyless search source per platform. account_list + * expands to one author source per account entry. + */ +export function sourceIdentities(type: StreamColumnType, spec: StreamColumnSpec): SourceIdentity[] { + switch (type) { + case "platform_feed": { + const feed = spec as import("@tactical-media-broadcast-hub/core").PlatformFeedSpec; + const kinds = feed.kinds.length > 0 ? feed.kinds : (["home"] as const); + return kinds.map((kind): SourceIdentity => ({ + provider: "", connectionId: feed.connectionId, sourceKind: kind, query: null, actorRef: null, + })); + } + case "keyword": + case "saved_search": { + const keyword = spec as import("@tactical-media-broadcast-hub/core").KeywordSpec; + return keyword.platforms.map((provider): SourceIdentity => ({ + provider, connectionId: null, sourceKind: "search", query: keyword.query, actorRef: null, + })); + } + case "account_list": { + const list = spec as import("@tactical-media-broadcast-hub/core").AccountListSpec; + return list.accounts.map((account): SourceIdentity => ({ + provider: account.provider, connectionId: null, sourceKind: "author", query: null, actorRef: account.handle, + })); + } + } +} + +/** Providers referenced by a spec (empty for platform_feed: the connection supplies it). */ +export function columnProviders(type: StreamColumnType, spec: StreamColumnSpec): string[] { + switch (type) { + case "platform_feed": return []; + case "keyword": + case "saved_search": return (spec as import("@tactical-media-broadcast-hub/core").KeywordSpec).platforms; + case "account_list": return (spec as import("@tactical-media-broadcast-hub/core").AccountListSpec).accounts.map((account) => account.provider); + } +} + +/** The title today's implicit column renders: "Display Name · handle" (web streams-data liveColumns). */ +export function seedTitle(row: ConnectionSummary): string { + const displayName = String(row.fields.displayName ?? "").trim() || row.channelId; + const handle = String(row.fields.handle ?? "").trim() || row.channelId; + return clampTitle(`${displayName} · ${handle}`); +} + +interface ColumnRow { + id: string; type: StreamColumnType; title: string; position: number; + spec: unknown; refresh_policy: unknown; created_at: Date; updated_at: Date; +} + +const COLUMN_SELECT = "id, type, title, position, spec, refresh_policy, created_at, updated_at"; + +function toRecord(row: ColumnRow): ColumnRecord { + return { + id: row.id, + type: row.type, + title: row.title, + position: row.position, + spec: mapValidationError(() => parseStreamColumnSpec(row.type, row.spec)), + refreshPolicy: parseRefreshPolicyClamped(row.refresh_policy ?? {}), + createdAt: row.created_at.toISOString(), + updatedAt: row.updated_at.toISOString(), + }; +} + +const SOURCE_SELECT = `SELECT s.id, s.column_id, s.provider, s.connection_id, s.source_kind, s.query, s.actor_ref, + s.state, s.last_error, s.last_polled_at, s.next_poll_at, s.consecutive_failures + FROM column_sources s WHERE s.workspace_id = $1`; + +function sourceFromRow(row: Record): ColumnSourceView { + return { + id: String(row.id), + columnId: String(row.column_id), + provider: String(row.provider), + connectionId: row.connection_id === null || row.connection_id === undefined ? null : String(row.connection_id), + sourceKind: row.source_kind as ColumnSourceKind, + query: row.query === null || row.query === undefined ? null : String(row.query), + actorRef: row.actor_ref === null || row.actor_ref === undefined ? null : String(row.actor_ref), + state: row.state as ColumnSourceState, + lastError: row.last_error === null || row.last_error === undefined ? null : String(row.last_error), + lastPolledAt: isoOrNull(row.last_polled_at), + nextPollAt: isoOrNull(row.next_poll_at), + consecutiveFailures: Number(row.consecutive_failures ?? 0), + }; +} + +function isoOrNull(value: unknown): string | null { + return value === null || value === undefined ? null : (value as Date).toISOString(); +} + +export interface ColumnStoreDependencies { + /** Resolve a connection's provider for platform_feed source rows. */ + providerOf(workspaceId: string, connectionId: string): Promise; + /** Feed-capable connection summaries — the lazy seeding input. */ + connections(workspaceId: string): Promise; +} + +export const COLUMN_STORE_DEPENDENCIES = Symbol("COLUMN_STORE_DEPENDENCIES"); + +@Injectable() +export class ColumnStore { + constructor(@Optional() @Inject(COLUMN_STORE_DEPENDENCIES) private readonly deps?: ColumnStoreDependencies) {} + + private async providerOf(workspaceId: string, connectionId: string): Promise { + if (this.deps) return this.deps.providerOf(workspaceId, connectionId); + return withTenant(workspaceId, async (client): Promise => { + const result = await client.query<{ provider: string }>( + "SELECT provider FROM connections WHERE id = $1 AND workspace_id = $2 AND disconnected_at IS NULL", [connectionId, workspaceId]); + const row = result.rows[0]; + if (row === undefined) throw new NotFoundException(`connection ${connectionId} not found`); + return row.provider; + }); + } + + private async connections(workspaceId: string): Promise { + if (this.deps) return this.deps.connections(workspaceId); + const { PgConnectionStore } = await import("../connections/connection-store.js"); + return new PgConnectionStore().list(workspaceId); + } + + async list(workspaceId: string): Promise { + return withTenant(workspaceId, async (client) => { + const result = await client.query( + `SELECT ${COLUMN_SELECT} FROM stream_columns WHERE deleted_at IS NULL ORDER BY position, created_at`); + return result.rows.map((row) => toRecord(row as ColumnRow)); + }); + } + + async get(workspaceId: string, id: string): Promise { + return withTenant(workspaceId, async (client) => this.read(client, workspaceId, id)); + } + + async sources(workspaceId: string, columnId?: string): Promise { + return withTenant(workspaceId, async (client) => { + const result = columnId === undefined + ? await client.query(`${SOURCE_SELECT} ORDER BY s.created_at`, [workspaceId]) + : await client.query(`${SOURCE_SELECT} AND s.column_id = $2 ORDER BY s.created_at`, [workspaceId, columnId]); + return result.rows.map((row) => sourceFromRow(row)); + }); + } + + async create(session: SessionPayload, input: CreateColumn): Promise { + const spec = mapValidationError(() => parseStreamColumnSpec(input.type, input.spec)); + const policy = parseRefreshPolicyClamped(input.refreshPolicy ?? {}); + const title = clampTitle(input.title ?? ""); + if (title === "") throw new UnprocessableEntityException("title must be 1..80 characters after trimming"); + return withTenant(session.workspaceId, async (client) => { + await this.assertSpecReferenceable(client, session.workspaceId, input.type, spec); + const position = input.position === undefined + ? await this.nextPosition(client, session.workspaceId) + : clampPosition(input.position); + const result = await client.query( + `INSERT INTO stream_columns (workspace_id, type, title, position, spec, refresh_policy, created_by) + VALUES ($1,$2,$3,$4,$5,$6,$7) RETURNING ${COLUMN_SELECT}`, + [session.workspaceId, input.type, title, position, JSON.stringify(spec), JSON.stringify(policy), session.userId], + ); + const record = toRecord(result.rows[0] as ColumnRow); + await this.writeSources(client, session.workspaceId, record.id, input.type, spec); + await appendTenantAudit(session.workspaceId, { + actorUserId: session.userId, actorRole: session.role, action: "columns.created", + targetType: "stream_column", targetId: record.id, diff: { type: input.type, title }, + }); + return record; + }); + } + + async update(session: SessionPayload, id: string, input: UpdateColumn): Promise { + if (Object.keys(input).length === 0) throw new BadRequestException("no recognized fields to update"); + return withTenant(session.workspaceId, async (client) => { + await client.query("SELECT id FROM stream_columns WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NULL FOR UPDATE", [session.workspaceId, id]); + const current = await this.read(client, session.workspaceId, id); + const title = input.title === undefined ? current.title : clampTitle(input.title); + if (title === "") throw new UnprocessableEntityException("title must be 1..80 characters after trimming"); + // Column type is immutable (design §1.4: PATCH carries title/position/ + // spec/refresh_policy); a different type is a new column. + const spec = input.spec === undefined ? current.spec : mapValidationError(() => parseStreamColumnSpec(current.type, input.spec)); + const policy = input.refreshPolicy === undefined ? current.refreshPolicy : parseRefreshPolicyClamped(input.refreshPolicy); + const position = input.position === undefined ? current.position : clampPosition(input.position); + if (input.spec !== undefined) await this.assertSpecReferenceable(client, session.workspaceId, current.type, spec); + const result = await client.query( + `UPDATE stream_columns SET title = $3, position = $4, spec = $5, refresh_policy = $6, updated_at = now() + WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NULL RETURNING ${COLUMN_SELECT}`, + [session.workspaceId, id, title, position, JSON.stringify(spec), JSON.stringify(policy)], + ); + if (result.rows[0] === undefined) throw new NotFoundException(`column ${id} not found`); + if (input.spec !== undefined) await this.rewriteSources(client, session.workspaceId, id, current.type, spec); + await appendTenantAudit(session.workspaceId, { + actorUserId: session.userId, actorRole: session.role, action: "columns.updated", + targetType: "stream_column", targetId: id, + diff: { titleChanged: title !== current.title, positionChanged: position !== current.position, specChanged: input.spec !== undefined }, + }); + return toRecord(result.rows[0] as ColumnRow); + }); + } + + private async writeSources(client: PoolClient, workspaceId: string, columnId: string, type: StreamColumnType, spec: StreamColumnSpec): Promise { + for (const identity of sourceIdentities(type, spec)) { + const provider = identity.provider !== "" ? identity.provider : await this.providerOf(workspaceId, identity.connectionId ?? ""); + await client.query( + `INSERT INTO column_sources (workspace_id, column_id, provider, connection_id, source_kind, query, actor_ref) + VALUES ($1,$2,$3,$4,$5,$6,$7) + ON CONFLICT (workspace_id, column_id, provider, source_kind, connection_id, query, actor_ref) DO NOTHING`, + [workspaceId, columnId, provider, identity.connectionId, identity.sourceKind, identity.query, identity.actorRef], + ); + } + } + + /** Spec identity changes rewrite the source set; identities no longer referenced by this column are dropped. */ + private async rewriteSources(client: PoolClient, workspaceId: string, columnId: string, type: StreamColumnType, spec: StreamColumnSpec): Promise { + const wanted = new Set(sourceIdentities(type, spec).map(identityKey)); + const existing = await client.query<{ id: string; provider: string; connection_id: string | null; source_kind: ColumnSourceKind; query: string | null; actor_ref: string | null }>( + "SELECT id, provider, connection_id, source_kind, query, actor_ref FROM column_sources WHERE workspace_id = $1 AND column_id = $2", [workspaceId, columnId]); + for (const row of existing.rows) { + const identity: SourceIdentity = { provider: row.provider, connectionId: row.connection_id, sourceKind: row.source_kind, query: row.query, actorRef: row.actor_ref }; + if (!wanted.has(identityKey(identity))) { + await client.query("DELETE FROM column_sources WHERE workspace_id = $1 AND id = $2", [workspaceId, row.id]); + } + } + await this.writeSources(client, workspaceId, columnId, type, spec); + } + + async remove(session: SessionPayload, id: string): Promise { + await withTenant(session.workspaceId, async (client) => { + await client.query("SELECT id FROM stream_columns WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NULL FOR UPDATE", [session.workspaceId, id]); + const result = await client.query( + "UPDATE stream_columns SET deleted_at = now(), updated_at = now() WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NULL RETURNING id", [session.workspaceId, id]); + if (result.rows[0] === undefined) throw new NotFoundException(`column ${id} not found`); + await appendTenantAudit(session.workspaceId, { + actorUserId: session.userId, actorRole: session.role, action: "columns.deleted", + targetType: "stream_column", targetId: id, + }); + }); + } + + /** + * Explicit order (design §1.4 POST /reorder): the client sends the full + * ordered id list; positions are rewritten densely. Unknown, foreign or + * missing ids are a 400 — a partial reorder would corrupt the order. + */ + async reorder(session: SessionPayload, ids: string[]): Promise { + if (ids.length === 0) return this.list(session.workspaceId); + if (new Set(ids).size !== ids.length) throw new BadRequestException("reorder ids must be unique"); + return withTenant(session.workspaceId, async (client) => { + const existing = await client.query<{ id: string }>( + "SELECT id FROM stream_columns WHERE workspace_id = $1 AND deleted_at IS NULL", [session.workspaceId]); + if (ids.length !== existing.rows.length || !existing.rows.every((row) => ids.includes(row.id))) { + throw new BadRequestException("reorder must list every column in the workspace exactly once"); + } + // Two-phase write: the unique (workspace_id, position) index would fire + // mid-swap if positions were assigned in place (e.g. moving column 0 to + // the end collides with the row already at that slot). Parking every row + // at a high negative offset first leaves the final range untouched, so + // the second pass can never collide. Offsets are outside the CHECK + // (position >= 0) only transiently... so instead park at (n + i), which + // is >= 0, unique within the transaction, and freed before phase two. + // Phase 1 — park every row at a unique high slot (n, n+1, …) so the + // unique (workspace_id, position) index can't fire mid-reorder. + let park = ids.length; + for (const id of ids) { + await client.query("UPDATE stream_columns SET position = $3 WHERE workspace_id = $1 AND id = $2", [session.workspaceId, id, park]); + park += 1; + } + // Phase 2 — assign final dense ranks in the REQUESTED order. The parked + // range is disjoint from [0, n), so no intermediate state collides. + const byId = new Map(ids.map((id, index) => [id, index] as const)); + const current = await client.query( + `SELECT ${COLUMN_SELECT} FROM stream_columns WHERE workspace_id = $1 AND deleted_at IS NULL`, [session.workspaceId]); + const records = current.rows + .map((row) => toRecord(row as ColumnRow)) + .sort((a, b) => (byId.get(a.id) ?? 0) - (byId.get(b.id) ?? 0)); + let rank = 0; + for (const record of records) { + await client.query("UPDATE stream_columns SET position = $3, updated_at = now() WHERE workspace_id = $1 AND id = $2", [session.workspaceId, record.id, rank]); + record.position = rank; + rank += 1; + } + await appendTenantAudit(session.workspaceId, { + actorUserId: session.userId, actorRole: session.role, action: "columns.reordered", + targetType: "workspace", targetId: session.workspaceId, diff: { count: ids.length }, + }); + return records; + }); + } + + /** + * Lazy seeding (design §6 P1): on the first column-API touch for a workspace + * with no columns at all, materialize TODAY's implicit /feeds columns — one + * platform_feed column per feed-capable connection, titled exactly like the + * implicit column the UI renders today ("Display · handle", kinds ["home"], + * position by connection creation order), so the Streams page is unchanged. + * Idempotent by construction: a second call sees a non-empty column list and + * seeds nothing — including for connections added later. + */ + async ensureSeeded(workspaceId: string, connections: ConnectionSummary[], capable: (row: ConnectionSummary) => boolean): Promise<{ seeded: number }> { + const rows = connections.filter(capable); + if (rows.length === 0) return { seeded: 0 }; + return withTenant(workspaceId, async (client) => { + // Serialize concurrent first touches without taking a row lock on + // workspaces itself: a FOR UPDATE here deadlocks with any FK insert + // into audit_log/column_sources from a nested withTenant client, + // because FK validation takes FOR KEY SHARE on the parent workspace + // row. Advisory xact locks never block key-share, so seeding + // serializes while audits and FK checks proceed. + await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [`columns-seed:${workspaceId}`]); + const existing = await client.query<{ count: string }>( + "SELECT count(*)::text AS count FROM stream_columns WHERE workspace_id = $1 AND deleted_at IS NULL", [workspaceId]); + if (Number(existing.rows[0]?.count ?? 0) > 0) return { seeded: 0 }; + const seeder = await client.query<{ user_id: string }>( + "SELECT m.user_id FROM memberships m JOIN users u ON u.id = m.user_id WHERE m.workspace_id = $1 ORDER BY m.created_at, u.created_at LIMIT 1", [workspaceId]); + const createdBy = seeder.rows[0]?.user_id; + if (createdBy === undefined) return { seeded: 0 }; // no member to attribute + let seeded = 0; + let position = 0; + for (const row of rows) { + const spec = { connectionId: row.id, kinds: ["home"] } as const; + const inserted = await client.query<{ id: string }>( + `INSERT INTO stream_columns (workspace_id, type, title, position, spec, refresh_policy, created_by) + VALUES ($1,'platform_feed',$2,$3,$4,'{}',$5) RETURNING id`, + [workspaceId, seedTitle(row), position, JSON.stringify(spec), createdBy], + ); + const columnId = inserted.rows[0]?.id; + if (columnId === undefined) continue; + await client.query( + `INSERT INTO column_sources (workspace_id, column_id, provider, connection_id, source_kind) + VALUES ($1,$2,$3,$4,'home') + ON CONFLICT (workspace_id, column_id, provider, source_kind, connection_id, query, actor_ref) DO NOTHING`, + [workspaceId, columnId, row.provider, row.id], + ); + seeded += 1; + position += 1; + } + if (seeded > 0) await appendTenantAudit(workspaceId, { + actorUserId: null, actorRole: "system", action: "columns.seeded", + targetType: "workspace", targetId: workspaceId, diff: { seeded }, + }); + return { seeded }; + }); + } + + /** + * import-saved-searches (design §1.4): the localStorage saved-searches + * contract moves server-side. Each query becomes a saved_search column over + * the platforms the workspace can actually search, defaulting to every + * connected feed provider when the client sends none. Idempotent per query: + * a live saved_search column with the same query is skipped. + */ + async importSavedSearches( + session: SessionPayload, queries: string[], platforms: string[], + ): Promise<{ imported: number; skipped: number }> { + if (platforms.length === 0) return { imported: 0, skipped: queries.length }; + return withTenant(session.workspaceId, async (client) => { + let imported = 0; + let skipped = 0; + for (const raw of queries) { + const query = raw.trim(); + if (query === "") { skipped += 1; continue; } + const duplicate = await client.query<{ id: string }>( + `SELECT id FROM stream_columns WHERE workspace_id = $1 AND deleted_at IS NULL AND type = 'saved_search' + AND spec->>'query' = $2 LIMIT 1`, [session.workspaceId, query]); + if (duplicate.rows[0] !== undefined) { skipped += 1; continue; } + const spec = { query, platforms, matchMode: "any" } as const; + const title = clampTitle(query); + if (title === "") { skipped += 1; continue; } + const position = await this.nextPosition(client, session.workspaceId); + const inserted = await client.query<{ id: string }>( + `INSERT INTO stream_columns (workspace_id, type, title, position, spec, refresh_policy, created_by) + VALUES ($1,'saved_search',$2,$3,$4,'{}',$5) RETURNING id`, + [session.workspaceId, title, position, JSON.stringify(spec), session.userId], + ); + const columnId = inserted.rows[0]?.id; + if (columnId === undefined) { skipped += 1; continue; } + await this.writeSources(client, session.workspaceId, columnId, "saved_search", + parseStreamColumnSpec("saved_search", spec)); + imported += 1; + } + if (imported > 0) await appendTenantAudit(session.workspaceId, { + actorUserId: session.userId, actorRole: session.role, action: "columns.saved_searches_imported", + targetType: "workspace", targetId: session.workspaceId, diff: { imported, skipped }, + }); + return { imported, skipped }; + }); + } + + /** + * Poll-time item persistence (§3.3): first writer wins per dedup key (the + * cross-column dedup), then fan-out rows link the item to every column + * whose source shares this source's identity — which is how "same account + * in two columns costs one poll" still surfaces items in both. + */ + async storeItems(workspaceId: string, sourceId: string, items: NormalizedFeedItem[]): Promise<{ stored: number; fanned: number }> { + if (items.length === 0) return { stored: 0, fanned: 0 }; + return withTenant(workspaceId, async (client) => { + const found = await client.query<{ id: string; provider: string; connection_id: string | null; source_kind: ColumnSourceKind; query: string | null; actor_ref: string | null }>( + "SELECT id, provider, connection_id, source_kind, query, actor_ref FROM column_sources WHERE workspace_id = $1 AND id = $2", [workspaceId, sourceId]); + const source = found.rows[0]; + if (source === undefined) return { stored: 0, fanned: 0 }; + const sharing = await client.query<{ column_id: string }>( + `SELECT column_id FROM column_sources + WHERE workspace_id = $1 AND provider = $2 AND source_kind = $3 + AND connection_id IS NOT DISTINCT FROM $4 AND query IS NOT DISTINCT FROM $5 AND actor_ref IS NOT DISTINCT FROM $6 + AND state <> 'disabled'`, + [workspaceId, source.provider, source.source_kind, source.connection_id, source.query, source.actor_ref]); + const columnIds = sharing.rows.map((row) => row.column_id); + let stored = 0; + let fanned = 0; + for (const item of items) { + const dedupKey = dedupKeyFor(workspaceId, item.provider, item.remoteId); + // First writer wins per dedup key (the workspace-wide cross-column + // dedup, §3.3) — the returning row resolves the item id either way. + const inserted = await client.query<{ id: string }>( + `INSERT INTO feed_items (workspace_id, dedup_key, provider, remote_id, item_id, author_handle, author_ref, url, + created_at_remote, fetched_at, content, plain_text, media, metrics, source_id, expires_at) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,now(),$10,$11,$12,$13,$14,$15) + ON CONFLICT (workspace_id, dedup_key) DO UPDATE SET fetched_at = feed_items.fetched_at RETURNING id`, + [workspaceId, dedupKey, item.provider, item.remoteId, item.id, item.authorHandle, item.authorRef, item.url, + item.createdAt.toISOString(), JSON.stringify(item.content), item.plainText, JSON.stringify(item.media), + JSON.stringify(item.metrics), sourceId, item.expiresAt === null || item.expiresAt === undefined ? null : item.expiresAt.toISOString()], + ); + const itemId = inserted.rows[0]?.id; + if (itemId === undefined) continue; + stored += 1; + // Fan out regardless of who inserted: an item ingested for one source + // identity is visible to EVERY column sharing that identity, including + // items an earlier poll already stored (§3.3) — otherwise the second + // column's dedup-hit items would never get their link rows. + for (const columnId of columnIds) { + const linked = await client.query( + "INSERT INTO feed_item_columns (workspace_id, item_id, column_id) VALUES ($1,$2,$3) ON CONFLICT DO NOTHING", + [workspaceId, itemId, columnId]); + fanned += linked.rowCount ?? 0; + } + } + return { stored, fanned }; + }); + } + + /** + * Keyset page for GET /feeds/columns/:id/items (§1.4). Cursor rule: + * (created_at_remote DESC, dedup_key DESC) — a pure position in a total + * order, so paging survives cache flushes and later insertions alike. + */ + async columnItems( + workspaceId: string, columnId: string, options: { cursor?: { at: number; key: string }; limit: number }, + ): Promise<{ items: StoredFeedItem[]; nextCursor: string | null }> { + return withTenant(workspaceId, async (client) => { + const column = await client.query("SELECT id FROM stream_columns WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NULL", [workspaceId, columnId]); + if (column.rows[0] === undefined) throw new NotFoundException(`column ${columnId} not found`); + const params: unknown[] = [workspaceId, columnId]; + const filter = options.cursor === undefined ? "" : " AND (i.created_at_remote, i.dedup_key) < ($3::timestamptz, $4)"; + if (options.cursor !== undefined) params.push(new Date(options.cursor.at).toISOString(), options.cursor.key); + params.push(options.limit + 1); + const result = await client.query( + `SELECT i.id, i.dedup_key, i.created_at_remote, i.provider, i.remote_id, i.item_id, i.author_handle, + i.author_ref, i.url, i.content, i.plain_text, i.media, i.metrics, i.expires_at + FROM feed_items i + JOIN feed_item_columns fic ON fic.item_id = i.id AND fic.column_id = $2 + WHERE i.workspace_id = $1${filter} + ORDER BY i.created_at_remote DESC, i.dedup_key DESC + LIMIT $${params.length}`, + params, + ); + const rows = result.rows; + const hasMore = rows.length > options.limit; + const items = (hasMore ? rows.slice(0, options.limit) : rows).map((row) => storedItemFromRow(row)); + const last = items[items.length - 1]; + return { + items, + nextCursor: hasMore && last + ? Buffer.from(JSON.stringify({ at: Date.parse(last.createdAtRemote), key: last.dedupKey })).toString("base64url") + : null, + }; + }); + } + + private async nextPosition(client: PoolClient, workspaceId: string): Promise { + const result = await client.query<{ max: number | null }>( + "SELECT max(position) AS max FROM stream_columns WHERE workspace_id = $1 AND deleted_at IS NULL", [workspaceId]); + return (result.rows[0]?.max ?? -1) + 1; + } + + /** A spec's references must exist in the workspace before the column can persist. */ + private async assertSpecReferenceable(client: PoolClient, workspaceId: string, type: StreamColumnType, spec: StreamColumnSpec): Promise { + if (type === "platform_feed") { + const connectionId = (spec as import("@tactical-media-broadcast-hub/core").PlatformFeedSpec).connectionId; + const result = await client.query( + "SELECT 1 FROM connections WHERE workspace_id = $1 AND id = $2 AND disconnected_at IS NULL", [workspaceId, connectionId]); + if (result.rows[0] === undefined) throw new UnprocessableEntityException("platform_feed: connectionId does not reference a connection in this workspace"); + return; + } + const providers = columnProviders(type, spec); + if (providers.length === 0) return; + const result = await client.query<{ provider: string }>( + "SELECT DISTINCT provider FROM connections WHERE workspace_id = $1 AND disconnected_at IS NULL AND provider = ANY($2)", [workspaceId, providers]); + const present = new Set(result.rows.map((row) => row.provider)); + const missing = [...new Set(providers)].filter((provider) => !present.has(provider)); + if (missing.length > 0) throw new UnprocessableEntityException(`no connection for provider(s): ${missing.join(", ")}`); + } + + private async read(client: PoolClient, workspaceId: string, id: string): Promise { + const result = await client.query( + `SELECT ${COLUMN_SELECT} FROM stream_columns WHERE workspace_id = $1 AND id = $2 AND deleted_at IS NULL`, [workspaceId, id]); + const row = result.rows[0]; + if (row === undefined) throw new NotFoundException(`column ${id} not found`); + return toRecord(row as ColumnRow); + } +} + +interface FeedItemRow { + id: string; dedup_key: string; created_at_remote: Date; provider: string; remote_id: string; item_id: string; + author_handle: string; author_ref: string | null; url: string | null; content: unknown; plain_text: string; + media: unknown; metrics: unknown; expires_at: Date | null; +} + +function storedItemFromRow(row: FeedItemRow): StoredFeedItem { + const metrics = (typeof row.metrics === "object" && row.metrics !== null ? row.metrics : {}) as Record; + const item: NormalizedFeedItem = { + id: row.item_id, + remoteId: row.remote_id, + provider: row.provider, + // The store page does not carry a live connection id; the column's source + // rows hold it, so the connectionId field is filled by the caller when it + // matters (the items endpoint derives it from the column spec). + connectionId: "", + authorHandle: row.author_handle, + authorRef: row.author_ref, + url: row.url, + text: row.plain_text, + content: row.content as NormalizedFeedItem["content"], + plainText: row.plain_text, + createdAt: row.created_at_remote, + mediaUrls: [], + media: Array.isArray(row.media) ? row.media as NormalizedFeedItem["media"] : [], + metrics, + expiresAt: row.expires_at, + }; + return { id: row.id, dedupKey: row.dedup_key, createdAtRemote: new Date(row.created_at_remote).toISOString(), item }; +} diff --git a/apps/api/src/feeds/columns.controller.ts b/apps/api/src/feeds/columns.controller.ts new file mode 100644 index 0000000..8800064 --- /dev/null +++ b/apps/api/src/feeds/columns.controller.ts @@ -0,0 +1,300 @@ +/** + * Columns API (P1 — design §1.4). + * + * Routes (all under /feeds, the surface the Streams page already calls): + * GET /feeds/columns list (+ lazy seeding on first touch) + * POST /feeds/columns create + * GET /feeds/columns/:id one column + * PATCH /feeds/columns/:id update (title/position/spec/refreshPolicy) + * DELETE /feeds/columns/:id soft delete + * POST /feeds/columns/reorder explicit full-list order + * POST /feeds/columns/import-saved-searches localStorage contract migration + * GET /feeds/columns/:id/items keyset-cursored items + * POST /feeds/columns/:id/refresh force a poll (cooldown-gated) + * POST /feeds/columns/:id/sources/:sourceId/disable|enable + * + * Writes are gated on `streams.columns.manage` (026 seed: Owner/Admin/Editor); + * reads stay on `draft.view` — the same permission /feeds uses, so every + * current reader keeps reading. GET /feeds is untouched except the additive + * `meta.columns` block. + */ +import { BadRequestException, Body, Controller, Delete, Get, Inject, NotFoundException, Param, Patch, Post, Query, Req, UnprocessableEntityException, UseGuards } from "@nestjs/common"; +import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import { parseId, parseLimit, requireObject, requireSession, requireString } from "../http/validation.js"; +import type { SessionPayload } from "../auth/session.js"; +import { + ColumnStore, mapValidationError, type ColumnRecord, type ColumnSourceView, type CreateColumn, type StoredFeedItem, type UpdateColumn, +} from "./column-store.js"; +import { ColumnScheduler } from "./column-scheduler.js"; +import { FeedsService } from "./feeds.service.js"; +import type { ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk"; + +export interface ColumnResponse extends ColumnRecord { + sources: ColumnSourceView[]; +} + +export interface ColumnItemsResponse { + items: StoredFeedItem[]; + nextCursor: string | null; +} + +/** DI token for the shared connector registry (single process-wide instance). */ +export const COLUMNS_REGISTRY = Symbol("COLUMNS_REGISTRY"); + +@Controller("feeds/columns") +@UseGuards(RbacGuard) +export class ColumnsController { + constructor( + @Inject(ColumnStore) private readonly store: ColumnStore, + @Inject(ColumnScheduler) private readonly scheduler: ColumnScheduler, + @Inject(FeedsService) private readonly feeds: FeedsService, + @Inject(COLUMNS_REGISTRY) private readonly registry: ConnectorRegistry, + ) {} + + @Get() + @RequirePermission("draft.view") + async list(@Req() request: AuthenticatedRequest): Promise { + const session = requireSession(request); + await this.ensureSeeded(session); + return this.views(session.workspaceId, await this.store.list(session.workspaceId)); + } + + @Post() + @RequirePermission("streams.columns.manage") + async create(@Req() request: AuthenticatedRequest, @Body() body: unknown): Promise { + const session = requireSession(request); + const input = parseCreate(body); + await this.ensureSeeded(session); + const record = await this.store.create(session, input); + return this.view(record, await this.store.sources(session.workspaceId, record.id)); + } + + @Get(":id") + @RequirePermission("draft.view") + async get(@Req() request: AuthenticatedRequest, @Param("id") id: string): Promise { + const session = requireSession(request); + await this.ensureSeeded(session); + const record = await this.store.get(session.workspaceId, parseId(id)); + return this.view(record, await this.store.sources(session.workspaceId, record.id)); + } + + @Patch(":id") + @RequirePermission("streams.columns.manage") + async update(@Req() request: AuthenticatedRequest, @Param("id") id: string, @Body() body: unknown): Promise { + const session = requireSession(request); + const record = await this.store.update(session, parseId(id), parseUpdate(body)); + return this.view(record, await this.store.sources(session.workspaceId, record.id)); + } + + @Delete(":id") + @RequirePermission("streams.columns.manage") + async remove(@Req() request: AuthenticatedRequest, @Param("id") id: string): Promise<{ id: string; deleted: true }> { + const session = requireSession(request); + const parsed = parseId(id); + await this.store.remove(session, parsed); + return { id: parsed, deleted: true }; + } + + @Post("reorder") + @RequirePermission("streams.columns.manage") + async reorder(@Req() request: AuthenticatedRequest, @Body() body: unknown): Promise { + const session = requireSession(request); + const ids = parseIds(requireObject(body, "body").ids); + return this.views(session.workspaceId, await this.store.reorder(session, ids)); + } + + @Post("import-saved-searches") + @RequirePermission("streams.columns.manage") + async importSavedSearches(@Req() request: AuthenticatedRequest, @Body() body: unknown): Promise<{ imported: number; skipped: number }> { + const session = requireSession(request); + const payload = requireObject(body, "body"); + const rawQueries = payload.queries; + if (!Array.isArray(rawQueries) || rawQueries.length === 0) { + throw new BadRequestException("queries must be a non-empty array of saved search strings"); + } + for (const query of rawQueries) { + if (typeof query !== "string") throw new BadRequestException("queries must contain strings only"); + } + if (rawQueries.length > 100) throw new BadRequestException("queries supports at most 100 entries per call"); + const rawPlatforms = payload.platforms; + let platforms: string[]; + if (rawPlatforms === undefined || rawPlatforms === null) { + platforms = this.searchCapableProviders(await this.connectedProviders(session)); + } else { + if (!Array.isArray(rawPlatforms) || rawPlatforms.some((provider) => typeof provider !== "string")) { + throw new BadRequestException("platforms must be an array of provider ids"); + } + platforms = [...new Set(rawPlatforms as string[])]; + // Provider ids are opaque to the P0a schema by design; the registry is + // the app-side source of truth, so an unknown id is a 422 here rather + // than a saved_search column that can never resolve a source. + const unknown = platforms.filter((provider) => !this.registry.has(provider)); + if (unknown.length > 0) { + throw new UnprocessableEntityException(`unknown provider id(s): ${unknown.join(", ")}`); + } + } + return this.store.importSavedSearches(session, rawQueries, platforms); + } + + @Get(":id/items") + @RequirePermission("draft.view") + async items( + @Req() request: AuthenticatedRequest, + @Param("id") id: string, + @Query("cursor") cursor?: string, + @Query("limit") limit?: string, + ): Promise { + const session = requireSession(request); + return this.store.columnItems(session.workspaceId, parseId(id), { + cursor: cursor === undefined || cursor === "" ? undefined : decodeColumnCursor(cursor), + limit: parseLimit(limit, 40, 200), + }); + } + + @Post(":id/refresh") + @RequirePermission("streams.columns.manage") + async refresh(@Req() request: AuthenticatedRequest, @Param("id") id: string): Promise<{ id: string; refreshed: number }> { + const session = requireSession(request); + const parsed = parseId(id); + await this.store.get(session.workspaceId, parsed); // 404 before polling + // Force-due this column's sources; the scheduler's cooldown/lease still + // decides whether a call actually goes out (refresh cannot bypass them). + const forced = await this.forceDue(session.workspaceId, parsed); + const outcome = await this.scheduler.pollWorkspaceDue(session.workspaceId); + return { id: parsed, refreshed: forced > 0 ? outcome.polled : 0 }; + } + + @Post(":id/sources/:sourceId/disable") + @RequirePermission("streams.columns.manage") + async disableSource(@Req() request: AuthenticatedRequest, @Param("id") id: string, @Param("sourceId") sourceId: string): Promise<{ id: string; state: string }> { + const session = requireSession(request); + return this.setSourceState(session, parseId(id), parseId(sourceId), true); + } + + @Post(":id/sources/:sourceId/enable") + @RequirePermission("streams.columns.manage") + async enableSource(@Req() request: AuthenticatedRequest, @Param("id") id: string, @Param("sourceId") sourceId: string): Promise<{ id: string; state: string }> { + const session = requireSession(request); + return this.setSourceState(session, parseId(id), parseId(sourceId), false); + } + + // -- helpers ------------------------------------------------------------- + + private async setSourceState(session: SessionPayload, columnId: string, sourceId: string, disabled: boolean): Promise<{ id: string; state: string }> { + const sources = await this.store.sources(session.workspaceId, columnId); + const source = sources.find((row) => row.id === sourceId); + if (source === undefined) throw new NotFoundException(`source ${sourceId} not found on column ${columnId}`); + await this.scheduler.setSourceDisabled(session.workspaceId, sourceId, disabled); + return { id: sourceId, state: disabled ? "disabled" : "pending" }; + } + + private async ensureSeeded(session: SessionPayload): Promise { + const connections = await this.feeds.capableConnections(session.workspaceId); + await this.store.ensureSeeded(session.workspaceId, connections, () => true); + } + + /** Bulk view assembly: one source query, grouped per column. */ + private async views(workspaceId: string, records: ColumnRecord[]): Promise { + if (records.length === 0) return []; + const byColumn = new Map(); + for (const source of await this.store.sources(workspaceId)) { + const list = byColumn.get(source.columnId); + if (list === undefined) byColumn.set(source.columnId, [source]); + else list.push(source); + } + return records.map((record) => ({ ...record, sources: byColumn.get(record.id) ?? [] })); + } + + private async view(record: ColumnRecord, sources: ColumnSourceView[]): Promise { + return { ...record, sources }; + } + + private async connectedProviders(session: SessionPayload): Promise { + const rows = await this.feeds.capableConnections(session.workspaceId); + return [...new Set(rows.map((row) => row.provider))]; + } + + /** Providers whose manifest declares the P0b search capability. */ + private searchCapableProviders(providers: string[]): string[] { + return providers.filter((provider) => { + if (!this.registry.has(provider)) return false; + return this.registry.get(provider).manifest().search === true; + }); + } + + private async forceDue(workspaceId: string, columnId: string): Promise { + return this.scheduler.forceColumnDue(workspaceId, columnId); + } +} + + +// --------------------------------------------------------------------------- +// Request parsing (unknown body in, typed DTO out; 400 on shape errors) +// --------------------------------------------------------------------------- + +function parseCreate(body: unknown): CreateColumn { + const payload = requireObject(body, "body"); + const type = requireString(payload.type, "type"); + if (type !== "platform_feed" && type !== "keyword" && type !== "account_list" && type !== "saved_search") { + throw new BadRequestException("type must be one of platform_feed|keyword|account_list|saved_search"); + } + const title = requireString(payload.title, "title"); + const position = payload.position === undefined ? undefined : requirePosition(payload.position); + return { + type, + title, + position, + spec: payload.spec === undefined ? {} : payload.spec, + refreshPolicy: payload.refreshPolicy, + }; +} + +function parseUpdate(body: unknown): UpdateColumn { + const payload = requireObject(body, "body"); + const update: UpdateColumn = {}; + if (payload.title !== undefined) update.title = requireString(payload.title, "title"); + if (payload.position !== undefined) update.position = requirePosition(payload.position); + if (payload.spec !== undefined) update.spec = payload.spec; + if (payload.refreshPolicy !== undefined) { + if (typeof payload.refreshPolicy !== "object" || payload.refreshPolicy === null || Array.isArray(payload.refreshPolicy)) { + throw new BadRequestException("refreshPolicy must be an object"); + } + update.refreshPolicy = payload.refreshPolicy; + } + return update; +} + +function requirePosition(value: unknown): number { + if (typeof value !== "number" || !Number.isInteger(value) || value < 0 || value > 2_147_483_647) { + throw new BadRequestException("position must be an integer between 0 and 2147483647"); + } + return value; +} + +function parseIds(value: unknown): string[] { + if (!Array.isArray(value) || value.length === 0) throw new BadRequestException("ids must be a non-empty array of column ids"); + const ids: string[] = []; + for (const id of value) { + if (typeof id !== "string") throw new BadRequestException("ids must contain strings only"); + ids.push(parseId(id)); + } + return ids; +} + +/** + * The keyset cursor: base64url JSON `{ at, key }` where `at` is the item's + * created_at_remote epoch ms and `key` its dedup_key. Mirrors the /feeds + * cursor contract (opaque, ≤4096 chars, position in a total order). + */ +export function decodeColumnCursor(cursor: string): { at: number; key: string } { + try { + if (cursor.length > 4096) throw new Error(); + const parsed: unknown = JSON.parse(Buffer.from(cursor, "base64url").toString()); + if (typeof parsed !== "object" || parsed === null || !("at" in parsed) || !("key" in parsed) || + typeof (parsed as { at: unknown }).at !== "number" || !Number.isFinite((parsed as { at: number }).at) || + typeof (parsed as { key: unknown }).key !== "string") throw new Error(); + return { at: (parsed as { at: number }).at, key: (parsed as { key: string }).key }; + } catch { + throw new BadRequestException("Invalid column cursor"); + } +} diff --git a/apps/api/src/feeds/columns.dependencies.ts b/apps/api/src/feeds/columns.dependencies.ts new file mode 100644 index 0000000..f5af93b --- /dev/null +++ b/apps/api/src/feeds/columns.dependencies.ts @@ -0,0 +1,165 @@ +/** + * Columns wiring (P1): production dependency providers for the ColumnStore + * and ColumnScheduler, mirroring feeds.dependencies.ts. + * + * The scheduler's per-host budget draws from `@tactical-media-broadcast-hub/egress` + * HostBudgets — the SAME `host_budgets` rows every other egress caller uses + * (monitor, bulk import), keyed by the egress `hostBudgetKey` rule (IDNA + * lowercased host + effective port), so feeds polls and monitor polls to one + * host contend for one shared bucket. That is the §3.4 requirement: budgets + * are per-host totals, never per-process. + * + * The egress package is type-imported and loaded natively at runtime (the + * same pattern as live-monitor/poller/egress.ts and virtual-channels/ + * import-egress.ts) because it is ESM-only. + */ +import { Inject, Injectable, Optional } from "@nestjs/common"; +import type { ConnectorConnection } from "@tactical-media-broadcast-hub/core"; +import type { CredentialAccessor, ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk"; +import { existsSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { pool } from "../db/client.js"; +import { PgConnectionStore } from "../connections/connection-store.js"; +import { RespControlRedis, type ControlRedis } from "../media-control/redis.js"; +import { vault } from "../vault/vault.js"; +import { feedRegistry } from "./feed-registry.js"; +import { FeedsService } from "./feeds.service.js"; +import { + ColumnStore, COLUMN_STORE_DEPENDENCIES, type ColumnStoreDependencies, +} from "./column-store.js"; +import { + ColumnScheduler, COLUMN_SCHEDULER_DEPENDENCIES, type ColumnSchedulerDependencies, +} from "./column-scheduler.js"; +import { COLUMNS_REGISTRY } from "./columns.controller.js"; + +type EgressPackage = typeof import("../../../../packages/egress/src/index.js"); + +const nativeImport = new Function("specifier", "return import(specifier)") as (specifier: string) => Promise; + +function resolveEgressPackage(): string { + for (const start of [process.cwd(), __dirname]) { + for (let root = start; ; root = dirname(root)) { + const directory = resolve(root, "packages/egress"); + const files = process.env.NODE_ENV === "production" ? ["dist/index.js"] : ["src/index.ts", "dist/index.js"]; + for (const file of files) { + const candidate = resolve(directory, file); + if (existsSync(candidate)) return candidate; + } + if (dirname(root) === root) break; + } + } + throw new Error("Build packages/egress before starting the columns scheduler"); +} + +/** Process-wide egress binding shared by the columns scheduler. */ +export interface ColumnsEgress { + consume(hostKey: string): Promise; +} + +let sharedEgress: Promise | undefined; + +export function columnsEgress(): Promise { + sharedEgress ??= (async () => { + const specifier = pathToFileURL(resolveEgressPackage()).href; + const pkg = await nativeImport(specifier); + const budgets = new pkg.HostBudgets({ pool, hostClasses: { feeds: { capacity: 3, refillPerInterval: 10 } } }); + return { consume: (hostKey) => budgets.consume(hostKey, "feeds") }; + })(); + return sharedEgress; +} + +/** The API host key for a provider's feed calls, per the egress hostBudgetKey rule. */ +export function feedHostKey(provider: string, connection: ConnectorConnection): string { + const raw = typeof connection.fields.instanceUrl === "string" && connection.fields.instanceUrl !== "" + ? connection.fields.instanceUrl + : providerApiBaseUrl(provider); + try { + const url = new URL(raw); + const host = url.hostname.replace(/^\[|\]$/g, "").replace(/\.$/, "").toLowerCase(); + const port = url.port || (url.protocol === "https:" ? "443" : "80"); + return `${host}:${port}`; + } catch { + return `${provider}:443`; + } +} + +/** Fallback API endpoints per provider (used only when no instanceUrl exists). */ +const PROVIDER_API_HOSTS: Record = { + mastodon: "https://mastodon.social", + bluesky: "https://bsky.app", + youtube: "https://www.googleapis.com", + reddit: "https://www.reddit.com", + telegram: "https://api.telegram.org", + twitter: "https://api.twitter.com", + "mock-social": "https://mock.social", +}; + +function providerApiBaseUrl(provider: string): string { + return PROVIDER_API_HOSTS[provider] ?? `https://${provider}.example`; +} + +/** The credential accessor feedRegistry needs (shared with feeds.dependencies). */ +function credentials(): CredentialAccessor { + const store = new PgConnectionStore(); + return { + async getAccessToken(connection) { return credentials().getSecret(connection, "access_token"); }, + async getSecret(connection, key) { + const stored = await store.get(connection.workspaceId, connection.id); + if (!stored.credential) throw new Error("Credential missing"); + const bytes = await vault.open(connection.workspaceId, connection.id, stored.credential); + const secrets = JSON.parse(bytes.toString()) as Record; + const value = secrets[key]; + if (typeof value !== "string" || !value) throw new Error("Credential missing"); + return value; + }, + async put(connection, key, value) { + const stored = await store.get(connection.workspaceId, connection.id); + if (!stored.credential) throw new Error("Credential missing"); + const bytes = await vault.open(connection.workspaceId, connection.id, stored.credential); + const secrets = JSON.parse(bytes.toString()) as Record; + secrets[key] = value; + const sealed = await vault.seal(connection.workspaceId, connection.id, Buffer.from(JSON.stringify(secrets))); + await store.rotateCredential(connection.workspaceId, connection.id, stored.credential, sealed); + }, + }; +} + +let sharedRegistry: ConnectorRegistry | undefined; + +/** One registry per process, preserving per-host HTTP token buckets. */ +export function columnsRegistry(): ConnectorRegistry { + sharedRegistry ??= feedRegistry(credentials()); + return sharedRegistry; +} + +/** Provider wiring for FeedsModule (see feeds.module.ts). */ +export const columnsProviders = [ + { provide: COLUMN_STORE_DEPENDENCIES, useFactory: (): ColumnStoreDependencies => { + const store = new PgConnectionStore(); + return { + providerOf: async (workspaceId, connectionId) => { + const rows = await store.list(workspaceId); + const row = rows.find((candidate) => candidate.id === connectionId); + if (row === undefined) throw new Error(`connection ${connectionId} not found`); + return row.provider; + }, + connections: (workspaceId) => store.list(workspaceId), + }; + } }, + { provide: COLUMNS_REGISTRY, useFactory: () => columnsRegistry() }, + { provide: COLUMN_SCHEDULER_DEPENDENCIES, inject: [FeedsService], useFactory: (feeds: FeedsService): ColumnSchedulerDependencies => { + const redis: Pick = new RespControlRedis(process.env.REDIS_URL ?? "redis://127.0.0.1:6379"); + const connectionStore = new PgConnectionStore(); + return { + redis, + registry: columnsRegistry(), + connection: async (workspaceId, id) => (await connectionStore.get(workspaceId, id)).connection, + cacheHomeFeed: (workspaceId, connectionId, items) => feeds.cacheHomeFeed(workspaceId, connectionId, items), + hostKeyOf: feedHostKey, + consumeBudget: (hostKey) => columnsEgress().then((egress) => egress.consume(hostKey)), + random: Math.random, + now: Date.now, + }; + } }, +]; diff --git a/apps/api/src/feeds/columns.integration.test.ts b/apps/api/src/feeds/columns.integration.test.ts new file mode 100644 index 0000000..5860ddf --- /dev/null +++ b/apps/api/src/feeds/columns.integration.test.ts @@ -0,0 +1,376 @@ +/** + * Columns P1 integration tests — real Postgres (migrations ≤ 026 applied), + * RLS-faithful via the app pool, following the video.integration.test pattern. + * + * Covers the DELIVER test list: + * - CRUD + reorder, including RBAC 403 for an ungranted role + * - seed idempotency (re-touch creates no duplicates) + * - cooldown enforcement on the column endpoints + * - cursor survival across a cache flush + * - keyset items pagination (stable order, no skips/dupes across pages) + * - dedup: same remote item from two columns sharing an identity → one row, + * visible in both + */ +import "reflect-metadata"; +import { randomUUID } from "node:crypto"; +import { ForbiddenException, type CallHandler, type ExecutionContext, type INestApplication } from "@nestjs/common"; +import { NestFactory, Reflector } from "@nestjs/core"; +import { Pool } from "pg"; +import { afterAll, beforeAll, afterEach, describe, expect, it, vi } from "vitest"; +import { InMemoryCredentials, type ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk"; +import type { ConnectorConnection, FeedItem } from "@tactical-media-broadcast-hub/core"; +import type { SessionPayload } from "../auth/session.js"; +import { PERMISSION_KEY, RbacGuard } from "../rbac/rbac.guard.js"; +import { injectHttp } from "../connections/http-test-transport.js"; +import { feedRegistry } from "./feed-registry.js"; +import { FeedsService } from "./feeds.service.js"; +import { FeedsController } from "./feeds.controller.js"; +import { ColumnsController, COLUMNS_REGISTRY } from "./columns.controller.js"; +import { ColumnStore } from "./column-store.js"; +import { ColumnScheduler, type ColumnSchedulerDependencies } from "./column-scheduler.js"; +import { MemoryFeedRedis } from "./testing.js"; +import type { ConnectionSummary } from "../connections/connections.service.js"; + +const dbAvailable = process.env.DATABASE_URL !== undefined; +// Setup runs as the table owner (superuser in dev) because memberships has +// FORCE RLS and the seed path needs to insert without a tenant context. +const ownerPool = new Pool({ connectionString: process.env.SUPER_DATABASE_URL ?? process.env.DATABASE_URL }); + +// The app pool is imported lazily where needed; ColumnStore's own fallback +// deps use withTenant from db/client, but these tests always inject explicit +// deps, so no module-scope side-effectful import is required. + +const owner: SessionPayload = { userId: randomUUID(), workspaceId: randomUUID(), role: "Owner", authzEpoch: 1, issuedAt: 0, expiresAt: 9_999_999_999 }; +const viewer: SessionPayload = { ...owner, userId: randomUUID(), role: "Moderator" }; // no streams.columns.manage + +const item = (remoteId: string, minutesAgo: number): FeedItem => ({ + id: remoteId, remoteId, provider: "mock-social", authorHandle: "alice", text: `post ${remoteId}`, + mediaUrls: [], createdAt: new Date(Date.now() - minutesAgo * 60_000), metrics: {}, +}); + +function setupRegistry(items: FeedItem[]): ConnectorRegistry { + const registry = feedRegistry(new InMemoryCredentials()); + const mock = registry.get("mock-social"); + const original = mock.getFeed; + if (original === undefined) throw new Error("mock-social has no feed"); + mock.getFeed = vi.fn(async () => ({ items })); + return registry; +} + +describe.skipIf(!dbAvailable)("columns P1 (live DB)", () => { + let app: INestApplication; + let store: ColumnStore; + let scheduler: ColumnScheduler; + let feeds: FeedsService; + let redis: MemoryFeedRedis; + let connections: Array<{ summary: ConnectionSummary; connection: ConnectorConnection }>; + let items: FeedItem[]; + let granted = true; + let registry: ConnectorRegistry; + + beforeAll(async () => { + // Users first (ids are DB-generated and captured for memberships), then + // workspace + members, in the owner pool which bypasses RLS for setup. + const inserted = await ownerPool.query<{ id: string }>( + `INSERT INTO users (external_id, email, display_name, username) + VALUES ($1,$2,'Columns P1 Owner',$3) RETURNING id`, [`u-${randomUUID()}`, `o-${randomUUID()}@example.com`, `p1o${randomUUID().slice(0, 8)}`]); + (owner as { userId: string }).userId = String(inserted.rows[0]?.id); + const viewerRow = await ownerPool.query<{ id: string }>( + `INSERT INTO users (external_id, email, display_name, username) + VALUES ($1,$2,'Columns P1 Viewer',$3) RETURNING id`, [`u-${randomUUID()}`, `v-${randomUUID()}@example.com`, `p1v${randomUUID().slice(0, 8)}`]); + (viewer as { userId: string }).userId = String(viewerRow.rows[0]?.id); + const workspace = await ownerPool.query( + "INSERT INTO workspaces (name, slug) VALUES ('columns p1', $1) RETURNING id", [`cols-${randomUUID()}`]); + const workspaceId = String(workspace.rows[0]?.id); + // memberships is FORCE RLS: insert inside the workspace's tenant context. + const setupClient = await ownerPool.connect(); + try { + await setupClient.query("BEGIN"); + await setupClient.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]); + await setupClient.query( + "INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1,$2,'Owner'), ($1,$3,'Moderator')", + [workspaceId, owner.userId, viewer.userId]); + await setupClient.query("COMMIT"); + } finally { + setupClient.release(); + } + (owner as { workspaceId: string }).workspaceId = workspaceId; + (viewer as { workspaceId: string }).workspaceId = workspaceId; + + items = [item("i-1", 1), item("i-2", 5), item("i-3", 10), item("i-4", 30), item("i-5", 60)]; + registry = setupRegistry(items); + + // Two feed-capable connections in the workspace: the seeding input. + connections = []; + for (let index = 0; index < 2; index += 1) { + // The DB generates the id (connections has no id column we may set); + // RETURNING it keeps the FK target and the summary in sync. + const inserted = await ownerPool.query<{ id: string; created_at: Date }>( + "INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1,'mock-social',$2) RETURNING id, created_at", [owner.workspaceId, `ch-${index}`]); + const id = String(inserted.rows[0]?.id); + const createdAt = inserted.rows[0]?.created_at as Date; + const summary: ConnectionSummary = { + id, provider: "mock-social", channelId: `ch-${index}`, fields: {}, health: "healthy", + credentialRotatedAt: null, recentPublishes: [], createdAt: createdAt.toISOString(), + }; + const connection: ConnectorConnection = { + id, workspaceId: owner.workspaceId, provider: "mock-social", channelId: `ch-${index}`, + credentialId: { kind: "credential-ref", id }, fields: {}, createdAt, + }; + connections.push({ summary, connection }); + } + + redis = new MemoryFeedRedis(); + store = new ColumnStore({ + providerOf: async (_ws, id) => connections.find((row) => row.summary.id === id)?.summary.provider ?? "mock-social", + connections: async (ws) => ws === owner.workspaceId ? connections.map((row) => row.summary) : [], + }); + feeds = new FeedsService({ + redis, registry, random: () => 0.5, requestsPerSecond: () => 0.5, + workspaces: async () => [owner.workspaceId], + connections: async (ws) => ws === owner.workspaceId ? connections.map((row) => row.summary) : [], + connection: async (ws, id) => { const row = connections.find((c) => c.summary.id === id); if (!row || ws !== owner.workspaceId) throw new Error(); return row.connection; }, + visible: async (s) => s.workspaceId === owner.workspaceId ? connections.map((row) => row.summary.id) : [], + audit: async () => undefined, + }, store); + scheduler = new ColumnScheduler(store, { + redis, registry, + connection: async (ws, id) => { const row = connections.find((c) => row_summary(c) === id); if (!row || ws !== owner.workspaceId) throw new Error(); return row.connection; }, + cacheHomeFeed: async (ws, id, normalized) => { if (ws === owner.workspaceId) await feeds.cacheHomeFeed(ws, id, normalized); }, + hostKeyOf: () => "mock.social:443", + consumeBudget: async () => undefined, + random: () => 0.5, + now: () => Date.now(), + }); + + // Both poll loops stay off: the feeds discovery timer would contend with + // ensureSeeded's workspace row lock (this suite drives polls explicitly). + vi.spyOn(scheduler, "onApplicationBootstrap").mockImplementation(() => undefined); + vi.spyOn(feeds, "onApplicationBootstrap").mockImplementation(() => undefined); + vi.spyOn(RbacGuard.prototype, "canActivate").mockImplementation(async (context: ExecutionContext) => { + const request = context.switchToHttp().getRequest<{ headers: Record; session?: SessionPayload }>(); + if (request.headers.authorization !== "Bearer test-session") throw new ForbiddenException("unauthenticated"); + const session = request.headers["x-test-user"] === "viewer" ? viewer : owner; + const permission = new Reflector().get(PERMISSION_KEY, context.getHandler()); + // The seeded RBAC: streams.columns.manage is Owner/Admin/Editor only. + if (permission === "streams.columns.manage" && !granted) throw new ForbiddenException("missing permission"); + request.session = session; + return true; + }); + app = await NestFactory.create({ + module: class TestModule {}, + controllers: [ColumnsController, FeedsController], + providers: [ + { provide: FeedsService, useValue: feeds }, + { provide: ColumnStore, useValue: store }, + { provide: ColumnScheduler, useValue: scheduler }, + { provide: COLUMNS_REGISTRY, useValue: registry }, + ], + }, { logger: false }); + await app.init(); + }); + + afterEach(() => { granted = true; }); + + afterAll(async () => { + await app?.close(); + if (owner.workspaceId !== undefined) await ownerPool.query("DELETE FROM workspaces WHERE id = $1", [owner.workspaceId]); + await ownerPool.end(); + }); + + const request = (path: string, method = "GET", body?: unknown, user: "owner" | "viewer" = "owner") => + injectHttp(app.getHttpAdapter().getInstance(), `http://api.test${path}`, { + method, headers: { authorization: "Bearer test-session", "x-test-user": user, "content-type": "application/json" }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + + it("seeds one platform_feed column per feed connection, idempotently, with sources", async () => { + const first = await request("/feeds/columns"); + expect(first.status).toBe(200); + const seeded = await first.json() as Array<{ id: string; type: string; title: string; sources: Array<{ sourceKind: string; state: string }> }>; + expect(seeded).toHaveLength(2); + expect(seeded.every((column) => column.type === "platform_feed")).toBe(true); + expect(seeded[0]?.sources).toHaveLength(1); + expect(seeded[0]?.sources[0]?.sourceKind).toBe("home"); + // Titles reproduce the implicit column the UI renders today. + expect(seeded[0]?.title).toContain("·"); + + // Second touch seeds nothing. + const second = await request("/feeds/columns"); + expect(await second.json()).toHaveLength(2); + // And the DB agrees: two columns, two source rows, one per connection. + const rows = await ownerPool.query("SELECT count(*)::int AS c FROM stream_columns WHERE workspace_id = $1", [owner.workspaceId]); + expect(rows.rows[0]?.c).toBe(2); + const sources = await ownerPool.query("SELECT count(*)::int AS c FROM column_sources WHERE workspace_id = $1", [owner.workspaceId]); + expect(sources.rows[0]?.c).toBe(2); + }); + + it("enforces streams.columns.manage on writes while reads stay on draft.view", async () => { + granted = false; + expect((await request("/feeds/columns", "POST", { type: "keyword", title: "x", spec: { query: "q", platforms: ["mock-social"] } }, "viewer")).status).toBe(403); + granted = true; + const created = await request("/feeds/columns", "POST", { type: "keyword", title: "news", spec: { query: "anarch", platforms: ["mock-social"] } }); + expect(created.status).toBe(201); + const column = await created.json() as { id: string; sources: Array<{ sourceKind: string }> }; + expect(column.sources.map((source) => source.sourceKind)).toEqual(["search"]); + // Reads: viewer (no manage permission) still reads columns and items. + expect((await request(`/feeds/columns/${column.id}`, "GET", undefined, "viewer")).status).toBe(200); + granted = false; + expect((await request(`/feeds/columns/${column.id}`, "PATCH", { title: "z" })).status).toBe(403); + expect((await request(`/feeds/columns/${column.id}`, "DELETE")).status).toBe(403); + expect((await request("/feeds/columns/reorder", "POST", { ids: [] })).status).toBe(403); + granted = true; + await request(`/feeds/columns/${column.id}`, "DELETE"); + }); + + it("rejects invalid specs with 422 and never stores them", async () => { + const bad = await request("/feeds/columns", "POST", { type: "keyword", title: "bad", spec: { query: "", platforms: ["mock-social"] } }); + expect(bad.status).toBe(422); + const badConnection = await request("/feeds/columns", "POST", { type: "platform_feed", title: "bad", spec: { connectionId: randomUUID() } }); + expect(badConnection.status).toBe(422); + const count = await ownerPool.query( + "SELECT count(*)::int AS c FROM stream_columns WHERE workspace_id = $1 AND deleted_at IS NULL", [owner.workspaceId]); + expect(count.rows[0]?.c).toBe(2); // only the two seeded columns (the RBAC test's column was soft-deleted) + }); + + it("polls due sources, persists items, and fans out to columns sharing an identity", async () => { + const columns = await (await request("/feeds/columns")).json() as Array<{ id: string; sources: Array<{ id: string }> }>; + const first = columns[0]!; + const outcome = await scheduler.pollWorkspaceDue(owner.workspaceId); + expect(outcome.polled).toBeGreaterThanOrEqual(2); // both seeded home sources + + // Items landed in the store for the polled column. + const page = await store.columnItems(owner.workspaceId, first.id, { limit: 10 }); + expect(page.items.length).toBeGreaterThanOrEqual(1); + expect(page.items.every((stored) => stored.item.remoteId.startsWith("i-"))).toBe(true); + // The source row moved to ready with a next poll scheduled. + const sources = await store.sources(owner.workspaceId, first.id); + expect(sources[0]?.state).toBe("ready"); + expect(sources[0]?.nextPollAt).not.toBeNull(); + }); + + it("keyset pagination: pages in stable order, no skips or duplicates, cursor survives cache flush", async () => { + const columns = await (await request("/feeds/columns")).json() as Array<{ id: string }>; + const columnId = columns[0]!.id; + const seen: string[] = []; + let cursor: string | null = null; + // limit 2 over 5 items = 3 pages (2,2,1) + for (let page = 0; page < 3; page += 1) { + const response = await request(`/feeds/columns/${columnId}/items?limit=2${cursor === null ? "" : `&cursor=${cursor}`}`); + expect(response.status).toBe(200); + const body = await response.json() as { items: Array<{ item: { remoteId: string } }>; nextCursor: string | null }; + for (const stored of body.items) seen.push(stored.item.remoteId); + cursor = body.nextCursor; + if (cursor === null) break; + } + expect(seen.length).toBe(new Set(seen).size); // no duplicates + expect(seen).toHaveLength(items.length); // no skips + + // Cursor survival: flush every cache key, then continue from a cursor + // minted before the flush — the store page must pick up exactly where the + // cache left off because the cursor is a position in the stored order. + redis.entries.clear(); + const beforeFlush = await request(`/feeds/columns/${columnId}/items?limit=1`); + const firstCursor = ((await beforeFlush.json()) as { nextCursor: string | null }).nextCursor; + expect(firstCursor).not.toBeNull(); + redis.entries.clear(); + const afterFlush = await request(`/feeds/columns/${columnId}/items?limit=10&cursor=${firstCursor}`); + expect(afterFlush.status).toBe(200); + const pageItems = (await afterFlush.json()) as { items: Array<{ item: { remoteId: string } }> }; + // Everything strictly after the first item, in keyset order. + expect(pageItems.items.map((stored) => stored.item.remoteId)).toEqual(seen.slice(1)); + }); + + it("cooldown blocks a forced refresh from issuing a second poll", async () => { + const columns = await (await request("/feeds/columns")).json() as Array<{ id: string }>; + const columnId = columns[0]!.id; + const before = registry.get("mock-social").getFeed as ReturnType; + const callsBefore = before.mock.calls.length; + const outcome = await scheduler.pollWorkspaceDue(owner.workspaceId); + expect(outcome.polled).toBe(0); // sources just polled: next_poll_at is in the future + // The HTTP refresh endpoint cannot bypass the persisted floor: the poll + // happened seconds ago, so force-due refuses even though the Redis + // cooldown keys were flushed (cursor-survival test) — next_poll_at is the + // authoritative guard, not the cache. + const refreshed = await request(`/feeds/columns/${columnId}/refresh`, "POST"); + expect(refreshed.status).toBe(201); + const body = await refreshed.json() as { refreshed: number }; + expect(body.refreshed).toBe(0); // DB floor refused the forced poll + expect(before.mock.calls.length).toBe(callsBefore); + }); + + it("dedups the same remote item across columns sharing an identity", async () => { + // Second column on the same connection as the first: same identity. + const columns = await (await request("/feeds/columns")).json() as Array<{ id: string; sources: Array<{ id: string }> }>; + const [first, second] = columns; + expect(first && second).toBeTruthy(); + // Make both due again, then poll: one identity, one connector call. + await ownerPool.query("UPDATE column_sources SET next_poll_at = now() WHERE workspace_id = $1", [owner.workspaceId]); + const getFeed = registry.get("mock-social").getFeed as ReturnType; + const before = getFeed.mock.calls.length; + const outcome = await scheduler.pollWorkspaceDue(owner.workspaceId); + expect(outcome.polled).toBe(2); + expect(getFeed.mock.calls.length).toBe(before + 2); // two source rows, two calls (per-source semantics) + // One feed_items row per remote item — the dedup key is workspace-wide. + const stored = await ownerPool.query("SELECT count(*)::int AS c FROM feed_items WHERE workspace_id = $1", [owner.workspaceId]); + expect(stored.rows[0]?.c).toBe(items.length); + // Both columns surface the same items. + const left = await store.columnItems(owner.workspaceId, first!.id, { limit: 10 }); + const right = await store.columnItems(owner.workspaceId, second!.id, { limit: 10 }); + expect(left.items).toHaveLength(items.length); + expect(right.items).toHaveLength(items.length); + // And a cursor minted on one column pages identically on the other. + expect(left.items.map((row) => row.item.remoteId)).toEqual(right.items.map((row) => row.item.remoteId)); + }); + + it("reorder requires the full id list and rewrites positions densely", async () => { + const columns = await (await request("/feeds/columns")).json() as Array<{ id: string; position: number }>; + const ids = columns.map((column) => column.id); + const partial = await request("/feeds/columns/reorder", "POST", { ids: [ids[0]!] }); + expect(partial.status).toBe(400); + const reordered = await request("/feeds/columns/reorder", "POST", { ids: [...ids].reverse() }); + expect(reordered.status).toBe(201); + const after = await reordered.json() as Array<{ id: string; position: number }>; + expect(after.map((column) => column.id)).toEqual([...ids].reverse()); + expect(after.map((column) => column.position)).toEqual([0, 1]); + }); + + it("import-saved-searches creates idempotent saved_search columns", async () => { + // No search-capable provider is connected (mock-social declares none), so + // the implicit-platforms path correctly skips everything rather than + // inventing a platform the workspace cannot search. + const skipped = await request("/feeds/columns/import-saved-searches", "POST", { queries: ["anarch"] }); + expect(skipped.status).toBe(201); + expect(await skipped.json()).toEqual({ imported: 0, skipped: 1 }); + // Explicit platforms still import (the UI knows which it used locally). + const imported = await request("/feeds/columns/import-saved-searches", "POST", { queries: ["anarch", "mutual aid"], platforms: ["mock-social"] }); + expect(imported.status).toBe(201); + expect(await imported.json()).toEqual({ imported: 2, skipped: 0 }); + // Re-import: idempotent per query. + const again = await request("/feeds/columns/import-saved-searches", "POST", { queries: ["anarch", "squatting"], platforms: ["mock-social"] }); + expect(await again.json()).toEqual({ imported: 1, skipped: 1 }); + const columns = await (await request("/feeds/columns")).json() as Array<{ type: string; title: string }>; + const searches = columns.filter((column) => column.type === "saved_search"); + expect(searches.map((column) => column.title).sort()).toEqual(["anarch", "mutual aid", "squatting"]); + // Unknown platform ids are rejected up front (P0a schema rules). + const bad = await request("/feeds/columns/import-saved-searches", "POST", { queries: ["x"], platforms: ["not-a-provider"] }); + expect(bad.status).toBe(422); + }); + + it("GET /feeds keeps its external shape and gains additive meta.columns", async () => { + const response = await request("/feeds"); + expect(response.status).toBe(200); + const feed = await response.json() as { items: unknown[]; nextCursor: string | null; meta: { configuredFeeds: number; columns?: Array<{ id: string }> } }; + expect(Array.isArray(feed.items)).toBe(true); + expect(feed.meta.configuredFeeds).toBe(2); + expect(feed.meta.columns?.length).toBeGreaterThanOrEqual(2); // seeded columns surface + }); + + it("invalid cursor is a 400 and unknown column a 404", async () => { + expect((await request(`/feeds/columns/${randomUUID()}/items`)).status).toBe(404); + expect((await request("/feeds/columns/not-a-uuid/items")).status).toBe(400); + expect((await request("/feeds/columns/00000000-0000-0000-0000-000000000000/items?cursor=garbage")).status).toBe(400); + }); +}); + +function row_summary(row: { summary: ConnectionSummary }): string { return row.summary.id; } diff --git a/apps/api/src/feeds/feed-normalize.test.ts b/apps/api/src/feeds/feed-normalize.test.ts new file mode 100644 index 0000000..c59dea5 --- /dev/null +++ b/apps/api/src/feeds/feed-normalize.test.ts @@ -0,0 +1,366 @@ +import { randomUUID } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk"; +import type { ConnectorConnection, FeedItem } from "@tactical-media-broadcast-hub/core"; +import { createMockSocialConnector, createMockSocialState } from "../../../../packages/connector-mock-social/src/index.js"; +import { DEFAULT_MEDIA_PROXY, parseProviderHtml, type CustomEmoji, type LinkPreview, type MediaProxyPolicy, type RichTextDocument } from "../../../../packages/safe-content/src/index.js"; +import { astToPlainText, feedAuthorRef, feedItemMedia, normalizeFeedItem, type MediaRef, type NormalizedFeedItem } from "./feed-normalize.js"; +import { FeedsService, feedCacheKey } from "./feeds.service.js"; +import type { FeedDependencies } from "./feeds.dependencies.js"; +import { MemoryFeedRedis } from "./testing.js"; + +/** Per-connector fixtures (design §3.2 provider table) + hostile-HTML cases. */ +interface ProviderFixture { + provider: string; + contentKind?: "text" | "html"; + instanceUrl?: string; + emojis?: readonly CustomEmoji[]; + item: FeedItem; + url?: unknown; + expiresAt?: unknown; + linkPreview?: unknown; +} + +const PROXY: MediaProxyPolicy = DEFAULT_MEDIA_PROXY; +const connection = (fields: Record = {}): ConnectorConnection => ({ + id: "conn-1", workspaceId: "ws-1", provider: "mastodon", channelId: "conn-1", + credentialId: { kind: "credential-ref", id: "cred-1" }, fields, createdAt: new Date(), +}); + +const mastodonStatus: FeedItem = { + id: "1001", provider: "mastodon", remoteId: "1001", authorHandle: "alice@foreso.social", + // The exact HTML shape Mastodon emits: p/br/a, mention + hashtag + invisible + // ellipsis link spans, HTML-escaped ampersands, raw emoji tags. + text: '

A call for @bob & friends

# kicks off example.org/manifes :ok:
line two bolded

', + mediaUrls: ["https://cdn.foreso.social/media/1234.png", "https://cdn.foreso.social/media/clip.mp4"], + createdAt: new Date("2026-10-06T12:00:00.000Z"), + metrics: { reblogs: 2, replies: 1, favourites: 5, broken: Number.NaN, infinite: Number.POSITIVE_INFINITY }, +}; +const xssStatus: FeedItem = { + id: "2002", provider: "mastodon", remoteId: "2002", authorHandle: "mallory@evil.example", + text: '

hi clickpasstabbed

after

', + mediaUrls: ["javascript:alert(5)", "data:text/html,evil", "//protocol-relative.example/x.png", "https://cdn.example/ok.png"], + createdAt: new Date("2026-10-06T12:01:00.000Z"), + metrics: {}, +}; + +const fixturePlainText: ProviderFixture = { + // Plain-text provider source (Bluesky/Telegram-style): auto-linked, no HTML. + provider: "bluesky", + item: { + id: "1", provider: "bluesky", remoteId: "at://did:plc:abc/app.bsky.feed.post/1", + authorHandle: "acampada.bsky.social", mediaUrls: [], createdAt: new Date("2026-10-06T10:00:00.000Z"), + metrics: { likes: 3 }, + text: "Line one\n\n hub https://github.com/eche/acampada26/blob/main/core/search.ts #acampada26 @alice@foreso.social", + }, + url: "https://bsky.app/profile/acampada.bsky.social/post/1", +}; +const fixtureMastodonHtml: ProviderFixture = { + // Mastodon status HTML: allowlist parser, mention + ellipsis link + emoji. + provider: "mastodon", + contentKind: "html", + emojis: [{ shortcode: "ok", url: "https://foreso.social/emoji/ok.png" }], + item: mastodonStatus, + url: "https://foreso.social/@alice/1001", +}; +const fixtureHostileHtml: ProviderFixture = { + // Hostile HTML from a compromised/bogus federated peer. + provider: "mastodon", + contentKind: "html", + instanceUrl: "https://EVIL.example:443/", + item: xssStatus, + url: "javascript:alert(6)", + linkPreview: { url: "javascript:alert(7)", title: "javascript:alert(8)", image: "javascript:alert(9)" }, +}; +const fixtureMastodonLinkPreview: ProviderFixture = { + // Mastodon HTML + card (link-preview) payload as one source row. + provider: "mastodon", + contentKind: "html", + item: { ...mastodonStatus, id: "1002", remoteId: "1002", text: '

with a @bob card

', mediaUrls: [] }, + url: "https://foreso.social/@alice/1002", + linkPreview: { + url: "https://example.org/manifesto", title: "Manifesto <scripts> & plans", + description: "A plan", siteName: "example dot org", image: "https://cdn.example.org/cover.png", + }, +}; +const fixtures: Record<"plainText" | "mastodonHtml" | "hostileHtml" | "mastodonLinkPreview", ProviderFixture> = { + plainText: fixturePlainText, + mastodonHtml: fixtureMastodonHtml, + hostileHtml: fixtureHostileHtml, + mastodonLinkPreview: fixtureMastodonLinkPreview, +}; + +/** No provider CDN URL outside the media-proxy query string, ever. */ +function assertProxyOnly(media: MediaRef[]): void { + const serialized = JSON.stringify(media); + let index = serialized.indexOf("https://"); + while (index !== -1) { + expect(serialized.slice(0, index)).toContain(`${PROXY.proxyPath}?url=`); + index = serialized.indexOf("https://", index + 1); + } + for (const ref of media) expect(ref.proxyUrl.startsWith(`${PROXY.proxyPath}?url=`)).toBe(true); +} + +/** Raw-remnant scan applied to every fixture's AST, plain text and media. */ +function assertZeroRawRemnants(ast: RichTextDocument, plainText: string, media: MediaRef[]): void { + const astJson = JSON.stringify(ast); + const hostile: RegExp[] = [ + /<\s*(script|iframe|style|img|svg|math|template|form|object|embed|applet|frame)\b/i, + /<\s*\/\s*/, /||<\/( [a-zA-Z][a-zA-Z0-9:_-]* )\s*>|<([a-zA-Z][a-zA-Z0-9:_-]*)((?:\s+[^<>]*?)?)(\/?)>/g; +const NAME_RE = /^([a-zA-Z][a-zA-Z0-9:_-]*)\s*$/; + +/** Strip any namespace prefix: "dc:creator" -> "creator". */ +function localName(tag: string): string { + const at = tag.indexOf(":"); + return at > 0 ? tag.slice(at + 1) : tag; +} + +function parseAttrs(raw: string | undefined): Record { + const attrs: Record = {}; + if (!raw) return attrs; + for (const match of raw.matchAll(/([a-zA-Z_:][a-zA-Z0-9_:.-]*)\s*=\s*(?:"([^"]*)"|'([^']*)')/g)) { + attrs[match[1]!.toLowerCase()] = decodeEntities(match[2] ?? match[3] ?? ""); + } + return attrs; +} + +/** + * Tolerant single-pass scan into a flat tree: one regex walks every tag token + * in document order, a stack tracks open elements, text between tags + * accumulates on the current node. Malformed markup (unterminated tags, + * mismatched closes) degrades to whatever structure is recoverable — never + * an exception, because feeds in the wild are routinely broken. + */ +export function parseXml(input: string): XmlNode { + const root: XmlNode = { tag: "#root", attrs: {}, children: [], text: "" }; + const stack: XmlNode[] = [root]; + let cursor = 0; + let textChunk = ""; + const flush = (): void => { + if (textChunk !== "") { + stack[stack.length - 1]!.text += decodeEntities(textChunk); + textChunk = ""; + } + }; + const token = /||<\/([a-zA-Z][a-zA-Z0-9:_-]*)\s*>|<([a-zA-Z][a-zA-Z0-9:_-]*)((?:\s+[^<>]*?)?)(\/?)>/g; + for (const match of input.matchAll(token)) { + textChunk += input.slice(cursor, match.index); + cursor = match.index! + match[0].length; + if (match[1] !== undefined) { textChunk += match[1]; continue; } // CDATA: raw text + if (match[0].startsWith(" node.tag).lastIndexOf(close); + if (at > 0) stack.length = at; // pop to the matching open; stray closes are ignored + continue; + } + // Local-name only (dc:creator -> creator, yt:videoId -> videoid, + // media:thumbnail -> thumbnail): feed namespaces are decorative for the + // fields this extractor reads, and matching on the local name keeps every + // lookup below prefix-agnostic. + const tag = localName(match[3]!.toLowerCase()); + flush(); + const node: XmlNode = { tag, attrs: parseAttrs(match[4]), children: [], text: "" }; + stack[stack.length - 1]!.children.push(node); + if (match[5] !== "/") stack.push(node); // not self-closing: descend + } + textChunk += input.slice(cursor); + flush(); + return root; +} + +function findFirst(node: XmlNode, tag: string): XmlNode | undefined { + for (const child of node.children) { + if (child.tag === tag) return child; + const nested = findFirst(child, tag); + if (nested) return nested; + } + return undefined; +} + +function findAll(node: XmlNode, tag: string): XmlNode[] { + const out: XmlNode[] = []; + const walk = (current: XmlNode): void => { + for (const child of current.children) { + if (child.tag === tag) out.push(child); + walk(child); + } + }; + walk(node); + return out; +} + +/** text of a child tag by name, "" when absent — keeps author lookups flat. */ +function childText(node: XmlNode, tag: string): string { + return textOf(childTag(node, tag)); +} + +/** Raw inner text (entities decoded, whitespace preserved) — CDATA HTML. */ +function rawText(node: XmlNode): string { + return decodeEntities(node.text); +} + +/** Inner text of a node across nested markup, whitespace-collapsed. */ +function textOf(node: XmlNode | undefined): string { + if (!node) return ""; + let out = node.text; + for (const child of node.children) out += " " + textOf(child); + return out.replace(/\s+/g, " ").trim(); +} + +export interface FeedEntry { + id: string; + title: string; + link: string | null; + contentHtml: string | null; + contentText: string | null; + publishedAt: Date | null; + updatedAt: Date | null; + author: string | null; + enclosures: Array<{ url: string; type: string }>; + mediaThumbnails: string[]; + /** youtube atom: yt:videoId */ + videoId: string | null; +} + +export interface ParsedFeed { + type: "rss" | "atom"; + channelTitle: string | null; + channelLink: string | null; + entries: FeedEntry[]; +} + +function childTag(node: XmlNode, tag: string): XmlNode | undefined { + return node.children.find((child) => child.tag === tag); +} + +/** RSS … / Atom author/name text. */ +function authorNameOf(node: XmlNode): string { + const author = childTag(node, "author"); + return author === undefined ? "" : textOf(childTag(author, "name") ?? author); +} + +function linkOf(item: XmlNode, channelLink: string | null): string | null { + // Atom: wins; rel absent is alternate too. + // Self/service links (rel="self") are skipped. RSS: text. + const links = item.children.filter((child) => child.tag === "link"); + for (const link of links) { + const rel = link.attrs.rel; + const href = link.attrs.href; + if (href !== undefined && (rel === undefined || rel === "alternate")) return href; + } + const plain = links.map((link) => textOf(link)).find((text) => text !== ""); + if (plain) return plain; + const guid = childTag(item, "guid"); + if (guid && /^https?:\/\//i.test(guid.text.trim())) return guid.text.trim(); + return channelLink; +} + +function dateOf(...candidates: Array): Date | null { + for (const node of candidates) { + const raw = textOf(node); + if (raw === "") continue; + const parsed = new Date(raw); + if (Number.isFinite(parsed.getTime())) return parsed; + // RFC822 without timezone ("Thu, 09 Oct 2026 12:00:00") is common in the wild + const zoned = new Date(`${raw} GMT`); + if (Number.isFinite(zoned.getTime())) return zoned; + } + return null; +} + +export function parseFeed(xml: string): ParsedFeed { + const root = parseXml(xml); + const rssChannel = findFirst(root, "channel"); + const atomFeed = root.children.find((child) => child.tag === "feed"); + if (rssChannel) { + const channelTitle = textOf(childTag(rssChannel, "title")) || null; + const channelLink = textOf(childTag(rssChannel, "link")) || null; + const entries = findAll(rssChannel, "item").map((item): FeedEntry => { + const encoded = childTag(item, "encoded"); // content:encoded (local name) + const description = childTag(item, "description"); + // A description carrying markup IS provider HTML (CDATA-wrapped in + // practice); plain descriptions stay text so P0c parses them as text. + const descriptionHtml = encoded !== undefined ? textOf(encoded) + : description !== undefined && /<\s*[a-zA-Z]/.test(rawText(description)) ? rawText(description) : null; + const enclosures = item.children.filter((child) => child.tag === "enclosure") + .map((enclosure) => ({ url: enclosure.attrs.url ?? "", type: enclosure.attrs.type ?? "" })) + .filter((entry) => entry.url !== ""); + const group = findFirst(item, "group"); // media:group + const thumbnails = [...findAll(item, "thumbnail"), ...(group ? findAll(group, "thumbnail") : [])] + .map((thumb) => thumb.attrs.url ?? "").filter((url) => url !== ""); + return { + id: textOf(childTag(item, "guid")) || textOf(childTag(item, "link")) || "", + title: textOf(childTag(item, "title")), + link: linkOf(item, channelLink), + contentHtml: descriptionHtml, + contentText: descriptionHtml === null && description !== undefined ? textOf(description) : null, + publishedAt: dateOf(childTag(item, "pubdate"), childTag(item, "date"), childTag(item, "published")), + updatedAt: dateOf(childTag(item, "updated")), + author: textOf(childTag(item, "creator")) || childText(item, "author") || authorNameOf(item) || null, + enclosures, + mediaThumbnails: thumbnails, + videoId: null, + }; + }); + return { type: "rss", channelTitle, channelLink, entries }; + } + if (atomFeed) { + const channelTitle = textOf(childTag(atomFeed, "title")) || null; + const channelLink = linkOf(atomFeed, null); + const entries = findAll(atomFeed, "entry").map((entry): FeedEntry => { + const contentHtml = textOf(childTag(entry, "content")); + const summary = textOf(childTag(entry, "summary")); + const enclosures = entry.children.filter((child) => child.tag === "link" && child.attrs.rel === "enclosure") + .map((link) => ({ url: link.attrs.href ?? "", type: link.attrs.type ?? "" })) + .filter((item) => item.url !== ""); + const videoIdNode = entry.children.find((child) => child.tag === "videoid"); + // yt:videoId is a leaf: raw text only (no nested aggregation). + const group = findFirst(entry, "group"); + const thumbnails = [...findAll(entry, "thumbnail"), ...(group ? findAll(group, "thumbnail") : [])] + .map((thumb) => thumb.attrs.url ?? "").filter((url) => url !== ""); + return { + id: textOf(childTag(entry, "id")) || "", + title: textOf(childTag(entry, "title")), + link: linkOf(entry, channelLink), + contentHtml: contentHtml !== "" ? contentHtml : null, + contentText: contentHtml === "" && summary !== "" ? summary : null, + publishedAt: dateOf(childTag(entry, "published"), childTag(entry, "updated")), + updatedAt: dateOf(childTag(entry, "updated")), + author: authorNameOf(entry) || childText(entry, "name") || null, + enclosures, + mediaThumbnails: thumbnails, + videoId: videoIdNode ? videoIdNode.text.trim() || null : null, + }; + }); + return { type: "atom", channelTitle, channelLink, entries }; + } + return { type: "rss", channelTitle: null, channelLink: null, entries: [] }; +} diff --git a/apps/api/src/intake/adapters/rss.adapter.ts b/apps/api/src/intake/adapters/rss.adapter.ts new file mode 100644 index 0000000..8b4c2cc --- /dev/null +++ b/apps/api/src/intake/adapters/rss.adapter.ts @@ -0,0 +1,78 @@ +/** + * RSS/Atom adapter — conditional GET via ETag / Last-Modified. + * + * The feed is fetched once per poll through the egress client (one budget + * draw); when upstream answers 304 the poll is free and emits nothing, and + * the carried ETag/Last-Modified state rides in intake_sources.state. Text + * lands as sanitized HTML through the P0c allowlist parser (feed summaries + * are HTML in practice), enclosures/thumbnails become harvest candidates. + */ +import { createHash } from "node:crypto"; +import { fetchText, type AdapterContext, type AdapterPollResult, type IntakeAdapter, type RawIntakeItem } from "./adapter.js"; +import { parseFeed, type ParsedFeed } from "./rss-parse.js"; +import { hostClassFor } from "./adapter.js"; +import { KIND_RSS } from "../intake-kinds.js"; +import type { RssParams } from "../intake-params.js"; + +export const MAX_FEED_ITEMS = 100; + +function stateHeaders(state: Record): Record { + const headers: Record = {}; + if (typeof state.etag === "string" && state.etag !== "") headers["if-none-match"] = state.etag; + if (typeof state.lastModified === "string" && state.lastModified !== "") headers["if-modified-since"] = state.lastModified; + return headers; +} + +export function feedToItems(feed: ParsedFeed, publishedFallback: Date): RawIntakeItem[] { + return feed.entries.slice(0, MAX_FEED_ITEMS).map((entry) => { + const media: Array<{ url: string; kind: "image" | "video" }> = []; + for (const enclosure of entry.enclosures) { + if (/^video\//i.test(enclosure.type)) media.push({ url: enclosure.url, kind: "video" }); + else if (/^image\//i.test(enclosure.type) || /\.(?:png|jpe?g|webp|gif)(?:[?#]|$)/i.test(enclosure.url)) media.push({ url: enclosure.url, kind: "image" }); + else if (/^audio\//i.test(enclosure.type)) media.push({ url: enclosure.url, kind: "video" }); + } + for (const thumb of entry.mediaThumbnails.slice(0, 4)) media.push({ url: thumb, kind: "image" }); + const text = entry.contentHtml ?? entry.contentText ?? entry.title; + const scratch = createHash("sha256").update(text).digest("hex").slice(0, 32); + const ref = entry.id || entry.link || `sha256:${scratch}`; + return { + id: entry.id || entry.link || scratch, + remoteId: ref, + remoteRef: ref, + provider: "rss", + authorHandle: entry.author ?? feed.channelTitle ?? "feed", + text, + mediaUrls: media.map((entry_media) => entry_media.url), + createdAt: entry.publishedAt ?? entry.updatedAt ?? publishedFallback, + metrics: {}, + normalize: { contentKind: entry.contentHtml !== null ? "html" : "text", url: entry.link ?? undefined }, + attribution: { + title: entry.title || null, + url: entry.link ?? null, + author: entry.author ?? null, + channelTitle: feed.channelTitle, + provider: "rss", + ...(entry.publishedAt ? { publishedAt: entry.publishedAt.toISOString() } : {}), + }, + media, + }; + }); +} + +export class RssAdapter implements IntakeAdapter { + readonly kind = KIND_RSS; + + async poll(context: AdapterContext): Promise { + const params = context.params as RssParams; + const hostClass = hostClassFor(KIND_RSS); + const { text, response, unchanged } = await fetchText(context.fetch, params.url, hostClass, stateHeaders(context.state)); + const state: Record = { ...context.state }; + const etag = response.headers.get("etag"); + const lastModified = response.headers.get("last-modified"); + if (etag !== null) state.etag = etag; + if (lastModified !== null) state.lastModified = lastModified; + if (unchanged) return { items: [], state, unchanged: true }; + const feed = parseFeed(text); + return { items: feedToItems(feed, new Date()), state }; + } +} diff --git a/apps/api/src/intake/adapters/web-watch.adapter.ts b/apps/api/src/intake/adapters/web-watch.adapter.ts new file mode 100644 index 0000000..96881bb --- /dev/null +++ b/apps/api/src/intake/adapters/web-watch.adapter.ts @@ -0,0 +1,100 @@ +/** + * Page-watch adapter — the changedetection.io pattern in ~100 lines + * (research §1.8): fetch the page through the egress budget, normalize the + * text (strip scripts/styles, collapse whitespace, strip rotating + * timestamps/CSRF nonces), hash, and emit an item only when the hash moves. + * + * Plain fetch only — no headless browser (per the task's "no headless" + * bound); a page that needs JS renders as its static DOM, which is the right + * default for press pages and gazettes. + */ +import { createHash } from "node:crypto"; +import { fetchText, hostClassFor, type AdapterContext, type AdapterPollResult, type IntakeAdapter, type RawIntakeItem } from "./adapter.js"; +import { KIND_WEB_WATCH } from "../intake-kinds.js"; +import type { WebWatchParams } from "../intake-params.js"; + +/** + * Diff normalization: what changes on a page without changing its content — + * numbers (counts, timestamps), long hex/base64 tokens (CSRF, cache busters). + * Hashing the normalized text is what keeps phantom diffs out of the archive. + */ +export function normalizePageText(html: string, selector?: string): string { + let work = html + .replace(//gi, " ") + .replace(//gi, " ") + .replace(//gi, " ") + .replace(//g, " ") + .replace(/<[^>]+>/g, " ") + .replace(/ /gi, " ") + .replace(/&/gi, "&") + .replace(/</gi, "<") + .replace(/>/gi, ">") + .replace(/"/gi, '"') + .replace(/�?39;/g, "'"); + if (selector !== undefined && selector.trim() !== "") { + // Cheap id/class containment: keep only lines containing the selector's + // leaf token when the selector looks like #id or .class (a full CSS + // engine is out of scope for phase 1; a wrong selector degrades to the + // whole page's diff, which is still correct — just noisier). + const leaf = selector.trim().split(/[ >]+/).at(-1) ?? ""; + const match = /^([#.])([\w-]+)$/.exec(leaf); + if (match) { + // Extract the region around the marker token from the raw HTML instead. + const marker = match[1] === "#" ? `id="${match[2]}"` : `class="${match[2]}"`; + const at = html.toLowerCase().indexOf(marker.toLowerCase()); + if (at >= 0) { + const from = html.lastIndexOf("<", at); + const to = html.indexOf("", at); + work = normalizePageText(html.slice(from > 0 ? from : 0, to > 0 ? to : undefined)); + } + } + } + return work + .replace(/\b[0-9]{2,}\b/g, " ") // counts, epochs, timestamps + .replace(/\b[0-9a-f]{16,}\b/gi, " ") // CSRF tokens, digests + .replace(/\s+/g, " ") + .trim(); +} + +export function pageHash(normalized: string): string { + return createHash("sha256").update(normalized).digest("hex"); +} + +export class WebWatchAdapter implements IntakeAdapter { + readonly kind = KIND_WEB_WATCH; + + async poll(context: AdapterContext): Promise { + const params = context.params as WebWatchParams; + const { text, unchanged } = await fetchText(context.fetch, params.url, hostClassFor(KIND_WEB_WATCH), { + "cache-control": "no-cache", + }); + if (unchanged) return { items: [], state: context.state, unchanged: true }; + const normalized = normalizePageText(text, params.selector); + const hash = pageHash(normalized); + if (typeof context.state.contentHash === "string" && context.state.contentHash === hash) { + return { items: [], state: context.state, unchanged: true }; + } + const now = new Date(); + const items: RawIntakeItem[] = []; + if (typeof context.state.contentHash === "string") { + // First fetch seeds the baseline; only a CHANGED page (second fetch + // onward) emits — a new subscription never manufactures an item for a + // page that was already in that state. + items.push({ + id: hash.slice(0, 24), + remoteId: hash.slice(0, 24), + remoteRef: `${params.url}#${hash.slice(0, 16)}`, + provider: "web", + authorHandle: new URL(params.url).hostname, + text: normalized.slice(0, 8_000), + mediaUrls: [], + createdAt: now, + metrics: {}, + normalize: { contentKind: "text", url: params.url }, + attribution: { url: params.url, provider: "web", changedAt: now.toISOString(), diffHash: hash }, + media: [], + }); + } + return { items, state: { ...context.state, contentHash: hash, lastChangedAt: now.toISOString() } }; + } +} diff --git a/apps/api/src/intake/adapters/youtube.adapter.ts b/apps/api/src/intake/adapters/youtube.adapter.ts new file mode 100644 index 0000000..625a917 --- /dev/null +++ b/apps/api/src/intake/adapters/youtube.adapter.ts @@ -0,0 +1,59 @@ +/** + * YouTube channel adapter — the channel Atom feed, no key, no quota + * (verified live, research §1.1/§1.6: `feeds/videos.xml?channel_id=…` + * returns Atom with yt:videoId, media thumbnails; 15-entry window, ~15 min + * freshness → poll ≤ every 10 min). The Data API is deliberately NOT used + * here: search.list costs 100 units against a 10k/day project budget while + * the feed is free. + */ +import { fetchText, hostClassFor, type AdapterContext, type AdapterPollResult, type IntakeAdapter, type RawIntakeItem } from "./adapter.js"; +import { parseFeed } from "./rss-parse.js"; +import { KIND_YOUTUBE_CHANNEL, YOUTUBE_FEED_URL } from "../intake-kinds.js"; +import type { YouTubeChannelParams } from "../intake-params.js"; + +export class YouTubeChannelAdapter implements IntakeAdapter { + readonly kind = KIND_YOUTUBE_CHANNEL; + + async poll(context: AdapterContext): Promise { + const params = context.params as YouTubeChannelParams; + const { text, response, unchanged } = await fetchText(context.fetch, YOUTUBE_FEED_URL(params.channelId), hostClassFor(KIND_YOUTUBE_CHANNEL)); + const state: Record = { ...context.state }; + const etag = response.headers.get("etag"); + if (etag !== null) state.etag = etag; + if (unchanged) return { items: [], state, unchanged: true }; + const feed = parseFeed(text); + const seen = new Set(Array.isArray(state.seenVideoIds) + ? state.seenVideoIds.filter((id): id is string => typeof id === "string") : []); + const items: RawIntakeItem[] = []; + for (const entry of feed.entries) { + const videoId = entry.videoId ?? (entry.id.startsWith("yt:video:") ? entry.id.slice("yt:video:".length) : null); + if (videoId === null || seen.has(videoId)) continue; + const watchUrl = `https://www.youtube.com/watch?v=${videoId}`; + items.push({ + id: videoId, + remoteId: `yt:video:${videoId}`, + remoteRef: `yt:video:${videoId}`, + provider: "youtube", + authorHandle: entry.author ?? feed.channelTitle ?? params.channelId, + text: entry.title, + mediaUrls: entry.mediaThumbnails.slice(0, 1), + createdAt: entry.publishedAt ?? entry.updatedAt ?? new Date(), + metrics: {}, + normalize: { contentKind: "text", url: watchUrl }, + attribution: { + title: entry.title || null, + url: watchUrl, + channelTitle: feed.channelTitle, + channelId: params.channelId, + videoId, + provider: "youtube", + }, + // metadata-only default: the thumbnail is harvested, never the video. + media: entry.mediaThumbnails.slice(0, 1).map((url) => ({ url, kind: "image" as const })), + }); + seen.add(videoId); + } + state.seenVideoIds = [...seen].slice(-30); + return { items, state }; + } +} diff --git a/apps/api/src/intake/intake-egress.ts b/apps/api/src/intake/intake-egress.ts new file mode 100644 index 0000000..1bfbb51 --- /dev/null +++ b/apps/api/src/intake/intake-egress.ts @@ -0,0 +1,108 @@ +/** + * Intake egress binding — THE single outbound surface for every intake + * adapter (task requirement 2: all outbound through packages/egress per-host + * budgets). + * + * Same shape as live-monitor/poller/egress.ts and virtual-channels/ + * import-egress.ts: the egress package is ESM-only while the API emits + * CommonJS, so it is type-imported (erased at build) and loaded natively at + * runtime. Every request drawn by an adapter goes through `egressFetch`, + * which means SSRF policy (private-address denial, scheme/port policy), the + * global timeout, the response-size cap and — the point — the shared + * `host_budgets` rows all apply. Two replicas polling mastodon.social contend + * for one bucket; a 429 parks that host for every caller. + * + * `IntakeHttpError` carries the status + headers so adapters can classify + * 304-not-modified (free), 429/Retry-After (budget/park) and 5xx (backoff) + * the way the monitor poller's BackoffPollDriver does. + */ +import { existsSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { pool } from "../db/client.js"; +import { INTAKE_HOST_CLASSES } from "./intake-kinds.js"; +// Type-only (erased in the CommonJS build); loaded natively below. +import type { createEgressClient, EgressError, EgressPolicy } from "../../../../packages/egress/src/index.js"; + +type EgressPackage = typeof import("../../../../packages/egress/src/index.js"); + +export interface IntakeEgress { + egressFetch: ReturnType["egressFetch"]; + policy: EgressPolicy; + isEgressError(error: unknown): error is EgressError; +} + +/** Response body cap for feed/page/JSON fetches (not media downloads). */ +export const INTAKE_MAX_BYTES = 8 * 1024 * 1024; +export const INTAKE_TIMEOUT_MS = 20_000; +/** Media harvest ceiling per file (research §2.5 default image cap). */ +export const INTAKE_MEDIA_MAX_BYTES = 50 * 1024 * 1024; + +const nativeImport = new Function("specifier", "return import(specifier)") as (specifier: string) => Promise; +let shared: Promise | undefined; + +function resolveEgressPackage(): string { + for (const start of [process.cwd(), __dirname]) { + for (let root = start; ; root = dirname(root)) { + const directory = resolve(root, "packages/egress"); + const files = process.env.NODE_ENV === "production" ? ["dist/index.js"] : ["src/index.ts", "dist/index.js"]; + for (const file of files) { + const candidate = resolve(directory, file); + if (existsSync(candidate)) return candidate; + } + if (dirname(root) === root) break; + } + } + throw new Error("Build packages/egress before starting intake"); +} + +/** Process-wide egress client shared by every intake adapter. */ +export function intakeEgress(): Promise { + shared ??= (async () => { + const pkg = await nativeImport(pathToFileURL(resolveEgressPackage()).href); + const client = pkg.createEgressClient({ + pool, + timeoutMs: INTAKE_TIMEOUT_MS, + maxResponseBytes: INTAKE_MAX_BYTES, + hostClasses: INTAKE_HOST_CLASSES, + }); + return { ...client, isEgressError: (error: unknown): error is EgressError => error instanceof pkg.EgressError }; + })(); + return shared; +} + +/** Non-2xx/3xx egress response surfaced with its headers (Retry-After, ETag…). */ +export class IntakeHttpError extends Error { + constructor(readonly status: number, readonly headers: Headers, readonly url: string) { + super(`intake HTTP ${status}`); + this.name = "IntakeHttpError"; + } + get notModified(): boolean { return this.status === 304; } + get gone(): boolean { return this.status === 404 || this.status === 410; } + get rateLimited(): boolean { return this.status === 429; } + get transient(): boolean { return this.status === 429 || this.status >= 500; } +} + +/** Retry-After as a lower-bound delay in ms, including HTTP-date form. */ +export function intakeRetryAfterMs(headers: Headers, now: number): number { + const raw = headers.get("retry-after"); + if (raw === null) return 0; + if (/^\d+(?:\.\d+)?$/.test(raw.trim())) { + const delay = Number(raw) * 1000; + return Number.isFinite(delay) ? delay : 0; + } + const deadline = Date.parse(raw); + return Number.isFinite(deadline) ? Math.max(0, deadline - now) : 0; +} + +/** Reads a response body to a string under the shared size cap. */ +export async function readBodyText(response: Response): Promise { + return response.text(); +} + +/** Reads a response body to a Buffer under the media cap, before S3 put. */ +export async function readBodyBytes(response: Response): Promise { + const bytes = Buffer.from(await response.arrayBuffer()); + if (bytes.byteLength > INTAKE_MEDIA_MAX_BYTES) throw new Error("intake media exceeds size cap"); + return bytes; +} diff --git a/apps/api/src/intake/intake-ingest.ts b/apps/api/src/intake/intake-ingest.ts new file mode 100644 index 0000000..51b6a43 --- /dev/null +++ b/apps/api/src/intake/intake-ingest.ts @@ -0,0 +1,122 @@ +/** + * The intake ingest funnel: raw adapter items → P0c normalize → content-hash + * → idempotent upsert into intake_items (027). + * + * Every item passes `normalizeFeedItem` (task requirement 3), so `content` + * lands as the closed safe-content AST — never raw provider HTML — and media + * as proxy-minted refs. `text` (the FTS corpus) is derived from the AST via + * `astToPlainText`, never the raw provider string. + * + * `content_hash` is the schema's dedup key — UNIQUE (workspace_id, + * content_hash): sha256 of the NFKC + whitespace-collapsed AST plain text + * plus the sorted media origin identity. The same statement arriving via RSS + * and a Mastodon boost (or the same URL watched twice) hashes equal and + * stores once; first writer wins, later deliveries no-op. + */ +import { createHash } from "node:crypto"; +import type { PoolClient } from "pg"; +import { normalizeFeedItem, astToPlainText, type MediaRef, type NormalizedFeedItem } from "../feeds/feed-normalize.js"; +import { connectionFor, type RawIntakeItem } from "./adapters/adapter.js"; +import { PROVIDER_BY_KIND, type IntakeKind } from "./intake-kinds.js"; +import type { IntakeParams } from "./intake-params.js"; +import type { HarvestedMedia } from "./media-harvest.js"; + +export type IntakeMediaRef = MediaRef & { + /** S3 content-addressed key when the harvest stored the origin file. */ + storageKey?: string; + hash?: string; + sizeBytes?: number; + contentType?: string; + harvestStatus?: "stored" | "skipped" | "too_large" | "failed"; + originUrl?: string; +}; + +export interface NormalizeOutcome { + normalized: NormalizedFeedItem | null; + plainText: string; + contentHash: string; +} + +/** NFKC + whitespace-collapse, per the 027 content_hash note. */ +export function collapseText(plainText: string): string { + return plainText.normalize("NFKC").replace(/\s+/g, " ").trim(); +} + +/** sha256(collapsed plain text + sorted unique media origin URLs). */ +export function itemContentHash(plainText: string, mediaUrls: readonly string[]): string { + const origins = [...new Set(mediaUrls.filter((url) => typeof url === "string" && url !== ""))].sort(); + return createHash("sha256") + .update(JSON.stringify([collapseText(plainText), origins])) + .digest("hex"); +} + +/** Normalize one raw item; normalized = null when the funnel's validation drops it. */ +export function normalizeRawItem(raw: RawIntakeItem, kind: IntakeKind, params: IntakeParams["params"], + workspaceId: string, sourceId: string): NormalizeOutcome { + const connection = connectionFor(kind, params, sourceId, workspaceId); + const normalized = normalizeFeedItem(raw, connection, raw.normalize); + if (normalized === null) return { normalized: null, plainText: "", contentHash: "" }; + const plainText = astToPlainText(normalized.content); + return { normalized, plainText, contentHash: itemContentHash(plainText, raw.mediaUrls) }; +} + +/** Merge the P0c proxied media refs with the harvest outcome per origin URL. */ +export function mergeMediaRefs(normalized: NormalizedFeedItem, harvested: HarvestedMedia[]): IntakeMediaRef[] { + const byUrl = new Map(harvested.map((entry) => [entry.url, entry])); + return normalized.media.map((ref): IntakeMediaRef => { + const origin = originOfProxyUrl(ref.proxyUrl); + const hit = origin === null ? undefined : byUrl.get(origin); + if (!hit) return ref; + return { + ...ref, + originUrl: hit.url, + ...(hit.status === "stored" ? { + storageKey: hit.storageKey, + hash: hit.hash, + sizeBytes: hit.sizeBytes, + contentType: hit.contentType, + harvestStatus: "stored" as const, + } : { harvestStatus: hit.status }), + }; + }); +} + +/** Extracts the origin URL back out of a media-proxy URL. */ +export function originOfProxyUrl(proxyUrl: string): string | null { + const marker = "url="; + const at = proxyUrl.indexOf(marker); + if (at < 0) return null; + try { return decodeURIComponent(proxyUrl.slice(at + marker.length)); } catch { return null; } +} + +export interface UpsertResult { + inserted: number; + unchanged: number; +} + +/** + * Idempotent upsert inside the caller's tenant transaction. The conflict + * target is the content-addressed key — (workspace_id, content_hash) — so a + * re-poll that yields the same content writes nothing, and cross-source + * duplicates (RSS + Mastodon boost of the same statement) collapse too. + */ +export async function upsertItems(client: PoolClient, workspaceId: string, sourceId: string, kind: IntakeKind, + rows: Array<{ raw: RawIntakeItem; outcome: NormalizeOutcome; harvested: HarvestedMedia[] }>): Promise { + const result: UpsertResult = { inserted: 0, unchanged: 0 }; + for (const entry of rows) { + const { normalized, plainText, contentHash } = entry.outcome; + if (normalized === null) continue; + const mediaRefs = mergeMediaRefs(normalized, entry.harvested); + const attribution: Record = { ...entry.raw.attribution, provider: PROVIDER_BY_KIND[kind] }; + const inserted = await client.query( + `INSERT INTO intake_items (workspace_id, source_id, remote_ref, content_hash, published_at, + text, content, media_refs, attribution) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) + ON CONFLICT (workspace_id, content_hash) DO NOTHING`, + [workspaceId, sourceId, entry.raw.remoteRef, contentHash, normalized.createdAt.toISOString(), + plainText, JSON.stringify(normalized.content), JSON.stringify(mediaRefs), JSON.stringify(attribution)]); + if ((inserted.rowCount ?? 0) > 0) result.inserted += 1; + else result.unchanged += 1; + } + return result; +} diff --git a/apps/api/src/intake/intake-kinds.ts b/apps/api/src/intake/intake-kinds.ts new file mode 100644 index 0000000..dd9adaa --- /dev/null +++ b/apps/api/src/intake/intake-kinds.ts @@ -0,0 +1,128 @@ +/** + * Intake source kinds (migration 027 CHECK) and the per-kind maps the rest of + * the module keys off. + * + * This file is the ONE place provider-named string literals live outside the + * connector packages — the same standing as archive's ARCHIVE_PROVIDER + * boundary: the intake adapters ARE the provider integration surface for + * keyless public sources, so naming the platform they adapt is the boundary, + * not provider branching. Everything else in apps/api/src/intake references + * these constants; nothing else declares the literals. + */ +/* eslint-disable no-restricted-syntax -- intake kinds name the platform they adapt; this file is the provider boundary (archive ARCHIVE_PROVIDER pattern). */ + +export const INTAKE_KINDS = [ + "rss", "mastodon_tag", "mastodon_account", "bluesky_keyword", + "telegram_channel", "peertube_channel", "youtube_channel", "web_watch", +] as const; + +export type IntakeKind = (typeof INTAKE_KINDS)[number]; + +export function isIntakeKind(value: unknown): value is IntakeKind { + return typeof value === "string" && (INTAKE_KINDS as readonly string[]).includes(value); +} + +export const KIND_RSS: IntakeKind = "rss"; +export const KIND_MASTODON_TAG: IntakeKind = "mastodon_tag"; +export const KIND_MASTODON_ACCOUNT: IntakeKind = "mastodon_account"; +export const KIND_BLUESKY_KEYWORD: IntakeKind = "bluesky_keyword"; +export const KIND_TELEGRAM_CHANNEL: IntakeKind = "telegram_channel"; +export const KIND_PEERTUBE_CHANNEL: IntakeKind = "peertube_channel"; +export const KIND_YOUTUBE_CHANNEL: IntakeKind = "youtube_channel"; +export const KIND_WEB_WATCH: IntakeKind = "web_watch"; + +/** Kinds whose default posture is push-only (poll_interval_sec = 0, never due). */ +export const PUSH_ONLY_KINDS: readonly IntakeKind[] = [KIND_BLUESKY_KEYWORD, KIND_TELEGRAM_CHANNEL]; + +/** + * The provider id handed to the P0c `normalizeFeedItem` funnel, so intake + * items carry the same canonical `{provider}:{handle@host}` authorRef shape + * every other feed item does. + */ +export const PROVIDER_BY_KIND: Record = { + rss: "rss", + mastodon_tag: "mastodon", + mastodon_account: "mastodon", + bluesky_keyword: "bluesky", + telegram_channel: "telegram", + peertube_channel: "peertube", + youtube_channel: "youtube", + web_watch: "web", +}; + +/** + * Default poll cadence per kind (seconds), from the firehose research source + * inventory: RSS 5–15 min, YouTube channel RSS ≤10 min (15-entry window), + * Mastodon 300/5min budget comfortably supports 60 s hashtags, PeerTube 10 min, + * page-watch hourly. 0 = push-only (never due; the telegram webhook bridge and + * the Jetstream seam park there). + */ +export const DEFAULT_POLL_INTERVAL_SEC: Record = { + rss: 900, + mastodon_tag: 300, + mastodon_account: 300, + bluesky_keyword: 0, + telegram_channel: 0, + peertube_channel: 600, + youtube_channel: 600, + web_watch: 3_600, +}; + +/** + * Egress host class per kind — the budget row class every outbound request of + * that adapter draws from (packages/egress HostBudgets, shared host_budgets + * rows deployment-wide). intake-social runs at capacity 3 / refill 10 per 10 s + * = 1 req/s ceiling = exactly the verified Mastodon 300/5min-per-IP budget. + */ +export const HOST_CLASS_BY_KIND: Record = { + rss: "intake-rss", + mastodon_tag: "intake-social", + mastodon_account: "intake-social", + bluesky_keyword: "intake-jetstream", + telegram_channel: "intake-telegram", + peertube_channel: "intake-peertube", + youtube_channel: "intake-rss", + web_watch: "intake-web", +}; + +/** Host-class budget shapes handed to createEgressClient. */ +export const INTAKE_HOST_CLASSES = { + "intake-rss": { capacity: 3, refillPerInterval: 10 }, + "intake-social": { capacity: 3, refillPerInterval: 10 }, + "intake-peertube": { capacity: 3, refillPerInterval: 5 }, + "intake-web": { capacity: 2, refillPerInterval: 2 }, + "intake-telegram": { capacity: 3, refillPerInterval: 10 }, + "intake-jetstream": { capacity: 1, refillPerInterval: 1 }, +} as const; + +/** + * Media policy defaults per kind (research §2.5): images are cheap and safe to + * harvest; full video files are opt-in per source because YouTube-class full + * harvest is the storage wildcard. + */ +export type MediaPolicy = "metadata" | "images" | "full"; + +export const DEFAULT_MEDIA_POLICY: Record = { + rss: "images", + mastodon_tag: "images", + mastodon_account: "images", + bluesky_keyword: "metadata", + telegram_channel: "metadata", + peertube_channel: "images", + youtube_channel: "metadata", + web_watch: "images", +}; + +/** + * Public endpoints the adapters poll. Verified keyless/live in the firehose + * research appendix (2026-10-09): YouTube channel Atom feed (no key, no + * quota), the Telegram Bot API base. + */ +export const YOUTUBE_FEED_URL = (channelId: string): string => + `https://www.youtube.com/feeds/videos.xml?channel_id=${encodeURIComponent(channelId)}`; + +export const TELEGRAM_API_ORIGIN = "https://api.telegram.org"; + +export const YOUTUBE_ORIGIN = "https://www.youtube.com"; + +/* eslint-enable no-restricted-syntax */ diff --git a/apps/api/src/intake/intake-params.ts b/apps/api/src/intake/intake-params.ts new file mode 100644 index 0000000..bb1bc6c --- /dev/null +++ b/apps/api/src/intake/intake-params.ts @@ -0,0 +1,169 @@ +/** + * Per-kind source params: validation, canonicalization, and the params_hash + * identity half of UNIQUE (workspace_id, kind, params_hash) (migration 027). + * + * The DB only backstops "params is an object" — everything shape-related is + * validated here so a bad subscription is a 400/422 at the API edge, never a + * CHECK violation at insert time. Canonicalization is deterministic and total + * over the validated shape (sorted keys, trimmed strings, normalized URLs), so + * two subscriptions that differ only by `https://x/` vs `https://x` collide on + * the unique key instead of double-polling one feed. + */ +import { createHash } from "node:crypto"; +import { BadRequestException } from "@nestjs/common"; +import { + isIntakeKind, KIND_MASTODON_ACCOUNT, KIND_MASTODON_TAG, KIND_PEERTUBE_CHANNEL, + KIND_RSS, KIND_TELEGRAM_CHANNEL, KIND_WEB_WATCH, KIND_YOUTUBE_CHANNEL, + TELEGRAM_API_ORIGIN, YOUTUBE_ORIGIN, type IntakeKind, +} from "./intake-kinds.js"; + +export interface RssParams { url: string } +export interface MastodonTagParams { instance: string; tag: string } +export interface MastodonAccountParams { instance: string; accountId: string; handle?: string } +export interface BlueskyKeywordParams { keywords: string[] } +export interface TelegramChannelParams { channelId: string } +export interface PeerTubeChannelParams { instance: string; handle: string } +export interface YouTubeChannelParams { channelId: string } +export interface WebWatchParams { url: string; selector?: string } + +export type IntakeParams = + | { kind: typeof KIND_RSS; params: RssParams } + | { kind: typeof KIND_MASTODON_TAG; params: MastodonTagParams } + | { kind: typeof KIND_MASTODON_ACCOUNT; params: MastodonAccountParams } + | { kind: "bluesky_keyword"; params: BlueskyKeywordParams } + | { kind: typeof KIND_TELEGRAM_CHANNEL; params: TelegramChannelParams } + | { kind: typeof KIND_PEERTUBE_CHANNEL; params: PeerTubeChannelParams } + | { kind: typeof KIND_YOUTUBE_CHANNEL; params: YouTubeChannelParams } + | { kind: typeof KIND_WEB_WATCH; params: WebWatchParams }; + +const URL_MAX = 2_048; +const STRING_MAX = 512; +const TAG = /^[\p{L}\p{N}_]{1,64}$/u; +const ACCOUNT_ID = /^[0-9]{1,19}$/; +const TELEGRAM_CHANNEL_ID = /^-?\d{3,}$/; +const YOUTUBE_CHANNEL = /^UC[\p{L}\p{N}_-]{20,24}$/u; + +function isObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function requireString(value: unknown, field: string, max = STRING_MAX): string { + if (typeof value !== "string") throw new BadRequestException(`${field} must be a string`); + const trimmed = value.trim(); + if (trimmed === "") throw new BadRequestException(`${field} must not be empty`); + if (trimmed.length > max) throw new BadRequestException(`${field} must be at most ${max} characters`); + return trimmed; +} + +/** Origin-only instance URL (`https://mastodon.social`) — never a path/query. */ +function requireInstance(value: unknown, field: string): string { + const raw = requireString(value, field, 2_048); + const withScheme = /^https?:\/\//i.test(raw) ? raw : `https://${raw}`; + let url: URL; + try { url = new URL(withScheme); } catch { throw new BadRequestException(`${field} must be a valid host or URL`); } + if (url.protocol !== "http:" && url.protocol !== "https:") throw new BadRequestException(`${field} must be http(s)`); + if (!url.hostname.includes(".")) throw new BadRequestException(`${field} must be a public host`); + return `${url.protocol}//${url.hostname.toLowerCase()}${url.port ? `:${url.port}` : ""}`; +} + +function requireHttpUrl(value: unknown, field: string): string { + const raw = requireString(value, field, URL_MAX); + let url: URL; + try { url = new URL(raw); } catch { throw new BadRequestException(`${field} must be an absolute URL`); } + if (url.protocol !== "http:" && url.protocol !== "https:") throw new BadRequestException(`${field} must be an http(s) URL`); + if (url.username !== "" || url.password !== "") throw new BadRequestException(`${field} must not carry credentials`); + if (!url.hostname.includes(".")) throw new BadRequestException(`${field} must be a public host`); + url.hash = ""; + return url.toString(); +} + +/** Parse + canonicalize the per-kind params object. Throws BadRequest on shape. */ +export function parseIntakeParams(kind: unknown, raw: unknown): IntakeParams { + if (!isIntakeKind(kind)) throw new BadRequestException("kind must be a supported intake kind"); + if (!isObject(raw)) throw new BadRequestException("params must be an object"); + switch (kind) { + case KIND_RSS: + return { kind, params: { url: requireHttpUrl(raw.url, "params.url") } }; + case KIND_MASTODON_TAG: { + const tag = requireString(raw.tag, "params.tag", 64); + if (!TAG.test(tag)) throw new BadRequestException("params.tag must be 1-64 word characters"); + return { kind, params: { instance: requireInstance(raw.instance, "params.instance"), tag: tag.toLowerCase() } }; + } + case KIND_MASTODON_ACCOUNT: { + const accountId = requireString(raw.accountId, "params.accountId", 19); + if (!ACCOUNT_ID.test(accountId)) throw new BadRequestException("params.accountId must be the numeric account id"); + const params: MastodonAccountParams = { instance: requireInstance(raw.instance, "params.instance"), accountId }; + const handle = raw.handle; + if (handle !== undefined && handle !== null && handle !== "") { + params.handle = requireString(handle, "params.handle", 64).replace(/^@+/, "").toLowerCase(); + } + return { kind, params }; + } + case "bluesky_keyword": { + const keywords = raw.keywords; + if (!Array.isArray(keywords) || keywords.length === 0 || keywords.length > 32) { + throw new BadRequestException("params.keywords must be a non-empty array (max 32)"); + } + return { kind, params: { keywords: keywords.map((k) => requireString(k, "params.keywords[]", 100).toLowerCase()) } }; + } + case KIND_TELEGRAM_CHANNEL: { + const channelId = requireString(raw.channelId, "params.channelId", 32); + if (!TELEGRAM_CHANNEL_ID.test(channelId)) throw new BadRequestException("params.channelId must be a numeric Telegram chat id"); + return { kind, params: { channelId } }; + } + case KIND_PEERTUBE_CHANNEL: { + const handle = requireString(raw.handle, "params.handle", 253).replace(/^@+/, ""); + if (!/^[\p{L}\p{N}_.-]+(@[\p{L}\p{N}.-]+)?$/u.test(handle)) throw new BadRequestException("params.handle must be a channel handle"); + return { kind, params: { instance: requireInstance(raw.instance, "params.instance"), handle } }; + } + case KIND_YOUTUBE_CHANNEL: { + const channelId = requireString(raw.channelId, "params.channelId", 32); + if (!YOUTUBE_CHANNEL.test(channelId)) throw new BadRequestException("params.channelId must be a UC… channel id"); + return { kind, params: { channelId } }; + } + case KIND_WEB_WATCH: { + const params: WebWatchParams = { url: requireHttpUrl(raw.url, "params.url") }; + if (raw.selector !== undefined && raw.selector !== null && raw.selector !== "") { + params.selector = requireString(raw.selector, "params.selector", 512); + } + return { kind, params }; + } + default: { + throw new BadRequestException("kind must be a supported intake kind"); + } + } +} + +/** Deterministic key-sorted JSON — the canonical form hashing reads. */ +export function canonicalParams(input: IntakeParams): string { + const sorted = (value: unknown): unknown => { + if (Array.isArray(value)) return value.map(sorted); + if (isObject(value)) { + return Object.fromEntries(Object.keys(value).sort().map((key) => [key, sorted(value[key])])); + } + return value; + }; + return JSON.stringify(sorted({ kind: input.kind, params: input.params })); +} + +/** sha256 of the canonical serialization — migration 027's params_hash. */ +export function paramsHash(input: IntakeParams): string { + return createHash("sha256").update(canonicalParams(input)).digest("hex"); +} + +/** Parse params already stored on a row (trusted shape, still coerced). */ +export function storedParams(kind: IntakeKind, raw: unknown): IntakeParams["params"] { + return parseIntakeParams(kind, isObject(raw) ? raw : {}).params; +} + +/** The outbound host a kind+params pair polls — the egress budget key base. */ +export function egressOrigin(input: IntakeParams): string { + const params = input.params; + if ("instance" in params && typeof params.instance === "string") return params.instance; + if ("url" in params && typeof params.url === "string") return originOf(params.url); + return input.kind === KIND_YOUTUBE_CHANNEL ? YOUTUBE_ORIGIN : TELEGRAM_API_ORIGIN; +} + +function originOf(url: string): string { + try { return new URL(url).origin; } catch { return "https://invalid.example"; } +} diff --git a/apps/api/src/intake/intake-search.ts b/apps/api/src/intake/intake-search.ts new file mode 100644 index 0000000..79557b9 --- /dev/null +++ b/apps/api/src/intake/intake-search.ts @@ -0,0 +1,193 @@ +/** + * Intake search + listing (task requirement 5) — the 025 archive-search + * pattern applied to intake_items: Postgres FTS `websearch_to_tsquery` + * ORed with pg_trgm similarity per token for typo tolerance, RLS-scoped via + * withTenant (FORCE binds — there is no system path), keyset pagination with + * a base64url JSON cursor and an N+1 fetch. + * + * Ranking: ts_rank dominates (weight-A title hits outrank body hits via the + * 027 trigger's setweight), trigram similarity breaks ties. The no-query + * path keysets on (published_at, id). + */ +import { BadRequestException } from "@nestjs/common"; +import { withTenant } from "../db/client.js"; +import type { StorageService } from "../storage/storage.js"; +import { searchTokens } from "../archive/archive-search.js"; + +export interface IntakeQueryFilters { + q: string | null; + sourceId: string | null; + kind: string | null; + cursor: string | undefined; + limit: number; +} + +export const SEARCH_DEFAULT_LIMIT = 20; +export const SEARCH_MAX_LIMIT = 100; + +export interface IntakeItemRow { + id: string; + source_id: string; + remote_ref: string; + content_hash: string; + published_at: Date; + text: string; + content: Record; + media_refs: Array<{ storageKey?: string; proxyUrl?: string; kind?: string }>; + attribution: Record; + fetched_at: Date; + created_at: Date; + source_kind: string; + rank?: number; +} + +const UUID_SHAPE = /^[0-9a-f-]{36}$/i; +const ISO_TIMESTAMP = /^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?$/; + +interface Params { + values: unknown[]; + bind(value: unknown): number; +} + +function params(): Params { + const values: unknown[] = []; + return { + values, + bind(value: unknown): number { values.push(value); return values.length; }, + }; +} + +function decodeCursor(cursor: string | undefined, what: string, shape: (v: Record) => T): T { + if (cursor === undefined) throw new BadRequestException(`invalid cursor: ${what}`); + let value: unknown; + try { value = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8")); } + catch { throw new BadRequestException(`invalid cursor: ${what}`); } + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new BadRequestException(`invalid cursor: ${what}`); + } + return shape(value as Record); +} + +const ITEM_FIELDS = "intake_items.id, intake_items.source_id, intake_items.remote_ref, intake_items.content_hash, intake_items.published_at, intake_items.text, intake_items.content, intake_items.media_refs, intake_items.attribution, intake_items.fetched_at, intake_items.created_at"; + +function filterConds(p: Params, filters: IntakeQueryFilters): string[] { + const conds: string[] = []; + if (filters.sourceId !== null) conds.push(`source_id = $${p.bind(filters.sourceId)}::uuid`); + if (filters.kind !== null) conds.push(`s.kind = $${p.bind(filters.kind)}`); + return conds; +} + +/** + * Word-level fuzzy predicate. `text % $token` compares the WHOLE corpus + * string to the token, so a single misspelled word inside a long post scores + * ~0.1 and never clears pg_trgm's 0.3 threshold. Splitting first makes the + * comparison word-vs-word, where a one-letter typo still scores ~0.67 — + * that is the typo tolerance the spec asks pg_trgm for. + */ +function fuzzyPredicate(p: Params, tokens: string[]): string { + if (tokens.length === 0) return "false"; + return tokens.map((token) => { + const ti = p.bind(token); + return `(EXISTS (SELECT 1 FROM regexp_split_to_table(intake_items.text, ' ') w WHERE w % $${ti})` + + ` OR coalesce(intake_items.attribution->>'title', '') % $${ti})`; + }).join(" OR "); +} + +function rankExpression(p: Params, q: string, tokens: string[]): string { + const qi = p.bind(q); + const textSims = tokens.map((token) => `(SELECT max(similarity(w, $${p.bind(token)})) FROM regexp_split_to_table(intake_items.text, ' ') w)`); + const titleSims = tokens.map((token) => `similarity(coalesce(intake_items.attribution->>'title',''), $${p.bind(token)})`); + return `(ts_rank(intake_items.search_vector, websearch_to_tsquery('simple', $${qi})) * 2` + + (textSims.length > 0 ? ` + GREATEST(${textSims.join(", ")})` : "") + + (titleSims.length > 0 ? ` + GREATEST(${titleSims.join(", ")})` : "") + + `)`; +} + +export function searchIntake(workspaceId: string, filters: IntakeQueryFilters, + storage: StorageService): Promise<{ items: Record[]; nextCursor: string | null }> { + return runQuery(workspaceId, filters, storage, true); +} + +export function listIntakeItems(workspaceId: string, filters: IntakeQueryFilters, + storage: StorageService): Promise<{ items: Record[]; nextCursor: string | null }> { + // Listing without q is the plain keyset path; a q present upgrades it to search. + return runQuery(workspaceId, filters, storage, filters.q !== null); +} + +async function runQuery(workspaceId: string, filters: IntakeQueryFilters, + storage: StorageService, ranked: boolean): Promise<{ items: Record[]; nextCursor: string | null }> { + const p = params(); + const tokens = filters.q !== null ? searchTokens(filters.q) : []; + let sql: string; + let cursorValue: { rk: number; id: string } | null = null; + let cursorPosted: { at: string; id: string } | null = null; + if (ranked && filters.q !== null) { + const rank = rankExpression(p, filters.q, tokens); + const conds = filterConds(p, filters); + const qi = p.bind(filters.q); + conds.push(`(intake_items.search_vector @@ websearch_to_tsquery('simple', $${qi}) OR ${fuzzyPredicate(p, tokens)})`); + if (filters.cursor !== undefined) { + cursorValue = decodeCursor(filters.cursor, "rank", (v) => { + if (typeof v.rk !== "number" || !Number.isFinite(v.rk) || typeof v.id !== "string" || !UUID_SHAPE.test(v.id)) { + throw new BadRequestException("invalid cursor: rank"); + } + return { rk: v.rk, id: v.id }; + }); + conds.push(`(${rank}, intake_items.id) < ($${p.bind(cursorValue.rk)}::real, $${p.bind(cursorValue.id)}::uuid)`); + } + sql = `SELECT ranked.*, s.kind AS source_kind FROM + (SELECT ${ITEM_FIELDS}, ${rank} AS rank + FROM intake_items JOIN intake_sources s ON s.id = intake_items.source_id + WHERE ${conds.join(" AND ")}) ranked + JOIN intake_sources s ON s.id = ranked.source_id + ORDER BY ranked.rank DESC, ranked.id ASC LIMIT $${p.bind(filters.limit + 1)}`; + } else { + const conds = filterConds(p, filters); + if (filters.cursor !== undefined) { + cursorPosted = decodeCursor(filters.cursor, "published", (v) => { + if (typeof v.at !== "string" || !ISO_TIMESTAMP.test(v.at) || !Number.isFinite(Date.parse(v.at)) + || typeof v.id !== "string" || !UUID_SHAPE.test(v.id)) { + throw new BadRequestException("invalid cursor: published"); + } + return { at: v.at, id: v.id }; + }); + conds.push(`(intake_items.published_at, intake_items.id) < ($${p.bind(cursorPosted.at)}::timestamptz, $${p.bind(cursorPosted.id)}::uuid)`); + } + sql = `SELECT ${ITEM_FIELDS}, s.kind AS source_kind FROM intake_items + JOIN intake_sources s ON s.id = intake_items.source_id + WHERE ${conds.length > 0 ? conds.join(" AND ") : "true"} + ORDER BY intake_items.published_at DESC, intake_items.id DESC LIMIT $${p.bind(filters.limit + 1)}`; + } + const result = await withTenant(workspaceId, (client) => client.query(sql, p.values)); + const page = result.rows.slice(0, filters.limit); + const last = page.at(-1); + const items = await Promise.all(page.map(async (row) => { + const media = await Promise.all((row.media_refs ?? []).map(async (media) => ({ + ...media, + thumbUrl: media.storageKey ? await storage.presign(media.storageKey, 900) : null, + }))); + return { + id: row.id, + sourceId: row.source_id, + remoteRef: row.remote_ref, + url: typeof row.attribution.url === "string" ? row.attribution.url : null, + contentHash: row.content_hash, + publishedAt: row.published_at.toISOString(), + text: row.text, + content: row.content, + media, + attribution: row.attribution, + kind: row.source_kind, + fetchedAt: row.fetched_at.toISOString(), + ...(row.rank !== undefined ? { rank: row.rank } : {}), + }; + })); + const nextCursor = result.rows.length > filters.limit && last + ? Buffer.from(JSON.stringify(cursorShape(last, ranked && filters.q !== null))).toString("base64url") + : null; + return { items, nextCursor }; +} + +function cursorShape(last: IntakeItemRow, ranked: boolean): Record { + return ranked ? { rk: last.rank ?? 0, id: last.id } : { at: last.published_at.toISOString(), id: last.id }; +} diff --git a/apps/api/src/intake/intake.controller.ts b/apps/api/src/intake/intake.controller.ts new file mode 100644 index 0000000..559f63d --- /dev/null +++ b/apps/api/src/intake/intake.controller.ts @@ -0,0 +1,85 @@ +/** + * Intake API (task requirement 5). + * + * POST /intake/sources subscribe (kind + params) + * GET /intake/sources list subscriptions + * GET /intake/sources/:id one subscription (+ state) + * PATCH /intake/sources/:id pause / resume + * DELETE /intake/sources/:id unsubscribe + * GET /intake/search?q=… FTS + pg_trgm fuzzy, keyset + * GET /intake/items?sourceId=… per-source listing, keyset + * + * Writes gate on `connection.manage` (subscribing an outbound poller is + * connection-shaped: it spends a shared host budget); reads stay on + * `draft.view` like /feeds and /archive, so every current reader keeps + * reading. RLS scoping is in the service layer (withTenant, FORCE). + */ +import { BadRequestException, Body, Controller, Delete, Get, Param, Patch, Post, Query, Req, UseGuards } from "@nestjs/common"; +import { parseId, requireObject, requireSession } from "../http/validation.js"; +import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import { IntakeService } from "./intake.service.js"; +import { INTAKE_KINDS } from "./intake-kinds.js"; + +const KINDS_DOC = INTAKE_KINDS.join(", "); + +@Controller("intake") +@UseGuards(RbacGuard) +export class IntakeController { + constructor(private readonly intake: IntakeService) {} + + @Post("sources") + @RequirePermission("connection.manage") + async create(@Req() request: AuthenticatedRequest, @Body() body: unknown): Promise<{ id: string; kind: string; health: string }> { + const input = requireObject(body, "body"); + const kind = input.kind; + if (typeof kind !== "string" || kind === "") throw new BadRequestException(`kind must be one of: ${KINDS_DOC}`); + return this.intake.createSubscription(requireSession(request).workspaceId, kind, + input.params ?? {}, { + pollIntervalSec: input.pollIntervalSec, + mediaPolicy: input.mediaPolicy, + }); + } + + @Get("sources") + @RequirePermission("draft.view") + async list(@Req() request: AuthenticatedRequest): Promise<{ sources: Record[] }> { + return { sources: await this.intake.listSources(requireSession(request).workspaceId) }; + } + + @Get("sources/:id") + @RequirePermission("draft.view") + async get(@Req() request: AuthenticatedRequest, @Param("id") id: string): Promise> { + return this.intake.getSource(requireSession(request).workspaceId, parseId(id)); + } + + @Patch("sources/:id") + @RequirePermission("connection.manage") + async patch(@Req() request: AuthenticatedRequest, @Param("id") id: string, @Body() body: unknown): Promise<{ ok: true }> { + const input = requireObject(body, "body"); + const health = input.health; + if (health !== "active" && health !== "paused") throw new BadRequestException("health must be active or paused"); + await this.intake.setSourceHealth(requireSession(request).workspaceId, parseId(id), health); + return { ok: true }; + } + + @Delete("sources/:id") + @RequirePermission("connection.manage") + async remove(@Req() request: AuthenticatedRequest, @Param("id") id: string): Promise<{ ok: true }> { + await this.intake.deleteSource(requireSession(request).workspaceId, parseId(id)); + return { ok: true }; + } + + @Get("search") + @RequirePermission("draft.view") + async search(@Req() request: AuthenticatedRequest, + @Query() query: Record): Promise<{ items: Record[]; nextCursor: string | null }> { + return this.intake.search(requireSession(request).workspaceId, query); + } + + @Get("items") + @RequirePermission("draft.view") + async items(@Req() request: AuthenticatedRequest, + @Query() query: Record): Promise<{ items: Record[]; nextCursor: string | null }> { + return this.intake.listItems(requireSession(request).workspaceId, query); + } +} diff --git a/apps/api/src/intake/intake.integration.test.ts b/apps/api/src/intake/intake.integration.test.ts new file mode 100644 index 0000000..427d3a7 --- /dev/null +++ b/apps/api/src/intake/intake.integration.test.ts @@ -0,0 +1,467 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { join } from "node:path"; +import { Pool, type PoolClient } from "pg"; + +/** + * Intake phase-1 integration against a live PostgreSQL, as sp_app (FORCE RLS + * binds both tables). Own scratch database (INTAKE_IT_DATABASE_URL + + * INTAKE_IT_SUPER_DATABASE_URL) per the local-db-sol pattern, because the + * shared-DB suites wipe every workspace and vitest runs files in parallel. + * Skipped when the URLs are unset. + * + * Covers the task's integration list: content-hash dedup, RLS cross-tenant + * denial, budget draws via fake egress, FTS + fuzzy matching, keyset + * pagination (both the ranked and the time-ordered path), the scheduler's + * poll/write-back cycle, and the source lifecycle (subscribe → pause → + * delete). Storage is a fake S3 that content-addresses in memory. + */ +const appUrl = process.env.INTAKE_IT_DATABASE_URL; +const ownerUrl = process.env.INTAKE_IT_SUPER_DATABASE_URL; + +type Modules = { + service: typeof import("./intake.service.js"); + ingest: typeof import("./intake-ingest.js"); + search: typeof import("./intake-search.js"); + kinds: typeof import("./intake-kinds.js"); + adapter: typeof import("./adapters/adapter.js"); + rss: typeof import("./adapters/rss.adapter.js"); + db: typeof import("../db/client.js"); +}; + +class FakeStorage { + readonly objects = new Map(); + put(buffer: Buffer, contentType: string): { key: string; hash: string; sizeBytes: number } { + const { createHash } = require("node:crypto") as typeof import("node:crypto"); + const hash = createHash("sha256").update(buffer).digest("hex"); + const ext = contentType.split("/")[1]?.split("+")[0] ?? "bin"; + const key = `sha256/${hash}.${ext}`; + this.objects.set(key, { body: buffer, contentType }); + return { key, hash, sizeBytes: buffer.byteLength }; + } + async putContent(buffer: Buffer, contentType: string) { + const info = this.put(buffer, contentType); + const deduplicated = this.objects.has(info.key); + return { ...info, contentType, deduplicated }; + } + async presign(key: string): Promise { return `https://fake-s3/${key}?sig=test`; } +} + +/** Recording fake egress with fixture routes + Retry-After budget behavior. */ +class FakeEgress { + readonly draws: Array<{ url: string; hostClass: string }> = []; + constructor(private readonly routes: Record { status: number; headers?: Record; body?: string }>) {} + async egressFetch(url: string | URL, options: { headers?: Record }, config: { hostClass: string }): Promise { + const key = String(url); + this.draws.push({ url: key, hostClass: config.hostClass }); + const route = Object.entries(this.routes).find(([prefix]) => key.startsWith(prefix)); + if (route === undefined) return new Response("not found", { status: 404 }); + const r = route[1](); + return new Response(r.body ?? "", { status: r.status, headers: r.headers ?? {} }); + } + readonly policy = { timeoutMs: 1000, maxResponseBytes: 1024 * 1024 } as never; + isEgressError(): boolean { return false; } +} + +const RSS_BODY = `Water WG +Water rights are a community responsibilityhttps://example.org/w1w1 +Fri, 09 Oct 2026 12:00:00 GMTWater rights are a community responsibility, not a commodity. + +Bicycle repair nighthttps://example.org/w2w2 +Thu, 08 Oct 2026 09:00:00 GMTBring your bicycles to the tool library Thursday. +`; + +const MASTODON_BODY = JSON.stringify([ + { id: "s1", created_at: "2026-10-09T11:00:00.000Z", + content: "

Water rights update: the hearing is postponed

", + account: { acct: "alice", username: "alice" }, url: "https://social.example/@alice/s1", + media_attachments: [{ url: "https://files.social.example/s1.png", type: "image", mime_type: "image/png" }], visibility: "public" }, + { id: "s2", created_at: "2026-10-08T10:00:00.000Z", + content: "

Water rights are a community responsibility, not a commodity for markets.

", + account: { acct: "bob@example.social", username: "bob" }, visibility: "public" }, +]); + +const IMAGE_BYTES = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3, 4]); + +describe.skipIf(appUrl === undefined || ownerUrl === undefined)("intake phase 1 (live DB)", () => { + let m: Modules; + let ownerPool: Pool; + let storage: FakeStorage; + const suffix = `${Date.now()}${Math.floor(Math.random() * 1e6)}`; + let wsA: string; + let wsB: string; + + async function asOwner(fn: (client: PoolClient) => Promise): Promise { + const client = await ownerPool.connect(); + try { return await fn(client); } finally { client.release(); } + } + + let service: InstanceType; + + beforeAll(async () => { + process.env.DATABASE_URL = appUrl; + process.env.SUPER_DATABASE_URL = ownerUrl; + const { runMigrations } = await import("../db/migrate.js"); + await runMigrations(join(process.cwd(), "migrations")); + m = { + service: await import("./intake.service.js"), + ingest: await import("./intake-ingest.js"), + search: await import("./intake-search.js"), + kinds: await import("./intake-kinds.js"), + adapter: await import("./adapters/adapter.js"), + rss: await import("./adapters/rss.adapter.js"), + db: await import("../db/client.js"), + }; + ownerPool = new Pool({ connectionString: ownerUrl }); + + await asOwner(async (c) => { + const mk = async (name: string): Promise => + String((await c.query("INSERT INTO workspaces (name, slug) VALUES ($1, $2) RETURNING id", [name, `${name}-${suffix}`])).rows[0].id); + wsA = await mk("intake-a"); + wsB = await mk("intake-b"); + }); + + storage = new FakeStorage(); + process.env.S3_BUCKET = "intake-test"; + // Silence the sweep timer: this suite drives polls explicitly. + service = new m.service.IntakeService({ + egress: new FakeEgress({}) as never, + storage: storage as never, + random: () => 0.5, + now: () => new Date(), + sweepMs: 3_600_000, + }); + service.onApplicationShutdown().catch(() => undefined); + }); + + afterAll(async () => { + if (ownerPool !== undefined) { + await asOwner(async (c) => { await c.query("DELETE FROM workspaces WHERE id = ANY($1::uuid[])", [[wsA, wsB]]); }); + await ownerPool.end(); + } + await m?.db.pool.end(); + }); + + // ----------------------------------------------------------------------- + // Source lifecycle + params identity + // ----------------------------------------------------------------------- + + it("subscribes with canonical params identity; duplicates collapse; push-only parks unsupported", async () => { + const first = await service.createSubscription(wsA, "rss", { url: "https://example.org/feed.xml" }, { mediaPolicy: "images" }); + expect(first.kind).toBe("rss"); + expect(first.health).toBe("active"); + // hash-stripped variant collides on (workspace, kind, params_hash) + const dupe = await service.createSubscription(wsA, "rss", { url: "https://example.org/feed.xml#fragment" }); + expect(dupe.id).toBe(first.id); + const other = await service.createSubscription(wsA, "rss", { url: "https://other.example/feed.xml" }); + expect(other.id).not.toBe(first.id); + const tg = await service.createSubscription(wsA, "telegram_channel", { channelId: "-1001234567890" }); + expect(tg.health).toBe("unsupported"); // 027: push-only kinds park + const list = await service.listSources(wsA); + expect(list).toHaveLength(3); + // push-only kinds cannot opt into polling + await expect(service.createSubscription(wsA, "bluesky_keyword", { keywords: ["water"] }, { pollIntervalSec: 60 })) + .rejects.toThrow(/push-only/); + }); + + it("validates params shapes at the API edge", async () => { + await expect(service.createSubscription(wsA, "mastodon_account", { instance: "https://mastodon.social", accountId: "not-numeric" })).rejects.toThrow(); + await expect(service.createSubscription(wsA, "youtube_channel", { channelId: "XD" })).rejects.toThrow(); + await expect(service.createSubscription(wsA, "rss", { url: "file:///etc/passwd" })).rejects.toThrow(); + await expect(service.createSubscription(wsA, "nope", {})).rejects.toThrow(); + }); + + it("pauses, resumes and deletes sources", async () => { + const src = await service.createSubscription(wsA, "web_watch", { url: "https://example.org/watched" }); + await service.setSourceHealth(wsA, src.id, "paused"); + let row = await service.getSource(wsA, src.id) as { health: string }; + expect(row.health).toBe("paused"); + await service.setSourceHealth(wsA, src.id, "active"); + row = await service.getSource(wsA, src.id) as { health: string }; + expect(row.health).toBe("active"); + await service.deleteSource(wsA, src.id); + await expect(service.getSource(wsA, src.id)).rejects.toThrow(); + }); + + // ----------------------------------------------------------------------- + // Poll + ingest + dedup + media harvest + // ----------------------------------------------------------------------- + + it("polls an rss source end-to-end: budget draws, P0c AST, media to S3, dedup on re-poll", async () => { + const egress = new FakeEgress({ + "https://example.org/feed.xml": () => ({ status: 200, headers: { "content-type": "application/xml", etag: "\"v1\"" }, body: RSS_BODY }), + "https://cdn.example.org/water.jpg": () => ({ status: 200, headers: { "content-type": "image/png" }, body: IMAGE_BYTES.toString("latin1") }), + }); + const poller = new m.service.IntakeService({ + egress: egress as never, storage: storage as never, + random: () => 0.5, now: () => new Date(), sweepMs: 3_600_000, + }); + const src = await service.createSubscription(wsA, "rss", { url: "https://example.org/feed.xml" }); + const due = await poller.pollWorkspaceDue(wsA); + expect(due.polled).toBeGreaterThanOrEqual(1); + // Feed draw went through the egress client with the intake-rss host class. + expect(egress.draws.some((d) => d.url.startsWith("https://example.org/feed.xml") && d.hostClass === "intake-rss")).toBe(true); + // Media draw billed to the shared media class. + expect(egress.draws.some((d) => d.url.startsWith("https://cdn.example.org/water.jpg") && d.hostClass === "intake-web")).toBe(true); + const items = await service.listItems(wsA, { sourceId: src.id, limit: 50 }); + expect(items.items).toHaveLength(2); + const first = items.items[0] as Record; + // P0c AST landed — never raw HTML. + const content = first.content as { type: string; children: unknown[] }; + expect(content.type).toBe("doc"); + // text derived from the AST is the searchable corpus + expect(String(first.text)).toMatch(/community responsibility/); + // media ref carries the S3 storage key (content-addressed) + proxy URL + const media = first.media as Array>; + expect(media.length).toBeGreaterThanOrEqual(1); + expect(String(media[0]!.storageKey)).toMatch(/^sha256\/[0-9a-f]{64}\./); + // state carries the etag for conditional GET + const row = await service.getSource(wsA, src.id) as { state: { etag?: string }; nextPollAt: string | null }; + expect(row.state.etag).toBe("\"v1\""); + expect(row.nextPollAt).not.toBeNull(); + + // Re-poll with identical content: zero inserts (content-hash dedup). + const before = await service.listItems(wsA, { sourceId: src.id, limit: 50 }); + await poller.pollWorkspaceDue(wsA); + const after = await service.listItems(wsA, { sourceId: src.id, limit: 50 }); + expect(after.items).toHaveLength(before.items.length); + }); + + it("cross-source dedup: the same statement via rss and mastodon stores once", async () => { + const statement = "Water rights are a community responsibility, not a commodity for markets."; + const rssBody = `x + Statementhttps://example.org/xx1 + Fri, 09 Oct 2026 12:00:00 GMT${statement}`; + const mastodonBody = JSON.stringify([ + { id: "m1", created_at: "2026-10-09T13:00:00.000Z", content: `

${statement}

`, + account: { acct: "relay", username: "relay" }, visibility: "public" }, + ]); + const egress = new FakeEgress({ + "https://dedup.example.org/rss": () => ({ status: 200, headers: { "content-type": "application/xml" }, body: rssBody }), + "https://mastodon.social/api/v1/timelines/tag/water": () => ({ status: 200, headers: { "content-type": "application/json" }, body: mastodonBody }), + }); + const poller = new m.service.IntakeService({ + egress: egress as never, storage: storage as never, + random: () => 0.5, now: () => new Date(), sweepMs: 3_600_000, + }); + const rssSrc = await service.createSubscription(wsA, "rss", { url: "https://dedup.example.org/rss" }); + const mdSrc = await service.createSubscription(wsA, "mastodon_tag", { instance: "https://mastodon.social", tag: "water" }, { mediaPolicy: "metadata" }); + await poller.pollWorkspaceDue(wsA); + const rssItems = await service.listItems(wsA, { sourceId: rssSrc.id, limit: 10 }); + const mdItems = await service.listItems(wsA, { sourceId: mdSrc.id, limit: 10 }); + // First writer wins: the row is attributed to whichever source landed it. + expect(rssItems.items.length + mdItems.items.length).toBe(1); + // and the mastodon poll drew from the intake-social class + expect(egress.draws.some((d) => d.hostClass === "intake-social")).toBe(true); + }); + + it("mastodon items carry html-parsed AST + authorRef attribution", async () => { + const egress = new FakeEgress({ + "https://mastodon.social/api/v1/timelines/tag/hearing": () => ({ status: 200, headers: { "content-type": "application/json" }, body: MASTODON_BODY }), + "https://files.social.example/s1.png": () => ({ status: 200, headers: { "content-type": "image/png" }, body: IMAGE_BYTES.toString("latin1") }), + }); + const poller = new m.service.IntakeService({ + egress: egress as never, storage: storage as never, + random: () => 0.5, now: () => new Date(), sweepMs: 3_600_000, + }); + const src = await service.createSubscription(wsA, "mastodon_tag", { instance: "https://mastodon.social", tag: "hearing" }); + await poller.pollWorkspaceDue(wsA); + const items = await service.listItems(wsA, { sourceId: src.id, limit: 10 }); + expect(items.items.length).toBeGreaterThanOrEqual(1); + const newest = items.items.find((i) => (i as { remoteRef: string }).remoteRef === "s1") as Record; + const attribution = newest.attribution as Record; + expect(attribution.authorHandle).toBe("alice"); + expect(attribution.authorRef).toBe("mastodon:alice"); + expect(String(newest.url ?? "")).toContain("/@alice/s1"); + }); + + it("poll failure: 429 parks the source in backoff with retry-after honored, then recovers", async () => { + let fail = true; + const body = `x + Recoveredhttps://example.org/r1r1 + Fri, 09 Oct 2026 15:00:00 GMTrecovered content`; + const egress = new FakeEgress({ + "https://backoff.example.org/feed": (): { status: number; headers: Record; body?: string } => fail + ? { status: 429, headers: { "retry-after": "1" } } + : { status: 200, headers: { "content-type": "application/xml" }, body }, + }); + const poller = new m.service.IntakeService({ + egress: egress as never, storage: storage as never, + random: () => 0.5, now: () => new Date(), sweepMs: 3_600_000, + }); + const src = await service.createSubscription(wsA, "rss", { url: "https://backoff.example.org/feed" }); + const firstPass = await poller.pollWorkspaceDue(wsA); + expect(firstPass.failed).toBeGreaterThanOrEqual(1); + let row = await service.getSource(wsA, src.id) as { health: string; lastError: string | null; consecutiveFailures: number }; + expect(row.health).toBe("backoff"); + expect(row.lastError).toMatch(/transient: http 429/); + expect(row.consecutiveFailures).toBe(1); + // Retry-After (1 s) becomes the next_poll_at floor — not due immediately. + const again = await poller.pollWorkspaceDue(wsA); + expect(again.polled).toBe(0); + // force-recover: clear backoff + due immediately + await asOwner(async (c) => { await c.query("UPDATE intake_sources SET next_poll_at = now(), health='active' WHERE id = $1", [src.id]); }); + fail = false; + const third = await poller.pollWorkspaceDue(wsA); + expect(third.polled).toBeGreaterThanOrEqual(1); + row = await service.getSource(wsA, src.id) as { health: string; consecutiveFailures: number; lastError: string | null }; + expect(row.health).toBe("active"); + expect(row.consecutiveFailures).toBe(0); + }); + + it("permanent 410 parks the source in error, not backoff", async () => { + const egress = new FakeEgress({ "https://gone.example.org/feed": () => ({ status: 410 }) }); + const poller = new m.service.IntakeService({ + egress: egress as never, storage: storage as never, + random: () => 0.5, now: () => new Date(), sweepMs: 3_600_000, + }); + const src = await service.createSubscription(wsA, "rss", { url: "https://gone.example.org/feed" }); + await poller.pollWorkspaceDue(wsA); + const row = await service.getSource(wsA, src.id) as { health: string; lastError: string | null }; + expect(row.health).toBe("error"); + expect(row.lastError).toMatch(/permanent: http 410/); + }); + + // ----------------------------------------------------------------------- + // RLS + // ----------------------------------------------------------------------- + + it("RLS: cross-tenant reads see nothing; writes are scoped", async () => { + // Distinct content from earlier feeds: dedup is content-addressed per + // workspace, so a re-delivery of RSS_BODY would (correctly) no-op. + const body = `rls + RLS probehttps://example.org/rls1rls1 + Fri, 09 Oct 2026 16:00:00 GMTTenant isolation probe statement.`; + const egress = new FakeEgress({ + "https://rls.example.org/feed": () => ({ status: 200, headers: { "content-type": "application/xml" }, body }), + }); + const poller = new m.service.IntakeService({ + egress: egress as never, storage: storage as never, + random: () => 0.5, now: () => new Date(), sweepMs: 3_600_000, + }); + const srcA = await service.createSubscription(wsA, "rss", { url: "https://rls.example.org/feed" }); + await poller.pollWorkspaceDue(wsA); + // wsB never subscribed; its listing + search are empty even though wsA has rows. + const inB = await service.listItems(wsB, { limit: 100 }); + expect(inB.items).toHaveLength(0); + const searchB = await service.search(wsB, { q: "community responsibility" }); + expect(searchB.items).toHaveLength(0); + await expect(service.getSource(wsB, srcA.id)).rejects.toThrow(); + await expect(service.deleteSource(wsB, srcA.id)).rejects.toThrow(); + const inA = await service.listItems(wsA, { sourceId: srcA.id, limit: 10 }); + expect(inA.items.length).toBeGreaterThan(0); + }); + + // ----------------------------------------------------------------------- + // Search: FTS, fuzzy, keyset + // ----------------------------------------------------------------------- + + // Seeds the corpus the three suites below read: a timestamped feed so each + // run's rows are unique (content-hash dedup is workspace-wide and this + // database is recreated per CI run, but re-runs against a kept database + // stay honest too). + let seeded = false; + it("seeds the search corpus via full polls", async () => { + if (seeded) return; + const stamp = Date.now(); + const rssA = `Corpus + Community responsibility statement ${stamp}https://example.org/c1c1-${stamp} + Fri, 09 Oct 2026 12:00:00 GMTWater rights are a community responsibility held in trust for the watershed ${stamp}. + Hearing update ${stamp}https://example.org/c2c2-${stamp} + Fri, 09 Oct 2026 13:00:00 GMTThe eviction hearing is postponed until February for the tenants union ${stamp}. + Tool library ${stamp}https://example.org/c3c3-${stamp} + Fri, 09 Oct 2026 14:00:00 GMTBicycle repair night at the tool library on Thursday evening ${stamp}.`; + const egress = new FakeEgress({ + "https://corpus.example.org/feed": () => ({ status: 200, headers: { "content-type": "application/xml" }, body: rssA }), + }); + const poller = new m.service.IntakeService({ + egress: egress as never, storage: storage as never, + random: () => 0.5, now: () => new Date(), sweepMs: 3_600_000, + }); + await service.createSubscription(wsA, "rss", { url: "https://corpus.example.org/feed" }); + const pass = await poller.pollWorkspaceDue(wsA); + expect(pass.polled).toBeGreaterThanOrEqual(1); + const seededRows = await service.listItems(wsA, { limit: 100 }); + expect(seededRows.items.length).toBeGreaterThanOrEqual(3); + seeded = true; + }); + + it("FTS matches websearch_to_tsquery; fuzzy tolerates typos via pg_trgm", async () => { + const exact = await service.search(wsA, { q: "watershed" }); + expect(exact.items.some((item) => /watershed/i.test(String(item.text)))).toBe(true); + // Typo tolerance: "responsability" matches the corpus item whose text has + // "responsibility" (word-level trigram), alongside the FTS hit above. + const fuzzy = await service.search(wsA, { q: "responsability" }); + expect(fuzzy.items.some((item) => /responsibility/i.test(String(item.text)))).toBe(true); + const phrase = await service.search(wsA, { q: '"hearing is postponed"' }); + expect(phrase.items.length).toBeGreaterThanOrEqual(1); + const nothing = await service.search(wsA, { q: "zzzzqqqq" }); + expect(nothing.items).toHaveLength(0); + }); + + it("search filters by kind and rejects bad cursors", async () => { + const byKind = await service.search(wsA, { q: null, kind: "rss" }); + expect(byKind.items.length).toBeGreaterThan(0); + for (const item of byKind.items) expect(item.kind).toBe("rss"); + await expect(service.search(wsA, { q: "x", cursor: "not-a-cursor!" })).rejects.toThrow(); + await expect(service.search(wsA, { q: "x".repeat(300) })).rejects.toThrow(); + await expect(service.search(wsA, { limit: 1000 })).rejects.toThrow(); + }); + + it("keyset pagination walks time-ordered listings without overlap", async () => { + let cursor: string | null = null; + const seen = new Set(); + let pages = 0; + for (;;) { + const page = await service.listItems(wsA, { limit: 2, ...(cursor ? { cursor } : {}) }); + pages += 1; + for (const item of page.items) expect(seen.add(String(item.id))).toBe(true); + cursor = page.nextCursor; + if (cursor === null) break; + expect(pages).toBeLessThan(20); + } + const all = await service.listItems(wsA, { limit: 100 }); + expect(seen.size).toBe(all.items.length); + }); + + it("keyset pagination on the ranked search path", async () => { + let cursor: string | null = null; + const seen = new Set(); + let pages = 0; + for (;;) { + const page = await service.search(wsA, { q: "water rights", limit: 2, ...(cursor ? { cursor } : {}) }); + pages += 1; + for (const item of page.items) expect(seen.add(String(item.id))).toBe(true); + cursor = page.nextCursor; + if (cursor === null) break; + expect(pages).toBeLessThan(20); + } + expect(seen.size).toBeGreaterThanOrEqual(2); + }); + + // ----------------------------------------------------------------------- + // Push-only bridge + // ----------------------------------------------------------------------- + + it("telegram bridge: archive item becomes an intake row for the subscribed channel", async () => { + const channelId = "-1009876543210"; + const src = await service.createSubscription(wsA, "telegram_channel", { channelId }); + expect(src.health).toBe("unsupported"); // never polled + // Seed an archive item directly (the webhook path owns that write). + let archiveId = ""; + await asOwner(async (c) => { + c.query("SELECT set_config('app.workspace_id', $1, true)", [wsA]); + const row = await c.query( + `INSERT INTO archived_feed_items + (workspace_id, remote_ref, channel_id, message_id, text_content, content_hash, posted_at, attribution) + VALUES ($1, $2, $3, 5001, $4, encode(sha256($5::bytea), 'hex'), now(), $6::jsonb) RETURNING id`, + [wsA, `${channelId}:5001`, channelId, "Telegram harvest: water rights hearing moved.", + `tg-${suffix}`, JSON.stringify({ channelUsername: "water_wg", author: "Water WG" })]); + archiveId = String(row.rows[0].id); + }); + await service.ingestArchiveItem(wsA, archiveId); + const items = await service.listItems(wsA, { sourceId: src.id, limit: 10 }); + expect(items.items).toHaveLength(1); + const found = await service.search(wsA, { q: "harvest water" }); + expect(found.items.length).toBeGreaterThanOrEqual(1); + }); +}); diff --git a/apps/api/src/intake/intake.module.ts b/apps/api/src/intake/intake.module.ts new file mode 100644 index 0000000..09436a8 --- /dev/null +++ b/apps/api/src/intake/intake.module.ts @@ -0,0 +1,35 @@ +/** + * Intake module wiring: production providers bind the process-wide egress + * client, the S3 storage service and the real Redis lease surface. + * + * The egress package is ESM-only while the API emits CommonJS, so the client + * is constructed through the native-import binding in intake-egress.ts (same + * pattern as live-monitor/poller/egress.ts) inside a factory — the DI token + * carries the already-loaded surface. + */ +import { Module } from "@nestjs/common"; +import { IntakeController } from "./intake.controller.js"; +import { INTAKE_DEPENDENCIES, IntakeService } from "./intake.service.js"; +import { intakeEgress } from "./intake-egress.js"; +import { S3Storage, s3ConfigFromEnv } from "../storage/storage.js"; +import { RespControlRedis } from "../media-control/redis.js"; + +@Module({ + controllers: [IntakeController], + providers: [ + IntakeService, + { + provide: INTAKE_DEPENDENCIES, + useFactory: async () => ({ + egress: await intakeEgress(), + storage: new S3Storage(s3ConfigFromEnv()), + redis: new RespControlRedis(process.env.REDIS_URL ?? "redis://127.0.0.1:6379"), + random: Math.random, + now: () => new Date(), + sweepMs: 15_000, + }), + }, + ], + exports: [IntakeService], +}) +export class IntakeModule {} diff --git a/apps/api/src/intake/intake.service.ts b/apps/api/src/intake/intake.service.ts new file mode 100644 index 0000000..1b39e5d --- /dev/null +++ b/apps/api/src/intake/intake.service.ts @@ -0,0 +1,432 @@ +/** + * IntakeService — subscriptions, the poll scheduler, ingest, and the search + * surface (firehose phase 1). + * + * Scheduler = the live-monitor pattern (task requirement 1): one serialized + * sweep, per-source state in the DB (`next_poll_at` / `consecutive_failures` + * / `health`), exponential backoff 2^n capped at 15 min, +/−20 % jitter on + * every schedule so a fleet of replicas never thundering-herds one host, and + * a Redis lease + cooldown per source so replicas never double-poll. Sources + * self-schedule via next_poll_at; the sweep only has to notice. + * + * Failures classify exactly like BackoffPollDriver: transient (429/5xx/ + * budget/DNS) → backoff with Retry-After as a lower bound, health='backoff'; + * policy denials and permanent shapes (404/410, bad params) → health='error' + * with the much longer cap; push-only kinds (poll_interval_sec = 0) are + * never due and park in 'unsupported' when first seen (027 note). + */ +import { BadRequestException, Inject, Injectable, Logger, NotFoundException, Optional, type OnApplicationBootstrap, type OnApplicationShutdown } from "@nestjs/common"; +import { randomUUID } from "node:crypto"; +import { pool, withTenant } from "../db/client.js"; +import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js"; +import type { ControlRedis } from "../media-control/redis.js"; +import { MemoryFeedRedis } from "../feeds/testing.js"; +import { intakeEgress, IntakeHttpError, intakeRetryAfterMs, type IntakeEgress } from "./intake-egress.js"; +import { DEFAULT_MEDIA_POLICY, DEFAULT_POLL_INTERVAL_SEC, HOST_CLASS_BY_KIND, isIntakeKind, KIND_TELEGRAM_CHANNEL, PROVIDER_BY_KIND, type IntakeKind, type MediaPolicy } from "./intake-kinds.js"; +import { parseIntakeParams, paramsHash, storedParams, type IntakeParams } from "./intake-params.js"; +import { adapterFor } from "./adapters/index.js"; +import type { AdapterState } from "./adapters/adapter.js"; +import { MediaHarvester, type HarvestedMedia } from "./media-harvest.js"; +import { itemContentHash, normalizeRawItem, upsertItems, type UpsertResult } from "./intake-ingest.js"; +import { searchIntake, listIntakeItems, type IntakeQueryFilters } from "./intake-search.js"; + +export const INTAKE_DEPENDENCIES = Symbol("INTAKE_DEPENDENCIES"); + +export interface IntakeDependencies { + egress: IntakeEgress; + storage: StorageService; + redis: Pick; + random(): number; + now(): Date; + /** Sweep period; defaults to 15 s (min cadence is 60 s, so this is ample). */ + sweepMs?: number; +} + +/** Backoff curve caps (live-monitor: 2^n; intake polls are slower, cap 15 min). */ +export const BACKOFF_CAP_MS = 15 * 60_000; +export const PERMANENT_ERROR_MS = 6 * 60 * 60_000; +export const LEASE_MS = 60_000; + +const RENEW_LEASE = "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('PEXPIRE', KEYS[1], ARGV[2]) else return 0 end"; +const RELEASE_LEASE = "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) else return 0 end"; + +interface SourceRow { + id: string; + workspace_id: string; + kind: IntakeKind; + params: Record; + poll_interval_sec: number; + state: AdapterState; + health: string; + consecutive_failures: number; +} + +export function backoffDelayMs(failures: number, intervalMs: number): number { + return Math.min(BACKOFF_CAP_MS, Math.max(intervalMs, 1_000) * 2 ** Math.min(6, Math.max(0, failures - 1))); +} + +/** +/−20 % jitter (live-monitor jitteredInterval shape). */ +export function jitteredIntervalMs(intervalMs: number, random: number): number { + if (!Number.isFinite(intervalMs) || intervalMs <= 0) throw new Error("interval must be positive"); + if (!Number.isFinite(random) || random < 0 || random > 1) throw new Error("random must be between 0 and 1"); + return Math.round(intervalMs * (0.8 + random * 0.4)); +} + +function defaults(): IntakeDependencies { + return { + egress: undefined as never, // bound lazily by ensureDeps + storage: new S3Storage(s3ConfigFromEnv()), + redis: new MemoryFeedRedis(), + random: Math.random, + now: () => new Date(), + sweepMs: 15_000, + }; +} + +@Injectable() +export class IntakeService implements OnApplicationBootstrap, OnApplicationShutdown { + private readonly logger = new Logger(IntakeService.name); + private readonly deps: IntakeDependencies; + private readonly egress: IntakeEgress | undefined; + private running = false; + private timer?: ReturnType; + private active?: Promise; + private harvester?: MediaHarvester; + + constructor(@Optional() @Inject(INTAKE_DEPENDENCIES) dependencies?: Partial) { + const base = defaults(); + this.deps = { ...base, ...dependencies }; + // The egress client binds lazily (native ESM import); tests inject a fake. + if (dependencies?.egress !== undefined) this.egress = dependencies.egress; + if (this.egress !== undefined) { + const nowMs = (): number => this.deps.now().getTime(); + this.harvester = new MediaHarvester(this.egress, this.deps.storage, nowMs); + } + } + + onApplicationBootstrap(): void { + this.running = true; + this.schedule(0); + } + + async onApplicationShutdown(): Promise { + this.running = false; + if (this.timer) clearTimeout(this.timer); + await this.active; + } + + get isRunning(): boolean { return this.running; } + + private schedule(delayMs: number): void { + if (!this.running) return; + const sweep = this.deps.sweepMs ?? 15_000; + this.timer = setTimeout(() => { + this.timer = undefined; + this.active = this.pollDueSources().catch((error: unknown) => { + this.logger.error(`intake sweep failed: ${String(error)}`); + }).then(() => undefined).finally(() => { + this.active = undefined; + this.schedule(jitteredIntervalMs(sweep, this.deps.random())); + }); + }, delayMs); + this.timer.unref(); + } + + // ------------------------------------------------------------------------- + // Subscriptions + // ------------------------------------------------------------------------- + + async createSubscription(workspaceId: string, kind: unknown, paramsRaw: unknown, + options: { pollIntervalSec?: unknown; mediaPolicy?: unknown } = {}): Promise<{ id: string; kind: IntakeKind; health: string }> { + const parsed = parseIntakeParams(kind, paramsRaw); + const hash = paramsHash(parsed); + const kindDefaults = DEFAULT_POLL_INTERVAL_SEC[parsed.kind]; + const mediaPolicy = parseMediaPolicy(options.mediaPolicy, DEFAULT_MEDIA_POLICY[parsed.kind]); + let interval = kindDefaults; + const rawInterval: unknown = options.pollIntervalSec; + if (rawInterval !== undefined && rawInterval !== null && rawInterval !== "") { + const n = typeof rawInterval === "number" ? rawInterval : typeof rawInterval === "string" ? Number(rawInterval) : NaN; + if (!Number.isInteger(n) || n < 0 || n > 86_400) throw new BadRequestException("pollIntervalSec must be an integer 0..86400 (0 = push-only)"); + if (kindDefaults === 0 && n > 0) throw new BadRequestException("this kind is push-only; pollIntervalSec must be 0"); + interval = n > 0 ? Math.max(60, Math.min(n, 86_400)) : 0; + } + return withTenant(workspaceId, async (client) => { + const result = await client.query<{ id: string; health: string }>( + `INSERT INTO intake_sources (workspace_id, kind, params, params_hash, poll_interval_sec, state, health) + VALUES ($1,$2,$3,$4,$5,$6,$7) + ON CONFLICT (workspace_id, kind, params_hash) DO UPDATE SET updated_at = now() + RETURNING id, health`, + [workspaceId, parsed.kind, JSON.stringify({ ...parsed.params, mediaPolicy }), hash, interval, JSON.stringify(mediaPolicy ? { mediaPolicy } : {}), + kindDefaults === 0 ? "unsupported" : "active"]); + const row = result.rows[0]; + if (!row) throw new Error("intake source insert failed"); + return { id: String(row.id), kind: parsed.kind, health: row.health }; + }); + } + + async listSources(workspaceId: string): Promise[]> { + return withTenant(workspaceId, async (client) => { + const result = await client.query( + `SELECT id, kind, params, poll_interval_sec, health, last_error, last_polled_at, next_poll_at, + consecutive_failures, created_at, updated_at + FROM intake_sources WHERE workspace_id = $1 ORDER BY created_at`, [workspaceId]); + return result.rows.map((row) => ({ + id: String(row.id), kind: row.kind, params: row.params, + pollIntervalSec: Number(row.poll_interval_sec), health: row.health, + lastError: row.last_error, lastPolledAt: row.last_polled_at?.toISOString() ?? null, + nextPollAt: row.next_poll_at?.toISOString() ?? null, + consecutiveFailures: Number(row.consecutive_failures), + createdAt: row.created_at.toISOString(), updatedAt: row.updated_at.toISOString(), + })); + }); + } + + async getSource(workspaceId: string, sourceId: string): Promise> { + const rows = await withTenant(workspaceId, async (client) => client.query( + "SELECT * FROM intake_sources WHERE workspace_id = $1 AND id = $2", [workspaceId, sourceId])); + const row = rows.rows[0]; + if (!row) throw new NotFoundException("intake source not found"); + return { + id: String(row.id), kind: row.kind, params: row.params, + pollIntervalSec: Number(row.poll_interval_sec), health: row.health, + lastError: row.last_error, lastPolledAt: row.last_polled_at?.toISOString() ?? null, + nextPollAt: row.next_poll_at?.toISOString() ?? null, state: row.state, + consecutiveFailures: Number(row.consecutive_failures), + createdAt: row.created_at.toISOString(), updatedAt: row.updated_at.toISOString(), + }; + } + + async setSourceHealth(workspaceId: string, sourceId: string, health: "active" | "paused"): Promise { + if (health !== "active" && health !== "paused") throw new BadRequestException("health must be active or paused"); + await withTenant(workspaceId, async (client) => { + const result = await client.query( + `UPDATE intake_sources SET health = $3, updated_at = now(), + next_poll_at = CASE WHEN $3 = 'active' THEN now() ELSE next_poll_at END + WHERE workspace_id = $1 AND id = $2`, [workspaceId, sourceId, health]); + if (result.rowCount === 0) throw new NotFoundException("intake source not found"); + }); + } + + async deleteSource(workspaceId: string, sourceId: string): Promise { + await withTenant(workspaceId, async (client) => { + const result = await client.query( + "DELETE FROM intake_sources WHERE workspace_id = $1 AND id = $2", [workspaceId, sourceId]); + if (result.rowCount === 0) throw new NotFoundException("intake source not found"); + }); + } + + // ------------------------------------------------------------------------- + // Polling + // ------------------------------------------------------------------------- + + /** One sweep over every workspace's due sources; per-source failures never abort the sweep. */ + async pollDueSources(): Promise<{ polled: number; skipped: number; failed: number }> { + if (this.egress === undefined) return { polled: 0, skipped: 0, failed: 0 }; + const workspaces = await pool.query<{ id: string }>("SELECT id FROM workspaces ORDER BY id"); + const totals = { polled: 0, skipped: 0, failed: 0 }; + for (const { id } of workspaces.rows) { + const pass = await this.pollWorkspaceDue(id); + totals.polled += pass.polled; + totals.skipped += pass.skipped; + totals.failed += pass.failed; + } + return totals; + } + + async pollWorkspaceDue(workspaceId: string): Promise<{ polled: number; skipped: number; failed: number }> { + const due = await withTenant(workspaceId, async (client) => client.query( + `SELECT id, workspace_id, kind, params, poll_interval_sec, state, health, consecutive_failures + FROM intake_sources + WHERE workspace_id = $1 + AND health NOT IN ('paused','unsupported') + AND poll_interval_sec > 0 + AND (next_poll_at IS NULL OR next_poll_at <= now()) + ORDER BY COALESCE(next_poll_at, created_at), created_at`, [workspaceId])); + const totals = { polled: 0, skipped: 0, failed: 0 }; + for (const row of due.rows) { + const outcome = await this.pollSource(workspaceId, row); + totals[outcome] += 1; + } + return totals; + } + + /** + * Poll one source under its Redis lease. The lease makes a multi-replica + * deployment safe (one owner at a time; renewed on a heartbeat, released on + * completion, expires on crash). Spacing between polls is enforced solely + * by next_poll_at in the DB — the single source of truth the scheduler + * reads — so a manual due-reset (or an ops force-poll) is honored rather + * than silently eaten by a side-channel cooldown. + */ + async pollSource(workspaceId: string, row: SourceRow): Promise<"polled" | "skipped" | "failed"> { + if (this.egress === undefined || this.harvester === undefined) return "skipped"; + const intervalMs = Math.max(60_000, row.poll_interval_sec * 1_000); + const leaseKey = `intake:${workspaceId}:${row.id}:lease`; + const owner = randomUUID(); + if (!await this.deps.redis.set(leaseKey, owner, { nx: true, pxMs: LEASE_MS })) return "skipped"; + let renew: Promise = Promise.resolve(); + let heartbeat: ReturnType | undefined; + try { + heartbeat = setInterval(() => { + renew = this.deps.redis.eval(RENEW_LEASE, [leaseKey], [owner, String(LEASE_MS)]).catch(() => { + this.logger.warn("intake poll lease renewal unavailable"); + }); + }, LEASE_MS / 3); + heartbeat.unref(); + + const params = storedParams(row.kind, row.params); + const adapter = adapterFor(row.kind); + const fetcher = { + fetch: (url: string | URL, options: { headers?: Record; hostClass: string }) => + this.egress!.egressFetch(url, { headers: options.headers ?? {} }, { hostClass: options.hostClass }), + }; + const result = await adapter.poll({ + workspaceId, sourceId: row.id, params, kind: row.kind, state: row.state ?? {}, fetch: fetcher, + }); + + // Harvest media per the source's policy, then normalize + upsert. + const policy = sourceMediaPolicy(row); + const harvested: HarvestedMedia[][] = []; + for (const item of result.items) { + harvested.push(await this.harvester.harvestAll(policy, item.media ?? [])); + } + const rows = result.items.map((raw, index) => ({ + raw, + outcome: normalizeRawItem(raw, row.kind, params, workspaceId, row.id), + harvested: harvested[index] ?? [], + })); + const upsert: UpsertResult = await withTenant(workspaceId, async (client) => + upsertItems(client, workspaceId, row.id, row.kind, rows)); + void upsert; + await withTenant(workspaceId, async (client) => client.query( + `UPDATE intake_sources SET state = $3, health = 'active', last_error = NULL, last_polled_at = now(), + consecutive_failures = 0, next_poll_at = now() + ($4::bigint * interval '1 millisecond'), updated_at = now() + WHERE workspace_id = $1 AND id = $2`, + [workspaceId, row.id, JSON.stringify(result.state), jitteredIntervalMs(intervalMs, this.deps.random())])); + return "polled"; + } catch (error) { + await this.recordFailure(workspaceId, row, error, intervalMs); + return "failed"; + } finally { + if (heartbeat) clearInterval(heartbeat); + await renew; + await this.deps.redis.eval(RELEASE_LEASE, [leaseKey], [owner]).catch(() => undefined); + } + } + + private async recordFailure(workspaceId: string, row: SourceRow, error: unknown, intervalMs: number): Promise { + const now = this.deps.now().getTime(); + const http = error instanceof IntakeHttpError ? error : undefined; + const egress = this.egress?.isEgressError(error) === true ? error : undefined; + const retryAfter = http !== undefined + ? intakeRetryAfterMs(http.headers, now) + : egress !== undefined ? egress.retryAfterMs ?? 0 : 0; + const transient = (http !== undefined && http.transient) + || egress !== undefined && ["BUDGET_EXHAUSTED", "BUDGET_UNAVAILABLE", "TIMEOUT", "NETWORK_ERROR", "ABORTED", "DNS_FAILED"].includes(egress.code) + || http === undefined && egress === undefined; // network shape unknown → benefit of the doubt + const permanent = (http !== undefined && http.gone) + || egress !== undefined && ["DENIED_PRIVATE_ADDRESS", "SCHEME_DENIED", "PORT_DENIED", "ORIGIN_DENIED", "INVALID_URL"].includes(egress.code); + const failures = row.consecutive_failures + 1; + const base = permanent ? PERMANENT_ERROR_MS : transient ? backoffDelayMs(failures, intervalMs) : PERMANENT_ERROR_MS; + const delay = Math.max(retryAfter, jitteredIntervalMs(base, this.deps.random())); + const reason = permanent ? "permanent" : transient ? "transient" : "unknown"; + const message = http !== undefined ? `http ${http.status}` + : egress !== undefined ? `egress:${egress.code}` + : error instanceof Error ? error.message.slice(0, 240) : "poll failed"; + await withTenant(workspaceId, async (client) => client.query( + `UPDATE intake_sources SET health = $3, last_error = $4, last_polled_at = now(), + consecutive_failures = $5, next_poll_at = now() + ($6::bigint * interval '1 millisecond'), updated_at = now() + WHERE workspace_id = $1 AND id = $2`, + [workspaceId, row.id, permanent ? "error" : "backoff", `${reason}: ${message}`, failures, delay], + )).catch(() => this.logger.warn("intake source state write-back unavailable")); + } + + // ------------------------------------------------------------------------- + // Telegram archive bridge (task requirement: reuse the existing harvest) + // ------------------------------------------------------------------------- + + /** + * Bridges an archived_feed_items row into intake_items. The archive webhook + * path (apps/api/src/archive) already normalized the post, harvested media + * into S3 content-addressed keys and enforced revisions — intake stores the + * reference, it never re-fetches. + */ + async ingestArchiveItem(workspaceId: string, archiveItemId: string): Promise { + const found = await withTenant(workspaceId, async (client) => client.query<{ + id: string; remote_ref: string; channel_id: string; message_id: string; text_content: string; + media_refs: Array<{ storageKey?: string; contentType?: string; sizeBytes?: number | null }>; + attribution: Record; posted_at: Date; + }>("SELECT * FROM archived_feed_items WHERE id = $1", [archiveItemId])); + const item = found.rows[0]; + if (!item) throw new NotFoundException("archive item not found"); + const channelId = String(item.channel_id); + await withTenant(workspaceId, async (client) => { + const source = await client.query<{ id: string; params: Record }>( + `SELECT id, params FROM intake_sources WHERE workspace_id = $1 AND kind = $2 + AND params->>'channelId' = $3`, [workspaceId, KIND_TELEGRAM_CHANNEL, channelId]); + const sourceRow = source.rows[0]; + if (!sourceRow) return; // not subscribed in intake — nothing to do + const mediaKeys = (item.media_refs ?? []).map((media) => media.storageKey ?? "").filter((key) => key !== ""); + const bridgeHash = itemContentHash(item.text_content, mediaKeys); + await client.query( + `INSERT INTO intake_items (workspace_id, source_id, remote_ref, content_hash, published_at, + text, content, media_refs, attribution) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) + ON CONFLICT (workspace_id, content_hash) DO NOTHING`, + [workspaceId, sourceRow.id, item.remote_ref, bridgeHash, + item.posted_at.toISOString(), item.text_content, + JSON.stringify({ type: "doc", version: 1, children: [{ type: "paragraph", children: [{ type: "text", text: item.text_content }] }], truncated: false }), + JSON.stringify(item.media_refs.map((media) => ({ kind: "image", proxyUrl: "", storageKey: media.storageKey ?? null, contentType: media.contentType ?? null, sizeBytes: media.sizeBytes ?? null }))), + JSON.stringify({ ...item.attribution, provider: PROVIDER_BY_KIND[KIND_TELEGRAM_CHANNEL], archiveItemId: item.id })]); + }); + } + + // ------------------------------------------------------------------------- + // Read surface (search + per-source listing) + // ------------------------------------------------------------------------- + + search(workspaceId: string, query: Record): Promise<{ items: Record[]; nextCursor: string | null }> { + return searchIntake(workspaceId, parseSearchQuery(query), this.deps.storage); + } + + listItems(workspaceId: string, query: Record): Promise<{ items: Record[]; nextCursor: string | null }> { + return listIntakeItems(workspaceId, parseSearchQuery(query), this.deps.storage); + } +} + +function parseMediaPolicy(value: unknown, fallback: MediaPolicy): MediaPolicy { + if (value === undefined || value === null || value === "") return fallback; + if (value !== "metadata" && value !== "images" && value !== "full") { + throw new BadRequestException("mediaPolicy must be metadata, images, or full"); + } + return value; +} + +function sourceMediaPolicy(row: SourceRow): MediaPolicy { + const fromParams = row.params.mediaPolicy; + if (fromParams === "metadata" || fromParams === "images" || fromParams === "full") return fromParams; + return DEFAULT_MEDIA_POLICY[row.kind]; +} + +/** Shared query parsing for /intake/search and /intake/items. */ +export function parseSearchQuery(query: Record): IntakeQueryFilters { + const rawQ = query.q; + const q = typeof rawQ === "string" && rawQ.trim() !== "" ? rawQ.trim() : null; + if (q !== null && q.length > 256) throw new BadRequestException("q must be at most 256 characters"); + const rawSource = query.sourceId; + const sourceId = rawSource === undefined || rawSource === null || rawSource === "" ? null + : typeof rawSource === "string" && /^[0-9a-f-]{36}$/i.test(rawSource) ? rawSource : (() => { throw new BadRequestException("sourceId must be a UUID"); })(); + const rawKind = query.kind; + const kind = rawKind === undefined || rawKind === null || rawKind === "" ? null + : isIntakeKind(rawKind) ? rawKind : (() => { throw new BadRequestException("kind is not a supported intake kind"); })(); + const cursor = typeof query.cursor === "string" && query.cursor !== "" ? query.cursor : undefined; + if (cursor !== undefined && (cursor.length > 256 || !/^[A-Za-z0-9_-]+$/.test(cursor))) throw new BadRequestException("invalid cursor"); + let limit = 20; + if (query.limit !== undefined && query.limit !== "") { + const n = typeof query.limit === "string" ? Number(query.limit) + : typeof query.limit === "number" ? query.limit : NaN; + if (!Number.isInteger(n) || n < 1 || n > 100) throw new BadRequestException("limit must be an integer between 1 and 100"); + limit = n; + } + return { q, sourceId, kind, cursor, limit }; +} diff --git a/apps/api/src/intake/media-harvest.ts b/apps/api/src/intake/media-harvest.ts new file mode 100644 index 0000000..f1694d0 --- /dev/null +++ b/apps/api/src/intake/media-harvest.ts @@ -0,0 +1,107 @@ +/** + * Intake media harvest — the archive StorageService pattern verbatim + * (task requirement 4): direct GET through the egress budget → + * `putContent(bytes, contentType)` → content-addressed `sha256/.` + * key, AES-256-GCM envelope encryption, dedup by plaintext hash. A blob + * already in the bucket is a no-op (`deduplicated: true`), so the same image + * seen via RSS and Mastodon stores once. + * + * Policy gate (research §2.5): `metadata` harvests nothing; `images` accepts + * image candidates only; `full` accepts video/audio too. Size caps are + * enforced twice (declared content-length and actual bytes). + */ +import { HOST_CLASS_BY_KIND, type IntakeKind, type MediaPolicy } from "./intake-kinds.js"; +import { INTAKE_MEDIA_MAX_BYTES, IntakeHttpError, intakeRetryAfterMs, type IntakeEgress } from "./intake-egress.js"; +import type { StorageService, StoredObject } from "../storage/storage.js"; + +export interface HarvestRequest { + url: string; + kind: "image" | "video"; +} + +export interface HarvestedMedia { + url: string; + kind: "image" | "video"; + status: "stored" | "skipped" | "too_large" | "failed"; + storageKey?: string; + hash?: string; + sizeBytes?: number; + contentType?: string; + deduplicated?: boolean; + error?: string; +} + +const IMAGE_TYPES = /^image\//i; +/** Download budget for media files is the shared intake-web class (polite). */ +const MEDIA_HOST_CLASS = "intake-web"; + +export function accepts(policy: MediaPolicy, kind: "image" | "video"): boolean { + if (policy === "metadata") return false; + if (policy === "images") return kind === "image"; + return true; +} + +export class MediaHarvester { + constructor( + private readonly egress: IntakeEgress, + private readonly storage: StorageService, + private readonly now: () => number = Date.now, + private readonly maxBytes: number = INTAKE_MEDIA_MAX_BYTES, + ) {} + + async harvestAll(policy: MediaPolicy, requests: HarvestRequest[]): Promise { + const out: HarvestedMedia[] = []; + for (const request of requests) { + out.push(await this.harvest(policy, request)); + } + return out; + } + + async harvest(policy: MediaPolicy, request: HarvestRequest): Promise { + if (!accepts(policy, request.kind)) { + return { url: request.url, kind: request.kind, status: "skipped" }; + } + try { + const response = await this.egress.egressFetch(request.url, { headers: { accept: "*/*" } }, { hostClass: MEDIA_HOST_CLASS }); + if (!response.ok) throw new IntakeHttpError(response.status, response.headers, request.url); + const declared = Number(response.headers.get("content-length") ?? "0"); + if (Number.isFinite(declared) && declared > this.maxBytes) { + return { url: request.url, kind: request.kind, status: "too_large", sizeBytes: declared }; + } + const contentType = normalizeContentType(response.headers.get("content-type"), request.kind); + const bytes = Buffer.from(await response.arrayBuffer()); + if (bytes.byteLength > this.maxBytes) { + return { url: request.url, kind: request.kind, status: "too_large", sizeBytes: bytes.byteLength }; + } + const stored: StoredObject & { deduplicated: boolean } = await this.storage.putContent(bytes, contentType); + return { + url: request.url, kind: request.kind, status: "stored", + storageKey: stored.key, hash: stored.hash, sizeBytes: stored.sizeBytes, + contentType: stored.contentType, deduplicated: stored.deduplicated, + }; + } catch (error) { + const retryAfter = error instanceof IntakeHttpError ? intakeRetryAfterMs(error.headers, this.now()) : 0; + return { + url: request.url, kind: request.kind, status: "failed", + error: retryAfter > 0 ? `http ${error instanceof IntakeHttpError ? error.status : "network"} (retry-after ${Math.round(retryAfter / 1000)}s)` : error instanceof Error ? error.message.slice(0, 120) : "harvest failed", + }; + } + } +} + +function normalizeContentType(raw: string | null, kind: "image" | "video"): string { + const value = (raw ?? "").split(";")[0]?.trim().toLowerCase() ?? ""; + if (value !== "") { + if (kind === "image" && IMAGE_TYPES.test(value)) return value; + if (kind === "video" && (value.startsWith("video/") || value === "application/octet-stream" || value.startsWith("audio/"))) return value; + if (value === "application/octet-stream") return value; + if (IMAGE_TYPES.test(value)) return value; + if (value.startsWith("video/") || value.startsWith("audio/")) return value; + } + return kind === "image" ? "image/jpeg" : "video/mp4"; +} + +/** The host class a kind's feed fetch draws on (re-exported for the service). */ +export function feedHostClassFor(kind: IntakeKind): string { + return HOST_CLASS_BY_KIND[kind]; +} diff --git a/apps/api/src/live-chat/adapters.test.ts b/apps/api/src/live-chat/adapters.test.ts new file mode 100644 index 0000000..4de36ef --- /dev/null +++ b/apps/api/src/live-chat/adapters.test.ts @@ -0,0 +1,66 @@ +import { randomUUID } from "node:crypto"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createMockLivestreamConnector } from "@tactical-media-broadcast-hub/connector-mock-livestream"; +import { TwitchChatTransport, twitchToEvent } from "./twitch-adapter.js"; +import { SharedChatPollers } from "./shared-poller.js"; +import { SendGovernors } from "./governors.js"; +import { LiveChatHubService } from "./live-chat.service.js"; + +afterEach(() => vi.useRealTimers()); +describe("Twitch adapter with injected Helix", () => { + it("maps the existing connector event and preserves reply/moderation arguments", async () => { + const ws = randomUUID(), channelId = randomUUID(), connectionId = randomUUID(); + const options = { workspaceId: ws, channelId, connectionId, liveRemoteId: "remote" }; + expect(twitchToEvent(options, { id: "provider-msg", provider: "twitch", liveStreamId: "remote", authorHandle: "viewer", text: "hi", sentAt: new Date(0), isModerated: false })).toMatchObject({ channelId, connectionId, providerMessageId: "provider-msg", author: { handle: "viewer" } }); + const helix = { send: vi.fn(async (_request: unknown) => ({ providerMessageId: "sent", sentAt: new Date(1) })), moderate: vi.fn(async (_request: unknown) => undefined) }; + const transport = new TwitchChatTransport(createMockLivestreamConnector(), helix); + await expect(transport.send({ connectionId, liveRemoteId: "remote", text: "reply", replyTo: "provider-msg" })).resolves.toMatchObject({ providerMessageId: "sent" }); + expect(helix.send).toHaveBeenCalledWith({ connectionId, liveRemoteId: "remote", text: "reply", replyTo: "provider-msg" }); + await transport.moderate({ connectionId, liveRemoteId: "remote", messageRemoteId: "provider-msg", authorHandle: "viewer", action: "timeout", durationSeconds: 600 }); + expect(helix.moderate).toHaveBeenCalledWith(expect.objectContaining({ action: "timeout", durationSeconds: 600, authorHandle: "viewer" })); + }); +}); +describe("shared provider long poll", () => { + it("shares one poll chain, charges list=1, and honors pollingIntervalMillis and cursor", async () => { + vi.useFakeTimers(); + const pollers = new SharedChatPollers(), fetch = vi.fn(async (_cursor?: string) => ({ events: [], nextPageToken: "next", pollingIntervalMillis: 5000 })), ingest = vi.fn(async () => ({ inserted: [], duplicates: [] })); + const governors = new SendGovernors({ quotas: { youtube: { budget: 100, costPerCall: 50 } } }); + const options = { workspaceId: "ws", broadcastId: "live", provider: "youtube", fetch, ingest, governors }; + const one = pollers.acquire(options), two = pollers.acquire(options); + await vi.advanceTimersByTimeAsync(0); expect(fetch).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(4999); expect(fetch).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1); expect(fetch).toHaveBeenCalledTimes(2); expect(fetch).toHaveBeenLastCalledWith("next"); + expect(governors.take("youtube").allowed).toBe(true); // two list units leave 98 units for a 50-unit send + one(); two(); await vi.advanceTimersByTimeAsync(10000); expect(fetch).toHaveBeenCalledTimes(2); pollers.close(); + }); + it("never polls below quota and exposes paused reason", async () => { + vi.useFakeTimers(); const fetch = vi.fn(), paused = vi.fn(), pollers = new SharedChatPollers(); + const release = pollers.acquire({ workspaceId: "ws", broadcastId: "live", provider: "youtube", fetch, paused, ingest: async () => ({ inserted: [], duplicates: [] }), governors: new SendGovernors({ quotas: { youtube: { budget: 0 } } }) }); + await vi.advanceTimersByTimeAsync(0); expect(fetch).not.toHaveBeenCalled(); expect(paused).toHaveBeenCalledWith("daily quota exhausted"); release(); + }); +}); +describe("hub display filters", () => { + it("applies commands, bots, muted provider handles and words before sequencing", async () => { + const transaction = vi.fn(); + const hub = new LiveChatHubService({ transaction, filters: { hideCommands: true, hideBots: true, mutedHandles: ["mock-livestream:muted"], words: ["blocked"] } }); + const base = { provider: "mock-livestream", connectionId: "connection", channelId: "channel", providerMessageId: "msg", author: { handle: "viewer" }, text: "hi", sentAt: new Date() }; + expect(await hub.ingest("ws", [{ ...base, text: "!command" }, { ...base, author: { handle: "bot", isBot: true } }, { ...base, author: { handle: "muted" } }, { ...base, text: "BLOCKED word" }])).toEqual({ inserted: [], duplicates: [] }); + expect(transaction).not.toHaveBeenCalled(); + }); +}); + +describe("registry capability probing and comments fallback", () => { + it("degrades to read-only embed, then labelled comments, without claiming XMPP support", async () => { + const { probeChatCapability } = await import("./capability-probe.js"); + const { MockLivestreamDriver } = await import("./mock-driver.js"); + const connector = createMockLivestreamConnector(), manifest = connector.manifest(); + connector.manifest = () => ({ ...manifest, capabilities: { ...manifest.capabilities, liveChatRead: false, comments: true } }); + connector.getComments = async () => ({ items: [{ id: "comment", provider: manifest.id, remoteId: "comment", onRemoteId: "live", authorHandle: "viewer", text: "comment text", createdAt: new Date(0) }] }); + const connection = { id: randomUUID(), workspaceId: randomUUID(), provider: manifest.id, channelId: "remote", credentialId: { kind: "credential-ref" as const, id: "ref" }, fields: {}, createdAt: new Date() }; + expect(probeChatCapability(connector, connection, "https://chat.example/embed")).toMatchObject({ mode: "readonly-embed", capability: { reply: false, moderate: false } }); + expect(probeChatCapability(connector, connection)).toMatchObject({ mode: "comments-only", capability: { read: true, state: "degraded", reason: "comments only" } }); + const driver = new MockLivestreamDriver(connector, { workspaceId: connection.workspaceId, channelId: "live", connectionId: connection.id, liveRemoteId: "live", connection, commentsOnly: true }); + const ingest = vi.fn(async () => ({ inserted: [], duplicates: [] })); + await driver.start(ingest); expect(ingest).toHaveBeenCalledWith(connection.workspaceId, [expect.objectContaining({ source: "social-reply", text: "comment text" })]); await driver.stop(); + }); +}); diff --git a/apps/api/src/live-chat/auth.ts b/apps/api/src/live-chat/auth.ts new file mode 100644 index 0000000..740f907 --- /dev/null +++ b/apps/api/src/live-chat/auth.ts @@ -0,0 +1,37 @@ +import type { PoolClient } from "pg"; +import { verifyDeviceToken } from "./devices.js"; +import { verifySession, type SessionPayload } from "../auth/session.js"; +import { isUuid } from "../http/validation.js"; + +export function chatSession(token: string): SessionPayload | null { try { return verifySession(token); } catch { return null; } } + +export function tokenWorkspace(token: string): string | null { + return chatSession(token)?.workspaceId ?? verifyDeviceToken(token)?.workspaceId ?? null; +} +/** Current membership and workspace epoch always fence both credential kinds. */ +export async function authorizeChat(client: PoolClient, token: string, workspaceId: string, permission: string): Promise { + const session = chatSession(token); + const device = session ? null : verifyDeviceToken(token); + const principal = session ?? device; + if (!principal || principal.workspaceId !== workspaceId || !isUuid(workspaceId) || !isUuid(principal.userId)) return null; + await client.query("BEGIN"); + try { + await client.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]); + const member = await client.query<{ role: string; authz_epoch: string | number }>( + `SELECT m.role, w.authz_epoch FROM memberships m JOIN workspaces w ON w.id=m.workspace_id + JOIN role_permissions rp ON rp.role=m.role AND rp.permission=$3 + WHERE m.workspace_id=$1 AND m.user_id=$2 AND m.status='active'`, [workspaceId, principal.userId, permission]); + const row = member.rows[0]; + if (!row || !Number.isSafeInteger(Number(row.authz_epoch))) return null; + if (session && session.authzEpoch !== Number(row.authz_epoch)) return null; + if (device) { + if (!device.scopes.includes(permission)) return null; + const found = await client.query<{ epoch: string | number; scopes: string[]; authz_epoch: string | number }>( + "SELECT epoch, scopes, authz_epoch FROM devices WHERE workspace_id=$1 AND id=$2 AND user_id=$3 AND revoked_at IS NULL", + [workspaceId, device.deviceId, device.userId]); + const stored = found.rows[0]; + if (!stored || Number(stored.epoch) !== device.epoch || Number(stored.authz_epoch) !== Number(row.authz_epoch) || !stored.scopes.includes(permission)) return null; + } + return { userId: principal.userId, workspaceId, role: row.role, authzEpoch: Number(row.authz_epoch), issuedAt: principal.issuedAt, expiresAt: principal.expiresAt }; + } finally { await client.query("ROLLBACK"); } +} diff --git a/apps/api/src/live-chat/bridge.ts b/apps/api/src/live-chat/bridge.ts new file mode 100644 index 0000000..ec72eed --- /dev/null +++ b/apps/api/src/live-chat/bridge.ts @@ -0,0 +1,29 @@ +import { Injectable, Inject, Optional, OnModuleDestroy } from "@nestjs/common"; +import { RespControlRedis, type ControlRedis } from "../media-control/redis.js"; +import type { LiveChatPublish } from "./live-chat.service.js"; + +export const CHAT_SINK = Symbol("CHAT_SINK"); +export type ChatSink = LiveChatPublish; +export const chatTopic = (workspaceId: string, liveId: string): string => `chat:${workspaceId}:${liveId}`; +export const chatWindowKey = (workspaceId: string, liveId: string): string => `${chatTopic(workspaceId, liveId)}:window`; +// Atomic durable resume window + notification. Scores preserve hub ingress order. +export const CHAT_PUBLISH_SCRIPT = ` +redis.call('ZADD', KEYS[1], ARGV[1], ARGV[2]) +local count = redis.call('ZCARD', KEYS[1]) +if count > 500 then redis.call('ZREMRANGEBYRANK', KEYS[1], 0, count - 501) end +redis.call('EXPIRE', KEYS[1], 86400) +return redis.call('PUBLISH', KEYS[2], ARGV[2])`; +export const CHAT_WINDOW_SCRIPT = `return redis.call('ZRANGE', KEYS[1], 0, -1)`; + +@Injectable() +export class ChatRealtimeBridge implements OnModuleDestroy { + private readonly redis: ControlRedis; + constructor(@Optional() @Inject(CHAT_SINK) private readonly sink?: ChatSink, redis?: ControlRedis) { + this.redis = redis ?? new RespControlRedis(process.env.REDIS_URL ?? "redis://localhost:6379"); + } + publish: LiveChatPublish = async (workspaceId, channelId, frame): Promise => { + if (this.sink) { await this.sink(workspaceId, channelId, frame); return; } + await this.redis.eval(CHAT_PUBLISH_SCRIPT, [chatWindowKey(workspaceId, channelId), chatTopic(workspaceId, channelId)], [String(frame.seq), JSON.stringify(frame)]); + }; + onModuleDestroy(): void { this.redis.close(); } +} diff --git a/apps/api/src/live-chat/capability-probe.ts b/apps/api/src/live-chat/capability-probe.ts new file mode 100644 index 0000000..bbf32b3 --- /dev/null +++ b/apps/api/src/live-chat/capability-probe.ts @@ -0,0 +1,22 @@ +import type { ConnectorConnection, LiveChatCapability } from "@tactical-media-broadcast-hub/core"; +import type { PlatformConnector } from "@tactical-media-broadcast-hub/connector-sdk"; + +export interface ChatCapabilityProbe { + capability: LiveChatCapability; + mode: "native" | "readonly-embed" | "comments-only" | "none"; + embedUrl?: string; +} +/** Registry-based probing covers PeerTube degradation without provider branches. + * XMPP is intentionally reserved for a later registered native adapter. */ +export function probeChatCapability(connector: PlatformConnector, connection: ConnectorConnection, embedUrl?: string): ChatCapabilityProbe { + const manifest = connector.manifest(), caps = manifest.capabilities; + const base = { connectionId: connection.id, provider: manifest.id }; + if (caps.liveChatRead && connector.getLiveChat) return { mode: "native", capability: { ...base, read: true, reply: Boolean(caps.liveChatWrite && connector.sendLiveChatMessage), moderate: Boolean(caps.liveChatModeration && connector.moderateLiveChat), state: "ok" } }; + if (embedUrl) { + const url = new URL(embedUrl); + if (url.protocol !== "https:") throw new Error("read-only chat embed must use HTTPS"); + return { mode: "readonly-embed", embedUrl, capability: { ...base, read: true, reply: false, moderate: false, state: "degraded", reason: "read-only web embed" } }; + } + if (caps.comments && connector.getComments) return { mode: "comments-only", capability: { ...base, read: true, reply: Boolean(connector.reply), moderate: false, state: "degraded", reason: "comments only" } }; + return { mode: "none", capability: { ...base, read: false, reply: false, moderate: false, state: "degraded", reason: "chat unavailable" } }; +} diff --git a/apps/api/src/live-chat/chat-fanout.ts b/apps/api/src/live-chat/chat-fanout.ts new file mode 100644 index 0000000..ab05e93 --- /dev/null +++ b/apps/api/src/live-chat/chat-fanout.ts @@ -0,0 +1,14 @@ +/** + * Gateway-side chat sink: lets the live-chat hub push `kind:"chat"` frames to + * the /realtime gateway WITHOUT the hub importing the gateway (S3 owns the + * gateway; the slice boundary stays clean and no cycle forms). + * + * The realtime module registers an implementation at bootstrap. Chat frames + * ride the same per-channel topic as state events: the same subscribe/ack + * handshake, the same bounded resume ring, the same authz eviction. This file + * only declares the port. + */ +import type { LiveChatPublish } from "../live-chat/live-chat.service.js"; + +export const CHAT_FANOUT = Symbol("CHAT_FANOUT"); +export type ChatFanout = LiveChatPublish; diff --git a/apps/api/src/live-chat/devices.test.ts b/apps/api/src/live-chat/devices.test.ts new file mode 100644 index 0000000..8e8d955 --- /dev/null +++ b/apps/api/src/live-chat/devices.test.ts @@ -0,0 +1,69 @@ +import { randomUUID } from "node:crypto"; +import type { PoolClient } from "pg"; +import { describe, expect, it } from "vitest"; +import { DEVICE_SCOPES } from "@tactical-media-broadcast-hub/core"; +import { DeviceService, issueDeviceToken, verifyDeviceToken } from "./devices.js"; +import { authorizeChat } from "./auth.js"; + +const WS = randomUUID(), USER = randomUUID(); +function fixture() { + const pairings: Array<{ workspace_id: string; user_id: string; code: string; scopes: string[]; expires_at: Date; redeemed_at: Date | null; device_id: string | null }> = []; + const devices = new Map(); + let now = Date.now(); + const client = { query: async (sql: string, args: unknown[]) => { + if (sql.includes("INSERT INTO device_pairings")) { + pairings.push({ workspace_id: String(args[0]), user_id: String(args[1]), code: String(args[2]), scopes: args[5] as string[], expires_at: args[4] as Date, redeemed_at: null, device_id: null }); return { rows: [], rowCount: 1 }; + } + if (sql.includes("SELECT * FROM device_pairings")) return { rows: pairings.filter(row => row.workspace_id === args[0] && row.code === args[1]) }; + if (sql.includes("INSERT INTO devices")) { const id = randomUUID(); devices.set(id, { epoch: 1, revoked_at: null }); return { rows: [{ id }] }; } + if (sql.includes("UPDATE device_pairings")) { const row = pairings.find(row => row.code === args[1]); if (row) { row.redeemed_at = new Date(now); row.device_id = String(args[2]); } return { rows: [] }; } + if (sql.includes("UPDATE devices")) { const row = devices.get(String(args[1])); if (!row || row.revoked_at) return { rows: [], rowCount: 0 }; row.epoch += 1; row.revoked_at = new Date(now); return { rows: [row], rowCount: 1 }; } + if (sql.includes("SELECT epoch")) { const row = devices.get(String(args[1])); return { rows: row ? [row] : [] }; } + throw new Error(`unexpected device SQL: ${sql}`); + } } as unknown as PoolClient; + let pending = Promise.resolve(); + const service = new DeviceService({ serverUrl: "https://studio.example", now: () => new Date(now), transaction: async (_ws, fn) => { + const previous = pending; let release: () => void = () => undefined; + pending = new Promise(resolve => { release = resolve; }); await previous; + try { return await fn(client); } finally { release(); } + } }); + return { service, advance: (ms: number) => { now += ms; } }; +} +describe("device pairing and revocation", () => { + it("code → signed scoped token, wrong workspace rejected, single-use code, revoke fences epoch", async () => { + const f = fixture(), issued = await f.service.issuePairing(WS, USER); + expect(JSON.parse(issued.qrPayload)).toEqual({ serverUrl: "https://studio.example", pairingCode: issued.pairingCode }); + await expect(f.service.redeem(randomUUID(), issued.pairingCode, "phone")).rejects.toThrow("invalid pairing code"); + const result = await f.service.redeem(WS, issued.pairingCode, "phone"); + const claims = verifyDeviceToken(result.deviceToken); + expect(claims).toMatchObject({ workspaceId: WS, userId: USER, scopes: [...DEVICE_SCOPES], epoch: 1 }); + if (!claims) throw new Error("missing token claims"); + expect(await f.service.epochOf(WS, claims.deviceId)).toBe(1); + await expect(f.service.redeem(WS, issued.pairingCode, "other")).rejects.toThrow("already redeemed"); + await f.service.revoke(WS, claims.deviceId); expect(await f.service.epochOf(WS, claims.deviceId)).toBeNull(); + }); + it("expires at five minutes, including the boundary", async () => { + const f = fixture(), issued = await f.service.issuePairing(WS, USER); + f.advance(300000); await expect(f.service.redeem(WS, issued.pairingCode, "phone")).rejects.toThrow("expired"); + }); + it("only one concurrent redemption succeeds", async () => { + const f = fixture(), issued = await f.service.issuePairing(WS, USER); + const results = await Promise.allSettled([f.service.redeem(WS, issued.pairingCode, "one"), f.service.redeem(WS, issued.pairingCode, "two")]); + expect(results.filter(result => result.status === "fulfilled")).toHaveLength(1); + }); + it("rejects tampering, expired tokens, and malformed signed claims", () => { + const token = issueDeviceToken({ deviceId: randomUUID(), workspaceId: WS, userId: USER, scopes: [...DEVICE_SCOPES], epoch: 1 }); + expect(verifyDeviceToken(`${token}x`)).toBeNull(); + expect(verifyDeviceToken(issueDeviceToken({ deviceId: randomUUID(), workspaceId: WS, userId: USER, scopes: [...DEVICE_SCOPES], epoch: 1 }, -1))).toBeNull(); + expect(verifyDeviceToken(issueDeviceToken({ deviceId: "bad", workspaceId: WS, userId: USER, scopes: [...DEVICE_SCOPES], epoch: 1 }))).toBeNull(); + }); + it("auth checks device epoch, current membership/scopes, and the workspace role epoch", async () => { + const deviceId = randomUUID(), token = issueDeviceToken({ deviceId, workspaceId: WS, userId: USER, scopes: [...DEVICE_SCOPES], epoch: 1 }); + let deviceEpoch = 1, workspaceEpoch = 7, active = true; + const client = { query: async (sql: string) => ({ rows: sql.includes("FROM memberships") ? (active ? [{ role: "Operator", authz_epoch: workspaceEpoch }] : []) : sql.includes("FROM devices") ? [{ epoch: deviceEpoch, scopes: [...DEVICE_SCOPES], authz_epoch: 7 }] : [] }) } as unknown as PoolClient; + expect(await authorizeChat(client, token, WS, "livechat.view")).toMatchObject({ userId: USER }); + deviceEpoch += 1; expect(await authorizeChat(client, token, WS, "livechat.view")).toBeNull(); + deviceEpoch = 1; workspaceEpoch += 1; expect(await authorizeChat(client, token, WS, "livechat.view")).toBeNull(); + workspaceEpoch = 7; active = false; expect(await authorizeChat(client, token, WS, "livechat.view")).toBeNull(); + }); +}); diff --git a/apps/api/src/live-chat/devices.ts b/apps/api/src/live-chat/devices.ts new file mode 100644 index 0000000..994e894 --- /dev/null +++ b/apps/api/src/live-chat/devices.ts @@ -0,0 +1,180 @@ +/** + * Device pairing (§3.4): the web UI mints a single-use, TTL'd code; the app + * redeems it for an epoch-gated device token bound to (workspace, user). + * + * Tokens use a separate HMAC domain and current device/workspace epochs fence + * every REST request and active socket. Pairing codes expire after five minutes. + */ +import { createHash, createHmac, randomBytes, timingSafeEqual } from "node:crypto"; +import { Injectable, Inject, Optional, UnauthorizedException, NotFoundException, ConflictException } from "@nestjs/common"; +import type { PoolClient } from "pg"; +import type { DeviceTokenPayload } from "@tactical-media-broadcast-hub/core"; +import { idempotentChat } from "./idempotency.js"; +import { pool, withTenant } from "../db/client.js"; +import { isUuid } from "../http/validation.js"; +import { DEVICE_SCOPES } from "@tactical-media-broadcast-hub/core"; + +export const DEVICE_DEPENDENCIES = Symbol("DEVICE_DEPENDENCIES"); + +export interface DeviceTransaction { + (workspaceId: string, fn: (client: PoolClient) => Promise): Promise; +} + +export interface DeviceDependencies { + transaction: DeviceTransaction; + now?: () => Date; + /** TTL of an unredeemed pairing code, ms (default 5 min, §3.4). */ + pairingTtlMs?: number; + /** Lifetime of a device token, seconds (default 30 days). */ + tokenTtlSeconds?: number; + /** Base URL the QR payload points at. */ + serverUrl?: string; + lookupWorkspace?: (code: string) => Promise; +} + +export interface PairingIssued { + pairingCode: string; + qrPayload: string; + expiresAt: string; +} + +export interface RedeemResult { + deviceToken: string; + workspaceId: string; +} + +interface PairingRow { + id: string; workspace_id: string; user_id: string; code: string; qr_payload: string; + scopes: string[]; expires_at: Date; redeemed_at: Date | null; device_id: string | null; +} + +function pairingCode(): string { + const alphabet = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789"; + const group = (): string => Array.from(randomBytes(4), b => alphabet[b % alphabet.length]).join(""); + return `${group()}-${group()}`; +} + +function deviceSecret(): string { + const s = process.env.SESSION_SECRET; + if (s !== undefined && s !== "") return s; + if (process.env.NODE_ENV === "production") throw new Error("SESSION_SECRET required in production"); + return "dev-session-secret"; +} + +function sign(body: string): string { + return createHmac("sha256", `${deviceSecret()}:device-v1`).update(body).digest("base64url"); +} + +export function issueDeviceToken(payload: Omit, ttlSeconds = 30 * 24 * 3600): string { + const full: DeviceTokenPayload = { ...payload, issuedAt: Math.floor(Date.now() / 1000), expiresAt: Math.floor(Date.now() / 1000) + ttlSeconds }; + const body = Buffer.from(JSON.stringify(full)).toString("base64url"); + return `${body}.${sign(body)}`; +} + +export function verifyDeviceToken(token: string): DeviceTokenPayload | null { + const dot = token.lastIndexOf("."); + if (dot === -1) return null; + const body = token.slice(0, dot); + const mac = token.slice(dot + 1); + const expected = sign(body); + if (Buffer.byteLength(mac) !== Buffer.byteLength(expected) || !timingSafeEqual(Buffer.from(mac), Buffer.from(expected))) return null; + try { + const payload = JSON.parse(Buffer.from(body, "base64url").toString("utf-8")) as DeviceTokenPayload; + if (!isUuid(payload.deviceId) || !isUuid(payload.workspaceId) || !isUuid(payload.userId) || + !Number.isSafeInteger(payload.epoch) || payload.epoch < 1 || !Number.isSafeInteger(payload.expiresAt) || + payload.expiresAt <= Math.floor(Date.now() / 1000) || !Array.isArray(payload.scopes) || + !payload.scopes.every(scope => typeof scope === "string" && DEVICE_SCOPES.some(allowed => allowed === scope))) return null; + return payload; + } catch { return null; } +} + +@Injectable() +export class DeviceService { + private readonly deps: Required> & { now: () => Date }; + constructor(@Optional() @Inject(DEVICE_DEPENDENCIES) dependencies?: DeviceDependencies) { + const now = dependencies?.now ?? (() => new Date()); + this.deps = { + transaction: dependencies?.transaction ?? withTenant, + pairingTtlMs: dependencies?.pairingTtlMs ?? 5 * 60 * 1000, + tokenTtlSeconds: dependencies?.tokenTtlSeconds ?? 30 * 24 * 3600, + serverUrl: dependencies?.serverUrl ?? process.env.PUBLIC_ORIGIN ?? "", + lookupWorkspace: dependencies?.lookupWorkspace ?? (async code => { + const lookup = await pool.query<{ workspace_id: string }>("SELECT device_pairing_workspace($1) AS workspace_id", [code]); + return lookup.rows[0]?.workspace_id ?? null; + }), + now, + }; + } + + /** Mint a pairing code for (workspace, user). TTL 5 minutes, single use. */ + async issuePairing(workspaceId: string, userId: string, scopes: readonly string[] = DEVICE_SCOPES): Promise { + if (!scopes.every(scope => DEVICE_SCOPES.some(allowed => allowed === scope))) throw new UnauthorizedException("invalid device scopes"); + const code = pairingCode(); + const expiresAt = new Date(this.deps.now().getTime() + this.deps.pairingTtlMs); + const qrPayload = JSON.stringify({ serverUrl: this.deps.serverUrl, pairingCode: code }); + await this.deps.transaction(workspaceId, async client => { + await client.query( + `INSERT INTO device_pairings (workspace_id, user_id, code, qr_payload, expires_at, scopes) VALUES ($1,$2,$3,$4,$5,$6)`, + [workspaceId, userId, code, qrPayload, expiresAt, [...scopes]]); + }); + return { pairingCode: code, qrPayload, expiresAt: expiresAt.toISOString() }; + } + + async redeemCode(code: string, deviceName: string, key: string | undefined, expectedWorkspace?: string): Promise { + if (!/^[A-Z0-9]{4}-[A-Z0-9]{4}$/.test(code)) throw new UnauthorizedException("invalid pairing code"); + const workspaceId = await this.deps.lookupWorkspace(code); + if (!workspaceId || (expectedWorkspace !== undefined && expectedWorkspace !== workspaceId)) throw new UnauthorizedException("invalid pairing code"); + const owner = await this.deps.transaction(workspaceId, async client => { + const result = await client.query<{ user_id: string }>("SELECT user_id FROM device_pairings WHERE workspace_id=$1 AND code=$2", [workspaceId, code]); + return result.rows[0]?.user_id; + }); + if (!owner) throw new UnauthorizedException("invalid pairing code"); + return idempotentChat(this.deps.transaction, workspaceId, owner, `redeem:${createHash("sha256").update(code).digest("hex")}`, key, { deviceName }, () => this.redeem(workspaceId, code, deviceName)); + } + + /** Redeem within an explicit tenant, rejecting codes from other workspaces. */ + async redeem(workspaceId: string, code: string, deviceName: string): Promise { + if (!/^[A-Z0-9]{4}-[A-Z0-9]{4}$/.test(code)) throw new UnauthorizedException("invalid pairing code"); + return this.deps.transaction(workspaceId, async client => { + const found = await client.query( + `SELECT * FROM device_pairings WHERE workspace_id=$1 AND code=$2 FOR UPDATE`, [workspaceId, code]); + const row = found.rows[0]; + if (!row) throw new UnauthorizedException("invalid pairing code"); + if (row.redeemed_at !== null) throw new ConflictException("pairing code already redeemed"); + if (row.expires_at.getTime() <= this.deps.now().getTime()) throw new UnauthorizedException("pairing code expired"); + const device = await client.query<{ id: string }>( + `INSERT INTO devices (workspace_id, user_id, name, scopes, authz_epoch) SELECT $1,$2,$3,$4,authz_epoch FROM workspaces WHERE id=$1 RETURNING id`, + [workspaceId, row.user_id, deviceName, row.scopes]); + const deviceId = device.rows[0]?.id; + if (!deviceId) throw new UnauthorizedException("pairing workspace no longer exists"); + await client.query(`UPDATE device_pairings SET redeemed_at = now(), device_id = $3 WHERE workspace_id=$1 AND code=$2`, + [workspaceId, code, deviceId]); + const token = issueDeviceToken({ deviceId, workspaceId, userId: row.user_id, scopes: row.scopes, epoch: 1 }, this.deps.tokenTtlSeconds); + return { deviceToken: token, workspaceId }; + }); + } + + /** Revoke: bumps the epoch, so every outstanding token fails closed. */ + async revoke(workspaceId: string, deviceId: string): Promise { + const result = await this.deps.transaction(workspaceId, async client => { + const updated = await client.query<{ epoch: string | number }>( + `UPDATE devices SET revoked_at = now(), epoch = epoch + 1 WHERE workspace_id=$1 AND id=$2 AND revoked_at IS NULL RETURNING epoch`, + [workspaceId, deviceId]); + return updated.rowCount ?? 0; + }); + if (result === 0) throw new NotFoundException("device not found or already revoked"); + } + + /** Epoch check for a presented device token. */ + async epochOf(workspaceId: string, deviceId: string): Promise { + return this.deps.transaction(workspaceId, async client => { + const row = await client.query<{ epoch: string | number; revoked_at: Date | null }>( + `SELECT epoch, revoked_at FROM devices WHERE workspace_id=$1 AND id=$2`, [workspaceId, deviceId]); + const found = row.rows[0]; + if (!found || found.revoked_at !== null) return null; + return Number(found.epoch); + }); + } +} + +export const deviceTokenFingerprint = (token: string): string => createHash("sha256").update(token).digest("hex").slice(0, 16); diff --git a/apps/api/src/live-chat/governors.ts b/apps/api/src/live-chat/governors.ts new file mode 100644 index 0000000..2c7387a --- /dev/null +++ b/apps/api/src/live-chat/governors.ts @@ -0,0 +1,102 @@ +/** + * Send governors (§3.2): per-provider token buckets in front of every + * outbound chat call, plus a YouTube-style daily quota ledger. + * + * A send below budget is REFUSED with a paused reason — it never queues and + * never throws, so a fan-out keeps going for the targets that still have + * budget (§3.5 "an incapable target is refused with its reason"). + */ +export interface GovernorVerdict { + allowed: boolean; + /** Present only when refused. */ + reason?: string; + state?: "ok" | "paused:rate" | "paused:quota" | "degraded"; +} + +export interface TokenBucketOptions { + /** Burst capacity (tokens). */ + capacity: number; + /** Refill per second. */ + refillPerSecond: number; + /** Virtual clock, overridable in tests. */ + now?: () => number; +} + +/** Classic token bucket: `capacity` burst, continuous refill. */ +export class TokenBucket { + private tokens: number; + private last: number; + constructor(private readonly options: TokenBucketOptions) { + if (!Number.isInteger(options.capacity) || options.capacity < 1) throw new Error("capacity must be a positive integer"); + if (!(options.refillPerSecond > 0)) throw new Error("refillPerSecond must be positive"); + this.tokens = options.capacity; + this.last = options.now?.() ?? Date.now(); + } + /** Peek without consuming. */ + available(): number { this.refill(); return this.tokens; } + /** Try to consume one token. */ + tryTake(): boolean { this.refill(); if (this.tokens < 1) return false; this.tokens -= 1; return true; } + private refill(): void { + const now = this.options.now?.() ?? Date.now(); + const elapsed = (now - this.last) / 1000; + if (elapsed > 0) { this.tokens = Math.min(this.options.capacity, this.tokens + elapsed * this.options.refillPerSecond); this.last = now; } + } +} + +/** Daily unit ledger (YouTube quota: list=1, insert/delete/ban=50, 10 000/day). */ +export class QuotaLedger { + private spent = 0; + private day: string; + constructor(private readonly dailyBudget: number, private readonly costPerCall = 1, private readonly now: () => number = Date.now) { + this.day = this.currentDay(); + if (!Number.isFinite(dailyBudget) || dailyBudget < 0 || !Number.isFinite(costPerCall) || costPerCall < 0) throw new Error("invalid quota budget/cost"); + } + private currentDay(): string { return new Date(this.now()).toISOString().slice(0, 10); } + private reset(): void { const day = this.currentDay(); if (day !== this.day) { this.spent = 0; this.day = day; } } + get used(): number { this.reset(); return this.spent; } + get budget(): number { return this.dailyBudget; } + canAfford(cost = this.costPerCall): boolean { this.reset(); return Number.isFinite(cost) && cost >= 0 && this.spent + cost <= this.dailyBudget; } + /** Record a spend; returns false (and records nothing) when over budget. */ + charge(cost = this.costPerCall): boolean { + if (!this.canAfford(cost)) return false; + this.spent += cost; + return true; + } +} + +export interface SendGovernorOptions { + /** Tokens per provider key, e.g. `twitch`, `youtube`, `mock-livestream`. */ + buckets?: Record; + /** Daily unit budgets per provider key (quota ledger). */ + quotas?: Record; + now?: () => number; +} + +/** Per-provider send governor: token bucket + optional daily quota. */ +export class SendGovernors { + private readonly buckets = new Map(); + private readonly quotas = new Map(); + constructor(options: SendGovernorOptions = {}) { + for (const [key, bucket] of Object.entries(options.buckets ?? {})) this.buckets.set(key, new TokenBucket({ ...bucket, now: bucket.now ?? options.now })); + for (const [key, quota] of Object.entries(options.quotas ?? {})) this.quotas.set(key, new QuotaLedger(quota.budget, quota.costPerCall ?? 1, options.now)); + } + /** Probe a provider's outbound capacity without consuming anything. */ + probe(provider: string): GovernorVerdict { + const quota = this.quotas.get(provider); + if (quota && !quota.canAfford()) return { allowed: false, reason: "daily quota exhausted", state: "paused:quota" }; + const bucket = this.buckets.get(provider); + if (bucket && bucket.available() < 1) return { allowed: false, reason: "send rate budget exhausted", state: "paused:rate" }; + return { allowed: true, state: "ok" }; + } + /** Consume one send's worth of budget for `provider`. */ + take(provider: string, cost?: number): GovernorVerdict { + const quota = this.quotas.get(provider); + if (quota && !quota.canAfford(cost)) return { allowed: false, reason: "daily quota exhausted", state: "paused:quota" }; + const bucket = this.buckets.get(provider); + if (bucket && !bucket.tryTake()) return { allowed: false, reason: "send rate budget exhausted", state: "paused:rate" }; + if (quota) quota.charge(cost); + return { allowed: true, state: "ok" }; + } + /** A provider with no configured governor is unlimited (mock, local hubs). */ + configured(provider: string): boolean { return this.buckets.has(provider) || this.quotas.has(provider); } +} diff --git a/apps/api/src/live-chat/hub.test.ts b/apps/api/src/live-chat/hub.test.ts new file mode 100644 index 0000000..a9fae71 --- /dev/null +++ b/apps/api/src/live-chat/hub.test.ts @@ -0,0 +1,439 @@ +import type { PoolClient } from "pg"; +import { randomUUID } from "node:crypto"; +import { describe, expect, it, vi } from "vitest"; +import { ChatRingBuffer } from "./ring-buffer.js"; +import { SendGovernors } from "./governors.js"; +import { Duplex } from "node:stream"; +import type { Socket } from "node:net"; +import { createMockLivestreamConnector } from "@tactical-media-broadcast-hub/connector-mock-livestream"; +import { RealtimeService } from "../realtime/realtime.service.js"; +import { RealtimeSocket } from "../realtime/websocket.js"; +import { RespRedisSubscriptions } from "../realtime/subscribe.js"; +import { encode, parse, type ControlRedis } from "../media-control/redis.js"; +import { MediaControlService } from "../media-control/media-control.service.js"; +import { ChatRealtimeBridge, CHAT_PUBLISH_SCRIPT } from "./bridge.js"; +import { MockLivestreamDriver } from "./mock-driver.js"; +import { issueSession } from "../auth/session.js"; +import { LiveChatHubService, type LiveChatPublish } from "./live-chat.service.js"; +import type { ChatTarget, ChatTransport, NormalizedChatEvent } from "./types.js"; + +const WS = randomUUID(), CH = randomUUID(); + +/** + * SQL-semantics fake for the hub's three statements: + * - live_chat_seq: FOR UPDATE serializes per (workspace, channel); the + * create-or-lock pattern runs exactly as written in the service. + * - live_chat_messages: UNIQUE(workspace_id, provider, provider_msg_id) + * turns a replay into a no-op that returns nothing. + * - cursor paging: seq > after ORDER BY seq ASC LIMIT n. + * The fake implements the CONSTRAINTS, not the text — the live-DB variant of + * these tests is live-chat.integration.test.ts. + */ +class SqlFake { + readonly messages: Array & { workspace_id: string; channel_id: string; seq: number; provider: string; provider_msg_id: string }> = []; + readonly seen = new Set(); + readonly seqRows = new Map(); + readonly locks = new Map>(); + readonly moderated: string[] = []; + + async transaction(workspaceId: string, fn: (client: PoolClient) => Promise): Promise { + const key = workspaceId; // one lock per workspace fakes the row lock + const prior = [...this.locks.values()]; + let unlock: () => void = () => undefined; + const gate = new Promise(resolve => { unlock = resolve; }); + this.locks.set(key, gate); + await Promise.all(prior); + try { return await fn(new Client(this) as unknown as PoolClient); } finally { unlock(); this.locks.delete(key); } + } +} + +class Client { + constructor(private readonly db: SqlFake) {} + async query(sql: string, params: unknown[]): Promise<{ rows: Array>; rowCount: number }> { + if (sql.includes("INSERT INTO live_chat_seen_events")) { + const key = JSON.stringify(params); + if (this.db.seen.has(key)) return { rows: [], rowCount: 0 }; + this.db.seen.add(key); return { rows: [{ provider_event_id: params[2] }], rowCount: 1 }; + } + if (sql.includes("live_chat_seq") && sql.includes("INSERT")) { + const ws = String(params[0]), ch = String(params[1]); + if (!this.db.seqRows.has(`${ws}:${ch}`)) this.db.seqRows.set(`${ws}:${ch}`, 0); + return { rows: [], rowCount: 0 }; + } + if (sql.includes("live_chat_seq") && sql.includes("FOR UPDATE")) { + const ws = String(params[0]), ch = String(params[1]); + return { rows: [{ seq: this.db.seqRows.get(`${ws}:${ch}`) ?? 0 }], rowCount: 1 }; + } + if (sql.includes("live_chat_seq")) { // the GREATEST update + const ws = String(params[0]), ch = String(params[1]), seq = Number(params[2]); + const key = `${ws}:${ch}`; + this.db.seqRows.set(key, Math.max(this.db.seqRows.get(key) ?? 0, seq)); + return { rows: [], rowCount: 1 }; + } + if (sql.includes("INSERT INTO live_chat_messages")) { + const columns = Array.from({ length: 16 }, (_, i) => params[i] as unknown[]); + const inserted: Array> = []; + for (let i = 0; i < (columns[0]?.length ?? 0); i += 1) { + const row = { + id: randomUUID(), workspace_id: String(columns[0]?.[i]), channel_id: String(columns[1]?.[i]), connection_id: columns[2]?.[i], + provider: String(columns[3]?.[i]), provider_msg_id: String(columns[4]?.[i]), author_handle: columns[5]?.[i], + author_display_name: columns[6]?.[i], author_avatar_url: columns[7]?.[i], author_badges: JSON.parse(String(columns[8]?.[i])) as unknown, + author_color: columns[9]?.[i], body: columns[10]?.[i], reply_to: columns[11]?.[i], seq: Number(columns[12]?.[i]), + sent_at: columns[13]?.[i], received_at: columns[14]?.[i], source: columns[15]?.[i], is_moderated: false, + }; + const dup = this.db.messages.some(m => m.workspace_id === row.workspace_id && m.provider === row.provider && m.provider_msg_id === row.provider_msg_id); + if (!dup) { this.db.messages.push(row); inserted.push(row); } + } + return { rows: inserted, rowCount: inserted.length }; + } + if (sql.includes("SELECT") && sql.includes("live_chat_messages") && sql.includes("provider=$5")) { + const items = this.db.messages.filter(m => m.workspace_id === params[0] && m.channel_id === params[1] && Number(m.seq) > Number(params[2]) && m.provider === params[4]) + .sort((a, b) => a.seq - b.seq).slice(0, Number(params[3])); + return { rows: items, rowCount: items.length }; + } + if (sql.includes("SELECT") && sql.includes("live_chat_messages")) { + const after = sql.includes("id=$3") + ? this.db.messages.filter(m => m.workspace_id === params[0] && m.channel_id === params[1] && m.id === params[2]) + : this.db.messages.filter(m => m.workspace_id === params[0] && m.channel_id === params[1] && Number(m.seq) > Number(params[2])) + .sort((a, b) => a.seq - b.seq).slice(0, Number(params[3])); + return { rows: after, rowCount: after.length }; + } + if (sql.includes("UPDATE live_chat_messages")) { + const [ws, , provider, providerMsgId] = params; + let count = 0; + for (const m of this.db.messages) { + if (m.workspace_id === ws && m.provider === provider && m.provider_msg_id === providerMsgId) { m.is_moderated = true; count += 1; } + } + return { rows: [], rowCount: count }; + } + throw new Error(`SqlFake: unexpected statement ${sql.slice(0, 60)}`); + } +} + +function fixture(options?: { + targets?: ChatTarget[]; transports?: Record; governors?: SendGovernors; capacity?: number; publish?: LiveChatPublish; +}) { + const db = new SqlFake(); + const buffer = new ChatRingBuffer(options?.capacity ?? 500); + const published: Array<{ workspaceId: string; channelId: string; frame: Record }> = []; + const transports = options?.transports ?? {}; + const targets = options?.targets ?? []; + let clock = Date.parse("2026-10-07T10:00:00Z"); + const hub = new LiveChatHubService({ + transaction: (workspaceId, fn) => db.transaction(workspaceId, fn), + publish: async (workspaceId, channelId, frame) => { published.push({ workspaceId, channelId, frame }); await options?.publish?.(workspaceId, channelId, frame); }, + buffer, + governors: options?.governors ?? new SendGovernors(), + targets: async () => targets, + transports, + now: () => new Date(clock += 7), + }); + return { db, buffer, published, hub, transports, targets }; +} + +function event(overrides: Partial = {}): NormalizedChatEvent { + return { + provider: "mock-livestream", connectionId: "conn-1", channelId: CH, providerMessageId: `pm-${randomUUID().slice(0, 8)}`, + author: { handle: "viewer@example" }, text: "hello", sentAt: new Date("2026-10-07T10:00:00Z"), source: "live-chat", ...overrides, + }; +} + +function target(overrides: Partial = {}): ChatTarget { + return { connectionId: "conn-1", provider: "mock-livestream", liveRemoteId: "live-1", read: true, reply: true, moderate: true, state: "ok", ...overrides }; +} + +describe("ingestion: seq assignment, dedupe, fan-out", () => { + it("assigns monotonic per-channel seqs, stores rows, publishes one frame per insert", async () => { + const f = fixture(); + const first = await f.hub.ingest(WS, [event({ text: "one" }), event({ text: "two" })]); + expect(first.inserted.map(m => m.seq)).toEqual([1, 2]); + expect(first.inserted.map(m => m.text)).toEqual(["one", "two"]); + expect(first.duplicates).toEqual([]); + expect(f.published).toHaveLength(2); + expect(f.published[0]?.frame).toMatchObject({ kind: "chat", seq: 1 }); + const second = await f.hub.ingest(WS, [event({ text: "three" })]); + expect(second.inserted[0]?.seq).toBe(3); // continues the channel sequence + }); + + it("same provider id ingested twice = one row, one frame, and the replay is reported", async () => { + const f = fixture(); + const original = event({ providerMessageId: "pm-dup" }); + const first = await f.hub.ingest(WS, [original]); + const replay = await f.hub.ingest(WS, [original]); + expect(first.inserted).toHaveLength(1); + expect(replay.inserted).toHaveLength(0); + expect(replay.duplicates).toEqual(["pm-dup"]); + expect(f.db.messages).toHaveLength(1); // one durable row + expect(f.published).toHaveLength(1); // replay publishes nothing + expect(f.buffer.latest(CH)).toHaveLength(1); // and does not enter the window + }); + + it("dedupe is scoped to (workspace, provider): same id, different provider, both stored", async () => { + const f = fixture(); + await f.hub.ingest(WS, [event({ provider: "twitch", providerMessageId: "shared", connectionId: "conn-tw" })]); + const second = await f.hub.ingest(WS, [event({ provider: "youtube", providerMessageId: "shared", connectionId: "conn-yt" })]); + expect(second.inserted).toHaveLength(1); + expect(f.db.messages).toHaveLength(2); + }); + + it("cursor paging returns strictly-after rows in seq order and a next cursor", async () => { + const f = fixture(); + await f.hub.ingest(WS, [event(), event(), event(), event(), event()]); + const page1 = await f.hub.messages(WS, CH, { limit: 3 }); + expect(page1.items.map(m => m.seq)).toEqual([1, 2, 3]); + expect(page1.nextCursor).toBe(3); + const page2 = await f.hub.messages(WS, CH, { after: page1.nextCursor, limit: 3 }); + expect(page2.items.map(m => m.seq)).toEqual([4, 5]); + expect(page2.nextCursor).toBeUndefined(); + }); + + it("resume serves from the window while lastSeq is inside it, resync past its edge", async () => { + const f = fixture(); + for (let i = 0; i < 5; i += 1) await f.hub.ingest(WS, [event()]); + const inside = f.hub.resume(CH, 3); + expect(inside.resync).toBe(false); + expect(inside.messages.map(m => m.seq)).toEqual([4, 5]); + expect(f.hub.resume(CH, 0).messages).toHaveLength(5); + const small = fixture({ capacity: 4 }); + for (let i = 0; i < 6; i += 1) await small.hub.ingest(WS, [event()]); + expect(small.hub.resume(CH, 1).resync).toBe(true); + expect(small.hub.resume(CH, 2).messages.map(m => m.seq)).toEqual([3, 4, 5, 6]); + }); +}); + +describe("system gap notices", () => { + it("stores one system notice and emits a distinct chat.gap sequence, deduping repeated gaps", async () => { + const f = fixture(), gap = event({ providerMessageId: "disconnect-1" }); + await f.hub.gap(WS, gap, "provider disconnected"); + expect(f.db.messages[0]?.source).toBe("system"); + expect(f.published.at(-1)?.frame).toMatchObject({ seq: 2, provider: "mock-livestream", reason: "provider disconnected" }); + await f.hub.gap(WS, gap, "provider disconnected"); expect(f.published).toHaveLength(2); + }); +}); + +describe("reply fan-out with per-target Delivery rows", () => { + const transportsFor = (send: ChatTransport["send"]): Record => ({ + "mock-livestream": { provider: "mock-livestream", send, moderate: async () => undefined }, + }); + + it("fans out to every capable target with one Delivery row each and stores the echo", async () => { + const send = vi.fn(async () => ({ providerMessageId: "pm-ours", sentAt: new Date("2026-10-07T10:01:00Z") })); + const f = fixture({ + targets: [target({ connectionId: "conn-a" }), target({ connectionId: "conn-b", provider: "twitch", liveRemoteId: "tw-1" })], + transports: { ...transportsFor(send), twitch: { provider: "twitch", send, moderate: async () => undefined } }, + }); + const outcome = await f.hub.send(WS, CH, { text: "hello from the studio", targets: "all", author: { handle: "studio" } }); + expect(outcome.delivery).toHaveLength(2); + expect(outcome.delivery.every(d => d.status === "sent")).toBe(true); + expect(outcome.message?.text).toBe("hello from the studio"); + expect(outcome.message?.providerMessageId).toBe("pm-ours"); + expect(f.published.some(p => p.frame.delivery !== undefined)).toBe(true); // chat.delivery frame + }); + + it("an unsupported target never fails the batch (§3.5)", async () => { + const send = vi.fn(async () => ({ providerMessageId: "pm-ok", sentAt: new Date() })); + const f = fixture({ + targets: [ + target({ connectionId: "conn-ok" }), + target({ connectionId: "conn-no-reply", reply: false, reason: "live chat write not supported" }), + ], + transports: transportsFor(send as ChatTransport["send"]), + }); + const outcome = await f.hub.send(WS, CH, { text: "hi", targets: "all", author: { handle: "studio" } }); + expect(outcome.delivery).toHaveLength(2); + expect(outcome.delivery.find(d => d.connectionId === "conn-ok")).toMatchObject({ status: "sent" }); + expect(outcome.delivery.find(d => d.connectionId === "conn-no-reply")).toMatchObject({ status: "unsupported", error: "live chat write not supported" }); + expect(send).toHaveBeenCalledTimes(1); + expect(outcome.message).not.toBeNull(); + }); + + it("a transport failure is a failed Delivery row; the other target still sends", async () => { + const send = vi.fn(async (request: { connectionId: string }) => { + if (request.connectionId === "conn-bad") throw new Error("Helix 503"); + return { providerMessageId: "pm-ok", sentAt: new Date() }; + }); + const f = fixture({ + targets: [target({ connectionId: "conn-good" }), target({ connectionId: "conn-bad" })], + transports: transportsFor(send as ChatTransport["send"]), + }); + const outcome = await f.hub.send(WS, CH, { text: "hi", targets: "all", author: { handle: "studio" } }); + expect(outcome.delivery.find(d => d.connectionId === "conn-bad")).toMatchObject({ status: "failed", error: "Helix 503" }); + expect(outcome.delivery.find(d => d.connectionId === "conn-good")).toMatchObject({ status: "sent" }); + }); + + it("a reply below budget is REFUSED with a paused reason and nothing is sent", async () => { + const send = vi.fn(async () => ({ providerMessageId: "pm-x", sentAt: new Date() })); + const f = fixture({ + targets: [target()], + transports: transportsFor(send as ChatTransport["send"]), + governors: new SendGovernors({ quotas: { "mock-livestream": { budget: 1, costPerCall: 2 } } }), // first take already over budget + }); + const outcome = await f.hub.send(WS, CH, { text: "hi", targets: "all", author: { handle: "studio" } }); + expect(send).not.toHaveBeenCalled(); + expect(outcome.message).toBeNull(); + expect(outcome.refused).toMatchObject({ state: "paused:quota", reason: expect.stringContaining("quota") }); + expect(outcome.delivery[0]).toMatchObject({ status: "unsupported", error: expect.stringContaining("quota") }); + expect(f.db.messages).toHaveLength(0); // refused means no echo row either + }); + + it("target selection honours an explicit target list", async () => { + const send = vi.fn(async (_request: Parameters[0]) => ({ providerMessageId: "pm-1", sentAt: new Date() })); + const f = fixture({ + targets: [target({ connectionId: "conn-a" }), target({ connectionId: "conn-b" })], + transports: transportsFor(send as ChatTransport["send"]), + }); + await f.hub.send(WS, CH, { text: "hi", targets: ["conn-b"], author: { handle: "studio" } }); + expect(send).toHaveBeenCalledTimes(1); + expect(send.mock.calls[0]?.[0]?.connectionId).toBe("conn-b"); + expect(outcomeOf(f)).toHaveLength(1); + }); + + it("resolves a hub replyTo id to the provider parent id before sending", async () => { + const send = vi.fn(async (_request: Parameters[0]) => ({ providerMessageId: "reply", sentAt: new Date() })); + const f = fixture({ targets: [target()], transports: transportsFor(send) }); + const parent = (await f.hub.ingest(WS, [event({ providerMessageId: "parent-provider-id" })])).inserted[0]; + if (!parent) throw new Error("missing parent"); + await f.hub.send(WS, CH, { text: "threaded", targets: "all", replyTo: parent.id, author: { handle: "studio" } }); + expect(send).toHaveBeenCalledWith(expect.objectContaining({ replyTo: "parent-provider-id" })); + }); + + it("no capable targets at all is a clean refusal, not an exception", async () => { + const f = fixture({ targets: [] }); + const outcome = await f.hub.send(WS, CH, { text: "hi", targets: "all", author: { handle: "studio" } }); + expect(outcome.refused).toMatchObject({ state: "degraded", reason: "no capable target" }); + expect(outcome.delivery).toEqual([]); + }); + + function outcomeOf(f: { published: Array<{ frame: Record }> }): unknown[] { + return f.published.filter(p => p.frame.delivery !== undefined).flatMap(p => p.frame.delivery as unknown[]); + } +}); + +describe("moderation fan-out", () => { + it("delete fans out, flags the local row and publishes a chat.moderated frame", async () => { + const moderate = vi.fn(async (_request: Parameters[0]) => undefined); + const f = fixture({ + targets: [target(), target({ connectionId: "conn-no-mod", moderate: false })], + transports: { "mock-livestream": { provider: "mock-livestream", send: async () => ({ providerMessageId: "x", sentAt: new Date() }), moderate: moderate as ChatTransport["moderate"] } }, + }); + const stored = (await f.hub.ingest(WS, [event({ providerMessageId: "pm-victim" })])).inserted[0]; + const outcome = await f.hub.moderate(WS, CH, { + action: "delete", + message: { provider: "mock-livestream", providerMessageId: "pm-victim", authorHandle: "viewer@example" }, + }); + expect(moderate).toHaveBeenCalledWith(expect.objectContaining({ messageRemoteId: "pm-victim", action: "delete" })); + expect(outcome.delivery.find(d => d.connectionId === "conn-no-mod")?.status).toBe("unsupported"); + expect(outcome.delivery.find(d => d.connectionId === "conn-1")?.status).toBe("sent"); + expect(f.db.messages[0]?.is_moderated).toBe(true); + const frame = f.published.at(-1)?.frame; + expect(frame).toMatchObject({ kind: "chat", action: "delete", targetMessageId: "pm-victim" }); + void stored; + }); + + it("a failed deletion preserves the row and emits no moderation success frame", async () => { + const f = fixture({ targets: [target()], transports: { "mock-livestream": { provider: "mock-livestream", send: async () => ({ providerMessageId: "x", sentAt: new Date() }), moderate: async () => { throw new Error("provider refused"); } } } }); + await f.hub.ingest(WS, [event({ providerMessageId: "victim" })]); + const outcome = await f.hub.moderate(WS, CH, { action: "delete", message: { provider: "mock-livestream", providerMessageId: "victim", authorHandle: "viewer" } }); + expect(outcome.delivery[0]?.status).toBe("failed"); expect(f.db.messages[0]?.is_moderated).toBe(false); + expect(f.published).toHaveLength(1); + }); + + it("timeout passes durationSeconds to the transport", async () => { + const moderate = vi.fn(async (_request: Parameters[0]) => undefined); + const f = fixture({ + targets: [target()], + transports: { "mock-livestream": { provider: "mock-livestream", send: async () => ({ providerMessageId: "x", sentAt: new Date() }), moderate: moderate as ChatTransport["moderate"] } }, + }); + await f.hub.moderate(WS, CH, { action: "timeout", durationSeconds: 600, message: { provider: "mock-livestream", providerMessageId: "pm-1", authorHandle: "a" } }); + expect(moderate.mock.calls[0]?.[0]).toMatchObject({ action: "timeout", durationSeconds: 600 }); + }); +}); + +describe("capabilities endpoint shape", () => { + it("mirrors target rows with state and optional reason", async () => { + const f = fixture({ targets: [target(), target({ connectionId: "c2", reply: false, state: "paused:quota", reason: "daily quota exhausted" })] }); + const caps = await f.hub.capabilities(WS, CH); + expect(caps).toEqual([ + expect.objectContaining({ connectionId: "conn-1", read: true, reply: true, moderate: true, state: "ok" }), + expect.objectContaining({ connectionId: "c2", reply: false, state: "paused:quota", reason: "daily quota exhausted" }), + ]); + }); +}); + +class RespChatSocket extends Duplex { + constructor() { super(); queueMicrotask(() => this.emit("connect")); } + override _read(): void {} + override _write(bytes: Buffer, _encoding: BufferEncoding, done: (error?: Error | null) => void): void { + const parts = parse(bytes)?.reply; + if (!Array.isArray(parts)) { done(new Error("invalid RESP")); return; } + queueMicrotask(() => this.emit("data", encode([String(parts[0]).toLowerCase(), String(parts[1]), "1"]))); + done(); + } + publish(topic: string, raw: string): void { const bytes = encode(["message", topic, raw]); this.emit("data", bytes.subarray(0, 7)); this.emit("data", bytes.subarray(7)); } +} +class ClientWire extends Duplex { + readonly frames: Array> = []; + override _read(): void {} + override _write(bytes: Buffer, _encoding: BufferEncoding, done: (error?: Error | null) => void): void { + if ((bytes[0] ?? 0) === 129) this.frames.push(JSON.parse(bytes.subarray((bytes[1] ?? 0) === 126 ? 4 : 2).toString()) as Record); + done(); + } + send(value: unknown): void { + const body = Buffer.from(JSON.stringify(value)), header = Buffer.alloc(body.length < 126 ? 6 : 8); + header[0] = 129; + if (body.length < 126) header[1] = 128 | body.length; + else { header[1] = 128 | 126; header.writeUInt16BE(body.length, 2); } + // A zero mask is valid; fragmentation exercises the incremental decoder. + const bytes = Buffer.concat([header, body]); this.emit("data", bytes.subarray(0, 5)); this.emit("data", bytes.subarray(5)); + } +} +async function settle(): Promise { await new Promise(resolve => setTimeout(resolve, 10)); } + +describe("mock connector → hub rows → Redis RESP → WS chat contract", () => { + it("dedupes, preserves ingress seq, resumes on a fresh gateway, and resyncs after 500-frame eviction", async () => { + let respSocket: RespChatSocket | null = null; + const subscriptions = new RespRedisSubscriptions("redis://localhost", () => undefined, () => { respSocket = new RespChatSocket(); return respSocket as unknown as Socket; }); + const window: Record[] = []; + const redis: ControlRedis = { + get: async () => null, mget: async () => [], set: async () => true, incr: async () => 1, sadd: async () => undefined, smembers: async () => [], publish: async () => 1, close: () => undefined, + eval: async (script, keys, args) => { + expect(script).toBe(CHAT_PUBLISH_SCRIPT); + const frame = JSON.parse(args[1] ?? "{}") as Record; + window.push(frame); if (window.length > 500) window.shift(); + respSocket?.publish(keys[1] ?? "", args[1] ?? ""); return 1; + }, + }; + const bridge = new ChatRealtimeBridge(undefined, redis), f = fixture({ publish: bridge.publish }); + const connector = createMockLivestreamConnector(); + const connection = { id: randomUUID(), workspaceId: WS, provider: connector.manifest().id, channelId: CH, credentialId: { kind: "credential-ref" as const, id: "mock" }, fields: {}, createdAt: new Date() }; + const live = await connector.createLiveStream?.(connection, { channelId: CH, title: "test", permanent: false, saveReplay: false }); + if (!live) throw new Error("mock live missing"); + await connector.sendLiveChatMessage?.(connection, { liveRemoteId: live.remoteId, text: "hello through the wire" }); + const driver = new MockLivestreamDriver(connector, { workspaceId: WS, channelId: CH, connectionId: connection.id, liveRemoteId: live.remoteId }); + await driver.start((ws, events) => f.hub.ingest(ws, events)); + await driver.poll(); expect(f.db.messages).toHaveLength(1); expect(window).toHaveLength(1); + const service = new RealtimeService(new MediaControlService({ signer: null, assignments: [] }), { + subscriptions, authorizeChat: async () => true, channelExists: async () => true, chatWindow: async () => window.slice(), status: async () => ({ channels: [] }), + }); + const wires: ClientWire[] = []; + const open = (lastSeq: number) => { + const wire = new ClientWire(), peer = new RealtimeSocket(wire), handlers = service.attach(peer); + peer.onMessage = handlers.message; peer.onClose = handlers.close; wires.push(wire); + wire.send({ op: "auth", token: issueSession({ userId: randomUUID(), workspaceId: WS, role: "Operator", authzEpoch: 1 }) }); + wire.send({ op: "subscribe", id: "chat", topic: "livechat", liveId: CH, lastSeq }); return wire; + }; + try { + const first = open(0); await settle(); + expect(first.frames).toContainEqual({ op: "ready" }); + expect(first.frames).toContainEqual({ op: "ack", id: "chat", fromSeq: 1 }); + expect(first.frames).toContainEqual(expect.objectContaining({ op: "msg", topic: "livechat", type: "chat.message", seq: 1, payload: expect.objectContaining({ seq: 1, message: expect.objectContaining({ seq: 1, text: "hello through the wire" }) }) })); + await f.hub.ingest(WS, [event()]); + expect(first.frames.filter(frame => frame.op === "msg").map(frame => frame.seq)).toEqual([1, 2]); + const resume = open(1); await settle(); expect(resume.frames.filter(frame => frame.op === "msg").map(frame => frame.seq)).toEqual([2]); + for (let i = 0; i < 500; i += 1) await f.hub.ingest(WS, [event()]); + const stale = open(1); await settle(); expect(stale.frames).toContainEqual({ op: "resync", topic: "livechat" }); + expect(window).toHaveLength(500); + const page = await f.hub.messages(WS, CH, { after: 1, limit: 100 }); expect(page.items[0]?.seq).toBe(2); + } finally { await driver.stop(); service.onModuleDestroy(); bridge.onModuleDestroy(); for (const wire of wires) wire.destroy(); } + }); +}); diff --git a/apps/api/src/live-chat/idempotency.ts b/apps/api/src/live-chat/idempotency.ts new file mode 100644 index 0000000..5634ca9 --- /dev/null +++ b/apps/api/src/live-chat/idempotency.ts @@ -0,0 +1,34 @@ +import { createHash } from "node:crypto"; +import { BadRequestException, ConflictException } from "@nestjs/common"; +import type { LiveChatDependencies } from "./live-chat.service.js"; + +/** Reserve before contacting providers. A crash leaves a pending row, so an + * ambiguous external send is never repeated automatically under the same key. */ +export async function idempotentChat(transaction: LiveChatDependencies["transaction"], workspaceId: string, userId: string, route: string, key: string | undefined, body: unknown, action: () => Promise): Promise { + if (!key || !/^[\x21-\x7e]{1,128}$/.test(key)) throw new BadRequestException("Idempotency-Key must contain 1..128 printable characters"); + const hash = createHash("sha256").update(JSON.stringify(body)).digest("hex"); + const existing = await transaction(workspaceId, async client => { + const claim = await client.query( + `INSERT INTO live_chat_requests (workspace_id,user_id,route,request_key,request_hash) VALUES ($1,$2,$3,$4,$5) + ON CONFLICT DO NOTHING RETURNING request_key`, [workspaceId, userId, route, key, hash]); + if (claim.rowCount) return null; + const row = await client.query<{ request_hash: string; status: string; response: T }>( + "SELECT request_hash,status,response FROM live_chat_requests WHERE workspace_id=$1 AND user_id=$2 AND route=$3 AND request_key=$4", [workspaceId, userId, route, key]); + return row.rows[0]; + }); + if (existing) { + if (existing.request_hash !== hash) throw new ConflictException("Idempotency-Key reused with different payload"); + if (existing.status !== "complete") throw new ConflictException("chat request pending or failed; reconcile before retrying"); + return existing.response; + } + try { + const result = await action(); + await transaction(workspaceId, async client => { await client.query( + "UPDATE live_chat_requests SET status='complete',response=$5 WHERE workspace_id=$1 AND user_id=$2 AND route=$3 AND request_key=$4", [workspaceId, userId, route, key, JSON.stringify(result)]); }); + return result; + } catch (error) { + await transaction(workspaceId, async client => { await client.query( + "UPDATE live_chat_requests SET status='failed' WHERE workspace_id=$1 AND user_id=$2 AND route=$3 AND request_key=$4", [workspaceId, userId, route, key]); }).catch(() => undefined); + throw error; + } +} diff --git a/apps/api/src/live-chat/live-chat.controller.test.ts b/apps/api/src/live-chat/live-chat.controller.test.ts new file mode 100644 index 0000000..f253d73 --- /dev/null +++ b/apps/api/src/live-chat/live-chat.controller.test.ts @@ -0,0 +1,28 @@ +import "reflect-metadata"; +import { NestFactory } from "@nestjs/core"; +import { PATH_METADATA } from "@nestjs/common/constants.js"; +import { describe, expect, it, vi } from "vitest"; +import { LiveChatModule } from "./live-chat.module.js"; +import { LiveChatController } from "./live-chat.controller.js"; +import { LiveChatHubService } from "./live-chat.service.js"; +import { DeviceService } from "./devices.js"; + +describe("standalone S5 module and REST routes", () => { + it("boots with injected drivers and shuts them down without central AppModule wiring", async () => { + const driver = { provider: "mock-livestream", start: vi.fn(async () => undefined), stop: vi.fn(async () => undefined) }; + const app = await NestFactory.createApplicationContext(LiveChatModule.register({ drivers: [driver] }), { logger: false, abortOnError: false }); + try { expect(app.get(LiveChatController)).toBeInstanceOf(LiveChatController); expect(driver.start).toHaveBeenCalledTimes(1); } + finally { await app.close(); } + expect(driver.stop).toHaveBeenCalledTimes(1); + }); + it("matches §3.5 routes and rejects malformed redeem bodies before storage", async () => { + const proto = LiveChatController.prototype; + expect(Reflect.getMetadata(PATH_METADATA, proto.lives)).toBe("live-chat/lives"); + expect(Reflect.getMetadata(PATH_METADATA, proto.messages)).toBe("live-chat/lives/:channelId/messages"); + expect(Reflect.getMetadata(PATH_METADATA, proto.moderate)).toBe("live-chat/lives/:channelId/messages/:messageId/moderation"); + expect(Reflect.getMetadata(PATH_METADATA, proto.capabilities)).toBe("live-chat/lives/:channelId/capabilities"); + expect(Reflect.getMetadata(PATH_METADATA, proto.redeem)).toBe("devices/pairings/redeem"); + const controller = new LiveChatController(new LiveChatHubService(), new DeviceService()); + await expect(controller.redeem({ pairingCode: "ABCD-1234", deviceName: "" }, "key")).rejects.toThrow("deviceName"); + }); +}); diff --git a/apps/api/src/live-chat/live-chat.controller.ts b/apps/api/src/live-chat/live-chat.controller.ts new file mode 100644 index 0000000..057d8c6 --- /dev/null +++ b/apps/api/src/live-chat/live-chat.controller.ts @@ -0,0 +1,169 @@ +import { BadRequestException, Body, Controller, Delete, Get, Headers, HttpCode, Inject, Param, Post, Query, Req, UseGuards, NotFoundException, UnauthorizedException } from "@nestjs/common"; +import type { Request } from "express"; +import { idempotentChat } from "./idempotency.js"; +import { withTenant } from "../db/client.js"; +import { appendTenantAudit } from "../audit/audit.service.js"; +import { DEVICE_SCOPES } from "@tactical-media-broadcast-hub/core"; +import { parseId, parseLimit, requireObject, requireSession, requireString } from "../http/validation.js"; +import { RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import { LiveChatGuard, SessionOnly } from "./live-chat.guard.js"; +import { tokenWorkspace } from "./auth.js"; +import { DeviceService } from "./devices.js"; +import { LiveChatHubService } from "./live-chat.service.js"; + +/** + * §3.5 endpoints. No `/workspaces/{wid}` prefix — the workspace comes from the + * session (or device token) per repo convention. + */ +@Controller() +export class LiveChatController { + constructor(@Inject(LiveChatHubService) private readonly hub: LiveChatHubService, @Inject(DeviceService) private readonly devices: DeviceService) {} + + @Get("live-chat/lives") + @UseGuards(LiveChatGuard) + @RequirePermission("stream.view") + async lives(@Req() request: Request & AuthenticatedRequest) { + return this.hub.lives(requireSession(request).workspaceId); + } + + @Get("live-chat/lives/:channelId/messages") + @UseGuards(LiveChatGuard) + @RequirePermission("livechat.view") + async messages( + @Req() request: Request & AuthenticatedRequest, + @Param("channelId") channelId: string, + @Query("after") after: string | undefined, + @Query("limit") limit: string | undefined, + @Query("platform") platform: string | undefined, + ) { + const session = requireSession(request); + return this.hub.messages(session.workspaceId, parseId(channelId, "channelId"), { + limit: parseLimit(limit, 100, 500), + ...(after === undefined || after === "" ? {} : { after: parseSeq(after) }), + ...(platform === undefined || platform === "" ? {} : { provider: platform }), + }); + } + + @Post("live-chat/lives/:channelId/messages") + @UseGuards(LiveChatGuard) + @RequirePermission("livechat.send") + @HttpCode(202) + async send( + @Req() request: Request & AuthenticatedRequest, + @Param("channelId") channelId: string, + @Body() body: unknown, + @Headers("idempotency-key") key?: string, + ) { + const session = requireSession(request); + const parsed = requireObject(body, "body"); + const text = requireString(parsed.text, "text"); + if (text.length > 500) throw new BadRequestException("text must be 1..500 characters"); + const targets = parseTargets(parsed.targets); + const liveId = parseId(channelId, "channelId"); + const replyTo = parsed.replyTo === undefined ? undefined : parseId(requireString(parsed.replyTo, "replyTo"), "replyTo"); + return idempotentChat(withTenant, session.workspaceId, session.userId, `send:${channelId}`, key, body, async () => { + const outcome = await this.hub.send(session.workspaceId, liveId, { + text, targets, ...(replyTo === undefined ? {} : { replyTo }), + author: { handle: `${session.role.toLowerCase()}:${session.userId.slice(0, 8)}` }, + }); + return { message: outcome.message, delivery: outcome.delivery, ...(outcome.refused ? { refused: outcome.refused } : {}) }; + }); + } + + @Post("live-chat/lives/:channelId/messages/:messageId/moderation") + @UseGuards(LiveChatGuard) + @RequirePermission("livechat.moderate") + @HttpCode(202) + async moderate( + @Req() request: Request & AuthenticatedRequest, + @Param("channelId") channelId: string, + @Param("messageId") messageId: string, + @Body() body: unknown, + @Headers("idempotency-key") key?: string, + ) { + const session = requireSession(request); + const parsed = requireObject(body, "body"); + const action = parsed.action; + if (action !== "delete" && action !== "timeout" && action !== "ban") throw new BadRequestException("action must be delete, timeout or ban"); + let durationSeconds: number | undefined; + if (parsed.durationSeconds !== undefined) { + if (typeof parsed.durationSeconds !== "number") throw new BadRequestException("durationSeconds must be a number"); + durationSeconds = parsed.durationSeconds; + if (!Number.isInteger(durationSeconds) || durationSeconds < 1 || durationSeconds > 1209600) throw new BadRequestException("durationSeconds must be 1..1209600"); + } + if (action === "timeout" && durationSeconds === undefined) throw new BadRequestException("timeout requires durationSeconds"); + return idempotentChat(withTenant, session.workspaceId, session.userId, `moderate:${channelId}:${messageId}`, key, body, async () => { + const outcome = await this.hub.moderate(session.workspaceId, parseId(channelId, "channelId"), { + action, + ...(durationSeconds === undefined ? {} : { durationSeconds }), + message: await this.hubTarget(session.workspaceId, parseId(channelId, "channelId"), parseId(messageId, "messageId")), + }); + await appendTenantAudit(session.workspaceId, { actorUserId: session.userId, actorRole: session.role, action: `livechat.${action}`, + targetType: "live_chat_message", targetId: messageId, diff: { delivery: outcome.delivery, durationSeconds } }); + return { delivery: outcome.delivery }; + }); + } + + @Get("live-chat/lives/:channelId/capabilities") + @UseGuards(LiveChatGuard) + @RequirePermission("livechat.view") + async capabilities(@Req() request: Request & AuthenticatedRequest, @Param("channelId") channelId: string) { + const session = requireSession(request); + return this.hub.capabilities(session.workspaceId, parseId(channelId, "channelId")); + } + + // -- Device pairing (§3.4) ------------------------------------------------ + + @Post("devices/pairings") + @SessionOnly() + @UseGuards(LiveChatGuard) + @RequirePermission("livechat.send") + async issuePairing(@Req() request: Request & AuthenticatedRequest, @Headers("idempotency-key") key?: string): Promise<{ pairingCode: string; qrPayload: string; expiresAt: string }> { + const session = requireSession(request); + return idempotentChat(withTenant, session.workspaceId, session.userId, "pairing", key, {}, () => this.devices.issuePairing(session.workspaceId, session.userId, DEVICE_SCOPES)); + } + + @Post("devices/pairings/redeem") + @HttpCode(200) + async redeem(@Body() body: unknown, @Headers("idempotency-key") key?: string, @Headers("authorization") auth?: string): Promise<{ deviceToken: string; workspaceId: string }> { + const parsed = requireObject(body, "body"); + const pairingCode = requireString(parsed.pairingCode, "pairingCode"); + const deviceName = requireString(parsed.deviceName, "deviceName"); + if (deviceName.length > 120) throw new BadRequestException("deviceName too long"); + // The bearer pairing code resolves its issuing workspace; no session is needed. + const workspace = auth?.startsWith("Bearer ") ? tokenWorkspace(auth.slice(7)) : undefined; + if (auth && !workspace) throw new UnauthorizedException(); + return this.devices.redeemCode(pairingCode, deviceName, key, workspace ?? undefined); + } + + @Delete("devices/:deviceId") + @UseGuards(LiveChatGuard) + @RequirePermission("livechat.send") + async revoke(@Req() request: Request & AuthenticatedRequest, @Param("deviceId") deviceId: string) { + const session = requireSession(request); + await this.devices.revoke(session.workspaceId, parseId(deviceId, "deviceId")); + return { revoked: true }; + } + + private async hubTarget(workspaceId: string, channelId: string, messageId: string) { + const found = await this.hub.findMessage(workspaceId, channelId, messageId); + if (!found) throw new NotFoundException("message not found"); + return { id: found.id, connectionId: found.connectionId, provider: found.provider, providerMessageId: found.providerMessageId, authorHandle: found.author.handle }; + } +} + +function parseSeq(value: string): number { + const n = Number(value); + if (!Number.isSafeInteger(n) || n < 0) throw new BadRequestException("after must be a non-negative integer seq"); + return n; +} + +function parseTargets(value: unknown): "all" | string[] { + if (value === "all") return "all"; + if (!Array.isArray(value) || value.length === 0 || value.length > 32) throw new BadRequestException('targets must be "all" or 1..32 connection ids'); + return value.map((entry): string => { + if (typeof entry !== "string") throw new BadRequestException("targets must contain connection ids"); + return parseId(entry, "connectionId"); + }); +} + diff --git a/apps/api/src/live-chat/live-chat.guard.ts b/apps/api/src/live-chat/live-chat.guard.ts new file mode 100644 index 0000000..182f4ff --- /dev/null +++ b/apps/api/src/live-chat/live-chat.guard.ts @@ -0,0 +1,26 @@ +import { CanActivate, ExecutionContext, Inject, Injectable, SetMetadata, UnauthorizedException, ForbiddenException } from "@nestjs/common"; +import { Reflector } from "@nestjs/core"; +import { PERMISSION_KEY, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import { pool } from "../db/client.js"; +import { authorizeChat, chatSession, tokenWorkspace } from "./auth.js"; + +export const SessionOnly = (): MethodDecorator => SetMetadata("livechat.session_only", true) as MethodDecorator; + +@Injectable() +export class LiveChatGuard implements CanActivate { + constructor(@Inject(Reflector) private readonly reflector: Reflector) {} + async canActivate(context: ExecutionContext): Promise { + const req = context.switchToHttp().getRequest<{ headers: Record } & AuthenticatedRequest>(); + const token = req.headers.authorization?.replace(/^Bearer /, ""); + const workspaceId = token ? tokenWorkspace(token) : null; + if (!token || !workspaceId || !req.headers.authorization?.startsWith("Bearer ")) throw new UnauthorizedException(); + if (this.reflector.get("livechat.session_only", context.getHandler()) && !chatSession(token)) throw new UnauthorizedException("session user required"); + const client = await pool.connect(); + try { + const session = await authorizeChat(client, token, workspaceId, this.reflector.get(PERMISSION_KEY, context.getHandler()) ?? "livechat.view"); + if (!session) throw new ForbiddenException("chat permission or authorization epoch invalid"); + req.session = session; + return true; + } finally { client.release(); } + } +} diff --git a/apps/api/src/live-chat/live-chat.integration.test.ts b/apps/api/src/live-chat/live-chat.integration.test.ts new file mode 100644 index 0000000..b1e8826 --- /dev/null +++ b/apps/api/src/live-chat/live-chat.integration.test.ts @@ -0,0 +1,97 @@ +import { randomUUID } from "node:crypto"; +import { join } from "node:path"; +import { Pool, type PoolClient } from "pg"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { runMigrations } from "../db/migrate.js"; +import { LiveChatHubService } from "./live-chat.service.js"; +import { DeviceService, verifyDeviceToken } from "./devices.js"; +import { idempotentChat } from "./idempotency.js"; +import type { NormalizedChatEvent } from "./types.js"; + +// Deliberately isolated from the user's configured runtime database. +const url = process.env.S5_TEST_DATABASE_URL; +const db = new Pool({ connectionString: url }); +const WS = randomUUID(), OTHER = randomUUID(), USER = randomUUID(), CH = randomUUID(), CONN = randomUUID(); +async function tenant(workspaceId: string, fn: (client: PoolClient) => Promise): Promise { + const client = await db.connect(); + try { + await client.query("BEGIN"); await client.query("SET LOCAL ROLE sp_app"); + await client.query("SELECT set_config('app.workspace_id',$1,true)", [workspaceId]); + const result = await fn(client); await client.query("COMMIT"); return result; + } catch (error) { await client.query("ROLLBACK"); throw error; } + finally { client.release(); } +} +const hub = new LiveChatHubService({ transaction: tenant, publish: () => undefined }); +function event(id: string, receivedAt = new Date()): NormalizedChatEvent { + return { provider: "mock-livestream", connectionId: CONN, channelId: CH, providerMessageId: id, author: { handle: "viewer" }, text: id, sentAt: receivedAt, receivedAt }; +} +describe.skipIf(!url)("S5 PostgreSQL partitioning, row locks, RLS, pairing, and retention", () => { + beforeAll(async () => { + process.env.SUPER_DATABASE_URL = url; + await runMigrations(join(__dirname, "../../migrations")); + await db.query("INSERT INTO workspaces(id,name,slug) VALUES ($1::uuid,'S5',$1::text),($2::uuid,'S5 other',$2::text)", [WS, OTHER]); + await db.query("INSERT INTO users(id,external_id,email,display_name) VALUES ($1::uuid,$1::text,'s5@example.test','S5')", [USER]); + await db.query("INSERT INTO memberships(workspace_id,user_id,role) VALUES ($1,$2,'Owner')", [WS, USER]); + await db.query("INSERT INTO connections(id,workspace_id,provider,channel_id) VALUES ($1,$2,'mock-livestream','remote')", [CONN, WS]); + await db.query("INSERT INTO virtual_channels(id,workspace_id,slug,name) VALUES ($1,$2,'s5','S5')", [CH, WS]); + await db.query("INSERT INTO virtual_channel_sources(workspace_id,virtual_channel_id,kind,label,connection_id,remote_ref,hls_url) VALUES ($1,$2,'hls','S5',$3,'remote','https://example.test/live.m3u8')", [WS, CH, CONN]); + }); + afterAll(async () => { + if (url) { await db.query("DELETE FROM workspaces WHERE id IN ($1,$2)", [WS, OTHER]); await db.query("DELETE FROM users WHERE id=$1", [USER]); } + await db.end(); + }); + it("concurrent ingress assigns unique seqs; duplicates claim exactly one durable row", async () => { + const results = await Promise.all([hub.ingest(WS, [event("same")]), hub.ingest(WS, [event("same")])]); + expect(results.flatMap(result => result.inserted)).toHaveLength(1); + expect(results.flatMap(result => result.duplicates)).toEqual(["same"]); + const more = await Promise.all(Array.from({ length: 8 }, (_, i) => hub.ingest(WS, [event(`unique-${i}`)]))); + expect(more.flatMap(result => result.inserted.map(message => message.seq)).sort((a,b) => a-b)).toEqual([2,3,4,5,6,7,8,9]); + expect((await hub.messages(OTHER, CH, { limit: 100 })).items).toEqual([]); + expect((await hub.lives(WS)).items).toContainEqual(expect.objectContaining({ liveStreamId: CH, connectionId: CONN })); + }); + it("RLS applies to direct partition access as well as parent access", async () => { + const partitions = await db.query<{ relname: string }>("SELECT c.relname FROM pg_inherits i JOIN pg_class c ON c.oid=i.inhrelid WHERE i.inhparent='live_chat_messages'::regclass"); + for (const row of partitions.rows) { + if (!/^live_chat_messages_(default|[0-9]{6})$/.test(row.relname)) throw new Error("unexpected partition name"); + const found = await tenant(OTHER, client => client.query(`SELECT * FROM ${row.relname}`)); expect(found.rowCount).toBe(0); + } + const client = await db.connect(); + try { await client.query("BEGIN"); await client.query("SET LOCAL ROLE sp_app"); expect((await client.query("SELECT * FROM live_chat_messages")).rowCount).toBe(0); } + finally { await client.query("ROLLBACK"); client.release(); } + }); + it("pairing lookup permits only a valid code; redemption is single use and workspace fenced", async () => { + const service = new DeviceService({ transaction: tenant, serverUrl: "https://studio.example", lookupWorkspace: async code => { + const row = await db.query<{ workspace_id: string }>("SELECT device_pairing_workspace($1) AS workspace_id", [code]); return row.rows[0]?.workspace_id ?? null; + } }); + const pairing = await service.issuePairing(WS, USER); + const lookup = await tenant(OTHER, client => client.query<{ workspace_id: string }>("SELECT device_pairing_workspace($1) AS workspace_id", [pairing.pairingCode])); + expect(lookup.rows[0]?.workspace_id).toBe(WS); + await expect(service.redeem(OTHER, pairing.pairingCode, "phone")).rejects.toThrow("invalid pairing code"); + await expect(service.redeemCode(pairing.pairingCode, "phone", "redeem-key", OTHER)).rejects.toThrow("invalid pairing code"); + const result = await service.redeemCode(pairing.pairingCode, "phone", "redeem-key"), payload = verifyDeviceToken(result.deviceToken); + expect(payload).toMatchObject({ userId: USER, workspaceId: WS, epoch: 1 }); + expect(await service.redeemCode(pairing.pairingCode, "phone", "redeem-key")).toEqual(result); + if (!payload) throw new Error("no token"); + await expect(service.redeem(WS, pairing.pairingCode, "phone")).rejects.toThrow("already redeemed"); + await service.revoke(WS, payload.deviceId); expect(await service.epochOf(WS, payload.deviceId)).toBeNull(); + }); + it("idempotency returns the recorded reply and refuses concurrent or changed requests", async () => { + let calls = 0; + const run = (body: unknown) => idempotentChat(tenant, WS, USER, "send:test", "retry-key", body, async () => ({ count: ++calls })); + expect(await run({ text: "hi" })).toEqual({ count: 1 }); + expect(await run({ text: "hi" })).toEqual({ count: 1 }); expect(calls).toBe(1); + await expect(run({ text: "changed" })).rejects.toThrow("different payload"); + const result = await Promise.allSettled([idempotentChat(tenant, WS, USER, "send:test", "concurrent", {}, async () => ({ count: ++calls })), idempotentChat(tenant, WS, USER, "send:test", "concurrent", {}, async () => ({ count: ++calls }))]); + expect(result.filter(item => item.status === "fulfilled").length).toBeGreaterThanOrEqual(1); expect(calls).toBe(2); + }); + it("retention prunes old messages without pruning another tenant", async () => { + const old = new Date(Date.now() - 100 * 86400000); + await hub.ingest(WS, [event("old-retention", old)]); + const before = (await hub.messages(WS, CH, { limit: 100 })).items.length; + await tenant(OTHER, client => client.query("SELECT live_chat_prune($1,$2)", [OTHER, new Date()])); + expect((await hub.messages(WS, CH, { limit: 100 })).items).toHaveLength(before); + const removed = await tenant(WS, client => client.query<{ count: number }>("SELECT live_chat_prune($1,$2) AS count", [WS, new Date(Date.now() - 30 * 86400000)])); + expect(removed.rows[0]?.count).toBe(1); + expect((await hub.messages(WS, CH, { limit: 100 })).items.some(message => message.providerMessageId === "old-retention")).toBe(false); + }); +}); diff --git a/apps/api/src/live-chat/live-chat.module.ts b/apps/api/src/live-chat/live-chat.module.ts new file mode 100644 index 0000000..4634afc --- /dev/null +++ b/apps/api/src/live-chat/live-chat.module.ts @@ -0,0 +1,53 @@ +import { Module, type DynamicModule } from "@nestjs/common"; +import { withTenant } from "../db/client.js"; +import { LIVE_CHAT_RUNTIME, LiveChatRuntime, type LiveChatRuntimeOptions } from "./runtime.js"; +import { ChatRealtimeBridge } from "./bridge.js"; +import { LiveChatGuard } from "./live-chat.guard.js"; +import { DeviceService } from "./devices.js"; +import { LiveChatController } from "./live-chat.controller.js"; +import { LiveChatHubService } from "./live-chat.service.js"; +import { ChatRingBuffer } from "./ring-buffer.js"; +import { RedisChatGovernors, type ChatGovernors } from "./redis-governors.js"; +import { SendGovernors } from "./governors.js"; +import type { ChatTarget } from "./types.js"; + +/** Central review imports this module with credential-bound transports, + * capability targets and ingestion drivers via LiveChatModule.register(). */ +function defaultTargets(): (workspaceId: string, channelId: string) => Promise { + return async (workspaceId, channelId) => { + const result = await withTenant(workspaceId, client => client.query<{ connection_id: string; provider: string; live_remote_id: string | null }>( + `SELECT vc.id AS connection_id, vc.provider, vcs.remote_ref AS live_remote_id + FROM virtual_channel_sources vcs + JOIN connections vc ON vc.id = vcs.connection_id AND vc.workspace_id = vcs.workspace_id + WHERE vcs.workspace_id = $1 AND vcs.virtual_channel_id = $2`, [workspaceId, channelId])); + return result.rows.map(row => ({ + connectionId: row.connection_id, + provider: row.provider, + liveRemoteId: row.live_remote_id ?? row.connection_id, + read: false, reply: false, moderate: false, state: "degraded" as const, + reason: "chat adapter not configured", + })); + }; +} + +@Module({ + controllers: [LiveChatController], + providers: [ + { provide: ChatRingBuffer, useValue: new ChatRingBuffer(500) }, + { provide: SendGovernors, useFactory: () => new RedisChatGovernors({ buckets: { twitch: { capacity: 100, refillPerSecond: 100 / 30 } }, quotas: { youtube: { budget: 10000, costPerCall: 50 } } }) }, + { provide: ChatRealtimeBridge, useFactory: () => new ChatRealtimeBridge() }, + { + provide: LiveChatHubService, + useFactory: (buffer: ChatRingBuffer, governors: ChatGovernors, bridge: ChatRealtimeBridge, runtime?: LiveChatRuntimeOptions) => + new LiveChatHubService({ buffer, governors, publish: bridge.publish, targets: defaultTargets(), ...runtime }), + inject: [ChatRingBuffer, SendGovernors, ChatRealtimeBridge, { token: LIVE_CHAT_RUNTIME, optional: true }], + }, + DeviceService, LiveChatGuard, LiveChatRuntime, + ], + exports: [LiveChatHubService, DeviceService, ChatRealtimeBridge], +}) +export class LiveChatModule { + static register(options: LiveChatRuntimeOptions): DynamicModule { + return { module: LiveChatModule, providers: [{ provide: LIVE_CHAT_RUNTIME, useValue: options }] }; + } +} diff --git a/apps/api/src/live-chat/live-chat.service.ts b/apps/api/src/live-chat/live-chat.service.ts new file mode 100644 index 0000000..12ce29b --- /dev/null +++ b/apps/api/src/live-chat/live-chat.service.ts @@ -0,0 +1,353 @@ +/** + * Chat hub service: ingestion → seq assignment → persistence (dedupe) → + * fan-out on the realtime gateway, plus reply/moderation fan-out with + * per-target Delivery rows and send governors in front. + * + * Sequencing is per (workspace, channel) under a row lock on `live_chat_seq` + * (§3.2): the ingest transaction takes FOR UPDATE on that row, so concurrent + * ingests cannot interleave seqs for one channel. Provider message ids dedupe + * through UNIQUE(workspace_id, provider, provider_msg_id) — a replay is a + * no-op that publishes nothing. + * + * Every port has a working default (see the module), and tests inject + * overrides through LIVE_CHAT_DEPENDENCIES — same pattern as RealtimeService. + */ +import { Inject, Injectable, Optional, NotFoundException } from "@nestjs/common"; +import type { PoolClient } from "pg"; +import { authorColor, type Delivery, type LiveChatCapability, type LiveSummary } from "@tactical-media-broadcast-hub/core"; +import { withTenant } from "../db/client.js"; +import { ChatRingBuffer } from "./ring-buffer.js"; +import type { ChatGovernors } from "./redis-governors.js"; +import { SendGovernors } from "./governors.js"; +import type { + ChatTarget, ChatTransport, IngestResult, MessagePage, MessageQuery, ModerateOutcome, NormalizedChatEvent, SendOutcome, StoredChatMessage, +} from "./types.js"; + +export const LIVE_CHAT_DEPENDENCIES = Symbol("LIVE_CHAT_DEPENDENCIES"); + +export type LiveChatPublish = (workspaceId: string, channelId: string, frame: Record) => void | Promise; + +export interface LiveChatDependencies { + transaction: (workspaceId: string, fn: (client: PoolClient) => Promise) => Promise; + publish: LiveChatPublish; + buffer: ChatRingBuffer; + governors: ChatGovernors; + /** Resolve the send/moderate targets for a channel (capability rows). */ + targets: (workspaceId: string, channelId: string) => Promise; + /** Transport per provider id. */ + transports: Record; + /** Virtual clock for deterministic tests. */ + now?: () => Date; + filters?: { hideCommands?: boolean; hideBots?: boolean; mutedHandles?: string[]; words?: string[] }; +} + +interface MessageRow { + id: string; workspace_id: string; channel_id: string; connection_id: string; provider: string; provider_msg_id: string; + author_handle: string; author_display_name: string | null; author_avatar_url: string | null; author_badges: unknown; + author_color: string; body: string; reply_to: string | null; seq: string | number; sent_at: Date; received_at: Date; + is_moderated: boolean; source: string; +} + +function toStored(row: MessageRow): StoredChatMessage { + return { + id: row.id, + seq: Number(row.seq), + liveStreamId: row.channel_id, + connectionId: row.connection_id, + provider: row.provider, + providerMessageId: row.provider_msg_id, + author: { + handle: row.author_handle, + ...(row.author_display_name === null ? {} : { displayName: row.author_display_name }), + ...(row.author_avatar_url === null ? {} : { avatarUrl: row.author_avatar_url }), + badges: Array.isArray(row.author_badges) ? (row.author_badges as string[]) : [], + color: row.author_color, + }, + text: row.body, + sentAt: new Date(row.sent_at).toISOString(), + receivedAt: new Date(row.received_at).toISOString(), + isModerated: row.is_moderated, + ...(row.reply_to === null ? {} : { replyTo: row.reply_to }), + source: row.source as StoredChatMessage["source"], + }; +} + +const SELECT_COLUMNS = `id, workspace_id, channel_id, connection_id, provider, provider_msg_id, author_handle, + author_display_name, author_avatar_url, author_badges, author_color, body, reply_to, seq, sent_at, + received_at, is_moderated, source`; + +const INSERT_SQL = `INSERT INTO live_chat_messages (workspace_id, channel_id, connection_id, provider, provider_msg_id, + author_handle, author_display_name, author_avatar_url, author_badges, author_color, body, reply_to, + seq, sent_at, received_at, source) + SELECT * FROM UNNEST($1::uuid[], $2::uuid[], $3::uuid[], $4::text[], $5::text[], $6::text[], $7::text[], + $8::text[], $9::jsonb[], $10::text[], $11::text[], $12::text[], $13::bigint[], $14::timestamptz[], + $15::timestamptz[], $16::text[]) + RETURNING ${SELECT_COLUMNS}`; + +@Injectable() +export class LiveChatHubService { + private readonly deps: LiveChatDependencies; + private readonly now: () => Date; + constructor(@Optional() @Inject(LIVE_CHAT_DEPENDENCIES) dependencies?: Partial) { + this.deps = { + transaction: dependencies?.transaction ?? withTenant, + publish: dependencies?.publish ?? (() => undefined), + buffer: dependencies?.buffer ?? new ChatRingBuffer(500), + governors: dependencies?.governors ?? new SendGovernors(), + targets: dependencies?.targets ?? (async () => []), + transports: dependencies?.transports ?? {}, + ...(dependencies?.now ? { now: dependencies.now } : {}), + ...(dependencies?.filters ? { filters: dependencies.filters } : {}), + }; + this.now = this.deps.now ?? (() => new Date()); + } + + async lives(workspaceId: string): Promise<{ items: LiveSummary[] }> { + return this.deps.transaction(workspaceId, async client => { + const rows = await client.query<{ channel_id: string; connection_id: string; provider: string; title: string; state: LiveSummary["state"] }>( + `SELECT DISTINCT vc.id AS channel_id, c.id AS connection_id, c.provider, vc.name AS title, + CASE WHEN vc.observed_status='RUNNING' THEN 'live' WHEN vc.desired_status='RUNNING' THEN 'waiting' ELSE 'ended' END AS state + FROM virtual_channels vc JOIN virtual_channel_sources s ON s.virtual_channel_id=vc.id AND s.workspace_id=vc.workspace_id + JOIN connections c ON c.id=s.connection_id AND c.workspace_id=s.workspace_id + WHERE vc.workspace_id=$1 ORDER BY vc.id, c.id`, [workspaceId]); + return { items: rows.rows.map(row => ({ liveStreamId: row.channel_id, connectionId: row.connection_id, provider: row.provider, title: row.title, state: row.state })) }; + }); + } + + private async nextSeq(workspaceId: string, channelId: string): Promise { + return this.deps.transaction(workspaceId, async client => { + await client.query("INSERT INTO live_chat_seq (workspace_id, channel_id, seq) VALUES ($1,$2,0) ON CONFLICT DO NOTHING", [workspaceId, channelId]); + const locked = await client.query<{ seq: string | number }>("SELECT seq FROM live_chat_seq WHERE workspace_id=$1 AND channel_id=$2 FOR UPDATE", [workspaceId, channelId]); + const seq = Number(locked.rows[0]?.seq ?? 0) + 1; + await client.query("UPDATE live_chat_seq SET seq = GREATEST(seq, $3), updated_at = now() WHERE workspace_id=$1 AND channel_id=$2", [workspaceId, channelId, seq]); + return seq; + }); + } + + /** Ingest a batch: assign seqs under the channel row lock, insert, publish. */ + async ingest(workspaceId: string, events: NormalizedChatEvent[]): Promise { + if (events.length === 0) return { inserted: [], duplicates: [] }; + const byChannel = new Map(); + for (const event of events) { + const filters = this.deps.filters; + if ((filters?.hideCommands && event.text.startsWith("!")) || (filters?.hideBots && event.author.isBot) || + filters?.mutedHandles?.includes(`${event.provider}:${event.author.handle}`) || + filters?.words?.some(word => word.length > 0 && event.text.toLowerCase().includes(word.toLowerCase()))) continue; + const list = byChannel.get(event.channelId) ?? []; + list.push(event); + byChannel.set(event.channelId, list); + } + const inserted: StoredChatMessage[] = []; + const duplicates: string[] = []; + for (const [channelId, batch] of byChannel) { + const rows = await this.deps.transaction(workspaceId, async client => { + // Create-or-take the channel's seq row. INSERT ... ON CONFLICT first so + // two first-ever ingests cannot both see "no row" and deadlock-fail. + await client.query( + `INSERT INTO live_chat_seq (workspace_id, channel_id, seq) VALUES ($1,$2,0) ON CONFLICT DO NOTHING`, + [workspaceId, channelId]); + const locked = await client.query<{ seq: string | number }>( + "SELECT seq FROM live_chat_seq WHERE workspace_id=$1 AND channel_id=$2 FOR UPDATE", [workspaceId, channelId]); + const next = Number(locked.rows[0]?.seq ?? 0); + const claimed: NormalizedChatEvent[] = []; + for (const event of batch) { + const claim = await client.query( + "INSERT INTO live_chat_seen_events (workspace_id, provider, provider_event_id) VALUES ($1,$2,$3) ON CONFLICT DO NOTHING RETURNING provider_event_id", + [workspaceId, event.provider, `message:${event.providerMessageId}`]); + if (claim.rowCount) claimed.push(event); + } + if (claimed.length === 0) return []; + const values = claimed.map((event, index) => [ + workspaceId, channelId, event.connectionId, event.provider, event.providerMessageId, + event.author.handle, event.author.displayName ?? null, event.author.avatarUrl ?? null, + JSON.stringify(event.author.badges ?? []), authorColor(event.provider, event.author.handle), + event.text, event.replyTo ?? null, next + index + 1, event.sentAt, event.receivedAt ?? this.now(), + event.source ?? "live-chat", + ]); + const columns = Array.from({ length: 16 }, (_, i) => values.map(row => row[i])); + const result = await client.query(INSERT_SQL, columns); + await client.query( + "UPDATE live_chat_seq SET seq = GREATEST(seq, $3), updated_at = now() WHERE workspace_id=$1 AND channel_id=$2", + [workspaceId, channelId, next + claimed.length]); + return result.rows.map(toStored); + }); + const insertedIds = new Map(); + for (const row of rows) { const key = `${row.provider}:${row.providerMessageId}`; insertedIds.set(key, (insertedIds.get(key) ?? 0) + 1); } + for (const message of rows) { this.deps.buffer.push(message); inserted.push(message); } + for (const event of batch) { + const key = `${event.provider}:${event.providerMessageId}`, count = insertedIds.get(key) ?? 0; + if (count === 0) duplicates.push(event.providerMessageId); else insertedIds.set(key, count - 1); + } + for (const message of rows) { + await this.deps.publish(workspaceId, channelId, { + kind: "chat", liveStreamId: channelId, channelId, seq: message.seq, message, + }); + } + } + return { inserted, duplicates }; + } + + /** A provider disconnect is accepted explicitly, with a shared system notice. */ + async gap(workspaceId: string, event: NormalizedChatEvent, reason: string): Promise { + const result = await this.ingest(workspaceId, [{ ...event, text: `Chat gap: ${reason}`, source: "system" }]); + if (result.inserted.length > 0) await this.deps.publish(workspaceId, event.channelId, { + kind: "chat", liveStreamId: event.channelId, channelId: event.channelId, + seq: await this.nextSeq(workspaceId, event.channelId), provider: event.provider, reason, + }); + } + + /** Cursor page from the durable store (REST fallback path). */ + async messages(workspaceId: string, channelId: string, query: MessageQuery): Promise { + return this.deps.transaction(workspaceId, async client => { + const params: unknown[] = [workspaceId, channelId, query.after ?? 0, query.limit]; + let filter = ""; + if (query.provider !== undefined) { params.push(query.provider); filter = " AND provider=$5"; } + const result = await client.query( + `SELECT ${SELECT_COLUMNS} FROM live_chat_messages WHERE workspace_id=$1 AND channel_id=$2 AND seq > $3${filter} + ORDER BY seq ASC LIMIT $4`, params); + const items = result.rows.map(toStored); + const nextCursor = items.length === query.limit ? items.at(-1)?.seq : undefined; + return { items, ...(nextCursor === undefined ? {} : { nextCursor }) }; + }); + } + + /** One message by id (moderation target lookup). */ + async findMessage(workspaceId: string, channelId: string, messageId: string): Promise { + return this.deps.transaction(workspaceId, async client => { + const result = await client.query( + `SELECT ${SELECT_COLUMNS} FROM live_chat_messages WHERE workspace_id=$1 AND channel_id=$2 AND id=$3`, + [workspaceId, channelId, messageId]); + return result.rows[0] ? toStored(result.rows[0]) : null; + }); + } + + /** Latest window straight from the ring buffer (resume path). */ + resume(liveStreamId: string, lastSeq: number, limit = 500): { messages: StoredChatMessage[]; resync: boolean } { + if (!this.deps.buffer.canResume(liveStreamId, lastSeq)) return { messages: [], resync: true }; + return { messages: this.deps.buffer.after(liveStreamId, lastSeq, limit), resync: false }; + } + + /** Per-connection capabilities for a channel (§3.5 capabilities endpoint). */ + async capabilities(workspaceId: string, channelId: string): Promise { + return Promise.all((await this.deps.targets(workspaceId, channelId)).map(async target => { + const governor = await Promise.resolve().then(() => this.deps.governors.probe(target.provider)).catch(() => ({ state: "degraded" as const, reason: "governor unavailable" })); + return { + connectionId: target.connectionId, provider: target.provider, read: target.read, reply: target.reply, + moderate: target.moderate, state: governor.state === "ok" ? target.state : governor.state ?? target.state, + ...(target.reason === undefined && governor.reason === undefined ? {} : { reason: governor.reason ?? target.reason }), + }; + })); + } + + /** Reply fan-out: governor per target, one Delivery row per target, and one + * stored hub message when at least one target accepted. An unsupported or + * failed target NEVER fails the batch (§3.5). */ + async send(workspaceId: string, channelId: string, request: { + text: string; targets: "all" | string[]; replyTo?: string; author: { handle: string }; + }): Promise { + const targets = await this.deps.targets(workspaceId, channelId); + const parent = request.replyTo ? await this.findMessage(workspaceId, channelId, request.replyTo) : null; + if (request.replyTo && !parent) throw new NotFoundException("reply target not found"); + const selected = request.targets === "all" ? targets : targets.filter(t => request.targets.includes(t.connectionId)); + if (selected.length === 0) return { message: null, delivery: [], refused: { reason: "no capable target", state: "degraded" } }; + const delivery = new Map(); + const accepted: ChatTarget[] = []; + let refused: SendOutcome["refused"]; + for (const target of selected) { + if (!target.reply || !this.deps.transports[target.provider]) { + delivery.set(target.connectionId, { connectionId: target.connectionId, provider: target.provider, status: "unsupported", error: target.reason ?? (!target.reply ? "reply not supported" : "no transport configured") }); + continue; + } + const verdict = await Promise.resolve().then(() => this.deps.governors.take(target.provider)).catch(() => ({ allowed: false, state: "degraded" as const, reason: "governor unavailable" })); + if (!verdict.allowed) { + refused ??= { state: verdict.state ?? "degraded", reason: verdict.reason ?? "refused" }; + delivery.set(target.connectionId, { connectionId: target.connectionId, provider: target.provider, status: "unsupported", error: verdict.reason ?? "refused" }); + continue; + } + delivery.set(target.connectionId, { connectionId: target.connectionId, provider: target.provider, status: "queued" }); + accepted.push(target); + } + if (accepted.length === 0) { + const first = [...delivery.values()].find(d => d.error); + return { message: null, delivery: [...delivery.values()], refused: { reason: refused?.reason ?? first?.error ?? "no target accepted", state: refused?.state ?? "degraded" } }; + } + const outcomes = new Map(); + for (const target of accepted) { + const transport = this.deps.transports[target.provider]; + if (!transport) { + delivery.set(target.connectionId, { connectionId: target.connectionId, provider: target.provider, status: "unsupported", error: "no transport configured" }); + continue; + } + try { + const result = await transport.send({ connectionId: target.connectionId, liveRemoteId: target.liveRemoteId, text: request.text, replyTo: parent?.connectionId === target.connectionId ? parent.providerMessageId : undefined }); + outcomes.set(target.connectionId, result); + delivery.set(target.connectionId, { connectionId: target.connectionId, provider: target.provider, status: "sent" }); + } catch (error) { + delivery.set(target.connectionId, { connectionId: target.connectionId, provider: target.provider, status: "failed", error: error instanceof Error ? error.message : String(error) }); + } + } + // Store our own message once (echoed to every viewer). + const primary = accepted.find(t => delivery.get(t.connectionId)?.status === "sent"); + if (!primary) return { message: null, delivery: [...delivery.values()], refused: { reason: "no transport accepted the send", state: "degraded" } }; + const primaryResult = outcomes.get(primary.connectionId); + if (!primaryResult) throw new Error("missing primary delivery result"); + const stored = await this.ingest(workspaceId, [{ + provider: primary.provider, + connectionId: primary.connectionId, + channelId, + providerMessageId: primaryResult.providerMessageId, + author: request.author, + text: request.text, + sentAt: primaryResult.sentAt, + receivedAt: this.now(), + ...(request.replyTo === undefined ? {} : { replyTo: request.replyTo }), + source: "live-chat", + }]).then(result => result.inserted[0] ?? null); + const deliveryRows = [...delivery.values()]; + if (stored) { + await this.deps.publish(workspaceId, channelId, { + kind: "chat", liveStreamId: channelId, channelId, seq: await this.nextSeq(workspaceId, channelId), messageId: stored.id, delivery: deliveryRows, + }); + } + return { message: stored, delivery: deliveryRows }; + } + + /** Moderation fan-out: delete/timeout/ban on every capable target; the local + * row is flagged so the panel drops it immediately. */ + async moderate(workspaceId: string, channelId: string, request: { + action: "delete" | "timeout" | "ban"; durationSeconds?: number; + message: { id?: string; connectionId?: string; provider: string; providerMessageId: string; authorHandle: string }; + }): Promise { + const targets = await this.deps.targets(workspaceId, channelId); + const delivery: Delivery[] = []; + for (const target of targets) { + if (!target.moderate) { delivery.push({ connectionId: target.connectionId, provider: target.provider, status: "unsupported", error: target.reason ?? "moderation not supported" }); continue; } + if (target.provider !== request.message.provider || (request.message.connectionId && target.connectionId !== request.message.connectionId)) { + delivery.push({ connectionId: target.connectionId, provider: target.provider, status: "unsupported", error: "message belongs to another target" }); continue; + } + const verdict = await Promise.resolve().then(() => this.deps.governors.take(target.provider)).catch(() => ({ allowed: false, state: "degraded" as const, reason: "governor unavailable" })); + if (!verdict.allowed) { delivery.push({ connectionId: target.connectionId, provider: target.provider, status: "unsupported", error: verdict.reason }); continue; } + const transport = this.deps.transports[target.provider]; + if (!transport) { delivery.push({ connectionId: target.connectionId, provider: target.provider, status: "unsupported", error: "no transport configured" }); continue; } + try { + await transport.moderate({ connectionId: target.connectionId, liveRemoteId: target.liveRemoteId, + messageRemoteId: request.message.providerMessageId, authorHandle: request.message.authorHandle, action: request.action, ...(request.durationSeconds === undefined ? {} : { durationSeconds: request.durationSeconds }) }); + delivery.push({ connectionId: target.connectionId, provider: target.provider, status: "sent" }); + } catch (error) { + delivery.push({ connectionId: target.connectionId, provider: target.provider, status: "failed", error: error instanceof Error ? error.message : String(error) }); + } + } + if (request.action === "delete" && delivery.some(d => d.status === "sent")) { + await this.deps.transaction(workspaceId, async client => { + await client.query( + "UPDATE live_chat_messages SET is_moderated = true WHERE workspace_id=$1 AND channel_id=$2 AND provider=$3 AND provider_msg_id=$4", + [workspaceId, channelId, request.message.provider, request.message.providerMessageId]); + }); + } + if (delivery.some(row => row.status === "sent")) await this.deps.publish(workspaceId, channelId, { + kind: "chat", liveStreamId: channelId, channelId, seq: await this.nextSeq(workspaceId, channelId), + action: request.action, targetMessageId: request.message.id ?? request.message.providerMessageId, targetHandle: request.message.authorHandle, + }); + return { delivery }; + } +} diff --git a/apps/api/src/live-chat/live-chat.units.test.ts b/apps/api/src/live-chat/live-chat.units.test.ts new file mode 100644 index 0000000..f0f16a0 --- /dev/null +++ b/apps/api/src/live-chat/live-chat.units.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from "vitest"; +import { authorColor } from "@tactical-media-broadcast-hub/core"; +import { ChatRingBuffer } from "./ring-buffer.js"; +import { SendGovernors, TokenBucket } from "./governors.js"; +import type { StoredChatMessage } from "./types.js"; + +describe("send governors", () => { + it("token bucket refuses beyond burst and refills over time", () => { + let t = 0; + const bucket = new TokenBucket({ capacity: 2, refillPerSecond: 1, now: () => t }); + expect(bucket.tryTake()).toBe(true); expect(bucket.tryTake()).toBe(true); + expect(bucket.tryTake()).toBe(false); + t += 1000; expect(bucket.tryTake()).toBe(true); + }); + it("quota ledger refuses a send below budget and reports paused:quota", () => { + const governors = new SendGovernors({ quotas: { youtube: { budget: 50, costPerCall: 50 } } }); + expect(governors.take("youtube")).toMatchObject({ allowed: true, state: "ok" }); // insert: 50 of 50 units + expect(governors.take("youtube")).toMatchObject({ allowed: false, state: "paused:quota", reason: expect.stringContaining("quota") }); + expect(governors.probe("youtube").state).toBe("paused:quota"); + }); + it("rate exhaustion surfaces paused:rate", () => { + const governors = new SendGovernors({ buckets: { twitch: { capacity: 1, refillPerSecond: 0.001 } } }); + expect(governors.take("twitch").allowed).toBe(true); + expect(governors.take("twitch")).toMatchObject({ allowed: false, state: "paused:rate" }); + }); +}); + +describe("ring buffer", () => { + it("serves resume from the window and demands resync past its edge", () => { + const buffer = new ChatRingBuffer(4); + for (let seq = 1; seq <= 12; seq += 1) { + buffer.push({ ...message(seq), seq }); + } + // The advertised capacity and the resumable window agree. + expect(buffer.latest("live-1", 100)).toHaveLength(4); + expect(buffer.latest("live-1", 100).at(0)?.seq).toBe(9); + expect(buffer.after("live-1", 10)).toHaveLength(2); + expect(buffer.canResume("live-1", 8)).toBe(true); // everything >8 is retained + expect(buffer.canResume("live-1", 7)).toBe(false); // fell out of the window + expect(buffer.canResume("live-1", 99)).toBe(false); // ahead of the hub + }); +}); + +function message(seq: number): StoredChatMessage { + return { + id: `m-${seq}`, seq, liveStreamId: "live-1", connectionId: "conn-1", provider: "mock-livestream", + providerMessageId: `pm-${seq}`, author: { handle: `viewer-${seq}`, color: authorColor("mock-livestream", `viewer-${seq}`) }, + text: `hello ${seq}`, sentAt: new Date(0).toISOString(), receivedAt: new Date(0).toISOString(), + isModerated: false, source: "live-chat", + }; +} + +describe("author identity colour", () => { + it("is a pure function of provider:handle", () => { + expect(authorColor("twitch", "ada")).toBe(authorColor("twitch", "ada")); + expect(authorColor("twitch", "ada")).not.toBe(authorColor("youtube", "ada")); + expect(authorColor("twitch", "ada")).toMatch(/^hsl\(\d+ 70% 62%\)$/); + }); +}); + + +describe("daily quota rollover", () => { + it("resets on the next UTC day without replacing the ledger", () => { + let now = Date.parse("2026-10-08T23:59:59Z"); + const governors = new SendGovernors({ quotas: { youtube: { budget: 50, costPerCall: 50 } }, now: () => now }); + expect(governors.take("youtube").allowed).toBe(true); + expect(governors.take("youtube").allowed).toBe(false); + now += 1000; + expect(governors.take("youtube").allowed).toBe(true); + }); + it("does not charge quota when a token bucket refuses the send", () => { + const governors = new SendGovernors({ quotas: { youtube: { budget: 100, costPerCall: 50 } }, buckets: { youtube: { capacity: 1, refillPerSecond: 1 } }, now: () => 0 }); + expect(governors.take("youtube").allowed).toBe(true); + expect(governors.take("youtube").state).toBe("paused:rate"); + expect(governors.probe("youtube").state).toBe("paused:rate"); + }); +}); diff --git a/apps/api/src/live-chat/mock-driver.ts b/apps/api/src/live-chat/mock-driver.ts new file mode 100644 index 0000000..a496b06 --- /dev/null +++ b/apps/api/src/live-chat/mock-driver.ts @@ -0,0 +1,91 @@ +/** + * Mock-livestream ingestion driver: the deterministic contract/E2E harness + * (§3.2). Pulls `getLiveChat` pages from the mock connector and feeds the hub + * with normalized events; the connector's own dedupe-free buffer is the + * "provider", so replaying a page is exactly the dedupe test. + */ +import type { PlatformConnector } from "@tactical-media-broadcast-hub/connector-sdk"; +import type { ConnectorConnection, LiveChatMessage } from "@tactical-media-broadcast-hub/core"; +import { authorColor } from "@tactical-media-broadcast-hub/core"; +import type { ChatIngestDriver, IngestHandler, NormalizedChatEvent } from "./types.js"; + +export interface MockDriverOptions { + workspaceId: string; + connectionId: string; + channelId: string; + liveRemoteId: string; + /** Poll interval; 0 disables the timer (tests call `poll()` by hand). */ + intervalMs?: number; + onError?: (error: unknown) => void; + commentsOnly?: boolean; + connection?: ConnectorConnection; + pageSize?: number; + schedule?: (callback: () => void, delayMs: number) => ReturnType; + cancel?: (timer: ReturnType) => void; +} + +export class MockLivestreamDriver implements ChatIngestDriver { + readonly provider: string; + private handler: IngestHandler | null = null; + private timer: ReturnType | null = null; + private cursor: string | undefined; + private stopped = false; + private inFlight: Promise | null = null; + constructor(private readonly connector: PlatformConnector, private readonly options: MockDriverOptions) { this.provider = connector.manifest().id; } + async start(handler: IngestHandler): Promise { + await this.stop(); + this.handler = handler; + this.stopped = false; + const intervalMs = this.options.intervalMs ?? 0; + if (intervalMs > 0) { + const schedule = this.options.schedule ?? ((cb, ms) => setInterval(cb, ms)); + this.timer = schedule(() => { void this.poll().catch(error => this.options.onError?.(error)); }, intervalMs); + this.timer.unref?.(); + } + await this.poll(); + } + /** One poll cycle → hub. Returns the events produced. */ + poll(): Promise { + if (this.inFlight) return this.inFlight; + this.inFlight = this.pollOnce().finally(() => { this.inFlight = null; }); + return this.inFlight; + } + private async pollOnce(): Promise { + if (!this.handler || this.stopped) return []; + const connection = this.options.connection ?? { id: this.options.connectionId, workspaceId: this.options.workspaceId, provider: this.provider, + channelId: this.options.channelId, credentialId: { kind: "credential-ref" as const, id: "mock" }, fields: {}, createdAt: new Date() }; + const page = this.options.commentsOnly ? await this.connector.getComments?.(connection, { onRemoteId: this.options.liveRemoteId, cursor: this.cursor, limit: this.options.pageSize ?? 100 }).then(page => ({ + ...page, items: page.items.map(comment => ({ id: comment.id, provider: comment.provider, liveStreamId: this.options.liveRemoteId, authorHandle: comment.authorHandle, text: comment.text, sentAt: comment.createdAt, isModerated: false })), + })) : await this.connector.getLiveChat?.( + connection, + { liveRemoteId: this.options.liveRemoteId, cursor: this.cursor, limit: this.options.pageSize ?? 100 }, + ); + if (!page || this.stopped || !this.handler) return []; + const events = page.items.map((message: LiveChatMessage) => ({ ...toEvent(this.options, message), source: this.options.commentsOnly ? "social-reply" as const : "live-chat" as const })); + if (events.length > 0) await this.handler(this.options.workspaceId, events); + this.cursor = page.nextCursor; + return events; + } + async stop(): Promise { + this.stopped = true; + if (this.timer) { (this.options.cancel ?? clearInterval)(this.timer); this.timer = null; } + this.handler = null; + } +} + +/** Map a connector `LiveChatMessage` to the hub's normalized event. */ +export function toEvent(options: Pick, message: LiveChatMessage): NormalizedChatEvent { + return { + provider: message.provider, + connectionId: options.connectionId, + channelId: options.channelId, + providerMessageId: message.id, + author: { handle: message.authorHandle }, + text: message.text, + sentAt: message.sentAt, + receivedAt: new Date(), + source: "live-chat", + }; +} + +export const mockAuthorColor = authorColor; diff --git a/apps/api/src/live-chat/redis-governors.ts b/apps/api/src/live-chat/redis-governors.ts new file mode 100644 index 0000000..0ac6167 --- /dev/null +++ b/apps/api/src/live-chat/redis-governors.ts @@ -0,0 +1,41 @@ +import type { OnModuleDestroy } from "@nestjs/common"; +import { RespControlRedis, type ControlRedis } from "../media-control/redis.js"; +import type { GovernorVerdict, SendGovernorOptions } from "./governors.js"; + +export interface ChatGovernors { + take(provider: string, cost?: number): GovernorVerdict | Promise; + probe(provider: string): GovernorVerdict | Promise; +} +/** One atomic check/charge for every API instance. UTC-day keys survive restarts. */ +export const CHAT_GOVERNOR_SCRIPT = ` +local now, capacity, refill, budget, cost, consume = tonumber(ARGV[1]), tonumber(ARGV[2]), tonumber(ARGV[3]), tonumber(ARGV[4]), tonumber(ARGV[5]), tonumber(ARGV[6]) +local spent = tonumber(redis.call('GET', KEYS[1]) or '0') +if budget >= 0 and spent + cost > budget then return {0, 'paused:quota', 'daily quota exhausted'} end +local tokens = capacity +if capacity > 0 then + local old = redis.call('HMGET', KEYS[2], 'tokens', 'at') + if old[1] then tokens = math.min(capacity, tonumber(old[1]) + math.max(0, now - tonumber(old[2])) * refill / 1000) end + if tokens < 1 then return {0, 'paused:rate', 'send rate budget exhausted'} end +end +if consume == 1 then + if capacity > 0 then redis.call('HSET', KEYS[2], 'tokens', tokens-1, 'at', now); redis.call('EXPIRE', KEYS[2], 172800) end + if budget >= 0 then redis.call('INCRBYFLOAT', KEYS[1], cost); redis.call('EXPIRE', KEYS[1], 172800) end +end +return {1, 'ok', ''}`; +export class RedisChatGovernors implements ChatGovernors, OnModuleDestroy { + constructor(private readonly options: SendGovernorOptions, private readonly redis: ControlRedis = new RespControlRedis(process.env.REDIS_URL ?? "redis://localhost:6379")) {} + take(provider: string, cost?: number): Promise { return this.check(provider, true, cost); } + probe(provider: string): Promise { return this.check(provider, false); } + private async check(provider: string, consume: boolean, cost?: number): Promise { + const bucket = this.options.buckets?.[provider], quota = this.options.quotas?.[provider]; + if (!bucket && !quota) return { allowed: true, state: "ok" }; + const now = this.options.now?.() ?? Date.now(), day = new Date(now).toISOString().slice(0,10), key = encodeURIComponent(provider); + const result = await this.redis.eval(CHAT_GOVERNOR_SCRIPT, [`chat:quota:${key}:${day}`, `chat:rate:${key}`], + [String(now), String(bucket?.capacity ?? 0), String(bucket?.refillPerSecond ?? 0), String(quota?.budget ?? -1), String(cost ?? quota?.costPerCall ?? 1), consume ? "1" : "0"]); + if (!Array.isArray(result)) throw new Error("invalid governor response"); + const allowed = result[0] === 1, state = result[1]; + if (state !== "ok" && state !== "paused:rate" && state !== "paused:quota") throw new Error("invalid governor state"); + return { allowed, state, ...(!allowed ? { reason: String(result[2]) } : {}) }; + } + onModuleDestroy(): void { this.redis.close(); } +} diff --git a/apps/api/src/live-chat/ring-buffer.ts b/apps/api/src/live-chat/ring-buffer.ts new file mode 100644 index 0000000..92ebe2c --- /dev/null +++ b/apps/api/src/live-chat/ring-buffer.ts @@ -0,0 +1,49 @@ +/** + * Bounded latest-message window per channel ("ring buffer", §3.3): the hub + * keeps the last N stored messages so a `lastSeq` resume can be served + * in-process, and the fallback REST cursor path reads the same window. + * + * The public resume window is exactly N messages, including at eviction edges. + */ +import type { StoredChatMessage } from "./types.js"; + +export class ChatRingBuffer { + private readonly map = new Map(); + constructor(private readonly capacity: number = 500) { + if (!Number.isInteger(capacity) || capacity < 1) throw new Error("ring buffer capacity must be a positive integer"); + } + push(message: StoredChatMessage): void { + const key = `${message.liveStreamId}`; + const items = this.map.get(key) ?? []; + items.push(message); + if (items.length > this.capacity) items.splice(0, items.length - this.capacity); + this.map.set(key, items); + } + /** Latest `limit` messages, oldest first. */ + latest(liveStreamId: string, limit = this.capacity): StoredChatMessage[] { + const items = this.map.get(liveStreamId) ?? []; + return items.slice(Math.max(0, items.length - limit)); + } + /** Messages strictly after `after`, oldest first, up to `limit`. */ + after(liveStreamId: string, after: number, limit = this.capacity): StoredChatMessage[] { + return (this.map.get(liveStreamId) ?? []).filter(m => m.seq > after).slice(0, limit); + } + /** Oldest retained seq; undefined when the window is empty. */ + oldestSeq(liveStreamId: string): number | undefined { + return this.map.get(liveStreamId)?.[0]?.seq; + } + highestSeq(liveStreamId: string): number { + return this.latest(liveStreamId, 1)[0]?.seq ?? 0; + } + /** Can a resume from `lastSeq` be served from the window? */ + canResume(liveStreamId: string, lastSeq: number): boolean { + if (lastSeq < 0) return false; + const items = this.map.get(liveStreamId) ?? []; + const highest = items.at(-1)?.seq ?? 0; + if (lastSeq > highest) return false; + const oldest = items[0]?.seq ?? highest + 1; + return lastSeq >= oldest - 1; + } + drop(liveStreamId: string): void { this.map.delete(liveStreamId); } + get size(): number { return this.map.size; } +} diff --git a/apps/api/src/live-chat/runtime.ts b/apps/api/src/live-chat/runtime.ts new file mode 100644 index 0000000..f3fed32 --- /dev/null +++ b/apps/api/src/live-chat/runtime.ts @@ -0,0 +1,17 @@ +import { Inject, Injectable, Optional, OnApplicationBootstrap, OnModuleDestroy } from "@nestjs/common"; +import type { LiveChatDependencies } from "./live-chat.service.js"; +import { LiveChatHubService } from "./live-chat.service.js"; +import type { ChatIngestDriver } from "./types.js"; + +export const LIVE_CHAT_RUNTIME = Symbol("LIVE_CHAT_RUNTIME"); +export type LiveChatRuntimeOptions = Partial> & { drivers?: ChatIngestDriver[] }; +@Injectable() +export class LiveChatRuntime implements OnApplicationBootstrap, OnModuleDestroy { + constructor(@Inject(LiveChatHubService) private readonly hub: LiveChatHubService, + @Optional() @Inject(LIVE_CHAT_RUNTIME) private readonly options?: LiveChatRuntimeOptions) {} + async onApplicationBootstrap(): Promise { + try { for (const driver of this.options?.drivers ?? []) await driver.start((workspaceId, events) => this.hub.ingest(workspaceId, events)); } + catch (error) { await this.onModuleDestroy(); throw error; } + } + async onModuleDestroy(): Promise { await Promise.all((this.options?.drivers ?? []).map(driver => driver.stop())); } +} diff --git a/apps/api/src/live-chat/shared-poller.ts b/apps/api/src/live-chat/shared-poller.ts new file mode 100644 index 0000000..62cb24b --- /dev/null +++ b/apps/api/src/live-chat/shared-poller.ts @@ -0,0 +1,49 @@ +import type { IngestHandler, NormalizedChatEvent } from "./types.js"; +import type { ChatGovernors } from "./redis-governors.js"; + +export interface SharedChatPollerOptions { + workspaceId: string; + broadcastId: string; + provider: string; + fetch: (cursor?: string) => Promise<{ events: NormalizedChatEvent[]; nextPageToken?: string; pollingIntervalMillis: number }>; + ingest: IngestHandler; + governors: ChatGovernors; + paused?: (reason: string) => void; +} +/** One poll chain per workspace/broadcast. Ref-counted consumers share list quota. */ +export class SharedChatPollers { + private readonly entries = new Map | null }>(); + acquire(options: SharedChatPollerOptions): () => void { + const key = `${options.workspaceId}:${options.broadcastId}`; + const existing = this.entries.get(key); + if (existing) { existing.refs += 1; return this.releaseOnce(key); } + const entry = { refs: 1, stopped: false, timer: null as ReturnType | null }; + this.entries.set(key, entry); + let cursor: string | undefined; + const poll = async (): Promise => { + if (entry.stopped) return; + let delay = 1000; + const verdict = await options.governors.take(options.provider, 1); + if (!verdict.allowed) { options.paused?.(verdict.reason ?? "polling paused"); delay = 60_000; } + else { + try { + const page = await options.fetch(cursor); + if (entry.stopped) return; + await options.ingest(options.workspaceId, page.events); + cursor = page.nextPageToken; + delay = Math.max(1, page.pollingIntervalMillis); + } catch (error) { options.paused?.(error instanceof Error ? error.message : "polling failed"); } + } + if (!entry.stopped) { entry.timer = setTimeout(() => { void poll(); }, delay); entry.timer.unref(); } + }; + void poll(); + return this.releaseOnce(key); + } + private releaseOnce(key: string): () => void { let active = true; return () => { if (active) { active = false; this.release(key); } }; } + private release(key: string): void { + const entry = this.entries.get(key); + if (!entry || --entry.refs > 0) return; + entry.stopped = true; if (entry.timer) clearTimeout(entry.timer); this.entries.delete(key); + } + close(): void { for (const [key, entry] of this.entries) { entry.refs = 1; this.release(key); } } +} diff --git a/apps/api/src/live-chat/twitch-adapter.ts b/apps/api/src/live-chat/twitch-adapter.ts new file mode 100644 index 0000000..5f88901 --- /dev/null +++ b/apps/api/src/live-chat/twitch-adapter.ts @@ -0,0 +1,103 @@ +/** + * Twitch adapter: maps the connector's existing chat event shape into the + * hub's normalized event (§3.2 read side), and wraps Helix send/moderate + * behind the `ChatTransport` port so the hub never branches on a provider. + * + * The connector buffers EventSub `channel.chat.message` notifications for + * `getLiveChat`; that buffer is the read path here. Send/moderate go through + * the connector's `sendLiveChatMessage`/`moderateLiveChat` when present, or an + * injected Helix transport (contract-tested with mocks — no real creds). + */ +import type { PlatformConnector } from "@tactical-media-broadcast-hub/connector-sdk"; +import { MockLivestreamDriver } from "./mock-driver.js"; +import type { ConnectorConnection, LiveChatMessage } from "@tactical-media-broadcast-hub/core"; +import { CapabilityNotSupportedError } from "@tactical-media-broadcast-hub/connector-sdk"; +import type { ChatTransport, NormalizedChatEvent } from "./types.js"; + +export interface TwitchAdapterOptions { + workspaceId: string; + connectionId: string; + channelId: string; + /** Provider's live (stream) id — the connector's `liveStreamId`. */ + liveRemoteId: string; +} + +/** Map a Twitch-shaped `LiveChatMessage` from the connector into the hub event. */ +export function twitchToEvent(options: TwitchAdapterOptions, message: LiveChatMessage): NormalizedChatEvent { + return { + provider: message.provider, + connectionId: options.connectionId, + channelId: options.channelId, + providerMessageId: message.id, + author: { handle: message.authorHandle }, + text: message.text, + sentAt: message.sentAt, + receivedAt: new Date(), + source: "live-chat", + }; +} + +export interface HelixChatTransport { + /** POST /helix/chat/messages (or an injected fake in tests). */ + send(request: { connectionId: string; liveRemoteId: string; text: string; replyTo?: string }): + Promise<{ providerMessageId: string; sentAt: Date }>; + /** DELETE /helix/moderation/chat, timeout/ban endpoints. */ + moderate(request: { + connectionId: string; + liveRemoteId: string; + messageRemoteId: string; + authorHandle?: string; + action: "delete" | "timeout" | "ban"; + durationSeconds?: number; + }): Promise; +} + +/** + * Twitch send/moderate transport. Prefers an injected Helix implementation + * (real creds, S5 phase 1 wiring); otherwise it delegates to the connector's + * own methods and surfaces `unsupported` when the connector declares the + * capability absent (its IRC adapter is not configured). + */ +export class TwitchChatTransport implements ChatTransport { + readonly provider: string; + constructor(private readonly connector: PlatformConnector, private readonly helix?: HelixChatTransport, + private readonly resolveConnection?: (connectionId: string) => Promise) { + this.provider = connector.manifest().id; + } + private async connection(connectionId: string): Promise { + if (!this.resolveConnection) throw new Error("chat adapter requires a connection resolver"); + return this.resolveConnection(connectionId); + } + async send(request: { connectionId: string; liveRemoteId: string; text: string; replyTo?: string }): + Promise<{ providerMessageId: string; sentAt: Date }> { + if (this.helix) return this.helix.send(request); + if (request.replyTo) throw new CapabilityNotSupportedError(this.provider, "threaded reply requires Helix transport"); + if (this.connector.sendLiveChatMessage) { + const sent = await this.connector.sendLiveChatMessage(await this.connection(request.connectionId), { liveRemoteId: request.liveRemoteId, text: request.text }); + return { providerMessageId: sent.id, sentAt: sent.sentAt }; + } + throw new CapabilityNotSupportedError(this.provider, "chat send"); + } + async moderate(request: { + connectionId: string; liveRemoteId: string; messageRemoteId: string; + authorHandle?: string; + action: "delete" | "timeout" | "ban"; durationSeconds?: number; + }): Promise { + if (this.helix) return this.helix.moderate(request); + if (request.action === "timeout") throw new CapabilityNotSupportedError(this.provider, "timed moderation requires Helix transport"); + if (this.connector.moderateLiveChat) { + await this.connector.moderateLiveChat(await this.connection(request.connectionId), { + liveRemoteId: request.liveRemoteId, messageRemoteId: request.messageRemoteId, action: request.action, + }); + return; + } + throw new CapabilityNotSupportedError(this.provider, "chat moderation"); + } +} + +/** The existing EventSub read buffer is consumed through the shared driver. */ +export class TwitchChatDriver extends MockLivestreamDriver { + constructor(connector: PlatformConnector, options: TwitchAdapterOptions, connection: ConnectorConnection) { + super(connector, { ...options, connection, intervalMs: 1000 }); + } +} diff --git a/apps/api/src/live-chat/types.ts b/apps/api/src/live-chat/types.ts new file mode 100644 index 0000000..ee57e01 --- /dev/null +++ b/apps/api/src/live-chat/types.ts @@ -0,0 +1,108 @@ +/** + * S5 chat hub — internal ports and normalized shapes. + * + * The wire contract itself lives in `@tactical-media-broadcast-hub/core` (chat-contract.ts) + * and is frozen. This module is the hub implementation: ingestion, sequencing, + * fan-out, send governors, device pairing. + */ +import type { Delivery, LiveChatCapability, LiveChatMessage, Page } from "@tactical-media-broadcast-hub/core"; + +/** A provider chat event normalized at the connector boundary. */ +export interface NormalizedChatEvent { + provider: string; + connectionId: string; + channelId: string; + /** Provider's own message id — the dedupe key with (workspace, provider). */ + providerMessageId: string; + author: { handle: string; isBot?: boolean; displayName?: string; avatarUrl?: string; badges?: string[] }; + text: string; + sentAt: Date; + receivedAt?: Date; + replyTo?: string; + source?: "live-chat" | "social-reply" | "system"; +} + +/** Row written by the hub, already in wire shape. */ +export interface StoredChatMessage { + id: string; + seq: number; + liveStreamId: string; + connectionId: string; + provider: string; + providerMessageId: string; + author: { handle: string; displayName?: string; avatarUrl?: string; badges?: string[]; color: string }; + text: string; + sentAt: string; + receivedAt: string; + isModerated: boolean; + replyTo?: string; + source: "live-chat" | "social-reply" | "system"; +} + +/** Result of ingesting one batch: only the rows actually inserted. */ +export interface IngestResult { + inserted: StoredChatMessage[]; + /** Provider ids already present (dedupe hits) — never re-published. */ + duplicates: string[]; +} + +export interface MessageQuery { + after?: number; + limit: number; + provider?: string; +} + +export interface MessagePage { + items: StoredChatMessage[]; + nextCursor?: number; +} + +export type IngestHandler = (workspaceId: string, events: NormalizedChatEvent[]) => Promise; + +/** Ingestion driver: a platform connection feeding the hub. */ +export interface ChatIngestDriver { + readonly provider: string; + start(handler: IngestHandler): Promise; + stop(): Promise; +} + +/** Per-target send/moderate transport (Helix for Twitch, mock for tests). */ +export interface ChatTransport { + readonly provider: string; + send(request: { connectionId: string; liveRemoteId: string; text: string; replyTo?: string }): + Promise<{ providerMessageId: string; sentAt: Date }>; + moderate(request: { + connectionId: string; + liveRemoteId: string; + messageRemoteId: string; + authorHandle?: string; + action: "delete" | "timeout" | "ban"; + durationSeconds?: number; + }): Promise; +} + +/** A capability-resolved target for a reply/moderation fan-out. */ +export interface ChatTarget { + connectionId: string; + provider: string; + liveRemoteId: string; + read: boolean; + reply: boolean; + moderate: boolean; + state: LiveChatCapability["state"]; + reason?: string; +} + +export interface SendOutcome { + message: StoredChatMessage | null; + delivery: Delivery[]; + /** Refused before any transport call (governor or no capable target). */ + refused?: { reason: string; state: LiveChatCapability["state"] }; +} + +export interface ModerateOutcome { + delivery: Delivery[]; +} + +/** Connector chat page shape (`PlatformConnector.getLiveChat`). */ +export type ConnectorChatPage = Page; diff --git a/apps/api/src/live-monitor/README.md b/apps/api/src/live-monitor/README.md new file mode 100644 index 0000000..ccb1c8f --- /dev/null +++ b/apps/api/src/live-monitor/README.md @@ -0,0 +1,64 @@ +# Monitor signal sources + +`POST /live-monitor/webhooks/:endpointId` authenticates providers independently +of browser sessions. The module installs a raw parser before Nest's JSON parser; +the request cap is 64 KiB and compressed bodies are refused. + +Configure `MONITOR_WEBHOOK_ENDPOINTS` as a server-controlled JSON array: + +```json +[ + { + "endpointId": "<32 server-generated random bytes, encoded as 64 lowercase hex characters>", + "workspaceId": "", + "connectionId": "", + "credentialId": "", + "subscriptionId": "" + } +] +``` + +The credential must belong to that workspace and connection, have kind +`monitor-webhook`, and contain a secret of at least 32 bytes sealed with +`vault.seal(workspaceId, connectionId, secret)`. Disconnection or credential +revocation disables the endpoint. Mappings contain references, never secrets. + +Send these headers: + +- `X-Monitor-Event-Id`: a unique provider message id (up to 256 characters). +- `X-Monitor-Timestamp`: an ISO UTC timestamp within five minutes of ingestion. +- `X-Monitor-Signature`: `sha256=` followed by lowercase hex HMAC-SHA256 over + `eventId + timestamp + rawBody`, using the vaulted secret. + +The JSON body is `{ "subscriptionId": "...", "sourceId": "", +"outcome": "ok|fail|error|unavailable" }`. A supplied workspace or connection +must also match the mapping. The source must belong to the mapped workspace +and connection. Successful delivery returns 200, replay 409, authentication +failure 401, unknown endpoint/source 404, invalid JSON 400, oversized body 413. + +Replay ids are namespaced by endpoint in `monitor_seen_events`. Verification, +ownership checks, replay claim, reducer fold, state write and audit share a +tenant transaction. Journal pruning must retain at least the full ten-minute +timestamp acceptance span plus a clock margin; this module does not prune it. + +`MEDIA_POLLER_SOURCES` entries support `playbackUrl`. Provider `hls` probes that +URL; provider `peertube` polls the video using the connection's `fields.instanceUrl` +and also probes `playbackUrl` when supplied. Canonical `video:@` refs +are supported. Existing mock configuration remains supported. + +HLS probes fetch one master and one variant, validate all child URIs through +WP7, and emit `hls-probe` observations. Decoder reports retain `media-health` +precedence. VOD/endlist/empty/missing playlists fail; stale segments fail after +three target durations. Without program dates, successive playlist progression +provides recency and an initial playlist has the same bounded grace period. + +Every outbound fetch uses the shared WP7 egress client and its database host +bucket (`monitor-probe` or `monitor-api`), including redirect hops. Requests +have a five-second total deadline, 256 KiB response cap, and 128-reference cap. +Per-source drivers back off with exponential jitter after failures, honoring +Retry-After seconds/dates and egress retry delays. Denied probe targets emit +`unavailable` with a policy reason so the existing reducer can fall back. + +Production requires the built `packages/egress` artifact and its host-budget +migration/grants. Those remain owned by WP7; this module performs no DDL and +implements no additional limiter. diff --git a/apps/api/src/live-monitor/live-monitor.controller.ts b/apps/api/src/live-monitor/live-monitor.controller.ts new file mode 100644 index 0000000..8fcb3da --- /dev/null +++ b/apps/api/src/live-monitor/live-monitor.controller.ts @@ -0,0 +1,84 @@ +import { BadRequestException, Body, Controller, Get, Inject, Param, Post, Query, Req, UseGuards } from "@nestjs/common"; +import type { Request } from "express"; +import { parseId, parseLimit, requireObject, requireSession, requireString } from "../http/validation.js"; +import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import { LiveMonitorService, type MonitorPage } from "./live-monitor.service.js"; +import { LiveMonitorPoller } from "./poller/poller.js"; +import type { MonitorObservation, MonitorRowState, ObservationOutcome, SignalTier } from "./reducer.js"; + +@Controller("live-monitor") +@UseGuards(RbacGuard) +export class LiveMonitorController { + constructor( + @Inject(LiveMonitorService) private readonly monitor: LiveMonitorService, + @Inject(LiveMonitorPoller) private readonly poller: LiveMonitorPoller, + ) {} + + @Post("poller/:action") + @RequirePermission("media.manage") + async controlPoller(@Param("action") action: string): Promise<{ running: boolean }> { + if (action === "start") return this.poller.start(); + if (action === "stop") return this.poller.stop(); + throw new BadRequestException("action must be start or stop"); + } + + @Post(":sourceId/observations") + @RequirePermission("media.manage") + async record( + @Req() request: Request & AuthenticatedRequest, + @Param("sourceId") sourceId: string, + @Body() body: unknown, + ): Promise { + const session = requireSession(request); + return this.monitor.recordObservation(session.workspaceId, parseId(sourceId, "sourceId"), + parseObservation(body), { userId: session.userId, role: session.role }); + } + + @Get() + @RequirePermission("media.view") + async list( + @Req() request: Request & AuthenticatedRequest, + @Query("cursor") cursor: string | undefined, + @Query("limit") limit: string | undefined, + ): Promise { + return this.monitor.listStates(requireSession(request).workspaceId, cursor, parseLimit(limit, 50, 200)); + } + + @Get(":id") + @RequirePermission("media.view") + async get( + @Req() request: Request & AuthenticatedRequest, + @Param("id") id: string, + ): Promise { + return this.monitor.getState(requireSession(request).workspaceId, parseId(id)); + } +} + +function parseObservation(value: unknown): MonitorObservation { + const body = requireObject(value, "body"); + const tier = body.tier; + if (tier !== "media-health" && tier !== "hls-probe" && tier !== "webhook" && tier !== "api-poll") { + throw new BadRequestException("tier is invalid"); + } + const observedAt = requireString(body.observedAt, "observedAt"); + if (!Number.isFinite(Date.parse(observedAt))) throw new BadRequestException("observedAt must be a timestamp"); + const payload = requireObject(body.payload, "payload"); + const outcome = payload.outcome; + if (outcome !== "ok" && outcome !== "fail" && outcome !== "error" && outcome !== "unavailable") { + throw new BadRequestException("outcome is invalid"); + } + if (payload.mediaPresent !== undefined && typeof payload.mediaPresent !== "boolean") { + throw new BadRequestException("mediaPresent must be boolean"); + } + return { + tier: tier as SignalTier, + observedAt, + receivedAt: new Date().toISOString(), + ...(body.providerEventId === undefined ? {} : { providerEventId: requireString(body.providerEventId, "providerEventId") }), + payload: { + outcome: outcome as ObservationOutcome, + ...(payload.reason === undefined ? {} : { reason: requireString(payload.reason, "reason") }), + ...(payload.mediaPresent === undefined ? {} : { mediaPresent: payload.mediaPresent as boolean }), + }, + }; +} diff --git a/apps/api/src/live-monitor/live-monitor.integration.test.ts b/apps/api/src/live-monitor/live-monitor.integration.test.ts new file mode 100644 index 0000000..19cc5ca --- /dev/null +++ b/apps/api/src/live-monitor/live-monitor.integration.test.ts @@ -0,0 +1,81 @@ +import { randomUUID } from "node:crypto"; +import { join } from "node:path"; +import { Pool, type PoolClient } from "pg"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { runMigrations } from "../db/migrate.js"; +import { LiveMonitorService, type LiveMonitorDependencies } from "./live-monitor.service.js"; + +const dbAvailable = process.env.DATABASE_URL !== undefined; +const ownerPool = new Pool({ connectionString: process.env.SUPER_DATABASE_URL ?? process.env.DATABASE_URL }); +const appPool = new Pool({ connectionString: process.env.DATABASE_URL }); + +async function asTenant(workspaceId: string, fn: (client: PoolClient) => Promise): Promise { + const client = await appPool.connect(); + try { + await client.query("BEGIN"); + await client.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]); + const result = await fn(client); + await client.query("COMMIT"); + return result; + } catch (error) { + await client.query("ROLLBACK").catch(() => undefined); + throw error; + } finally { + client.release(); + } +} + +describe.skipIf(!dbAvailable)("live monitor CAS (live DB)", () => { + let workspaceId: string; + let sourceId: string; + let service: LiveMonitorService; + + beforeAll(async () => { + await runMigrations(process.env.MIGRATIONS_DIR ?? join(process.cwd(), "migrations")); + const workspace = await ownerPool.query( + "INSERT INTO workspaces (name, slug) VALUES ('Monitor integration', $1) RETURNING id", + [`monitor-${randomUUID()}`], + ); + workspaceId = String(workspace.rows[0]?.id); + sourceId = await asTenant(workspaceId, async (client) => { + const connection = await client.query( + "INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-livestream', 'monitor') RETURNING id", + [workspaceId], + ); + const source = await client.query( + "INSERT INTO source_monitor_states (workspace_id, connection_id, remote_ref) VALUES ($1, $2, 'live:cas') RETURNING id", + [workspaceId, connection.rows[0]?.id], + ); + return String(source.rows[0]?.id); + }); + const dependencies: LiveMonitorDependencies = { tenantTransaction: asTenant }; + service = new LiveMonitorService(dependencies); + }); + + afterAll(async () => { + if (workspaceId !== undefined) await ownerPool.query("DELETE FROM workspaces WHERE id = $1", [workspaceId]); + await appPool.end(); + await ownerPool.end(); + }); + + it("serializes two recorders and keeps one transition per successful CAS", async () => { + const observedAt = new Date().toISOString(); + const [first, second] = await Promise.all([ + service.recordObservation(workspaceId, sourceId, { + tier: "api-poll", observedAt, receivedAt: observedAt, payload: { outcome: "ok" }, + }), + service.recordObservation(workspaceId, sourceId, { + tier: "hls-probe", observedAt, receivedAt: observedAt, payload: { outcome: "ok" }, + }), + ]); + expect([first.version, second.version].sort()).toEqual([1, 2]); + const current = await service.getState(workspaceId, sourceId); + expect(current.version).toBe(2); + expect(current.authoritativeTier).toBe("hls-probe"); + const audit = await asTenant(workspaceId, (client) => client.query( + "SELECT count(*)::int AS count FROM audit_log WHERE action = 'monitor.state_changed' AND target_id = $1", + [sourceId], + )); + expect(audit.rows[0]?.count).toBe(1); + }); +}); diff --git a/apps/api/src/live-monitor/live-monitor.module.ts b/apps/api/src/live-monitor/live-monitor.module.ts new file mode 100644 index 0000000..8931b4e --- /dev/null +++ b/apps/api/src/live-monitor/live-monitor.module.ts @@ -0,0 +1,19 @@ +import { Inject, Module } from "@nestjs/common"; +import { HttpAdapterHost } from "@nestjs/core"; +import type { ExpressAdapter } from "@nestjs/platform-express"; +import { LiveMonitorController } from "./live-monitor.controller.js"; +import { LiveMonitorService } from "./live-monitor.service.js"; +import { LiveMonitorPoller } from "./poller/poller.js"; +import { MonitorWebhookController } from "./webhook.controller.js"; +import { MonitorWebhookService, WEBHOOK_MAX_BYTES } from "./webhook.service.js"; + +@Module({ controllers: [LiveMonitorController, MonitorWebhookController], providers: [LiveMonitorService, LiveMonitorPoller, MonitorWebhookService], exports: [LiveMonitorService] }) +export class LiveMonitorModule { + constructor(@Inject(HttpAdapterHost) adapter: HttpAdapterHost) { + // Installed while Nest creates modules, before init registers its default JSON parser. + (adapter.httpAdapter as ExpressAdapter).useBodyParser("raw", false, { + type: (request: { url?: string }) => request.url?.startsWith("/live-monitor/webhooks/") === true, + limit: WEBHOOK_MAX_BYTES, inflate: false, + }); + } +} diff --git a/apps/api/src/live-monitor/live-monitor.service.ts b/apps/api/src/live-monitor/live-monitor.service.ts new file mode 100644 index 0000000..db6938e --- /dev/null +++ b/apps/api/src/live-monitor/live-monitor.service.ts @@ -0,0 +1,181 @@ +import { BadRequestException, ConflictException, Inject, Injectable, NotFoundException, Optional } from "@nestjs/common"; +import type { PoolClient } from "pg"; +import { withTenant } from "../db/client.js"; +import { isUuid } from "../http/validation.js"; +import { applyObservation, type MonitorObservation, type MonitorRowState, type MonitorState, + type SignalTier, type TierStatus, type MonitorPolicy, DEFAULT_MONITOR_POLICY } from "./reducer.js"; + +export interface MonitorActor { userId: string | null; role: string } +export interface MonitorPage { items: MonitorRowState[]; nextCursor?: string } +export interface LiveMonitorDependencies { + tenantTransaction: typeof withTenant; + policy?: Partial; +} +export const LIVE_MONITOR_DEPENDENCIES = Symbol("LIVE_MONITOR_DEPENDENCIES"); + +@Injectable() +export class LiveMonitorService { + private readonly dependencies: LiveMonitorDependencies; + private readonly policy: MonitorPolicy; + + constructor(@Optional() @Inject(LIVE_MONITOR_DEPENDENCIES) dependencies?: Partial) { + this.dependencies = { tenantTransaction: withTenant, ...dependencies }; + this.policy = { ...DEFAULT_MONITOR_POLICY, ...dependencies?.policy }; + } + + async recordObservation( + workspaceId: string, + sourceId: string, + observation: MonitorObservation, + actor: MonitorActor = { userId: null, role: "monitor" }, + ): Promise { + validateObservation(observation); + return this.dependencies.tenantTransaction(workspaceId, (client) => + this.recordObservationInTransaction(client, workspaceId, sourceId, observation, actor)); + } + + /** Webhook verification, durable replay claim and reduction share one transaction. */ + async recordObservationInTransaction( + client: PoolClient, workspaceId: string, sourceId: string, observation: MonitorObservation, + actor: MonitorActor = { userId: null, role: "monitor" }, rejectReplay = false, + ): Promise { + validateObservation(observation); + let seenInserted = false; + for (let attempt = 0; attempt < 5; attempt += 1) { + const locked = await client.query("SELECT * FROM source_monitor_states WHERE id = $1 FOR UPDATE", [sourceId]); + if (locked.rowCount !== 1) throw new NotFoundException(`live source ${sourceId} not found`); + const current = fromRow(locked.rows[0]); + if (observation.providerEventId !== undefined && !seenInserted) { + const seen = await client.query( + `INSERT INTO monitor_seen_events (workspace_id, provider_event_id, source_monitor_state_id) + VALUES ($1, $2, $3) ON CONFLICT (workspace_id, provider_event_id) DO NOTHING RETURNING provider_event_id`, + [workspaceId, observation.providerEventId, sourceId], + ); + if (seen.rowCount === 0) { + if (rejectReplay) throw new ConflictException("webhook event already received"); + return current; + } + seenInserted = true; + } + const reduction = applyObservation(current, observation, this.policy); + if (reduction.state === current) return current; + const next = reduction.state; + const changed = await client.query( + `UPDATE source_monitor_states SET state = $2, version = $3, authoritative_tier = $4, + last_media_health_at = $5, last_hls_probe_at = $6, last_webhook_at = $7, + last_api_poll_at = $8, tier_status = $9, ok_streak = $10, fail_streak = $11, + state_entered_at = $12, hold_until = $13, cooldown_until = $14, + previous_state = $15, last_transition_at = $16, last_transition_reason = $17, + updated_at = $18 + WHERE id = $1 AND version = $19 RETURNING *`, + [sourceId, next.state, next.version, next.authoritativeTier, next.lastMediaHealthAt, + next.lastHlsProbeAt, next.lastWebhookAt, next.lastApiPollAt, next.tierStatus, + next.okStreak, next.failStreak, next.stateEnteredAt, next.holdUntil, next.cooldownUntil, + next.previousState, next.lastTransitionAt, next.lastTransitionReason, next.updatedAt, current.version], + ); + if (changed.rowCount !== 1) continue; + if (reduction.transition !== undefined) { + await client.query( + "SELECT audit_append($1, $2, $3, 'monitor.state_changed', 'source_monitor_state', $4, $5)", + [workspaceId, actor.userId, actor.role, sourceId, { + from: reduction.transition.from, to: reduction.transition.to, + version: reduction.transition.version, reason: reduction.transition.reason, + tier: next.authoritativeTier, + }], + ); + } + return fromRow(changed.rows[0]); + } + throw new Error(`live source ${sourceId} CAS conflict did not settle`); + } + + async getState(workspaceId: string, sourceId: string): Promise { + return this.dependencies.tenantTransaction(workspaceId, async (client) => { + const found = await client.query("SELECT * FROM source_monitor_states WHERE id = $1", [sourceId]); + if (found.rowCount !== 1) throw new NotFoundException(`live source ${sourceId} not found`); + return fromRow(found.rows[0]); + }); + } + + async getStates(workspaceId: string, sourceIds: string[]): Promise { + if (sourceIds.length === 0) return []; + return this.dependencies.tenantTransaction(workspaceId, async (client) => { + const found = await client.query("SELECT * FROM source_monitor_states WHERE id = ANY($1::uuid[])", [sourceIds]); + return found.rows.map(fromRow); + }); + } + + async listStates(workspaceId: string, cursor?: string, limit = 50): Promise { + if (!Number.isInteger(limit) || limit < 1 || limit > 200) { + throw new BadRequestException("limit must be an integer between 1 and 200"); + } + const decoded = cursor === undefined || cursor === "" ? undefined : decodeCursor(cursor); + return this.dependencies.tenantTransaction(workspaceId, async (client) => { + const result = await client.query( + `SELECT * FROM source_monitor_states + WHERE ($1::timestamptz IS NULL OR (created_at, id) < ($1::timestamptz, $2::uuid)) + ORDER BY created_at DESC, id DESC LIMIT $3`, + [decoded?.createdAt ?? null, decoded?.id ?? null, limit + 1], + ); + const rows = result.rows.map(fromRow); + const items = rows.slice(0, limit); + const last = items[items.length - 1]; + return { items, ...(rows.length > limit && last !== undefined + ? { nextCursor: Buffer.from(JSON.stringify({ createdAt: last.createdAt, id: last.id })).toString("base64url") } + : {}) }; + }); + } +} + +function validateObservation(value: MonitorObservation): void { + if (!(["media-health", "hls-probe", "webhook", "api-poll"] as string[]).includes(value.tier)) { + throw new BadRequestException("invalid observation tier"); + } + if (!Number.isFinite(Date.parse(value.observedAt)) || !Number.isFinite(Date.parse(value.receivedAt))) { + throw new BadRequestException("observation timestamps must be valid"); + } + if (!(["ok", "fail", "error", "unavailable"] as string[]).includes(value.payload.outcome)) { + throw new BadRequestException("invalid observation outcome"); + } + if (value.providerEventId !== undefined && value.providerEventId.trim() === "") { + throw new BadRequestException("providerEventId must be non-empty"); + } +} + +function fromRow(raw: unknown): MonitorRowState { + if (typeof raw !== "object" || raw === null || Array.isArray(raw)) throw new Error("invalid monitor row"); + const row = raw as Record; + const version = Number(row.version); + if (!Number.isSafeInteger(version)) throw new Error("monitor version exceeds safe integer range"); + return { + id: String(row.id), workspaceId: String(row.workspace_id), connectionId: String(row.connection_id), + remoteRef: String(row.remote_ref), state: row.state as MonitorState, version, + authoritativeTier: row.authoritative_tier as SignalTier | null, + lastMediaHealthAt: dateOrNull(row.last_media_health_at), lastHlsProbeAt: dateOrNull(row.last_hls_probe_at), + lastWebhookAt: dateOrNull(row.last_webhook_at), lastApiPollAt: dateOrNull(row.last_api_poll_at), + tierStatus: row.tier_status as Partial>, + okStreak: Number(row.ok_streak), failStreak: Number(row.fail_streak), + stateEnteredAt: dateOrNull(row.state_entered_at) ?? "", + holdUntil: dateOrNull(row.hold_until), cooldownUntil: dateOrNull(row.cooldown_until), + previousState: row.previous_state as MonitorState | null, + lastTransitionAt: dateOrNull(row.last_transition_at), lastTransitionReason: row.last_transition_reason as string | null, + createdAt: dateOrNull(row.created_at) ?? "", updatedAt: dateOrNull(row.updated_at) ?? "", + }; +} + +function dateOrNull(value: unknown): string | null { + return value === null || value === undefined ? null : new Date(String(value)).toISOString(); +} + +function decodeCursor(value: string): { createdAt: string; id: string } { + try { + const parsed: unknown = JSON.parse(Buffer.from(value, "base64url").toString("utf8")); + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new Error("invalid cursor"); + const row = parsed as Record; + if (typeof row.createdAt !== "string" || !Number.isFinite(Date.parse(row.createdAt)) + || typeof row.id !== "string" || !isUuid(row.id)) throw new Error("invalid cursor"); + return { createdAt: row.createdAt, id: row.id }; + } catch { + throw new BadRequestException("cursor is invalid"); + } +} diff --git a/apps/api/src/live-monitor/poller/backoff-driver.ts b/apps/api/src/live-monitor/poller/backoff-driver.ts new file mode 100644 index 0000000..5676244 --- /dev/null +++ b/apps/api/src/live-monitor/poller/backoff-driver.ts @@ -0,0 +1,39 @@ +import type { MonitorObservation, SignalTier } from "../reducer.js"; +import type { PollDriver } from "./driver.js"; +import { MonitorHttpError, retryAfterMs } from "./egress.js"; + +/** Scheduling state only: all host admission/debits remain in packages/egress. */ +export class BackoffPollDriver implements PollDriver { + private failures = 0; + private retryAt = 0; + constructor( + private readonly driver: PollDriver, private readonly tier: SignalTier, + private readonly now: () => number = Date.now, private readonly random: () => number = Math.random, + ) {} + get nextAttemptAt(): number { return this.retryAt; } + + async poll(): Promise { + if (this.now() < this.retryAt) return []; + try { + const observations = await this.driver.poll(); + this.failures = 0; + this.retryAt = 0; + return observations; + } catch (error) { + const now = this.now(); + const metadata = error as { code?: string; retryAfterMs?: number } | null; + const transient = error instanceof MonitorHttpError && (error.status === 429 || error.status >= 500) + || metadata?.code === "BUDGET_EXHAUSTED" || metadata?.code === "BUDGET_UNAVAILABLE" + || metadata?.code === "NETWORK_ERROR" || metadata?.code === "TIMEOUT" || metadata?.code === "ABORTED"; + this.failures += 1; + const delay = Math.min(120_000, 1000 * 2 ** Math.min(7, this.failures - 1)); + const serverDelay = error instanceof MonitorHttpError ? retryAfterMs(error.headers, now) : metadata?.retryAfterMs ?? 0; + this.retryAt = now + Math.max(serverDelay, delay * (0.8 + this.random() * 0.4)); + const policyDenied = ["DENIED_PRIVATE_ADDRESS", "SCHEME_DENIED", "PORT_DENIED", "ORIGIN_DENIED", "INVALID_URL", "DNS_FAILED"].includes(metadata?.code ?? ""); + const timestamp = new Date(now).toISOString(); + return [{ tier: this.tier, observedAt: timestamp, receivedAt: timestamp, + payload: { outcome: transient || policyDenied ? "unavailable" : "error", + reason: policyDenied ? `egress-policy:${metadata?.code}` : transient ? "monitor-backoff" : "monitor-probe-error" } }]; + } + } +} diff --git a/apps/api/src/live-monitor/poller/driver.ts b/apps/api/src/live-monitor/poller/driver.ts new file mode 100644 index 0000000..3316095 --- /dev/null +++ b/apps/api/src/live-monitor/poller/driver.ts @@ -0,0 +1,7 @@ +import type { MonitorObservation } from "../reducer.js"; + +/** A provider probe emits observations ordered by its own source clock. */ +export interface PollDriver { + poll(): Promise; +} + diff --git a/apps/api/src/live-monitor/poller/egress.ts b/apps/api/src/live-monitor/poller/egress.ts new file mode 100644 index 0000000..fe3eddb --- /dev/null +++ b/apps/api/src/live-monitor/poller/egress.ts @@ -0,0 +1,96 @@ +import { existsSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { pool } from "../../db/client.js"; +import type { createEgressClient, EgressPolicy, rewriteSegmentUris, validateMasterPlaylist } from "../../../../../packages/egress/src/index.js"; + +type EgressPackage = typeof import("../../../../../packages/egress/src/index.js"); +export interface MonitorEgress { + egressFetch: ReturnType["egressFetch"]; + policy: EgressPolicy; + validateMasterPlaylist: typeof validateMasterPlaylist; + rewriteSegmentUris: typeof rewriteSegmentUris; +} +export const MONITOR_TIMEOUT_MS = 5_000; +export const MONITOR_MAX_BYTES = 256 * 1024; + +const nativeImport = new Function("specifier", "return import(specifier)") as (specifier: string) => Promise; +let shared: Promise | undefined; + +/** ESM package is loaded natively because the API emits CommonJS. No local limiter. */ +export function monitorEgress(): Promise { + shared ??= (async () => { + for (const start of [process.cwd(), __dirname]) { + for (let root = start; ; root = dirname(root)) { + const directory = resolve(root, "packages/egress"); + const files = process.env.NODE_ENV === "production" ? ["dist/index.js"] : ["src/index.ts", "dist/index.js"]; + for (const file of files) { + const path = resolve(directory, file); + if (!existsSync(path)) continue; + const pkg = await nativeImport(pathToFileURL(path).href); + const client = pkg.createEgressClient({ pool, timeoutMs: MONITOR_TIMEOUT_MS, maxResponseBytes: MONITOR_MAX_BYTES, + hostClasses: { + "monitor-probe": { capacity: 3, refillPerInterval: 10 }, + "monitor-api": { capacity: 3, refillPerInterval: 10 }, + } }); + return { ...client, validateMasterPlaylist: pkg.validateMasterPlaylist, rewriteSegmentUris: pkg.rewriteSegmentUris }; + } + if (dirname(root) === root) break; + } + } + throw new Error("Build packages/egress before starting the live monitor"); + })(); + return shared; +} + +export class MonitorHttpError extends Error { + constructor(readonly status: number, readonly headers: Headers) { super(`monitor HTTP ${status}`); } +} + +/** Retry-After remains a lower bound, including HTTP dates; never retry early. */ +export function retryAfterMs(headers: Headers, now: number): number { + const raw = headers.get("retry-after"); + if (raw === null) return 0; + if (/^\d+(?:\.\d+)?$/.test(raw.trim())) { + const delay = Number(raw) * 1000; + return Number.isFinite(delay) ? delay : 0; + } + const deadline = Date.parse(raw); + return Number.isFinite(deadline) ? Math.max(0, deadline - now) : 0; +} + +/** Caps body reads even for injected clients, which need not buffer like WP7. */ +export async function boundedText(response: Response, signal: AbortSignal): Promise { + if (!response.body) return ""; + const reader = response.body.getReader(); + const chunks: Buffer[] = []; + let size = 0; + const cancel = () => { void reader.cancel().catch(() => undefined); }; + signal.addEventListener("abort", cancel, { once: true }); + try { + for (;;) { + if (signal.aborted) throw new Error("monitor deadline exceeded"); + const part = await reader.read(); + if (part.done) break; + size += part.value.byteLength; + if (size > MONITOR_MAX_BYTES) throw new Error("monitor response exceeds size cap"); + chunks.push(Buffer.from(part.value)); + } + if (signal.aborted) throw new Error("monitor deadline exceeded"); + return Buffer.concat(chunks).toString("utf8"); + } finally { + signal.removeEventListener("abort", cancel); + void reader.cancel().catch(() => undefined); + } +} + +/** Covers DNS/URI validation as well as fetch and body consumption. */ +export async function monitorDeadline(run: (signal: AbortSignal) => Promise): Promise { + const controller = new AbortController(); + let timer: ReturnType | undefined; + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout(() => { controller.abort(); reject(new Error("monitor deadline exceeded")); }, MONITOR_TIMEOUT_MS); + }); + try { return await Promise.race([run(controller.signal), timeout]); } + finally { clearTimeout(timer); controller.abort(); } +} diff --git a/apps/api/src/live-monitor/poller/fixtures/live.m3u8 b/apps/api/src/live-monitor/poller/fixtures/live.m3u8 new file mode 100644 index 0000000..bdcd64a --- /dev/null +++ b/apps/api/src/live-monitor/poller/fixtures/live.m3u8 @@ -0,0 +1,8 @@ +#EXTM3U +#EXT-X-TARGETDURATION:6 +#EXT-X-MEDIA-SEQUENCE:20 +#EXT-X-PROGRAM-DATE-TIME:2026-01-01T00:00:00Z +#EXTINF:6, +segment20.ts +#EXTINF:6, +segment21.ts diff --git a/apps/api/src/live-monitor/poller/fixtures/malformed.m3u8 b/apps/api/src/live-monitor/poller/fixtures/malformed.m3u8 new file mode 100644 index 0000000..4c28046 --- /dev/null +++ b/apps/api/src/live-monitor/poller/fixtures/malformed.m3u8 @@ -0,0 +1,4 @@ +#EXTM3U +#EXT-X-TARGETDURATION:garbage +#EXTINF:6, +segment20.ts diff --git a/apps/api/src/live-monitor/poller/fixtures/master.m3u8 b/apps/api/src/live-monitor/poller/fixtures/master.m3u8 new file mode 100644 index 0000000..a0ab2f3 --- /dev/null +++ b/apps/api/src/live-monitor/poller/fixtures/master.m3u8 @@ -0,0 +1,5 @@ +#EXTM3U +#EXT-X-STREAM-INF:BANDWIDTH=800000 +live.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=2000000 +unused.m3u8 diff --git a/apps/api/src/live-monitor/poller/fixtures/offline.m3u8 b/apps/api/src/live-monitor/poller/fixtures/offline.m3u8 new file mode 100644 index 0000000..ba56782 --- /dev/null +++ b/apps/api/src/live-monitor/poller/fixtures/offline.m3u8 @@ -0,0 +1,6 @@ +#EXTM3U +#EXT-X-PLAYLIST-TYPE:VOD +#EXT-X-TARGETDURATION:6 +#EXTINF:6, +segment20.ts +#EXT-X-ENDLIST diff --git a/apps/api/src/live-monitor/poller/hls-driver.ts b/apps/api/src/live-monitor/poller/hls-driver.ts new file mode 100644 index 0000000..3e363ee --- /dev/null +++ b/apps/api/src/live-monitor/poller/hls-driver.ts @@ -0,0 +1,127 @@ +import type { MonitorObservation } from "../reducer.js"; +import type { PollDriver } from "./driver.js"; +import { boundedText, monitorDeadline, MonitorHttpError, type MonitorEgress } from "./egress.js"; + +export interface HlsMediaPlaylist { + targetDuration: number; + sequence: number; + segments: Array<{ uri: string; duration: number; endAt?: number }>; + ended: boolean; + vod: boolean; +} + +export function parseMediaPlaylist(text: string): HlsMediaPlaylist { + const lines = text.replace(/^\uFEFF/, "").split(/\r?\n/).map((line) => line.trim()); + if (lines[0] !== "#EXTM3U" || text.includes("\0")) throw new Error("malformed HLS header"); + let targetDuration = 0; + let sequence = 0; + let duration: number | undefined; + let programTime: number | undefined; + let ended = false; + let vod = false; + const segments: HlsMediaPlaylist["segments"] = []; + for (const line of lines.slice(1)) { + if (line.startsWith("#EXT-X-TARGETDURATION:")) { + const raw = line.slice(22); + if (targetDuration || !/^\d+$/.test(raw)) throw new Error("invalid HLS target duration"); + targetDuration = Number(raw); + } else if (line.startsWith("#EXT-X-MEDIA-SEQUENCE:")) { + const raw = line.slice(22); + if (!/^\d+$/.test(raw) || !Number.isSafeInteger(Number(raw))) throw new Error("invalid HLS sequence"); + sequence = Number(raw); + } else if (line.startsWith("#EXT-X-PLAYLIST-TYPE:")) { + const type = line.slice(21); + if (type !== "EVENT" && type !== "VOD") throw new Error("invalid HLS playlist type"); + vod = type === "VOD"; + } else if (line.startsWith("#EXT-X-PROGRAM-DATE-TIME:")) { + programTime = Date.parse(line.slice(25)); + if (!Number.isFinite(programTime)) throw new Error("invalid HLS program time"); + } else if (line.startsWith("#EXTINF:")) { + if (duration !== undefined) throw new Error("HLS segment URI missing"); + const raw = line.slice(8).split(",")[0] ?? ""; + if (!/^\d+(?:\.\d+)?$/.test(raw)) throw new Error("invalid HLS duration"); + duration = Number(raw); + if (!Number.isFinite(duration) || duration <= 0) throw new Error("invalid HLS duration"); + } else if (line === "#EXT-X-ENDLIST") ended = true; + else if (line && !line.startsWith("#")) { + if (duration === undefined) throw new Error("HLS segment duration missing"); + if (programTime !== undefined) programTime += duration * 1000; + segments.push({ uri: line, duration, ...(programTime === undefined ? {} : { endAt: programTime }) }); + duration = undefined; + } + } + if (duration !== undefined || !Number.isSafeInteger(targetDuration) || targetDuration <= 0 || targetDuration > 3600 + || segments.some((segment) => Math.round(segment.duration) > targetDuration) || segments.length > 128) { + throw new Error("invalid HLS media playlist"); + } + return { targetDuration, sequence, segments, ended, vod }; +} + +/** Playlist ground truth uses hls-probe, below decoder media-health and above provider signals. */ +export class HlsProbeDriver implements PollDriver { + private previous: { fingerprint: string; advancedAt: number; playlistUrl: string } | undefined; + constructor(private readonly playbackUrl: string, private readonly egress: MonitorEgress, + private readonly now: () => number = Date.now) {} + + async poll(): Promise { + return monitorDeadline(async (signal) => { + let url = this.playbackUrl; + let response = await this.fetch(url, signal); + if (response.status === 404 || response.status === 410) return [this.observation(false, "hls-missing")]; + let text = await boundedText(response, signal); + url = response.url || url; + // Bound URI/DNS work before the WP7 helper validates every reference (including keys/maps). + this.capReferences(text); + if (/^#EXT-X-STREAM-INF:/m.test(text)) { + await this.egress.validateMasterPlaylist(text, url, this.egress.policy); + const lines = text.split(/\r?\n/).map((line) => line.trim()); + const variantTag = lines.findIndex((line) => line.startsWith("#EXT-X-STREAM-INF:")); + const variant = lines.slice(variantTag + 1).find((line) => line && !line.startsWith("#")); + if (!variant) throw new Error("HLS master has no variant"); + if (signal.aborted) throw new Error("monitor deadline exceeded"); + url = new URL(variant, url).href; + response = await this.fetch(url, signal); + if (response.status === 404 || response.status === 410) return [this.observation(false, "hls-variant-missing")]; + text = await boundedText(response, signal); + url = response.url || url; + this.capReferences(text); + } + const validated = await this.egress.rewriteSegmentUris(text, url, this.egress.policy); + if (signal.aborted) throw new Error("monitor deadline exceeded"); + const playlist = parseMediaPlaylist(validated); + const now = this.now(); + const latest = playlist.segments.at(-1); + if (playlist.ended || playlist.vod || !latest) return [this.observation(false, "hls-ended-or-empty")]; + const fingerprint = JSON.stringify([playlist.sequence, playlist.segments.length, latest.uri]); + const old = this.previous; + if (!old || old.playlistUrl !== url || old.fingerprint !== fingerprint) { + this.previous = { fingerprint, advancedAt: now, playlistUrl: url }; + } + // Three target durations allow one missed reload; unchanged windows eventually fail even without PDT. + const graceMs = playlist.targetDuration * 3 * 1000; + const fresh = latest.endAt === undefined + ? now - this.previous!.advancedAt <= graceMs + : latest.endAt >= now - graceMs && latest.endAt <= now + graceMs; + return [this.observation(fresh, fresh ? "hls-recent-segments" : "hls-stale-segments")]; + }); + } + + private async fetch(url: string, signal: AbortSignal): Promise { + if (signal.aborted) throw new Error("monitor deadline exceeded"); + const response = await this.egress.egressFetch(url, { signal, headers: { accept: "application/vnd.apple.mpegurl" } }, + { hostClass: "monitor-probe" }); + if (!response.ok && response.status !== 404 && response.status !== 410) throw new MonitorHttpError(response.status, response.headers); + return response; + } + + private observation(live: boolean, reason: string): MonitorObservation { + const timestamp = new Date(this.now()).toISOString(); + return { tier: "hls-probe", observedAt: timestamp, receivedAt: timestamp, + payload: { outcome: live ? "ok" : "fail", mediaPresent: live, reason } }; + } + + private capReferences(text: string): void { + if (text.split(/\r?\n/).filter((line) => line.trim() && !line.trim().startsWith("#")).length + + (text.match(/(?:SERVER-)?URI=/g)?.length ?? 0) > 128) throw new Error("too many HLS references"); + } +} diff --git a/apps/api/src/live-monitor/poller/mock-driver.ts b/apps/api/src/live-monitor/poller/mock-driver.ts new file mode 100644 index 0000000..d741d48 --- /dev/null +++ b/apps/api/src/live-monitor/poller/mock-driver.ts @@ -0,0 +1,136 @@ +import { existsSync } from "node:fs"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import type { MonitorObservation } from "../reducer.js"; +import type { PollDriver } from "./driver.js"; + +interface ScenarioStep { + source: "A" | "B" | "C"; + action: "go-live" | "fail" | "recover" | "go-offline"; + holdSeconds: number; +} +interface Scenario { steps: ScenarioStep[] } +type MockLiveState = "created" | "waiting" | "live" | "ended"; +interface MockStream { remoteId: string; state: MockLiveState } +interface MockConnection { + id: string; + workspaceId: string; + provider: string; + channelId: string; + credentialId: { kind: "credential-ref"; id: string }; + fields: Record; + createdAt: Date; +} +interface MockConnector { + manifest(): { id: string }; + createLiveStream?(connection: MockConnection, request: { + channelId: string; title: string; permanent: boolean; saveReplay: boolean; + }): Promise; + getLiveStreams?(connection: MockConnection, request: { limit: number }): Promise<{ items: MockStream[] }>; +} +interface MockModule { + createMockLivestreamState(): unknown; + createMockLivestreamConnector(state: unknown): MockConnector; + transitionMockLive(state: unknown, remoteId: string, to: MockLiveState): void; +} +interface ScenarioModule { SCENARIOS: Record } +export interface MockDriverPackages { scenarios: ScenarioModule; mock: MockModule } + +export interface MockDriverOptions { + scenario: string; + source: "A" | "B" | "C"; + sourceRef: string; + workspaceId: string; + connectionId: string; + tickMs?: number; + epoch?: string; + ingestNow?: () => string; +} + +/** Native import keeps the ESM simulator and mock connector usable from the CJS API build. */ +const nativeImport = new Function("specifier", "return import(specifier)") as (specifier: string) => Promise; + +async function loadPackage(packageName: string, file: string): Promise { + const roots = [process.cwd(), resolve(process.cwd(), "../..")]; + for (const root of roots) { + const built = resolve(root, "packages", packageName, "dist", file); + const source = resolve(root, "packages", packageName, "src", file.replace(/\.js$/, ".ts")); + const candidates = process.env.NODE_ENV === "production" ? [built, source] : [source, built]; + for (const candidate of candidates) { + if (existsSync(candidate)) return await nativeImport(pathToFileURL(candidate).href) as T; + } + } + throw new Error(`missing ${packageName}/${file}; build the package before starting the poller`); +} + +export class MockPollDriver implements PollDriver { + private elapsedMs = 0; + private nextStep = 0; + private nextStepAt = 0; + private tick = 0; + private action: ScenarioStep["action"] | null = null; + + private constructor( + private readonly options: MockDriverOptions, + private readonly scenario: Scenario, + private readonly connector: MockConnector, + private readonly state: unknown, + private readonly transition: MockModule["transitionMockLive"], + private readonly connection: MockConnection, + private readonly stream: MockStream, + ) {} + + static async create(options: MockDriverOptions, packages?: MockDriverPackages): Promise { + const normalized = { ...options, epoch: options.epoch ?? new Date().toISOString() }; + if (!Number.isFinite(Date.parse(normalized.epoch))) throw new Error("mock epoch must be a timestamp"); + const [{ SCENARIOS }, mock] = packages === undefined ? await Promise.all([ + loadPackage("testing", "scenarios.js"), + loadPackage("connector-mock-livestream", "index.js"), + ]) : [packages.scenarios, packages.mock]; + const scenario = SCENARIOS[normalized.scenario]; + if (scenario === undefined) throw new Error(`unknown mock scenario ${normalized.scenario}`); + const state = mock.createMockLivestreamState(); + const connector = mock.createMockLivestreamConnector(state); + const connection: MockConnection = { + id: normalized.connectionId, workspaceId: normalized.workspaceId, provider: connector.manifest().id, + channelId: normalized.sourceRef, credentialId: { kind: "credential-ref", id: "mock" }, + fields: {}, createdAt: new Date(normalized.epoch), + }; + if (connector.createLiveStream === undefined) throw new Error("mock connector has no live stream factory"); + const stream = await connector.createLiveStream(connection, { + channelId: normalized.sourceRef, title: normalized.sourceRef, permanent: false, saveReplay: false, + }); + return new MockPollDriver(normalized, scenario, connector, state, mock.transitionMockLive, connection, stream); + } + + async poll(): Promise { + const tickMs = this.options.tickMs ?? 10_000; + if (!Number.isFinite(tickMs) || tickMs <= 0) throw new Error("mock tickMs must be positive"); + while (this.nextStep < this.scenario.steps.length && this.elapsedMs >= this.nextStepAt) { + const step = this.scenario.steps[this.nextStep]; + if (step === undefined) break; + if (step.source === this.options.source) { + this.action = step.action; + const state: MockLiveState = step.action === "go-live" || step.action === "recover" + ? "live" : step.action === "fail" ? "waiting" : "ended"; + this.transition(this.state, this.stream.remoteId, state); + } + this.nextStepAt += step.holdSeconds * 1000; + this.nextStep += 1; + } + if (this.connector.getLiveStreams === undefined) throw new Error("mock connector has no live stream query"); + const page = await this.connector.getLiveStreams(this.connection, { limit: 100 }); + const live = page.items.find((item) => item.remoteId === this.stream.remoteId); + if (live === undefined) throw new Error(`mock live ${this.stream.remoteId} disappeared`); + const observedAt = new Date(Date.parse(this.options.epoch ?? "") + this.elapsedMs).toISOString(); + const outcome = this.action === "fail" ? "error" : live.state === "live" ? "ok" : "fail"; + const observation: MonitorObservation = { + tier: "api-poll", observedAt, receivedAt: this.options.ingestNow?.() ?? new Date().toISOString(), + providerEventId: `mock:${this.options.workspaceId}:${this.options.connectionId}:${this.options.sourceRef}:${this.options.epoch}:${this.tick}`, + payload: { outcome, reason: this.action ?? "not-live", mediaPresent: live.state === "live" }, + }; + this.tick += 1; + this.elapsedMs += tickMs; + return [observation]; + } +} diff --git a/apps/api/src/live-monitor/poller/peertube-driver.ts b/apps/api/src/live-monitor/poller/peertube-driver.ts new file mode 100644 index 0000000..6bd8243 --- /dev/null +++ b/apps/api/src/live-monitor/poller/peertube-driver.ts @@ -0,0 +1,44 @@ +import type { MonitorObservation } from "../reducer.js"; +import type { PollDriver } from "./driver.js"; +import { boundedText, monitorDeadline, MonitorHttpError, type MonitorEgress } from "./egress.js"; + +/** Injectable provider surface; production uses the WP7-backed client below. */ +export interface PeerTubeProbeClient { + probe(videoId: string): Promise; +} + +export class EgressPeerTubeProbeClient implements PeerTubeProbeClient { + constructor(private readonly instanceUrl: string, private readonly egress: MonitorEgress, + private readonly now: () => number = Date.now) {} + + async probe(videoId: string): Promise { + return monitorDeadline(async (signal) => { + const url = new URL(`/api/v1/videos/${encodeURIComponent(videoId)}`, this.instanceUrl); + const response = await this.egress.egressFetch(url, { signal, headers: { accept: "application/json" } }, + { hostClass: "monitor-api", pinnedOrigin: true }); + const timestamp = new Date(this.now()).toISOString(); + if (response.status === 404 || response.status === 410) return [{ tier: "api-poll", observedAt: timestamp, + receivedAt: timestamp, payload: { outcome: "fail", reason: "api-video-missing" } }]; + if (!response.ok) throw new MonitorHttpError(response.status, response.headers); + const parsed: unknown = JSON.parse(await boundedText(response, signal)); + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) throw new Error("invalid PeerTube response"); + const video = parsed as { isLive?: unknown; state?: unknown }; + const state = typeof video.state === "object" && video.state !== null + ? (video.state as { id?: unknown }).id : video.state; + if (typeof video.isLive !== "boolean" || !Number.isInteger(state)) throw new Error("invalid PeerTube live state"); + // A running live remains PUBLISHED (1); WAITING_FOR_LIVE (4)/LIVE_ENDED (5) are offline. + const live = video.isLive && state === 1; + return [{ tier: "api-poll", observedAt: timestamp, receivedAt: timestamp, + payload: { outcome: live ? "ok" : "fail", reason: live ? "api-published-live" : "api-not-live" } }]; + }); + } +} + +export class PeerTubePollDriver implements PollDriver { + constructor(private readonly videoId: string, private readonly egress?: PeerTubeProbeClient) {} + + async poll(): Promise { + if (this.egress === undefined) throw new Error("NOT_IMPLEMENTED: PeerTube poll requires WP7 egress client"); + return await this.egress.probe(this.videoId); + } +} diff --git a/apps/api/src/live-monitor/poller/poller.test.ts b/apps/api/src/live-monitor/poller/poller.test.ts new file mode 100644 index 0000000..e3570ec --- /dev/null +++ b/apps/api/src/live-monitor/poller/poller.test.ts @@ -0,0 +1,166 @@ +import type { PoolClient } from "pg"; +import { describe, expect, it, vi } from "vitest"; +import { LiveMonitorService, type LiveMonitorDependencies } from "../live-monitor.service.js"; +import type { MonitorObservation, MonitorRowState } from "../reducer.js"; +import { MockPollDriver, type MockDriverPackages } from "./mock-driver.js"; +import { PeerTubePollDriver } from "./peertube-driver.js"; +import { jitteredInterval, LiveMonitorPoller, parsePollSources, type PollerDependencies } from "./poller.js"; + +const epoch = "2026-01-01T00:00:00.000Z"; + +async function mockPackages(): Promise { + const [mock, scenarios] = await Promise.all([ + vi.importActual("../../../../../packages/connector-mock-livestream/src/index.js"), + vi.importActual("../../../../../packages/testing/src/scenarios.js"), + ]); + return { mock, scenarios }; +} + +function fixture(): { service: LiveMonitorService; get: () => Promise; audits: () => number } { + const row: Record = { + id: "source", workspace_id: "workspace", connection_id: "connection", remote_ref: "live:1", + state: "OFFLINE", version: 0, authoritative_tier: null, + last_media_health_at: null, last_hls_probe_at: null, last_webhook_at: null, last_api_poll_at: null, + tier_status: {}, ok_streak: 0, fail_streak: 0, state_entered_at: epoch, + hold_until: null, cooldown_until: null, previous_state: null, last_transition_at: null, + last_transition_reason: null, created_at: epoch, updated_at: epoch, + }; + const seen = new Set(); + let auditCount = 0; + const client = { + async query(sql: string, params: unknown[] = []) { + if (sql.startsWith("SELECT * FROM source_monitor_states")) return { rowCount: 1, rows: [{ ...row }] }; + if (sql.startsWith("INSERT INTO monitor_seen_events")) { + const id = String(params[1]); + if (seen.has(id)) return { rowCount: 0, rows: [] }; + seen.add(id); + return { rowCount: 1, rows: [{ provider_event_id: id }] }; + } + if (sql.startsWith("UPDATE source_monitor_states")) { + if (Number(params[18]) !== Number(row.version)) return { rowCount: 0, rows: [] }; + const columns = ["state", "version", "authoritative_tier", "last_media_health_at", "last_hls_probe_at", + "last_webhook_at", "last_api_poll_at", "tier_status", "ok_streak", "fail_streak", "state_entered_at", + "hold_until", "cooldown_until", "previous_state", "last_transition_at", "last_transition_reason", "updated_at"]; + for (const [index, column] of columns.entries()) row[column] = params[index + 1]; + return { rowCount: 1, rows: [{ ...row }] }; + } + if (sql.startsWith("SELECT audit_append")) { + auditCount += 1; + return { rowCount: 1, rows: [{}] }; + } + throw new Error(`unexpected query: ${sql}`); + }, + } as unknown as PoolClient; + const dependencies: LiveMonitorDependencies = { tenantTransaction: async (_workspace, fn) => fn(client) }; + const service = new LiveMonitorService(dependencies); + return { service, get: () => service.getState("workspace", "source"), audits: () => auditCount }; +} + +describe("live monitor poller", () => { + it("drives dod-flap through the service and reducer without rapid terminal cuts", async () => { + const { service, get, audits } = fixture(); + const observations: MonitorObservation[] = []; + const driver = await MockPollDriver.create({ + scenario: "dod-flap", source: "A", sourceRef: "live:1", workspaceId: "workspace", + connectionId: "connection", epoch, + ingestNow: () => new Date(Date.parse(epoch) + observations.length * 10_000).toISOString(), + }, await mockPackages()); + const states: string[] = []; + for (let tick = 0; tick < 10; tick += 1) { + const [observation] = await driver.poll(); + if (observation === undefined) throw new Error("mock poll emitted no observation"); + observations.push(observation); + const state = await service.recordObservation("workspace", "source", observation); + expect(state.version).toBe(tick + 1); + states.push(state.state); + } + expect(states).toEqual(["PROBING", "LIVE", "LIVE", "LIVE", "ERROR", "ERROR", "ERROR", "PROBING", "LIVE", "LIVE"]); + expect(observations.map((item) => item.observedAt)).toEqual( + Array.from({ length: 10 }, (_, tick) => new Date(Date.parse(epoch) + tick * 10_000).toISOString()), + ); + expect(observations.every((item) => item.tier === "api-poll")).toBe(true); + expect((await get()).version).toBe(10); + expect(audits()).toBe(5); + const replay = observations[8]; + if (replay === undefined) throw new Error("missing replay event"); + expect((await service.recordObservation("workspace", "source", replay)).version).toBe(10); + expect(audits()).toBe(5); + }); + + it("plays the single-live scenario and keeps a replayed event from confirming LIVE", async () => { + const { service } = fixture(); + const driver = await MockPollDriver.create({ + scenario: "single-live", source: "A", sourceRef: "live:1", workspaceId: "workspace", + connectionId: "connection", epoch, + }, await mockPackages()); + const [first] = await driver.poll(); + if (first === undefined) throw new Error("missing first observation"); + expect((await service.recordObservation("workspace", "source", first)).state).toBe("PROBING"); + expect((await service.recordObservation("workspace", "source", first)).version).toBe(1); + const [second] = await driver.poll(); + if (second === undefined) throw new Error("missing second observation"); + const live = await service.recordObservation("workspace", "source", second); + expect(live.state).toBe("LIVE"); + expect(live.version).toBe(2); + }); + + it("applies the same provider event only once across two poll ticks", async () => { + const { service, get } = fixture(); + const observation: MonitorObservation = { + tier: "api-poll", observedAt: epoch, receivedAt: epoch, providerEventId: "same-event", + payload: { outcome: "ok" }, + }; + const poller = new LiveMonitorPoller(service, { + enabled: false, + sources: [{ provider: "mock-livestream", sourceRef: "live:1", workspaceId: "workspace", connectionId: "connection" }], + schedule: () => 1 as unknown as ReturnType, + cancel: () => undefined, + resolveSourceId: async () => "source", + makeDriver: async () => ({ poll: async () => [observation] }), + }); + poller.start(); + await poller.pollOnce(); + await poller.pollOnce(); + expect((await get()).version).toBe(1); + expect((await get()).state).toBe("PROBING"); + await poller.stop(); + }); + + it("uses a 10 second interval with bounded ±20% jitter and schedules the next tick", async () => { + expect([0, 0.5, 1].map((value) => jitteredInterval(10_000, value))).toEqual([8_000, 10_000, 12_000]); + const scheduled: Array<{ callback: () => void; delay: number }> = []; + const recorded: MonitorObservation[] = []; + const observation: MonitorObservation = { + tier: "api-poll", observedAt: epoch, receivedAt: epoch, providerEventId: "event-1", + payload: { outcome: "ok" }, + }; + const monitor = { recordObservation: async (_workspace: string, _source: string, value: MonitorObservation) => { + recorded.push(value); + } } as unknown as LiveMonitorService; + const overrides: Partial = { + enabled: false, intervalMs: 10_000, random: () => 0, + sources: [{ provider: "mock-livestream", sourceRef: "live:1", workspaceId: "workspace", connectionId: "connection" }], + schedule: (callback, delay) => { + scheduled.push({ callback, delay }); + return 1 as unknown as ReturnType; + }, + cancel: () => undefined, + resolveSourceId: async () => "source", + makeDriver: async () => ({ poll: async () => [observation] }), + }; + const poller = new LiveMonitorPoller(monitor, overrides); + poller.start(); + expect(scheduled[0]?.delay).toBe(0); + scheduled[0]?.callback(); + await new Promise((resolve) => setImmediate(resolve)); + expect(recorded).toEqual([observation]); + expect(scheduled[1]?.delay).toBe(8_000); + await poller.stop(); + }); + + it("parses source config and leaves the real probe stubbed", async () => { + expect(parsePollSources(JSON.stringify([{ provider: "mock-livestream", sourceRef: "live:1", + workspaceId: "workspace", connectionId: "connection" }]))).toHaveLength(1); + await expect(new PeerTubePollDriver("video-id").poll()).rejects.toThrow("NOT_IMPLEMENTED"); + }); +}); diff --git a/apps/api/src/live-monitor/poller/poller.ts b/apps/api/src/live-monitor/poller/poller.ts new file mode 100644 index 0000000..cb86f11 --- /dev/null +++ b/apps/api/src/live-monitor/poller/poller.ts @@ -0,0 +1,215 @@ +import { Inject, Injectable, Logger, NotFoundException, OnApplicationBootstrap, OnApplicationShutdown, Optional } from "@nestjs/common"; +import { withTenant } from "../../db/client.js"; +import { LiveMonitorService } from "../live-monitor.service.js"; +import type { PollDriver } from "./driver.js"; +import { MockPollDriver } from "./mock-driver.js"; +// eslint-disable-next-line no-restricted-syntax +import { EgressPeerTubeProbeClient, PeerTubePollDriver } from "./peertube-driver.js"; +import { monitorEgress } from "./egress.js"; +import { HlsProbeDriver } from "./hls-driver.js"; +import { BackoffPollDriver } from "./backoff-driver.js"; + +export interface PollSource { + provider: string; + sourceRef: string; + workspaceId: string; + connectionId: string; + scenario?: string; + scenarioSource?: "A" | "B" | "C"; + playbackUrl?: string; +} + +export interface PollerDependencies { + sources: PollSource[]; + intervalMs: number; + enabled: boolean; + random: () => number; + schedule: (callback: () => void, delayMs: number) => ReturnType; + cancel: (timer: ReturnType) => void; + resolveSourceId: (source: PollSource) => Promise; + makeDriver: (source: PollSource, intervalMs: number) => Promise; +} + +export const MEDIA_POLLER_DEPENDENCIES = Symbol("MEDIA_POLLER_DEPENDENCIES"); + +export function jitteredInterval(intervalMs: number, random: number): number { + if (!Number.isFinite(intervalMs) || intervalMs <= 0) throw new Error("poll interval must be positive"); + if (!Number.isFinite(random) || random < 0 || random > 1) throw new Error("random must be between 0 and 1"); + return Math.round(intervalMs * (0.8 + random * 0.4)); +} + +function requiredString(value: unknown, name: string): string { + if (typeof value !== "string" || value.trim() === "") throw new Error(`${name} must be a non-empty string`); + return value; +} + +export function parsePollSources(raw: string | undefined): PollSource[] { + if (raw === undefined || raw.trim() === "") return []; + const parsed: unknown = JSON.parse(raw); + if (!Array.isArray(parsed)) throw new Error("MEDIA_POLLER_SOURCES must be a JSON array"); + return parsed.map((entry: unknown, index): PollSource => { + if (typeof entry !== "object" || entry === null || Array.isArray(entry)) { + throw new Error(`MEDIA_POLLER_SOURCES[${index}] must be an object`); + } + const row = entry as Record; + const source: PollSource = { + provider: requiredString(row.provider, "provider"), + sourceRef: requiredString(row.sourceRef, "sourceRef"), + workspaceId: requiredString(row.workspaceId, "workspaceId"), + connectionId: requiredString(row.connectionId, "connectionId"), + }; + if (row.scenario !== undefined) source.scenario = requiredString(row.scenario, "scenario"); + if (row.playbackUrl !== undefined) source.playbackUrl = requiredString(row.playbackUrl, "playbackUrl"); + if (row.scenarioSource !== undefined) { + if (row.scenarioSource !== "A" && row.scenarioSource !== "B" && row.scenarioSource !== "C") { + throw new Error("scenarioSource must be A, B, or C"); + } + source.scenarioSource = row.scenarioSource; + } + return source; + }); +} + +async function resolveSourceId(source: PollSource): Promise { + return withTenant(source.workspaceId, async (client) => { + const result = await client.query( + "SELECT id FROM source_monitor_states WHERE connection_id = $1 AND remote_ref = $2", + [source.connectionId, source.sourceRef], + ); + if (result.rowCount !== 1) throw new NotFoundException(`monitored source ${source.sourceRef} not found`); + return String(result.rows[0]?.id); + }); +} + +async function makeDriver(source: PollSource, intervalMs: number): Promise { + // Provider routing lives here until the connector registry exposes monitor probes. + if (source.provider === "mock-livestream") { + return MockPollDriver.create({ + scenario: source.scenario ?? "single-live", source: source.scenarioSource ?? "A", + sourceRef: source.sourceRef, workspaceId: source.workspaceId, connectionId: source.connectionId, + tickMs: intervalMs, + }); + } + // eslint-disable-next-line no-restricted-syntax + if (source.provider === "peertube" || source.provider === "hls") { + const egress = await monitorEgress(); + const drivers: PollDriver[] = []; + // eslint-disable-next-line no-restricted-syntax + if (source.provider === "peertube") { + const fields = await withTenant(source.workspaceId, async (client) => { + const result = await client.query<{ fields: Record }>( + "SELECT fields FROM connections WHERE id = $1 AND workspace_id = $2 AND disconnected_at IS NULL", + [source.connectionId, source.workspaceId]); + if (result.rowCount !== 1) throw new NotFoundException("monitor connection not found"); + return result.rows[0]!.fields; + }); + const instanceUrl = requiredString(fields.instanceUrl, "connection instanceUrl"); + const videoId = source.sourceRef.startsWith("video:") + ? source.sourceRef.slice(6).split("@")[0]! : source.sourceRef; + drivers.push(new BackoffPollDriver(new PeerTubePollDriver(videoId, + new EgressPeerTubeProbeClient(instanceUrl, egress)), "api-poll")); + } + if (source.playbackUrl) drivers.push(new BackoffPollDriver(new HlsProbeDriver(source.playbackUrl, egress), "hls-probe")); + if (!drivers.length) throw new Error("HLS monitor source requires playbackUrl"); + return { poll: async () => { + const observations = []; + for (const driver of drivers) observations.push(...await driver.poll()); + return observations; + } }; + } + throw new Error(`no live poll driver for provider ${source.provider}`); +} + +function defaults(): PollerDependencies { + const interval = Number(process.env.MEDIA_POLLER_INTERVAL_MS ?? 10_000); + if (!Number.isFinite(interval) || interval <= 0) throw new Error("MEDIA_POLLER_INTERVAL_MS must be positive"); + const enabled = process.env.MEDIA_POLLER_ENABLED === undefined + ? process.env.NODE_ENV !== "production" : process.env.MEDIA_POLLER_ENABLED === "true"; + return { + sources: parsePollSources(process.env.MEDIA_POLLER_SOURCES), intervalMs: interval, enabled, + random: Math.random, schedule: setTimeout, cancel: clearTimeout, resolveSourceId, makeDriver, + }; +} + +@Injectable() +export class LiveMonitorPoller implements OnApplicationBootstrap, OnApplicationShutdown { + private readonly logger = new Logger(LiveMonitorPoller.name); + private readonly config: PollerDependencies; + private readonly drivers = new Map(); + private readonly sourceIds = new Map(); + private timer: ReturnType | undefined; + private active: Promise | undefined; + private running = false; + + constructor( + @Inject(LiveMonitorService) private readonly monitor: LiveMonitorService, + @Optional() @Inject(MEDIA_POLLER_DEPENDENCIES) dependencies?: Partial, + ) { + this.config = { ...defaults(), ...dependencies }; + } + + onApplicationBootstrap(): void { + if (this.config.enabled) this.start(); + } + + async onApplicationShutdown(): Promise { + await this.stop(); + } + + get isRunning(): boolean { return this.running; } + + start(): { running: boolean } { + if (this.running) return { running: true }; + this.running = true; + this.schedule(0); + return { running: true }; + } + + async stop(): Promise<{ running: boolean }> { + this.running = false; + if (this.timer !== undefined) this.config.cancel(this.timer); + this.timer = undefined; + await this.active; + return { running: false }; + } + + private schedule(delayMs: number): void { + if (!this.running) return; + this.timer = this.config.schedule(() => { + this.timer = undefined; + this.active = this.pollOnce().catch((error: unknown) => { + this.logger.error(`poll tick failed: ${String(error)}`); + }).finally(() => { + this.active = undefined; + this.schedule(jitteredInterval(this.config.intervalMs, this.config.random())); + }); + }, delayMs); + } + + /** One serialized tick; each source fails independently. */ + async pollOnce(): Promise { + for (const source of this.config.sources) { + if (!this.running) return; + const key = `${source.workspaceId}:${source.connectionId}:${source.sourceRef}`; + try { + let driver = this.drivers.get(key); + if (driver === undefined) { + driver = await this.config.makeDriver(source, this.config.intervalMs); + this.drivers.set(key, driver); + } + let sourceId = this.sourceIds.get(key); + if (sourceId === undefined) { + sourceId = await this.config.resolveSourceId(source); + this.sourceIds.set(key, sourceId); + } + const observations = await driver.poll(); + for (const observation of observations) { + if (!this.running) return; + await this.monitor.recordObservation(source.workspaceId, sourceId, observation); + } + } catch (error) { + this.logger.error(`poll failed for ${key}: ${String(error)}`); + } + } + } +} diff --git a/apps/api/src/live-monitor/poller/signals.test.ts b/apps/api/src/live-monitor/poller/signals.test.ts new file mode 100644 index 0000000..375f14a --- /dev/null +++ b/apps/api/src/live-monitor/poller/signals.test.ts @@ -0,0 +1,172 @@ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { EgressError, EgressPolicy, rewriteSegmentUris, validateMasterPlaylist } from "../../../../../packages/egress/src/index.js"; +import { BackoffPollDriver } from "./backoff-driver.js"; +import { MONITOR_MAX_BYTES, MONITOR_TIMEOUT_MS, type MonitorEgress } from "./egress.js"; +import { HlsProbeDriver, parseMediaPlaylist } from "./hls-driver.js"; +import { EgressPeerTubeProbeClient, PeerTubePollDriver } from "./peertube-driver.js"; + +const epoch = Date.parse("2026-01-01T00:00:12.000Z"); +const origin = "https://stream.example"; +const masterUrl = `${origin}/hls/master.m3u8`; +const liveUrl = `${origin}/hls/live.m3u8`; +const fixture = (name: string) => readFileSync(resolve(__dirname, "fixtures", name), "utf8"); +const live = fixture("live.m3u8"); + +/** A fake transport tracks each egress fetch/debit; real WP7 URI policy/helpers still run. */ +function fakeEgress(routes: Record Response | Promise)>) { + const draws: Array<{ host: string; hostClass: string; url: string }> = []; + const policy = new EgressPolicy({ resolver: async () => [{ address: "8.8.8.8", family: 4 }] }); + const egress: MonitorEgress = { + policy, validateMasterPlaylist: vi.fn(validateMasterPlaylist), rewriteSegmentUris: vi.fn(rewriteSegmentUris), + egressFetch: vi.fn(async (raw, _init, context) => { + const endpoint = await policy.validate(raw); + draws.push({ host: endpoint.hostKey, hostClass: context.hostClass, url: endpoint.url.href }); + const route = routes[endpoint.url.href]; + if (route === undefined) throw new Error("unexpected outbound URL"); + const response = typeof route === "function" ? await route() : typeof route === "string" ? new Response(route) : route.clone(); + if (!response.url) Object.defineProperty(response, "url", { value: endpoint.url.href }); + return response; + }), + }; + return { egress, draws }; +} + +afterEach(() => { vi.useRealTimers(); }); + +describe("HLS media ground truth", () => { + it("fetches a master and exactly one variant through monitor-probe egress, validates all URIs", async () => { + const f = fakeEgress({ [masterUrl]: fixture("master.m3u8"), [liveUrl]: live }); + const observation = (await new HlsProbeDriver(masterUrl, f.egress, () => epoch).poll())[0]; + expect(observation).toMatchObject({ tier: "hls-probe", payload: { outcome: "ok", mediaPresent: true } }); + expect(f.draws).toEqual([ + { url: masterUrl, host: "stream.example:443", hostClass: "monitor-probe" }, + { url: liveUrl, host: "stream.example:443", hostClass: "monitor-probe" }, + ]); + expect(f.egress.validateMasterPlaylist).toHaveBeenCalledOnce(); + expect(f.egress.rewriteSegmentUris).toHaveBeenCalledOnce(); + expect(f.egress.rewriteSegmentUris).toHaveBeenCalledWith(live, liveUrl, f.egress.policy); + }); + + it.each(["offline.m3u8", "empty", "404"])("reports %s as offline", async (name) => { + const body = name === "empty" ? "#EXTM3U\n#EXT-X-TARGETDURATION:6\n" + : name === "404" ? new Response(null, { status: 404 }) : fixture(name); + const f = fakeEgress({ [liveUrl]: body }); + expect((await new HlsProbeDriver(liveUrl, f.egress, () => epoch).poll())[0]?.payload) + .toMatchObject({ outcome: "fail", mediaPresent: false }); + }); + + it("reports stale program dates and unreasonable future dates as offline", async () => { + const f = fakeEgress({ [liveUrl]: live }); + expect((await new HlsProbeDriver(liveUrl, f.egress, () => epoch + 19_000).poll())[0]?.payload.outcome).toBe("fail"); + expect((await new HlsProbeDriver(liveUrl, f.egress, () => epoch - 19_000).poll())[0]?.payload.outcome).toBe("fail"); + }); + + it("uses media-sequence/last-segment progression and three target durations when PDT is absent", async () => { + let now = epoch; let body = live.replace(/#EXT-X-PROGRAM-DATE-TIME:.*\n/, ""); + const f = fakeEgress({ [liveUrl]: () => new Response(body) }); + const driver = new HlsProbeDriver(liveUrl, f.egress, () => now); + expect((await driver.poll())[0]?.payload.outcome).toBe("ok"); + now += 18_001; + expect((await driver.poll())[0]?.payload.outcome).toBe("fail"); + body = body.replace("SEQUENCE:20", "SEQUENCE:21").replace("segment21.ts", "segment22.ts"); + expect((await driver.poll())[0]?.payload.outcome).toBe("ok"); + }); + + it.each(["malformed.m3u8", "header", "missing-duration", "missing-uri"])("rejects malformed playlist %s", async (name) => { + const body = name === "header" ? "not a playlist" : name === "missing-duration" ? "#EXTM3U\n#EXT-X-TARGETDURATION:6\nseg.ts\n" + : name === "missing-uri" ? "#EXTM3U\n#EXT-X-TARGETDURATION:6\n#EXTINF:6,\n" : fixture(name); + const f = fakeEgress({ [liveUrl]: body }); + await expect(new HlsProbeDriver(liveUrl, f.egress, () => epoch).poll()).rejects.toThrow(); + }); + + it("accepts EVENT playlists and rejects invalid target-duration math", () => { + expect(parseMediaPlaylist(live.replace("#EXT-X-TARGETDURATION", "#EXT-X-PLAYLIST-TYPE:EVENT\n#EXT-X-TARGETDURATION")).vod).toBe(false); + expect(() => parseMediaPlaylist(live.replace("EXTINF:6", "EXTINF:7"))).toThrow(); + }); + + it.each(["variant", "unused-variant", "segment", "key"])("denies private %s references before any child request", async (kind) => { + const master = kind === "variant" ? "#EXTM3U\n#EXT-X-STREAM-INF:BANDWIDTH=800000\nhttp://169.254.169.254/private.m3u8\n" + : fixture("master.m3u8").replace("unused.m3u8", "http://127.0.0.1/unused.m3u8"); + const media = kind === "segment" ? live.replace("segment20.ts", "http://10.0.0.1/seg.ts") + : live.replace("#EXTINF:6,", '#EXT-X-KEY:METHOD=AES-128,URI="http://169.254.169.254/key"\n#EXTINF:6,'); + const isMaster = kind === "variant" || kind === "unused-variant"; + const f = fakeEgress({ [isMaster ? masterUrl : liveUrl]: isMaster ? master : media }); + await expect(new HlsProbeDriver(isMaster ? masterUrl : liveUrl, f.egress, () => epoch).poll()) + .rejects.toMatchObject({ code: "DENIED_PRIVATE_ADDRESS" }); + expect(f.draws).toHaveLength(1); + }); + + it("resolves variant and segment URIs against the final redirected playlist URL", async () => { + const response = new Response(fixture("master.m3u8")); + Object.defineProperty(response, "url", { value: `${origin}/redirected/master.m3u8` }); + const f = fakeEgress({ [masterUrl]: () => response, [`${origin}/redirected/live.m3u8`]: live }); + await new HlsProbeDriver(masterUrl, f.egress, () => epoch).poll(); + expect(f.draws[1]?.url).toBe(`${origin}/redirected/live.m3u8`); + }); + + it("enforces body size, URI count and whole-probe deadlines", async () => { + const oversized = fakeEgress({ [liveUrl]: "x".repeat(MONITOR_MAX_BYTES + 1) }); + await expect(new HlsProbeDriver(liveUrl, oversized.egress).poll()).rejects.toThrow("size cap"); + const references = fakeEgress({ [liveUrl]: `#EXTM3U\n${"#EXTINF:6,\nsegment.ts\n".repeat(129)}` }); + await expect(new HlsProbeDriver(liveUrl, references.egress).poll()).rejects.toThrow("too many HLS references"); + expect(references.egress.rewriteSegmentUris).not.toHaveBeenCalled(); + vi.useFakeTimers(); + const hanging = fakeEgress({ [liveUrl]: () => new Promise(() => undefined) }); + const pending = new HlsProbeDriver(liveUrl, hanging.egress).poll(); + const rejected = expect(pending).rejects.toThrow("deadline"); + await vi.advanceTimersByTimeAsync(MONITOR_TIMEOUT_MS); + await rejected; + expect(vi.getTimerCount()).toBe(0); + }); +}); + +describe("monitor API budgets and adaptive source backoff", () => { + it("counts API + master + variant draws on the same host, with the required host classes", async () => { + const f = fakeEgress({ [`${origin}/api/v1/videos/video-1`]: JSON.stringify({ isLive: true, state: { id: 1 } }), + [masterUrl]: fixture("master.m3u8"), [liveUrl]: live }); + const api = new PeerTubePollDriver("video-1", new EgressPeerTubeProbeClient(origin, f.egress, () => epoch)); + expect((await api.poll())[0]?.payload.outcome).toBe("ok"); + await new HlsProbeDriver(masterUrl, f.egress, () => epoch).poll(); + expect(f.draws.map((draw) => draw.hostClass)).toEqual(["monitor-api", "monitor-probe", "monitor-probe"]); + expect(new Set(f.draws.map((draw) => draw.host))).toEqual(new Set(["stream.example:443"])); + expect(f.egress.egressFetch).toHaveBeenNthCalledWith(1, expect.any(URL), expect.anything(), { hostClass: "monitor-api", pinnedOrigin: true }); + }); + + it.each([4, 5])("does not consider PeerTube state %s live even when isLive=true", async (state) => { + const f = fakeEgress({ [`${origin}/api/v1/videos/video-1`]: JSON.stringify({ isLive: true, state: { id: state } }) }); + expect((await new EgressPeerTubeProbeClient(origin, f.egress).probe("video-1"))[0]?.payload.outcome).toBe("fail"); + }); + + it.each([429, 503])("backs off source after HTTP %s, honors Retry-After and resets on recovery", async (status) => { + let now = epoch; let attempt = 0; + const f = fakeEgress({ [liveUrl]: () => ++attempt === 1 ? new Response(null, { status, headers: { "retry-after": "30" } }) : new Response(live) }); + const driver = new BackoffPollDriver(new HlsProbeDriver(liveUrl, f.egress, () => now), "hls-probe", () => now, () => 0.5); + expect((await driver.poll())[0]?.payload.outcome).toBe("unavailable"); + expect(driver.nextAttemptAt).toBe(epoch + 30_000); + now += 29_999; expect(await driver.poll()).toEqual([]); expect(f.draws).toHaveLength(1); + now += 1; await driver.poll(); expect(f.draws).toHaveLength(2); expect(driver.nextAttemptAt).toBe(0); + }); + + it("honors an HTTP-date Retry-After for API polls and grows source-local delays on repeated 5xx", async () => { + let now = epoch; let attempt = 0; + const f = fakeEgress({ [`${origin}/api/v1/videos/v`]: () => new Response(null, { status: 503, + headers: ++attempt === 1 ? { "retry-after": new Date(epoch + 60_000).toUTCString() } : {} }) }); + const driver = new BackoffPollDriver(new PeerTubePollDriver("v", new EgressPeerTubeProbeClient(origin, f.egress)), "api-poll", () => now, () => 0.5); + await driver.poll(); expect(driver.nextAttemptAt).toBe(epoch + 60_000); + now += 60_000; await driver.poll(); expect(driver.nextAttemptAt).toBe(now + 2000); + now += 2000; await driver.poll(); expect(driver.nextAttemptAt).toBe(now + 4000); + }); + + it("respects WP7 budget exhaustion and allows a different source to proceed", async () => { + let now = epoch; + const egress = fakeEgress({ [liveUrl]: live }); + const blocked = { ...egress.egress, egressFetch: vi.fn(async () => { throw new EgressError("BUDGET_EXHAUSTED", "budget", 10_000); }) }; + const one = new BackoffPollDriver(new HlsProbeDriver(liveUrl, blocked, () => now), "hls-probe", () => now, () => 0.5); + const two = new BackoffPollDriver(new HlsProbeDriver(liveUrl, egress.egress, () => now), "hls-probe", () => now, () => 0.5); + await one.poll(); now += 1000; expect(await one.poll()).toEqual([]); + expect((await two.poll())[0]?.payload.outcome).toBe("ok"); + expect(blocked.egressFetch).toHaveBeenCalledOnce(); expect(egress.draws).toHaveLength(1); + }); +}); diff --git a/apps/api/src/live-monitor/reducer.test.ts b/apps/api/src/live-monitor/reducer.test.ts new file mode 100644 index 0000000..b42cdc8 --- /dev/null +++ b/apps/api/src/live-monitor/reducer.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from "vitest"; +import { applyObservation, type MonitorObservation, type MonitorRowState, type SignalTier } from "./reducer.js"; + +const start = Date.parse("2026-01-01T00:00:00.000Z"); +const at = (seconds: number): string => new Date(start + seconds * 1000).toISOString(); + +function initial(): MonitorRowState { + return { + id: "source", workspaceId: "workspace", connectionId: "connection", remoteRef: "live:1", + state: "OFFLINE", version: 0, authoritativeTier: null, + lastMediaHealthAt: null, lastHlsProbeAt: null, lastWebhookAt: null, lastApiPollAt: null, + tierStatus: {}, okStreak: 0, failStreak: 0, stateEnteredAt: at(0), + holdUntil: null, cooldownUntil: null, previousState: null, + lastTransitionAt: null, lastTransitionReason: null, createdAt: at(0), updatedAt: at(0), + }; +} + +function obs(tier: SignalTier, outcome: MonitorObservation["payload"]["outcome"], seconds: number, + extra: Partial = {}): MonitorObservation { + return { tier, observedAt: at(seconds), receivedAt: at(seconds), payload: { outcome }, ...extra }; +} + +function fold(state: MonitorRowState, ...observations: MonitorObservation[]): MonitorRowState { + return observations.reduce((current, observation) => applyObservation(current, observation).state, state); +} + +describe("live source reducer", () => { + it("confirms LIVE after two ok signals from a poll-only source", () => { + const first = applyObservation(initial(), obs("api-poll", "ok", 1)); + expect(first.state.state).toBe("PROBING"); + const second = applyObservation(first.state, obs("api-poll", "ok", 2)); + expect(second.state.state).toBe("LIVE"); + expect(second.state.authoritativeTier).toBe("api-poll"); + expect(second.transition?.version).toBe(2); + }); + + it("confirms OFFLINE after three failures and waits out hysteresis", () => { + const live = fold(initial(), obs("api-poll", "ok", 1), obs("api-poll", "ok", 2)); + const one = applyObservation(live, obs("api-poll", "fail", 3)); + expect(one.state.state).toBe("LIVE"); + const two = applyObservation(one.state, obs("api-poll", "fail", 4)); + expect(two.state.state).toBe("UNSTABLE"); + const three = applyObservation(two.state, obs("api-poll", "fail", 5)); + expect(three.state.state).toBe("UNSTABLE"); + expect(applyObservation(three.state, obs("api-poll", "fail", 12)).state.state).toBe("OFFLINE"); + }); + + it("holds a flapping source without rapid LIVE/OFFLINE cuts", () => { + let state = fold(initial(), obs("api-poll", "ok", 1), obs("api-poll", "ok", 2)); + const terminal: string[] = [state.state]; + for (let second = 3; second <= 9; second += 1) { + const outcome = second % 2 === 0 ? "ok" : "fail"; + const result = applyObservation(state, obs("api-poll", outcome, second)); + state = result.state; + if (result.transition?.to === "LIVE" || result.transition?.to === "OFFLINE") terminal.push(result.transition.to); + } + expect(terminal).toEqual(["LIVE"]); + }); + + it("discards stale observations within a tier and repeated provider IDs", () => { + const first = fold(initial(), obs("webhook", "ok", 3, { providerEventId: "event-1" })); + expect(applyObservation(first, obs("webhook", "fail", 2)).state).toBe(first); + const second = fold(first, obs("webhook", "ok", 4, { providerEventId: "event-2" })); + expect(applyObservation(second, obs("webhook", "ok", 5, { providerEventId: "event-1" })).state).toBe(second); + expect(first.version).toBe(1); + }); + + it("keeps a newer HLS probe authoritative against a late API poll and clock-skewed webhook", () => { + const futureWebhook = obs("webhook", "fail", 1000, { receivedAt: at(1) }); + const state = fold(initial(), futureWebhook, + obs("hls-probe", "ok", 2), obs("hls-probe", "ok", 3), + obs("api-poll", "fail", 1, { receivedAt: at(6) })); + expect(state.authoritativeTier).toBe("hls-probe"); + expect(state.state).toBe("UNSTABLE"); + expect(state.tierStatus["hls-probe"]?.okStreak).toBe(2); + const confirmed = applyObservation(state, obs("hls-probe", "ok", 13)).state; + expect(confirmed.state).toBe("LIVE"); + }); + + it("raises and lowers authority when HLS becomes available then denied", () => { + const poll = fold(initial(), obs("api-poll", "ok", 1), obs("api-poll", "ok", 2)); + const raised = applyObservation(poll, obs("hls-probe", "ok", 3)).state; + expect(raised.authoritativeTier).toBe("hls-probe"); + const lowered = applyObservation(raised, obs("hls-probe", "unavailable", 4)).state; + expect(lowered.authoritativeTier).toBe("api-poll"); + expect(lowered.state).toBe("LIVE"); + }); + + it("uses ten seconds without media as a failure confirmation", () => { + const live = fold(initial(), + { ...obs("media-health", "ok", 1), payload: { outcome: "ok", mediaPresent: true } }, + { ...obs("media-health", "ok", 2), payload: { outcome: "ok", mediaPresent: true } }); + const failed = applyObservation(live, obs("media-health", "fail", 13)); + expect(failed.state.state).toBe("OFFLINE"); + expect(failed.transition?.reason).toBe("media-health:no-media-timeout"); + }); +}); diff --git a/apps/api/src/live-monitor/reducer.ts b/apps/api/src/live-monitor/reducer.ts new file mode 100644 index 0000000..5e04ea6 --- /dev/null +++ b/apps/api/src/live-monitor/reducer.ts @@ -0,0 +1,182 @@ +/** Pure per-source fold. Provider time is used only to order one tier's events. */ +export type MonitorState = "OFFLINE" | "PROBING" | "LIVE" | "UNSTABLE" | "ERROR"; +export type SignalTier = "media-health" | "hls-probe" | "webhook" | "api-poll"; +export type ObservationOutcome = "ok" | "fail" | "error" | "unavailable"; + +export interface MonitorObservation { + tier: SignalTier; + observedAt: string; + /** Ingest clock, never a provider clock. */ + receivedAt: string; + providerEventId?: string; + payload: { outcome: ObservationOutcome; reason?: string; mediaPresent?: boolean }; +} + +export interface TierStatus { + available: boolean; + lastOutcome?: ObservationOutcome; + lastReceivedAt?: string; + lastProviderEventId?: string; + /** Recent replay guard; the database journal provides durable full-history dedupe. */ + seenProviderEventIds?: string[]; + okStreak: number; + failStreak: number; + lastMediaAt?: string; +} + +export interface MonitorRowState { + id: string; + workspaceId: string; + connectionId: string; + remoteRef: string; + state: MonitorState; + version: number; + authoritativeTier: SignalTier | null; + lastMediaHealthAt: string | null; + lastHlsProbeAt: string | null; + lastWebhookAt: string | null; + lastApiPollAt: string | null; + tierStatus: Partial>; + okStreak: number; + failStreak: number; + stateEnteredAt: string; + holdUntil: string | null; + cooldownUntil: string | null; + previousState: MonitorState | null; + lastTransitionAt: string | null; + lastTransitionReason: string | null; + createdAt: string; + updatedAt: string; +} + +export interface MonitorPolicy { + liveConfirmations: number; + offlineConfirmations: number; + noMediaMs: number; + minHoldMs: number; + cooldownMs: number; +} + +export const DEFAULT_MONITOR_POLICY: MonitorPolicy = { + liveConfirmations: 2, + offlineConfirmations: 3, + noMediaMs: 10_000, + minHoldMs: 2_000, + cooldownMs: 10_000, +}; + +export interface MonitorTransition { + from: MonitorState; + to: MonitorState; + version: number; + reason: string; + at: string; +} + +export interface Reduction { + state: MonitorRowState; + transition?: MonitorTransition; +} + +const PRECEDENCE: readonly SignalTier[] = ["media-health", "hls-probe", "webhook", "api-poll"]; +const OBSERVED_FIELD: Record> = { + "media-health": "lastMediaHealthAt", + "hls-probe": "lastHlsProbeAt", + webhook: "lastWebhookAt", + "api-poll": "lastApiPollAt", +}; + +export function applyObservation( + state: MonitorRowState, + observation: MonitorObservation, + policy: MonitorPolicy = DEFAULT_MONITOR_POLICY, +): Reduction { + const now = Date.parse(observation.receivedAt); + const observed = Date.parse(observation.observedAt); + if (!Number.isFinite(now) || !Number.isFinite(observed)) throw new Error("observation timestamps must be valid"); + const field = OBSERVED_FIELD[observation.tier]; + const last = state[field]; + if (last !== null && observed <= Date.parse(last)) return { state }; + const oldTier = state.tierStatus[observation.tier]; + if (observation.providerEventId !== undefined && (oldTier?.lastProviderEventId === observation.providerEventId + || oldTier?.seenProviderEventIds?.includes(observation.providerEventId) === true)) { + return { state }; + } + + const outcome = observation.payload.outcome; + const available = outcome !== "unavailable"; + const okStreak = outcome === "ok" ? (oldTier?.okStreak ?? 0) + 1 : 0; + const failStreak = outcome === "fail" ? (oldTier?.failStreak ?? 0) + 1 : 0; + const tierStatus: MonitorRowState["tierStatus"] = { + ...state.tierStatus, + [observation.tier]: { + available, + lastOutcome: outcome, + lastReceivedAt: observation.receivedAt, + ...(observation.providerEventId === undefined + ? oldTier?.lastProviderEventId === undefined ? {} : { lastProviderEventId: oldTier.lastProviderEventId } + : { lastProviderEventId: observation.providerEventId }), + ...(observation.providerEventId === undefined && oldTier?.seenProviderEventIds === undefined + ? {} : { seenProviderEventIds: observation.providerEventId === undefined + ? oldTier?.seenProviderEventIds + : [...(oldTier?.seenProviderEventIds ?? []), observation.providerEventId].slice(-64) }), + okStreak, + failStreak, + ...(observation.payload.mediaPresent === true + ? { lastMediaAt: observation.receivedAt } + : oldTier?.lastMediaAt === undefined ? {} : { lastMediaAt: oldTier.lastMediaAt }), + }, + }; + const authoritativeTier = PRECEDENCE.find((tier) => tierStatus[tier]?.available === true) ?? null; + const authority = authoritativeTier === null ? undefined : tierStatus[authoritativeTier]; + let next: MonitorRowState = { + ...state, + [field]: observation.observedAt, + tierStatus, + authoritativeTier, + version: state.version + 1, + okStreak: authority?.okStreak ?? 0, + failStreak: authority?.failStreak ?? 0, + updatedAt: observation.receivedAt, + }; + + let target: MonitorState | undefined; + let reason = `${observation.tier}:${outcome}`; + if (outcome === "error" && observation.tier === authoritativeTier) { + target = "ERROR"; + } else if (authoritativeTier !== null) { + const lowerSignal = observation.tier !== authoritativeTier; + if (lowerSignal && outcome !== "unavailable") { + if (outcome === "fail" || outcome === "error") target = state.state === "LIVE" ? "UNSTABLE" : "PROBING"; + } else if (authority?.lastOutcome === "ok") { + if (authority.okStreak >= policy.liveConfirmations) target = "LIVE"; + else if (state.state !== "LIVE") target = "PROBING"; + } else if (authority?.lastOutcome === "fail") { + const noMedia = authoritativeTier === "media-health" && authority.lastMediaAt !== undefined + && now - Date.parse(authority.lastMediaAt) >= policy.noMediaMs; + if (authority.failStreak >= policy.offlineConfirmations || noMedia) target = "OFFLINE"; + else if (state.state === "LIVE") target = "UNSTABLE"; + else if (state.state !== "OFFLINE" && state.state !== "UNSTABLE") target = "PROBING"; + if (noMedia) reason = "media-health:no-media-timeout"; + } + } + + if (target === undefined || target === state.state) return { state: next }; + const terminal = target === "LIVE" || target === "OFFLINE"; + const held = target !== "ERROR" && state.state !== "PROBING" && state.state !== "UNSTABLE" + && state.holdUntil !== null && now < Date.parse(state.holdUntil); + const cooling = terminal && state.cooldownUntil !== null && now < Date.parse(state.cooldownUntil); + if (held || cooling) return { state: next }; + next = { + ...next, + state: target, + previousState: state.state, + stateEnteredAt: observation.receivedAt, + holdUntil: terminal ? new Date(now + policy.minHoldMs).toISOString() : state.holdUntil, + cooldownUntil: terminal ? new Date(now + policy.cooldownMs).toISOString() : state.cooldownUntil, + lastTransitionAt: observation.receivedAt, + lastTransitionReason: reason, + }; + return { state: next, transition: { from: state.state, to: target, version: next.version, reason, at: observation.receivedAt } }; +} diff --git a/apps/api/src/live-monitor/webhook.controller.ts b/apps/api/src/live-monitor/webhook.controller.ts new file mode 100644 index 0000000..d096d2c --- /dev/null +++ b/apps/api/src/live-monitor/webhook.controller.ts @@ -0,0 +1,19 @@ +import { Controller, HttpCode, Inject, Param, Post, Req } from "@nestjs/common"; +import type { Request } from "express"; +import { MonitorWebhookService } from "./webhook.service.js"; + +/** Provider authentication is performed over raw bytes; browser session guards do not apply. */ +@Controller("live-monitor/webhooks") +export class MonitorWebhookController { + constructor(@Inject(MonitorWebhookService) private readonly webhook: MonitorWebhookService) {} + + @Post(":endpointId") + @HttpCode(200) + async ingest(@Param("endpointId") endpointId: string, @Req() request: Request): Promise<{ accepted: true }> { + await this.webhook.ingest(endpointId, request.body as Buffer, { + signature: request.get("x-monitor-signature"), timestamp: request.get("x-monitor-timestamp"), + eventId: request.get("x-monitor-event-id"), + }); + return { accepted: true }; + } +} diff --git a/apps/api/src/live-monitor/webhook.integration.test.ts b/apps/api/src/live-monitor/webhook.integration.test.ts new file mode 100644 index 0000000..ebe3bfa --- /dev/null +++ b/apps/api/src/live-monitor/webhook.integration.test.ts @@ -0,0 +1,91 @@ +import { createHmac, randomBytes, randomUUID } from "node:crypto"; +import { join } from "node:path"; +import { Pool, type PoolClient } from "pg"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { runMigrations } from "../db/migrate.js"; +import { vault } from "../vault/vault.js"; +import { LiveMonitorService } from "./live-monitor.service.js"; +import { MonitorWebhookService, type MonitorWebhookEndpoint } from "./webhook.service.js"; + +const available = process.env.DATABASE_URL !== undefined && process.env.SUPER_DATABASE_URL !== undefined; +const owner = new Pool({ connectionString: process.env.SUPER_DATABASE_URL }); +const app = new Pool({ connectionString: process.env.DATABASE_URL }); +async function asTenant(workspaceId: string, fn: (client: PoolClient) => Promise): Promise { + const client = await app.connect(); + try { + await client.query("BEGIN"); + await client.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]); + const result = await fn(client); + await client.query("COMMIT"); + return result; + } catch (error) { await client.query("ROLLBACK"); throw error; } + finally { client.release(); } +} + +describe.skipIf(!available)("monitor webhook durable journal and RLS (PostgreSQL)", () => { + const secret = randomBytes(32); + const workspaces: string[] = []; + let endpoint: MonitorWebhookEndpoint; + let sourceId: string; + let foreignSourceId: string; + let monitor: LiveMonitorService; + const service = () => new MonitorWebhookService(monitor, { endpoints: [endpoint], tenantTransaction: asTenant }); + + beforeAll(async () => { + await runMigrations(process.env.MIGRATIONS_DIR ?? join(process.cwd(), "migrations")); + for (let index = 0; index < 2; index++) { + const workspace = await owner.query<{ id: string }>( + "INSERT INTO workspaces (name, slug) VALUES ('Webhook integration', $1) RETURNING id", [`monitor-webhook-${randomUUID()}`]); + const workspaceId = workspace.rows[0]!.id; + workspaces.push(workspaceId); + await asTenant(workspaceId, async (client) => { + const connection = await client.query<{ id: string }>( + "INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-livestream', 'webhook') RETURNING id", [workspaceId]); + const connectionId = connection.rows[0]!.id; + const source = await client.query<{ id: string }>( + "INSERT INTO source_monitor_states (workspace_id, connection_id, remote_ref) VALUES ($1, $2, 'live:webhook') RETURNING id", [workspaceId, connectionId]); + if (index === 0) { + sourceId = source.rows[0]!.id; + const sealed = await vault.seal(workspaceId, connectionId, secret); + const credential = await client.query<{ id: string }>( + `INSERT INTO credentials (workspace_id, connection_id, kind, ciphertext, key_version) + VALUES ($1, $2, 'monitor-webhook', $3, $4) RETURNING id`, [workspaceId, connectionId, sealed.ciphertext, sealed.keyVersion]); + endpoint = { endpointId: randomBytes(32).toString("hex"), workspaceId, connectionId, + credentialId: credential.rows[0]!.id, subscriptionId: "db-subscription" }; + } else foreignSourceId = source.rows[0]!.id; + }); + } + monitor = new LiveMonitorService({ tenantTransaction: asTenant }); + }); + + afterAll(async () => { + for (const workspace of workspaces) await owner.query("DELETE FROM workspaces WHERE id = $1", [workspace]); + await app.end(); await owner.end(); secret.fill(0); + }); + + function request(source = sourceId, eventId = randomUUID()) { + const raw = Buffer.from(JSON.stringify({ sourceId: source, subscriptionId: endpoint.subscriptionId, outcome: "ok" })); + const timestamp = new Date().toISOString(); + const signature = `sha256=${createHmac("sha256", secret).update(eventId).update(timestamp).update(raw).digest("hex")}`; + return { raw, headers: { eventId, timestamp, signature } }; + } + + it("accepts one concurrent delivery, rejects its replay after service restart, persists exactly one journal row", async () => { + const { raw, headers } = request(); + const results = await Promise.allSettled([service().ingest(endpoint.endpointId, raw, headers), service().ingest(endpoint.endpointId, raw, headers)]); + expect(results.filter((result) => result.status === "fulfilled")).toHaveLength(1); + expect(results.find((result) => result.status === "rejected")).toMatchObject({ reason: { status: 409 } }); + await expect(service().ingest(endpoint.endpointId, raw, headers)).rejects.toMatchObject({ status: 409 }); + expect((await monitor.getState(endpoint.workspaceId, sourceId)).version).toBe(1); + const journal = await asTenant(endpoint.workspaceId, (client) => client.query<{ count: number }>( + "SELECT count(*)::int AS count FROM monitor_seen_events WHERE source_monitor_state_id = $1", [sourceId])); + expect(journal.rows[0]!.count).toBe(1); + }); + + it("rejects an authenticated event targeting another workspace without touching either source", async () => { + const { raw, headers } = request(foreignSourceId); + await expect(service().ingest(endpoint.endpointId, raw, headers)).rejects.toMatchObject({ status: 404 }); + expect((await monitor.getState(workspaces[1]!, foreignSourceId)).version).toBe(0); + expect((await monitor.getState(endpoint.workspaceId, sourceId)).version).toBe(1); + }); +}); diff --git a/apps/api/src/live-monitor/webhook.service.ts b/apps/api/src/live-monitor/webhook.service.ts new file mode 100644 index 0000000..72dc00f --- /dev/null +++ b/apps/api/src/live-monitor/webhook.service.ts @@ -0,0 +1,136 @@ +import { createHash, createHmac, timingSafeEqual } from "node:crypto"; +import { BadRequestException, Inject, Injectable, NotFoundException, Optional, PayloadTooLargeException, UnauthorizedException } from "@nestjs/common"; +import { withTenant } from "../db/client.js"; +import { isUuid, requireObject, requireString } from "../http/validation.js"; +import { vault } from "../vault/vault.js"; +import { LiveMonitorService } from "./live-monitor.service.js"; +import type { MonitorRowState, ObservationOutcome } from "./reducer.js"; + +export const WEBHOOK_MAX_BYTES = 64 * 1024; +export const WEBHOOK_WINDOW_MS = 5 * 60 * 1000; +export interface MonitorWebhookEndpoint { + endpointId: string; + workspaceId: string; + connectionId: string; + credentialId: string; + subscriptionId: string; +} +export interface MonitorWebhookHeaders { signature?: string; timestamp?: string; eventId?: string } +export interface MonitorWebhookDependencies { + endpoints: readonly MonitorWebhookEndpoint[]; + tenantTransaction: typeof withTenant; + openSecret: typeof vault.open; + now: () => number; +} +export const MONITOR_WEBHOOK_DEPENDENCIES = Symbol("MONITOR_WEBHOOK_DEPENDENCIES"); + +/** Server-controlled mapping, never a workspace/connection supplied by the sender. */ +export function parseWebhookEndpoints(raw: string | undefined): MonitorWebhookEndpoint[] { + if (!raw) return []; + const value: unknown = JSON.parse(raw); + if (!Array.isArray(value)) throw new Error("MONITOR_WEBHOOK_ENDPOINTS must be an array"); + const endpoints = value.map((entry) => { + const row = requireObject(entry, "endpoint"); + const endpoint: MonitorWebhookEndpoint = { + endpointId: requireString(row.endpointId, "endpointId"), + workspaceId: requireString(row.workspaceId, "workspaceId"), + connectionId: requireString(row.connectionId, "connectionId"), + credentialId: requireString(row.credentialId, "credentialId"), + subscriptionId: requireString(row.subscriptionId, "subscriptionId"), + }; + if (!/^[a-f0-9]{64}$/.test(endpoint.endpointId) + || ![endpoint.workspaceId, endpoint.connectionId, endpoint.credentialId].every(isUuid)) { + throw new Error("invalid monitor webhook endpoint mapping (endpointId must be 32 random bytes in hex)"); + } + return endpoint; + }); + if (new Set(endpoints.map((entry) => entry.endpointId)).size !== endpoints.length) { + throw new Error("duplicate monitor webhook endpoint"); + } + return endpoints; +} + +/** Signature covers event id + ISO timestamp + exact bytes, including whitespace. */ +export function verifyMonitorSignature(raw: Buffer, headers: MonitorWebhookHeaders, secret: Buffer, now: number): void { + const { signature, timestamp, eventId } = headers; + if (!signature || !/^sha256=[a-f0-9]{64}$/.test(signature) || !timestamp + || !/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d{1,3})?Z$/.test(timestamp) + || !eventId || eventId.length > 256 || /[\s\p{Cc}]/u.test(eventId) + || !Number.isFinite(Date.parse(timestamp)) || Math.abs(now - Date.parse(timestamp)) > WEBHOOK_WINDOW_MS) { + throw new UnauthorizedException("invalid webhook authentication"); + } + const expected = createHmac("sha256", secret).update(eventId).update(timestamp).update(raw).digest(); + if (!timingSafeEqual(expected, Buffer.from(signature.slice(7), "hex"))) { + throw new UnauthorizedException("invalid webhook authentication"); + } +} + +@Injectable() +export class MonitorWebhookService { + private readonly config: MonitorWebhookDependencies; + constructor( + @Inject(LiveMonitorService) private readonly monitor: LiveMonitorService, + @Optional() @Inject(MONITOR_WEBHOOK_DEPENDENCIES) dependencies?: Partial, + ) { + this.config = { + endpoints: parseWebhookEndpoints(process.env.MONITOR_WEBHOOK_ENDPOINTS), + tenantTransaction: withTenant, openSecret: vault.open.bind(vault), now: Date.now, + ...dependencies, + }; + } + + async ingest(endpointId: string, raw: Buffer, headers: MonitorWebhookHeaders): Promise { + const endpoint = this.config.endpoints.find((entry) => entry.endpointId === endpointId); + if (!endpoint) throw new NotFoundException("webhook endpoint not found"); + if (!Buffer.isBuffer(raw)) throw new BadRequestException("raw webhook body required"); + if (raw.length > WEBHOOK_MAX_BYTES) throw new PayloadTooLargeException(); + return this.config.tenantTransaction(endpoint.workspaceId, async (client) => { + // A removed connection/revoked credential disables even a configured endpoint. + const credential = await client.query<{ ciphertext: Buffer; key_version: number }>( + `SELECT cr.ciphertext, cr.key_version FROM credentials cr + JOIN connections c ON c.id = cr.connection_id AND c.workspace_id = cr.workspace_id + WHERE cr.id = $1 AND cr.connection_id = $2 AND cr.workspace_id = $3 + AND cr.kind = 'monitor-webhook' AND cr.revoked_at IS NULL AND c.disconnected_at IS NULL + FOR SHARE OF cr, c`, [endpoint.credentialId, endpoint.connectionId, endpoint.workspaceId], + ); + const sealed = credential.rows[0]; + if (!sealed) throw new UnauthorizedException("invalid webhook authentication"); + let secret: Buffer; + try { + secret = await this.config.openSecret(endpoint.workspaceId, endpoint.connectionId, + { ciphertext: sealed.ciphertext, keyVersion: sealed.key_version }); + } catch { throw new UnauthorizedException("invalid webhook authentication"); } + const now = this.config.now(); + try { + if (secret.length < 32) throw new UnauthorizedException("invalid webhook authentication"); + verifyMonitorSignature(raw, headers, secret, now); + } finally { secret.fill(0); } + let parsed: unknown; + try { parsed = JSON.parse(raw.toString("utf8")); } + catch { throw new BadRequestException("invalid webhook JSON"); } + const body = requireObject(parsed, "event"); + if (body.subscriptionId !== endpoint.subscriptionId + || body.workspaceId !== undefined && body.workspaceId !== endpoint.workspaceId + || body.connectionId !== undefined && body.connectionId !== endpoint.connectionId) { + throw new UnauthorizedException("webhook binding mismatch"); + } + const sourceId = requireString(body.sourceId, "sourceId"); + if (!isUuid(sourceId)) throw new BadRequestException("sourceId must be a UUID"); + const outcome = body.outcome; + if (outcome !== "ok" && outcome !== "fail" && outcome !== "error" && outcome !== "unavailable") { + throw new BadRequestException("invalid webhook outcome"); + } + const source = await client.query( + `SELECT id FROM source_monitor_states WHERE id = $1 AND workspace_id = $2 AND connection_id = $3 FOR UPDATE`, + [sourceId, endpoint.workspaceId, endpoint.connectionId], + ); + if (source.rowCount !== 1) throw new NotFoundException("webhook source not found"); + // Journal retention exceeds the acceptance window. The tuple scopes ids per endpoint without ambiguity. + const providerEventId = `webhook:${createHash("sha256").update(JSON.stringify([endpointId, headers.eventId])).digest("hex")}`; + return this.monitor.recordObservationInTransaction(client, endpoint.workspaceId, sourceId, { + tier: "webhook", observedAt: headers.timestamp!, receivedAt: new Date(now).toISOString(), providerEventId, + payload: { outcome: outcome as ObservationOutcome, reason: "signed-webhook" }, + }, { userId: null, role: "monitor-webhook" }, true); + }); + } +} diff --git a/apps/api/src/live-monitor/webhook.test.ts b/apps/api/src/live-monitor/webhook.test.ts new file mode 100644 index 0000000..532c5fb --- /dev/null +++ b/apps/api/src/live-monitor/webhook.test.ts @@ -0,0 +1,199 @@ +import "reflect-metadata"; +import { createHmac } from "node:crypto"; +import { BadRequestException, ConflictException, NotFoundException, UnauthorizedException, type INestApplication } from "@nestjs/common"; +import { NestFactory } from "@nestjs/core"; +import type { PoolClient } from "pg"; +import { afterEach, describe, expect, it } from "vitest"; +import { injectHttp } from "../connections/http-test-transport.js"; +import { vault } from "../vault/vault.js"; +import { LiveMonitorModule } from "./live-monitor.module.js"; +import { LIVE_MONITOR_DEPENDENCIES, LiveMonitorService } from "./live-monitor.service.js"; +import { MEDIA_POLLER_DEPENDENCIES } from "./poller/poller.js"; +import { MONITOR_WEBHOOK_DEPENDENCIES, MonitorWebhookService, parseWebhookEndpoints, verifyMonitorSignature, + WEBHOOK_MAX_BYTES, WEBHOOK_WINDOW_MS, type MonitorWebhookDependencies, type MonitorWebhookHeaders } from "./webhook.service.js"; + +const workspaceId = "11111111-1111-4111-8111-111111111111"; +const connectionId = "22222222-2222-4222-8222-222222222222"; +const credentialId = "33333333-3333-4333-8333-333333333333"; +const sourceId = "44444444-4444-4444-8444-444444444444"; +const endpointId = "ab".repeat(32); +const endpoint = { endpointId, workspaceId, connectionId, credentialId, subscriptionId: "subscription-1" }; +const secret = Buffer.alloc(32, 42); +const epoch = Date.parse("2026-01-01T00:00:00.000Z"); +const event = { sourceId, subscriptionId: endpoint.subscriptionId, outcome: "ok" }; + +function signed(raw: Buffer, eventId = "event-1", time = epoch): MonitorWebhookHeaders { + const timestamp = new Date(time).toISOString(); + return { eventId, timestamp, signature: `sha256=${createHmac("sha256", secret).update(eventId).update(timestamp).update(raw).digest("hex")}` }; +} + +async function fixture() { + const row: Record = { + id: sourceId, workspace_id: workspaceId, connection_id: connectionId, remote_ref: "live:1", + state: "OFFLINE", version: 0, authoritative_tier: null, last_media_health_at: null, last_hls_probe_at: null, + last_webhook_at: null, last_api_poll_at: null, tier_status: {}, ok_streak: 0, fail_streak: 0, + state_entered_at: new Date(epoch).toISOString(), hold_until: null, cooldown_until: null, previous_state: null, + last_transition_at: null, last_transition_reason: null, created_at: new Date(epoch).toISOString(), updated_at: new Date(epoch).toISOString(), + }; + const sealed = await vault.seal(workspaceId, connectionId, secret); + const journal = new Set(); + let now = epoch; + let failUpdate = false; + let revoked = false; + let audits = 0; + const client = { async query(sql: string, params: unknown[] = []) { + if (sql.startsWith("SELECT cr.ciphertext")) { + expect(params).toEqual([credentialId, connectionId, workspaceId]); + return { rowCount: revoked ? 0 : 1, rows: revoked ? [] : [{ ciphertext: sealed.ciphertext, key_version: sealed.keyVersion }] }; + } + if (sql.startsWith("SELECT id FROM source_monitor_states")) { + const owns = params[0] === row.id && params[1] === row.workspace_id && params[2] === row.connection_id; + return { rowCount: owns ? 1 : 0, rows: owns ? [{ id: sourceId }] : [] }; + } + if (sql.startsWith("SELECT * FROM source_monitor_states")) return { rowCount: 1, rows: [{ ...row }] }; + if (sql.startsWith("INSERT INTO monitor_seen_events")) { + const key = JSON.stringify([params[0], params[1]]); + if (journal.has(key)) return { rowCount: 0, rows: [] }; + journal.add(key); + return { rowCount: 1, rows: [{}] }; + } + if (sql.startsWith("UPDATE source_monitor_states")) { + if (failUpdate) throw new Error("database update failed"); + const columns = ["state", "version", "authoritative_tier", "last_media_health_at", "last_hls_probe_at", + "last_webhook_at", "last_api_poll_at", "tier_status", "ok_streak", "fail_streak", "state_entered_at", "hold_until", + "cooldown_until", "previous_state", "last_transition_at", "last_transition_reason", "updated_at"]; + columns.forEach((column, index) => { row[column] = params[index + 1]; }); + return { rowCount: 1, rows: [{ ...row }] }; + } + if (sql.startsWith("SELECT audit_append")) { audits++; return { rowCount: 1, rows: [{}] }; } + throw new Error(`unexpected query ${sql}`); + } } as unknown as PoolClient; + let pending = Promise.resolve(); + const tenantTransaction: MonitorWebhookDependencies["tenantTransaction"] = async (workspace, fn) => { + expect(workspace).toBe(workspaceId); + const previous = pending; + let release!: () => void; + pending = new Promise((resolve) => { release = resolve; }); + await previous; + const snapshot = { row: structuredClone(row), journal: [...journal], audits }; + try { return await fn(client); } + catch (error) { + Object.assign(row, snapshot.row); journal.clear(); snapshot.journal.forEach((key) => journal.add(key)); + audits = snapshot.audits; + throw error; + } finally { release(); } + }; + const dependencies: MonitorWebhookDependencies = { + endpoints: [endpoint, { ...endpoint, endpointId: "cd".repeat(32) }], + tenantTransaction, now: () => now, openSecret: vault.open.bind(vault), + }; + const monitor = new LiveMonitorService({ tenantTransaction }); + const service = () => new MonitorWebhookService(monitor, dependencies); + return { service, dependencies, monitor, row, journal, audits: () => audits, + clock: (time: number) => { now = time; }, fail: (value: boolean) => { failUpdate = value; }, revoke: () => { revoked = true; } }; +} + +describe("monitor webhook authentication and ingest", () => { + it("verifies exact raw bytes, event id and timestamp in constant-time HMAC", () => { + const raw = Buffer.from(JSON.stringify(event)); + expect(() => verifyMonitorSignature(raw, signed(raw), secret, epoch)).not.toThrow(); + expect(() => verifyMonitorSignature(raw, signed(raw), Buffer.alloc(32, 43), epoch)).toThrow(UnauthorizedException); + expect(() => verifyMonitorSignature(Buffer.concat([raw, Buffer.from(" ")]), signed(raw), secret, epoch)).toThrow(UnauthorizedException); + expect(() => verifyMonitorSignature(raw, { ...signed(raw), eventId: "event-2" }, secret, epoch)).toThrow(UnauthorizedException); + expect(() => verifyMonitorSignature(raw, { ...signed(raw), timestamp: new Date(epoch + 1000).toISOString() }, secret, epoch)).toThrow(UnauthorizedException); + expect(() => verifyMonitorSignature(raw, { ...signed(raw), signature: "sha256=bad" }, secret, epoch)).toThrow(UnauthorizedException); + }); + + it.each([-1, 1])("rejects signed timestamps outside the window in direction %s", (direction) => { + const raw = Buffer.from(JSON.stringify(event)); + expect(() => verifyMonitorSignature(raw, signed(raw, "event-1", epoch + direction * (WEBHOOK_WINDOW_MS + 1)), secret, epoch)).toThrow(UnauthorizedException); + expect(() => verifyMonitorSignature(raw, signed(raw, "event-1", epoch + direction * WEBHOOK_WINDOW_MS), secret, epoch)).not.toThrow(); + }); + + it("folds authenticated events through the existing service and reducer; replay survives service recreation", async () => { + const f = await fixture(); const raw = Buffer.from(JSON.stringify(event)); + expect((await f.service().ingest(endpointId, raw, signed(raw))).state).toBe("PROBING"); + await expect(f.service().ingest(endpointId, raw, signed(raw))).rejects.toBeInstanceOf(ConflictException); + expect(f.row.version).toBe(1); expect(f.audits()).toBe(1); expect(f.journal.size).toBe(1); + }); + + it("retains dedupe across the full early-to-late timestamp acceptance span", async () => { + const f = await fixture(); const raw = Buffer.from(JSON.stringify(event)); + f.clock(epoch - WEBHOOK_WINDOW_MS); + await f.service().ingest(endpointId, raw, signed(raw)); + f.clock(epoch + 4 * 60 * 1000); + await expect(f.service().ingest(endpointId, raw, signed(raw))).rejects.toBeInstanceOf(ConflictException); + expect(f.row.version).toBe(1); + }); + + it("scopes event ids per endpoint, including under concurrent delivery", async () => { + const f = await fixture(); const raw = Buffer.from(JSON.stringify(event)); + const results = await Promise.allSettled([f.service().ingest(endpointId, raw, signed(raw)), f.service().ingest(endpointId, raw, signed(raw))]); + expect(results.filter((result) => result.status === "fulfilled")).toHaveLength(1); + await f.service().ingest("cd".repeat(32), raw, signed(raw)); + expect(f.journal.size).toBe(2); expect(f.row.version).toBe(1); // same-tier stale timestamp is still folded only once + }); + + it.each(["workspace_id", "connection_id"])("rejects a source owned by a different %s before journaling", async (column) => { + const f = await fixture(); f.row[column] = "55555555-5555-4555-8555-555555555555"; + const raw = Buffer.from(JSON.stringify(event)); + await expect(f.service().ingest(endpointId, raw, signed(raw))).rejects.toBeInstanceOf(NotFoundException); + expect(f.journal.size).toBe(0); expect(f.row.version).toBe(0); + }); + + it.each([{ subscriptionId: "other" }, { workspaceId: "other" }, { connectionId: "other" }])("rejects provider binding mismatch %j", async (binding) => { + const f = await fixture(); const raw = Buffer.from(JSON.stringify({ ...event, ...binding })); + await expect(f.service().ingest(endpointId, raw, signed(raw))).rejects.toBeInstanceOf(UnauthorizedException); + expect(f.journal.size).toBe(0); expect(f.row.version).toBe(0); + }); + + it("rolls back the replay claim when state persistence fails", async () => { + const f = await fixture(); const raw = Buffer.from(JSON.stringify(event)); f.fail(true); + await expect(f.service().ingest(endpointId, raw, signed(raw))).rejects.toThrow("database update failed"); + expect(f.journal.size).toBe(0); f.fail(false); + expect((await f.service().ingest(endpointId, raw, signed(raw))).version).toBe(1); + }); + + it("rejects unknown endpoints, revoked credentials, malformed JSON and bad authentication without writes", async () => { + const f = await fixture(); const raw = Buffer.from(JSON.stringify(event)); + await expect(f.service().ingest("unknown", raw, signed(raw))).rejects.toBeInstanceOf(NotFoundException); + await expect(f.service().ingest(endpointId, raw, {})).rejects.toBeInstanceOf(UnauthorizedException); + const malformed = Buffer.from("{"); + await expect(f.service().ingest(endpointId, malformed, signed(malformed))).rejects.toBeInstanceOf(BadRequestException); + f.revoke(); + await expect(f.service().ingest(endpointId, raw, signed(raw))).rejects.toBeInstanceOf(UnauthorizedException); + expect(f.journal.size).toBe(0); expect(f.row.version).toBe(0); + }); + + it("requires valid and unique server-controlled endpoint mappings", () => { + expect(parseWebhookEndpoints(JSON.stringify([endpoint]))).toEqual([endpoint]); + expect(() => parseWebhookEndpoints(JSON.stringify([endpoint, endpoint]))).toThrow("duplicate"); + expect(() => parseWebhookEndpoints(JSON.stringify([{ ...endpoint, endpointId: "guessable" }]))).toThrow("invalid"); + }); +}); + +describe("monitor webhook raw HTTP route", () => { + let app: INestApplication | undefined; + afterEach(async () => { await app?.close(); app = undefined; }); + + it("captures whitespace before Nest JSON parsing, needs no browser session, rejects replay/tampering/oversize", async () => { + const f = await fixture(); + app = await NestFactory.create({ module: LiveMonitorModule, providers: [ + { provide: LIVE_MONITOR_DEPENDENCIES, useValue: { tenantTransaction: f.dependencies.tenantTransaction } }, + { provide: MONITOR_WEBHOOK_DEPENDENCIES, useValue: f.dependencies }, + { provide: MEDIA_POLLER_DEPENDENCIES, useValue: { enabled: false } }, + ] }, { logger: false, abortOnError: false }); + await app.init(); + const body = ` ${JSON.stringify(event, null, 2)}\n`; + const auth = signed(Buffer.from(body)); + const headers = { "content-type": "application/json", "x-monitor-signature": auth.signature!, + "x-monitor-timestamp": auth.timestamp!, "x-monitor-event-id": auth.eventId! }; + const post = (value: string) => injectHttp(app!.getHttpAdapter().getInstance(), `http://api.test/live-monitor/webhooks/${endpointId}`, + { method: "POST", body: value, headers }); + expect((await post(body)).status).toBe(200); + expect((await post(body)).status).toBe(409); + expect((await post(`${body} `)).status).toBe(401); + expect((await post(" ".repeat(WEBHOOK_MAX_BYTES + 1))).status).toBe(413); + expect(f.row.version).toBe(1); + }); +}); diff --git a/apps/api/src/live/go-live.service.ts b/apps/api/src/live/go-live.service.ts new file mode 100644 index 0000000..5c999d5 --- /dev/null +++ b/apps/api/src/live/go-live.service.ts @@ -0,0 +1,171 @@ +import { BadRequestException, ConflictException, Inject, Injectable, NotFoundException, Optional } from "@nestjs/common"; +import type { PoolClient } from "pg"; +import { withTenant } from "../db/client.js"; +import { LiveDestinationRepository, type OutputSummary } from "./live-destination.repository.js"; +import { createGenericHlsResolver, GENERIC_HLS_PROVIDER, type LiveSourceResolver } from "./live-source-resolver.js"; + +export type TenantTransaction = (workspaceId: string, fn: (client: PoolClient) => Promise) => Promise; + +/** + * The media-plane target a go-live spawns: everything the relay needs, with + * raw secret material held only inside this service's resolution path. + */ +export interface ResolvedOutputTarget { + output: OutputSummary; + /** Ingest endpoint for the push (destination's, or null for a disabled/generic-missing output). */ + endpointUrl: string; + /** True when the credential came through a destination (vs the output's own handle). */ + throughDestination: boolean; + /** Vault handle the worker's VaultHandleBroker redeems at relay start. */ + credentialHandleId: string | null; +} + +/** Resolved program input for the take/start path. */ +export interface ResolvedProgramSource { + playbackHlsUrl: string; + live: boolean; + provider: string; +} + +export interface GoLiveDependencies { + transaction: TenantTransaction; + /** Provider-keyed resolvers; connectors register their own, generic-hls is built in. */ + sourceResolvers: Map; +} + +export const GO_LIVE_DEPENDENCIES = Symbol("GO_LIVE_DEPENDENCIES"); + +/** + * Go-live wiring (WP6/WP9): take/start resolves StreamOutput -> + * LiveStreamDestination -> credential via the vault (same pattern as the + * publish credential accessor), and resolves the program source through + * connector-level resolvers. Nothing here mutates Redis or signs commands; + * callers feed the result into MediaControlService.setDesired/director. + */ +@Injectable() +export class GoLiveService { + private readonly resolvers: Map; + private readonly transaction: TenantTransaction; + + constructor( + @Inject(LiveDestinationRepository) private readonly destinations: LiveDestinationRepository, + @Optional() @Inject(GO_LIVE_DEPENDENCIES) dependencies?: Partial, + ) { + const defaults: GoLiveDependencies = { + transaction: withTenant, + sourceResolvers: new Map([[GENERIC_HLS_PROVIDER, createGenericHlsResolver()]]), + }; + const merged = { ...defaults, ...dependencies }; + this.resolvers = merged.sourceResolvers; + this.transaction = merged.transaction; + } + + registerSourceResolver(provider: string, resolver: LiveSourceResolver): void { + this.resolvers.set(provider, resolver); + } + + /** + * Resolve every enabled output of a channel into media-plane targets. + * Destination-referenced outputs resolve their credential handle through the + * destination (aggregate boundary); handle outputs pass theirs through + * untouched. The raw stream key never enters this surface — the worker + * redeems the handle at relay start. + */ + async resolveOutputs(workspaceId: string, virtualChannelId: string): Promise { + const outputs = await this.listChannelOutputs(workspaceId, virtualChannelId); + const targets: ResolvedOutputTarget[] = []; + for (const output of outputs) { + if (!output.enabled) continue; + if (output.destinationId !== null) { + const destination = await this.destinations.get(workspaceId, output.destinationId); + if (destination.credentialId === null) { + throw new ConflictException(`destination ${destination.label} has no ingest credential`); + } + if (destination.credentialRevokedAt !== null) { + throw new ConflictException(`destination ${destination.label} ingest credential is revoked`); + } + targets.push({ + output, + endpointUrl: destination.endpointUrl, + throughDestination: true, + credentialHandleId: destination.credentialId, + }); + continue; + } + const handle = output.credentialHandleId; + if (handle === null) throw new ConflictException(`stream output ${output.id} has neither destination nor credential handle`); + targets.push({ output, endpointUrl: "", throughDestination: false, credentialHandleId: handle }); + } + return targets; + } + + /** + * Take/start resolution of the program source for a virtual channel source + * row: provider-keyed resolvers (twitch via Helix, generic-hls direct) map + * {channelId | externalUrl} to {playbackHlsUrl, live}. + */ + async resolveProgramSource(workspaceId: string, request: { provider: string; channelId?: string; externalUrl?: string }): Promise { + if (request.provider === "") throw new BadRequestException("provider is required"); + const resolver = this.resolvers.get(request.provider); + if (resolver === undefined) throw new NotFoundException(`no live source resolver for provider ${request.provider}`); + const resolution = await resolver({ channelId: request.channelId, externalUrl: request.externalUrl }); + if (resolution.playbackHlsUrl === null) { + throw new ConflictException(`live source for ${request.provider} is not live`); + } + return { playbackHlsUrl: resolution.playbackHlsUrl, live: resolution.live, provider: request.provider }; + } + + /** + * Redeem the destination ingest credential (vault.open with tenant + handle + * AAD) for a destination-referenced output. Called only by the relay spawn + * path; the returned secret never reaches logs, audit, or HTTP responses. + */ + async redeemDestinationCredential(workspaceId: string, destinationId: string): Promise<{ endpointUrl: string; streamKey: string; provider: string }> { + return this.destinations.resolveIngestCredential(workspaceId, destinationId); + } + + /** + * Take/start: build a DesiredSpec for the go-live flow from the channel's + * enabled StreamOutputs and a resolved program source. Destination-referenced + * outputs contribute the destination's endpoint + credential handle (resolved + * through the vault at relay start); a single-output channel maps onto the + * v1 spec shape MediaControlService.setDesired expects. Channels with + * multiple enabled outputs are a multi-fan-out concern of the worker relay + * (WP9) and resolve per-output there. + */ + async desiredSpecForGoLive(workspaceId: string, virtualChannelId: string, source: { provider: string; channelId?: string; externalUrl?: string }, + options?: { profile?: "720p30" | "1080p30" | "1080p60"; mode?: "passthrough" | "normalize" }): Promise<{ + desiredStatus: "RUNNING"; + source: { kind: "hls"; url: string }; + output: { endpointUrl: string; credentialHandleId: string }; + profile: "720p30" | "1080p30" | "1080p60"; + mode: "passthrough" | "normalize"; + }> { + const [program, outputs] = await Promise.all([ + this.resolveProgramSource(workspaceId, source), + this.resolveOutputs(workspaceId, virtualChannelId), + ]); + if (outputs.length === 0) throw new ConflictException("no enabled stream outputs for this channel"); + const target = outputs[0]!; + return { + desiredStatus: "RUNNING", + source: { kind: "hls", url: program.playbackHlsUrl }, + output: { endpointUrl: target.endpointUrl, credentialHandleId: target.credentialHandleId ?? "" }, + profile: options?.profile ?? (target.output.profile as "720p30" | "1080p30" | "1080p60"), + mode: options?.mode ?? (target.output.mode as "passthrough" | "normalize"), + }; + } + + private async listChannelOutputs(workspaceId: string, virtualChannelId: string): Promise { + return this.transaction(workspaceId, async (client) => { + const result = await client.query("SELECT * FROM stream_outputs WHERE workspace_id = $1 AND virtual_channel_id = $2 ORDER BY created_at", [workspaceId, virtualChannelId]); + return result.rows.map((row) => ({ + id: String(row.id), workspaceId: String(row.workspace_id), virtualChannelId: String(row.virtual_channel_id), + label: String(row.label), profile: String(row.profile), mode: String(row.mode), enabled: Boolean(row.enabled), + destinationId: row.destination_id === null ? null : String(row.destination_id), + credentialHandleId: row.credential_handle_id === null ? null : String(row.credential_handle_id), + createdAt: new Date(row.created_at), updatedAt: new Date(row.updated_at), + })); + }); + } +} diff --git a/apps/api/src/live/live-destination.controller.ts b/apps/api/src/live/live-destination.controller.ts new file mode 100644 index 0000000..4852f39 --- /dev/null +++ b/apps/api/src/live/live-destination.controller.ts @@ -0,0 +1,62 @@ +import { Body, Controller, Delete, Get, Inject, Param, Patch, Post, Req, UseGuards } from "@nestjs/common"; +import type { Request } from "express"; +import { parseId, requireSession } from "../http/validation.js"; +import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import { LiveDestinationRepository, parseCreateDestination, parseUpdateDestination, type DestinationRecord, type OutputSummary } from "./live-destination.repository.js"; + +@Controller("live/destinations") +@UseGuards(RbacGuard) +export class LiveDestinationController { + constructor(@Inject(LiveDestinationRepository) private readonly destinations: LiveDestinationRepository) {} + + @Post() + @RequirePermission("streamoutput.manage") + async create(@Req() request: Request & AuthenticatedRequest, @Body() body: unknown): Promise { + const session = requireSession(request); + return this.destinations.create(session.workspaceId, parseCreateDestination(body), + { userId: session.userId, role: session.role }); + } + + @Get() + @RequirePermission("streamoutput.manage") + async list(@Req() request: Request & AuthenticatedRequest): Promise { + return this.destinations.list(requireSession(request).workspaceId); + } + + @Get(":id") + @RequirePermission("streamoutput.manage") + async get(@Req() request: Request & AuthenticatedRequest, @Param("id") id: string): Promise { + return this.destinations.get(requireSession(request).workspaceId, parseId(id)); + } + + @Patch(":id") + @RequirePermission("streamoutput.manage") + async update( + @Req() request: Request & AuthenticatedRequest, + @Param("id") id: string, + @Body() body: unknown, + ): Promise { + const session = requireSession(request); + return this.destinations.update(session.workspaceId, parseId(id), parseUpdateDestination(body), + { userId: session.userId, role: session.role }); + } + + @Delete(":id") + @RequirePermission("streamoutput.manage") + async remove( + @Req() request: Request & AuthenticatedRequest, + @Param("id") id: string, + ): Promise<{ id: string; disabledOutputs: number; revokedCredential: boolean }> { + const session = requireSession(request); + const parsed = parseId(id); + const result = await this.destinations.remove(session.workspaceId, parsed, + { userId: session.userId, role: session.role }); + return { id: parsed, ...result }; + } + + @Get(":id/outputs") + @RequirePermission("streamoutput.manage") + async outputs(@Req() request: Request & AuthenticatedRequest, @Param("id") id: string): Promise { + return this.destinations.outputsForDestination(requireSession(request).workspaceId, parseId(id)); + } +} diff --git a/apps/api/src/live/live-destination.integration.test.ts b/apps/api/src/live/live-destination.integration.test.ts new file mode 100644 index 0000000..c4f0a7f --- /dev/null +++ b/apps/api/src/live/live-destination.integration.test.ts @@ -0,0 +1,171 @@ +import { randomUUID } from "node:crypto"; +import { join } from "node:path"; +import { Pool, type PoolClient } from "pg"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { runMigrations } from "../db/migrate.js"; + +/** + * Migration 022 (WP6 live destinations + stream outputs) against a live + * PostgreSQL: the exclusive-target CHECK, the FK-lifecycle rule, and FORCE + * RLS tenant isolation on both new tables. Assertions run as sp_app + * (DATABASE_URL); setup/cleanup as owner. Skipped without DATABASE_URL. + */ +const dbAvailable = process.env.DATABASE_URL !== undefined; +const ownerPool = new Pool({ connectionString: process.env.SUPER_DATABASE_URL ?? process.env.DATABASE_URL }); +const appPool = new Pool({ connectionString: process.env.DATABASE_URL }); + +const FK_VIOLATION = "23503"; +const CHECK_VIOLATION = "23514"; +const RLS_VIOLATION = "42501"; + +async function asTenant(workspaceId: string | null, fn: (client: PoolClient) => Promise): Promise { + const client = await appPool.connect(); + try { + await client.query("BEGIN"); + if (workspaceId !== null) await client.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]); + const result = await fn(client); + await client.query("COMMIT"); + return result; + } catch (error) { + await client.query("ROLLBACK").catch(() => undefined); + throw error; + } finally { + client.release(); + } +} + + +/** Each expected failure runs in its own savepoint so the surrounding tx survives. */ +async function expectQueryError(client: PoolClient, codes: string | string[], sql: string, params: unknown[]): Promise { + const accepted = Array.isArray(codes) ? codes : [codes]; + await client.query("SAVEPOINT probe"); + try { + await client.query(sql, params); + expect.fail(`expected ${sql} to fail with ${accepted.join("|")}`); + } catch (error) { + expect(accepted).toContain((error as { code?: string }).code); + } finally { + await client.query("ROLLBACK TO SAVEPOINT probe"); + } +} + +describe.skipIf(!dbAvailable)("live destinations + stream outputs (live DB)", () => { + let wsA: string; + let wsB: string; + let channelA: string; + + beforeAll(async () => { + await runMigrations(join(process.cwd(), "migrations")); + await ownerPool.query("BEGIN"); + const a = await ownerPool.query("INSERT INTO workspaces(name,slug) VALUES('Live A',$1) RETURNING id", [`la-${randomUUID().slice(0, 8)}`]); + const b = await ownerPool.query("INSERT INTO workspaces(name,slug) VALUES('Live B',$1) RETURNING id", [`lb-${randomUUID().slice(0, 8)}`]); + wsA = a.rows[0].id as string; + wsB = b.rows[0].id as string; + await ownerPool.query("SELECT set_config('app.workspace_id', $1, false)", [wsA]); + const ch = await ownerPool.query("INSERT INTO virtual_channels(workspace_id,slug,name) VALUES($1,'live','Live') RETURNING id", [wsA]); + channelA = ch.rows[0].id as string; + await ownerPool.query("COMMIT"); + }); + + afterAll(async () => { + await ownerPool.query("DELETE FROM workspaces WHERE id IN ($1,$2)", [wsA, wsB]); + await appPool.end(); + await ownerPool.end(); + }); + + async function destination(client: PoolClient, workspaceId: string, credentialId: string | null = null): Promise { + const result = await client.query( + "INSERT INTO live_stream_destinations(workspace_id,label,provider,endpoint_url,credential_id) VALUES($1,$2,'peertube','rtmps://live.example.net/app',$3) RETURNING id", + [workspaceId, `dest-${randomUUID().slice(0, 8)}`, credentialId], + ); + return result.rows[0].id as string; + } + + async function credential(client: PoolClient, workspaceId: string): Promise { + const result = await client.query( + "INSERT INTO credentials(workspace_id,kind,ciphertext,key_version) VALUES($1,'ingest-stream-key',gen_random_bytes(48),1) RETURNING id", + [workspaceId], + ); + return result.rows[0].id as string; + } + + it("holds the exclusive-target CHECK: neither or both targets rejected", async () => { + await asTenant(wsA, async (client) => { + const dest = await destination(client, wsA); + await expectQueryError(client, CHECK_VIOLATION, + "INSERT INTO stream_outputs(workspace_id,virtual_channel_id,label,destination_id,credential_handle_id) VALUES($1,$2,'none',NULL,NULL)", + [wsA, channelA]); + await expectQueryError(client, CHECK_VIOLATION, + "INSERT INTO stream_outputs(workspace_id,virtual_channel_id,label,destination_id,credential_handle_id) VALUES($1,$2,'both',$3,'vh_0123456789abcdef')", + [wsA, channelA, dest]); + await client.query( + "INSERT INTO stream_outputs(workspace_id,virtual_channel_id,label,destination_id,credential_handle_id) VALUES($1,$2,'dest',$3,NULL)", + [wsA, channelA, dest], + ); + }); + }); + + it("FK-lifecycle: deleting the destination while outputs reference it fails", async () => { + await asTenant(wsA, async (client) => { + const dest = await destination(client, wsA); + await client.query( + "INSERT INTO stream_outputs(workspace_id,virtual_channel_id,label,destination_id,credential_handle_id,enabled) VALUES($1,$2,'x',$3,NULL,false)", + [wsA, channelA, dest], + ); + await expect(client.query("DELETE FROM live_stream_destinations WHERE workspace_id=$1 AND id=$2", [wsA, dest])) + .rejects.toMatchObject({ code: FK_VIOLATION }); + }); + }); + + it("FK-lifecycle: the credential cannot be deleted before its destination", async () => { + await asTenant(wsA, async (client) => { + const cred = await credential(client, wsA); + const dest = await destination(client, wsA, cred); + await expectQueryError(client, FK_VIOLATION, + "DELETE FROM credentials WHERE workspace_id=$1 AND id=$2", [wsA, cred]); + // Orderly lifecycle: delete destination first, then the credential row. + await client.query("DELETE FROM live_stream_destinations WHERE workspace_id=$1 AND id=$2", [wsA, dest]); + await client.query("DELETE FROM credentials WHERE workspace_id=$1 AND id=$2", [wsA, cred]); + }); + }); + + it("FORCE RLS isolates both new tables across tenants", async () => { + const destA = await asTenant(wsA, async (client) => destination(client, wsA)); + await asTenant(wsB, async (client) => { + const rows = await client.query("SELECT count(*)::int AS n FROM live_stream_destinations"); + expect(rows.rows[0].n).toBe(0); + await expectQueryError(client, RLS_VIOLATION, + "INSERT INTO live_stream_destinations(workspace_id,label,provider,endpoint_url) VALUES($1,'cross','peertube','rtmps://live.example.net/app')", + [wsA]); + // A stream output of B referencing A's destination is denied — by the + // tenant policy and/or the tenant-composite FK, whichever fires first. + const channelB = await client.query( + "INSERT INTO virtual_channels(workspace_id,slug,name) VALUES($1,'live','Live') RETURNING id", [wsB]); + await expectQueryError(client, [RLS_VIOLATION, FK_VIOLATION], + "INSERT INTO stream_outputs(workspace_id,virtual_channel_id,label,destination_id,credential_handle_id) VALUES($1,$2,'cross',$3,NULL)", + [wsB, channelB.rows[0].id, destA]); + // And the denial is durable: B still has no outputs at all. + const bOutputs = await client.query("SELECT count(*)::int AS n FROM stream_outputs"); + expect(bOutputs.rows[0].n).toBe(0); + }); + const seen = await asTenant(wsA, async (client) => client.query("SELECT count(*)::int AS n FROM live_stream_destinations")); + expect(seen.rows[0].n).toBeGreaterThanOrEqual(1); + // No tenant context at all: zero rows, fail closed (FORCE binds sp_app). + const none = await asTenant(null, async (client) => client.query("SELECT count(*)::int AS n FROM live_stream_destinations")); + expect(none.rows[0].n).toBe(0); + }); + + it("rejects workspace_id changes on both tables (lifecycle immutability)", async () => { + await asTenant(wsA, async (client) => { + const dest = await destination(client, wsA); + await expectQueryError(client, CHECK_VIOLATION, + "UPDATE live_stream_destinations SET workspace_id=$2 WHERE id=$1", [dest, wsB]); + const out = (await client.query( + "INSERT INTO stream_outputs(workspace_id,virtual_channel_id,label,credential_handle_id) VALUES($1,$2,'g','vh_0123456789abcdef') RETURNING id", + [wsA, channelA], + )).rows[0].id as string; + await expectQueryError(client, CHECK_VIOLATION, + "UPDATE stream_outputs SET workspace_id=$2 WHERE id=$1", [out, wsB]); + }); + }); +}); diff --git a/apps/api/src/live/live-destination.repository.ts b/apps/api/src/live/live-destination.repository.ts new file mode 100644 index 0000000..4f3db7d --- /dev/null +++ b/apps/api/src/live/live-destination.repository.ts @@ -0,0 +1,348 @@ +import { BadRequestException, ConflictException, Inject, Injectable, NotFoundException, Optional } from "@nestjs/common"; +import type { PoolClient } from "pg"; +import { Vault } from "@tactical-media-broadcast-hub/connector-runtime"; +import { vault } from "../vault/vault.js"; +import { withTenant } from "../db/client.js"; +import type { LiveStreamDestination } from "@tactical-media-broadcast-hub/core"; +import { validateEndpointShape } from "../media-control/endpoints.js"; +import { scrub } from "../audit/audit.service.js"; + +export type DestinationPlatformState = LiveStreamDestination["lastPlatformState"]; + +export interface DestinationRecord { + id: string; + workspaceId: string; + label: string; + provider: string; + endpointUrl: string; + credentialId: string | null; + credentialRevokedAt: Date | null; + lastPlatformState: DestinationPlatformState; + createdAt: Date; + updatedAt: Date; +} + +export interface CreateDestination { + label: string; + provider: string; + endpointUrl: string; + credentialId?: string | null; +} + +export interface UpdateDestination extends Partial { + lastPlatformState?: DestinationPlatformState; +} + +export interface OutputSummary { + id: string; + workspaceId: string; + virtualChannelId: string; + label: string; + profile: string; + mode: string; + enabled: boolean; + destinationId: string | null; + credentialHandleId: string | null; + createdAt: Date; + updatedAt: Date; +} + +/** Credential material the go-live path resolves for a destination-referenced output. */ +export interface ResolvedIngestCredential { + endpointUrl: string; + /** Raw secret, only ever returned to the vault-backed relay path — never logged or audited. */ + streamKey: string; + provider: string; +} + +export type TenantTransaction = (workspaceId: string, fn: (client: PoolClient) => Promise) => Promise; + +export interface LiveDestinationDependencies { + transaction: TenantTransaction; + encryption: Vault; +} + +export const LIVE_DESTINATION_DEPENDENCIES = Symbol("LIVE_DESTINATION_DEPENDENCIES"); + +const DEFAULT_DEPENDENCIES: LiveDestinationDependencies = { transaction: withTenant, encryption: vault }; + +const LABEL = /^.{1,120}$/s; +const PROVIDER = /^[a-z0-9][a-z0-9._-]{0,63}$/; +const STATE_VALUES: readonly DestinationPlatformState[] = ["unknown", "created", "waiting", "live", "ended"]; + +function destinationFromRow(row: Record): DestinationRecord { + return { + id: String(row.id), + workspaceId: String(row.workspace_id), + label: String(row.label), + provider: String(row.provider), + endpointUrl: String(row.endpoint_url), + credentialId: row.credential_id === null ? null : String(row.credential_id), + credentialRevokedAt: row.credential_revoked_at === null || row.credential_revoked_at === undefined ? null : new Date(row.credential_revoked_at as string), + lastPlatformState: (row.last_platform_state ?? "unknown") as DestinationPlatformState, + createdAt: new Date(row.created_at as Date), + updatedAt: new Date(row.updated_at as Date), + }; +} + +function outputFromRow(row: Record): OutputSummary { + return { + id: String(row.id), + workspaceId: String(row.workspace_id), + virtualChannelId: String(row.virtual_channel_id), + label: String(row.label), + profile: String(row.profile), + mode: String(row.mode), + enabled: Boolean(row.enabled), + destinationId: row.destination_id === null ? null : String(row.destination_id), + credentialHandleId: row.credential_handle_id === null ? null : String(row.credential_handle_id), + createdAt: new Date(row.created_at as Date), + updatedAt: new Date(row.updated_at as Date), + }; +} + +export function parseCreateDestination(value: unknown): CreateDestination { + if (typeof value !== "object" || value === null) throw new BadRequestException("body must be an object"); + const body = value as Record; + const label = typeof body.label === "string" ? body.label.trim() : ""; + if (!LABEL.test(label)) throw new BadRequestException("label must be 1-120 characters"); + const provider = typeof body.provider === "string" ? body.provider.trim().toLowerCase() : ""; + if (!PROVIDER.test(provider)) throw new BadRequestException("provider must be 1-64 lowercase identifier characters"); + const endpointUrl = typeof body.endpointUrl === "string" ? body.endpointUrl : ""; + if (endpointUrl === "") throw new BadRequestException("endpointUrl is required"); + try { validateEndpointShape(endpointUrl, "output"); } catch (error: unknown) { + throw new BadRequestException(`endpointUrl: ${error instanceof Error ? error.message : "invalid endpoint"}`); + } + const credentialId = body.credentialId === undefined || body.credentialId === null ? null : String(body.credentialId); + return { label, provider, endpointUrl, credentialId }; +} + +export function parseUpdateDestination(value: unknown): UpdateDestination { + if (typeof value !== "object" || value === null) throw new BadRequestException("body must be an object"); + const body = value as Record; + const update: UpdateDestination = {}; + if (body.label !== undefined) { + const label = typeof body.label === "string" ? body.label.trim() : ""; + if (!LABEL.test(label)) throw new BadRequestException("label must be 1-120 characters"); + update.label = label; + } + if (body.provider !== undefined) { + const provider = typeof body.provider === "string" ? body.provider.trim().toLowerCase() : ""; + if (!PROVIDER.test(provider)) throw new BadRequestException("provider must be 1-64 lowercase identifier characters"); + update.provider = provider; + } + if (body.endpointUrl !== undefined) { + const endpointUrl = typeof body.endpointUrl === "string" ? body.endpointUrl : ""; + if (endpointUrl === "") throw new BadRequestException("endpointUrl cannot be empty"); + try { validateEndpointShape(endpointUrl, "output"); } catch (error: unknown) { + throw new BadRequestException(`endpointUrl: ${error instanceof Error ? error.message : "invalid endpoint"}`); + } + update.endpointUrl = endpointUrl; + } + if (body.credentialId !== undefined) update.credentialId = body.credentialId === null ? null : String(body.credentialId); + if (body.lastPlatformState !== undefined) { + if (!STATE_VALUES.includes(body.lastPlatformState as DestinationPlatformState)) throw new BadRequestException("lastPlatformState is invalid"); + update.lastPlatformState = body.lastPlatformState as DestinationPlatformState; + } + if (Object.keys(update).length === 0) throw new BadRequestException("no recognized fields to update"); + return update; +} + +@Injectable() +export class LiveDestinationRepository { + private readonly deps: LiveDestinationDependencies; + + constructor(@Optional() @Inject(LIVE_DESTINATION_DEPENDENCIES) dependencies?: Partial) { + this.deps = { ...DEFAULT_DEPENDENCIES, ...dependencies }; + } + + async list(workspaceId: string): Promise { + return this.deps.transaction(workspaceId, async (client) => { + const result = await client.query( + `SELECT d.*, c.revoked_at AS credential_revoked_at + FROM live_stream_destinations d + LEFT JOIN credentials c ON c.id = d.credential_id AND c.workspace_id = d.workspace_id + WHERE d.workspace_id = $1 ORDER BY d.label, d.id`, + [workspaceId], + ); + return result.rows.map(destinationFromRow); + }); + } + + async get(workspaceId: string, id: string): Promise { + return this.deps.transaction(workspaceId, async (client) => this.read(client, workspaceId, id)); + } + + private async read(client: PoolClient, workspaceId: string, id: string): Promise { + const result = await client.query( + `SELECT d.*, c.revoked_at AS credential_revoked_at + FROM live_stream_destinations d + LEFT JOIN credentials c ON c.id = d.credential_id AND c.workspace_id = d.workspace_id + WHERE d.workspace_id = $1 AND d.id = $2`, + [workspaceId, id], + ); + const row = result.rows[0]; + if (row === undefined) throw new NotFoundException("live stream destination not found"); + return destinationFromRow(row); + } + + async create(workspaceId: string, input: CreateDestination, actor: { userId: string | null; role: string }): Promise { + return this.deps.transaction(workspaceId, async (client) => { + await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [workspaceId]); + if (input.credentialId !== null) await this.assertLiveCredential(client, workspaceId, input.credentialId ?? ""); + const result = await client.query( + `INSERT INTO live_stream_destinations (workspace_id, label, provider, endpoint_url, credential_id) + VALUES ($1, $2, $3, $4, $5) RETURNING *`, + [workspaceId, input.label, input.provider, input.endpointUrl, input.credentialId], + ); + const record = destinationFromRow(result.rows[0] ?? {}); + await this.audit(client, workspaceId, actor, "live.destination.created", record.id, { + provider: record.provider, labeled: record.label, + }); + return record; + }); + } + + async update(workspaceId: string, id: string, input: UpdateDestination, actor: { userId: string | null; role: string }): Promise { + return this.deps.transaction(workspaceId, async (client) => { + await client.query("SELECT id FROM live_stream_destinations WHERE workspace_id = $1 AND id = $2 FOR UPDATE", [workspaceId, id]); + const before = await this.read(client, workspaceId, id); + if (input.credentialId !== undefined && input.credentialId !== null) { + await this.assertLiveCredential(client, workspaceId, input.credentialId); + } + const result = await client.query( + `UPDATE live_stream_destinations SET + label = COALESCE($3, label), provider = COALESCE($4, provider), + endpoint_url = COALESCE($5, endpoint_url), + credential_id = CASE WHEN $6 THEN $7 ELSE credential_id END, + last_platform_state = COALESCE($8, last_platform_state), + updated_at = now() + WHERE workspace_id = $1 AND id = $2 RETURNING *`, + [workspaceId, id, input.label ?? null, input.provider ?? null, input.endpointUrl ?? null, + input.credentialId !== undefined, input.credentialId ?? null, input.lastPlatformState ?? null], + ); + if (result.rows[0] === undefined) throw new NotFoundException("live stream destination not found"); + const record = destinationFromRow(result.rows[0]); + await this.audit(client, workspaceId, actor, "live.destination.updated", id, { + providerChanged: input.provider !== undefined && input.provider !== before.provider, + endpointChanged: input.endpointUrl !== undefined && input.endpointUrl !== before.endpointUrl, + credentialChanged: input.credentialId !== undefined, + }); + return record; + }); + } + + /** + * WP6 lifecycle rule: 409 while a referencing output is enabled; otherwise + * disable dependents, revoke the ingest credential, delete the row. + */ + async remove(workspaceId: string, id: string, actor: { userId: string | null; role: string }): Promise<{ disabledOutputs: number; revokedCredential: boolean }> { + return this.deps.transaction(workspaceId, async (client) => { + await client.query("SELECT id FROM live_stream_destinations WHERE workspace_id = $1 AND id = $2 FOR UPDATE", [workspaceId, id]); + const record = await this.read(client, workspaceId, id); + const referencing = await client.query<{ id: string; enabled: boolean }>( + "SELECT id, enabled FROM stream_outputs WHERE workspace_id = $1 AND destination_id = $2", + [workspaceId, id], + ); + const enabled = referencing.rows.filter((row) => row.enabled); + if (enabled.length > 0) { + throw new ConflictException( + `live stream destination is referenced by ${enabled.length} enabled stream output(s); disable them first`, + ); + } + const disabled = await client.query( + "UPDATE stream_outputs SET enabled = false, updated_at = now() WHERE workspace_id = $1 AND destination_id = $2 AND enabled RETURNING id", + [workspaceId, id], + ); + const deleted = await client.query("DELETE FROM live_stream_destinations WHERE workspace_id = $1 AND id = $2 RETURNING id", [workspaceId, id]); + if (deleted.rows[0] === undefined) throw new NotFoundException("live stream destination not found"); + let revoked = false; + if (record.credentialId !== null) { + await client.query("UPDATE credentials SET revoked_at = now() WHERE id = $1 AND workspace_id = $2 AND revoked_at IS NULL", [record.credentialId, workspaceId]); + revoked = true; + } + await this.audit(client, workspaceId, actor, "live.destination.deleted", id, { + disabledOutputs: disabled.rows.length, revokedCredential: revoked, + }); + return { disabledOutputs: disabled.rows.length, revokedCredential: revoked }; + }); + } + + async outputsForDestination(workspaceId: string, destinationId: string): Promise { + return this.deps.transaction(workspaceId, async (client) => { + const result = await client.query( + "SELECT * FROM stream_outputs WHERE workspace_id = $1 AND destination_id = $2 ORDER BY created_at", + [workspaceId, destinationId], + ); + return result.rows.map(outputFromRow); + }); + } + + /** + * Credential revoke disables dependents (WP9 lifecycle): a revoked ingest + * credential turns every destination-referencing output disabled so no new + * go-live can redeem it. Returns how many outputs were disabled. + */ + async disableDependentsOfCredential(workspaceId: string, credentialId: string): Promise { + return this.deps.transaction(workspaceId, async (client) => { + const result = await client.query( + `UPDATE stream_outputs o SET enabled = false, updated_at = now() + FROM live_stream_destinations d + WHERE o.workspace_id = $1 AND o.destination_id = d.id AND d.workspace_id = $1 + AND d.credential_id = $2 AND o.enabled + RETURNING o.id`, + [workspaceId, credentialId], + ); + return result.rows.length; + }); + } + + /** + * Go-live credential resolution (vault pattern of the publish credential + * accessor): destination -> credential row -> vault.open with tenant + handle + * AAD. Fails closed on a revoked credential, a cross-tenant handle, or a + * shape violation. Raw material never leaves this path. + */ + async resolveIngestCredential(workspaceId: string, destinationId: string): Promise { + return this.deps.transaction(workspaceId, async (client) => { + const result = await client.query<{ id: string; provider: string; endpoint_url: string; credential_id: string | null }>( + "SELECT id, provider, endpoint_url, credential_id FROM live_stream_destinations WHERE workspace_id = $1 AND id = $2", + [workspaceId, destinationId], + ); + const row = result.rows[0]; + if (row === undefined) throw new NotFoundException("live stream destination not found"); + if (row.credential_id === null) return { endpointUrl: row.endpoint_url, streamKey: "", provider: row.provider }; + const credential = await client.query<{ ciphertext: Buffer; key_version: number; revoked_at: Date | null; kind: string }>( + "SELECT ciphertext, key_version, revoked_at, kind FROM credentials WHERE id = $1 AND workspace_id = $2", + [row.credential_id, workspaceId], + ); + const cred = credential.rows[0]; + if (cred === undefined) throw new NotFoundException("destination credential not found"); + if (cred.revoked_at !== null) throw new ConflictException("destination credential is revoked"); + const plaintext = await this.deps.encryption.open(workspaceId, row.id, { ciphertext: cred.ciphertext, keyVersion: cred.key_version }); + const secrets: unknown = JSON.parse(plaintext.toString()); + if (typeof secrets !== "object" || secrets === null || Array.isArray(secrets)) { + throw new ConflictException("destination credential payload is malformed"); + } + const key = (secrets as Record).stream_key; + if (key !== undefined && typeof key !== "string") throw new ConflictException("destination credential payload is malformed"); + return { endpointUrl: row.endpoint_url, streamKey: typeof key === "string" ? key : "", provider: row.provider }; + }); + } + + private async assertLiveCredential(client: PoolClient, workspaceId: string, credentialId: string): Promise { + const result = await client.query<{ revoked_at: Date | null }>( + "SELECT revoked_at FROM credentials WHERE id = $1 AND workspace_id = $2", + [credentialId, workspaceId], + ); + const row = result.rows[0]; + if (row === undefined) throw new BadRequestException("credentialId does not reference a credential in this workspace"); + if (row.revoked_at !== null) throw new BadRequestException("credentialId references a revoked credential"); + } + + private async audit(client: PoolClient, workspaceId: string, actor: { userId: string | null; role: string }, + action: string, targetId: string, diff: Record): Promise { + await client.query("SELECT audit_append($1,$2,$3,$4,$5,$6,$7)", + [workspaceId, actor.userId, actor.role, action, "live_destination", targetId, scrub(diff)]); + } +} diff --git a/apps/api/src/live/live-destination.test.ts b/apps/api/src/live/live-destination.test.ts new file mode 100644 index 0000000..81a97d8 --- /dev/null +++ b/apps/api/src/live/live-destination.test.ts @@ -0,0 +1,331 @@ +import { randomUUID } from "node:crypto"; +import type { PoolClient } from "pg"; +import { ConflictException, NotFoundException } from "@nestjs/common"; +import { describe, expect, it } from "vitest"; +import { Vault } from "@tactical-media-broadcast-hub/connector-runtime"; +import { + LiveDestinationRepository, parseCreateDestination, parseUpdateDestination, type TenantTransaction, +} from "./live-destination.repository.js"; +import { GoLiveService } from "./go-live.service.js"; +import { createGenericHlsResolver, type LiveSourceResolver } from "./live-source-resolver.js"; + +/** + * Live-destination lifecycle (WP6/WP9 aggregate boundary) against an + * in-memory tenant-transaction double: delete-blocked-while-referenced, + * credential revoke disables dependents, and go-live resolution of + * StreamOutput -> LiveStreamDestination -> credential via the vault. + */ + +interface DestinationRow { + id: string; workspace_id: string; label: string; provider: string; endpoint_url: string; + credential_id: string | null; last_platform_state: string; created_at: Date; updated_at: Date; +} +interface OutputRow { + id: string; workspace_id: string; virtual_channel_id: string; label: string; profile: string; + mode: string; enabled: boolean; destination_id: string | null; credential_handle_id: string | null; + created_at: Date; updated_at: Date; +} +interface CredentialRow { id: string; workspace_id: string; ciphertext: Buffer; key_version: number; revoked_at: Date | null } + +const WS = randomUUID(); +const ACTOR = { userId: randomUUID(), role: "Owner" }; + +function store(): { + destinations: DestinationRow[]; outputs: OutputRow[]; credentials: CredentialRow[]; + transaction: TenantTransaction; client: PoolClient; +} { + const destinations: DestinationRow[] = []; + const outputs: OutputRow[] = []; + const credentials: CredentialRow[] = []; + const audit: unknown[][] = []; + const transaction: TenantTransaction = async (_ws: string, fn: (client: PoolClient) => Promise): Promise => { + const query = async (sql: string, args: unknown[]): Promise<{ rows: Record[]; rowCount: number }> => { + const table = sql.includes("FROM live_stream_destinations") || sql.includes("INTO live_stream_destinations") || sql.includes("UPDATE live_stream_destinations") || sql.includes("DELETE FROM live_stream_destinations") ? "destinations" + : sql.includes("stream_outputs") ? "outputs" + : sql.includes("FROM credentials") || sql.includes("UPDATE credentials") ? "credentials" : "other"; + if (sql.startsWith("SELECT")) { + if (table === "destinations" && sql.includes("LEFT JOIN credentials")) { + const rows = destinations.filter(d => sql.includes("$2") ? d.id === args[1] : d.workspace_id === args[0]) + .map(d => ({ ...d, credential_revoked_at: credentials.find(c => c.id === d.credential_id)?.revoked_at ?? null })); + return { rows: rows as unknown as Record[], rowCount: rows.length }; + } + if (table === "destinations") { + const rows = destinations.filter(d => d.id === args[1] && d.workspace_id === args[0]); + return { rows: rows as unknown as Record[], rowCount: rows.length }; + } + if (table === "outputs") { + const rows = outputs.filter(o => o.workspace_id === args[0] && (sql.includes("destination_id") ? o.destination_id === args[1] : o.virtual_channel_id === args[1])); + return { rows: rows as unknown as Record[], rowCount: rows.length }; + } + if (table === "credentials") { + const rows = credentials.filter(c => c.id === args[0] && c.workspace_id === args[1]); + return { rows: rows as unknown as Record[], rowCount: rows.length }; + } + if (sql.includes("workspaces") || sql.includes("virtual_channels") || sql.includes("audit_log")) { + return { rows: [{ id: args[0] ?? randomUUID() }], rowCount: 1 }; + } + return { rows: [], rowCount: 0 }; + } + if (sql.startsWith("INSERT INTO live_stream_destinations")) { + const row: DestinationRow = { id: randomUUID(), workspace_id: String(args[0]), label: String(args[1]), + provider: String(args[2]), endpoint_url: String(args[3]), credential_id: args[4] === null ? null : String(args[4]), + last_platform_state: "unknown", created_at: new Date(), updated_at: new Date() }; + destinations.push(row); + return { rows: [row as unknown as Record], rowCount: 1 }; + } + if (sql.startsWith("INSERT INTO stream_outputs")) { + const row: OutputRow = { id: randomUUID(), workspace_id: String(args[0]), virtual_channel_id: String(args[1]), + label: String(args[2]), profile: String(args[3]), mode: String(args[4]), enabled: Boolean(args[5]), + destination_id: args[6] === null ? null : String(args[6]), credential_handle_id: args[7] === null ? null : String(args[7]), + created_at: new Date(), updated_at: new Date() }; + outputs.push(row); + return { rows: [row as unknown as Record], rowCount: 1 }; + } + if (sql.startsWith("UPDATE stream_outputs SET enabled = false")) { + const ids = outputs.filter(o => o.destination_id === args[1] && o.enabled).map(o => { o.enabled = false; o.updated_at = new Date(); return { id: o.id }; }); + return { rows: ids, rowCount: ids.length }; + } + if (sql.startsWith("UPDATE stream_outputs o SET enabled = false")) { + const ids = outputs.filter(o => { const d = destinations.find(x => x.id === o.destination_id); return d !== undefined && d.credential_id === args[1] && o.enabled; }) + .map(o => { o.enabled = false; o.updated_at = new Date(); return { id: o.id }; }); + return { rows: ids, rowCount: ids.length }; + } + if (sql.startsWith("UPDATE stream_outputs")) { + const target = outputs.find(o => o.id === args[10] && o.workspace_id === args[0]); + if (target === undefined) return { rows: [], rowCount: 0 }; + if (args[2] !== null) target.label = String(args[2]); + if (args[3] !== null) target.enabled = Boolean(args[3]); + return { rows: [target as unknown as Record], rowCount: 1 }; + } + if (sql.startsWith("UPDATE live_stream_destinations")) { + const target = destinations.find(d => d.id === args[1] && d.workspace_id === args[0]); + if (target === undefined) return { rows: [], rowCount: 0 }; + if (args[2] !== null) target.label = String(args[2]); + if (args[3] !== null) target.provider = String(args[3]); + if (args[4] !== null) target.endpoint_url = String(args[4]); + if (args[6]) target.credential_id = args[7] === null ? null : String(args[7]); + if (args[7] !== null) target.last_platform_state = String(args[7]); + target.updated_at = new Date(); + return { rows: [target as unknown as Record], rowCount: 1 }; + } + if (sql.startsWith("UPDATE credentials SET revoked_at")) { + const target = credentials.find(c => c.id === args[0] && c.workspace_id === args[1] && c.revoked_at === null); + if (target !== undefined) target.revoked_at = new Date(); + return { rows: [], rowCount: 1 }; + } + if (sql.startsWith("DELETE FROM live_stream_destinations")) { + const index = destinations.findIndex(d => d.id === args[1] && d.workspace_id === args[0]); + if (index === -1) return { rows: [], rowCount: 0 }; + const [row] = destinations.splice(index, 1); + return { rows: [row as unknown as Record], rowCount: 1 }; + } + if (sql.startsWith("DELETE FROM stream_outputs")) { + const index = outputs.findIndex(o => o.id === args[2] && o.workspace_id === args[0]); + if (index === -1) return { rows: [], rowCount: 0 }; + outputs.splice(index, 1); + return { rows: [], rowCount: 1 }; + } + if (sql.startsWith("SELECT audit_append")) { audit.push(args); return { rows: [{ id: randomUUID() }], rowCount: 1 }; } + return { rows: [], rowCount: 0 }; + }; + return fn({ query } as unknown as PoolClient); + }; + return { destinations, outputs, credentials, transaction, client: {} as PoolClient }; +} + +function repository(over: Partial[0]> = {}): LiveDestinationRepository { + return new LiveDestinationRepository(over); +} + +describe("live stream destination CRUD + lifecycle", () => { + it("creates, lists, updates and reads destinations", async () => { + const s = store(); + const repo = repository({ transaction: s.transaction }); + const created = await repo.create(WS, parseCreateDestination({ label: "Twitch main", provider: "twitch", endpointUrl: "rtmps://live.example.net/app", credentialId: null }), ACTOR); + expect(created.label).toBe("Twitch main"); + expect(created.provider).toBe("twitch"); + const listed = await repo.list(WS); + expect(listed).toHaveLength(1); + const updated = await repo.update(WS, created.id, parseUpdateDestination({ label: "Twitch backup", lastPlatformState: "live" }), ACTOR); + expect(updated.label).toBe("Twitch backup"); + expect(updated.lastPlatformState).toBe("live"); + await expect(repo.get(WS, created.id)).resolves.toMatchObject({ label: "Twitch backup" }); + }); + + it("rejects bad bodies at parse time (label, provider, endpoint shape)", () => { + expect(() => parseCreateDestination({ label: "", provider: "twitch", endpointUrl: "rtmps://live.example.net/app" })).toThrow("label"); + expect(() => parseCreateDestination({ label: "x", provider: "Twitch!", endpointUrl: "rtmps://live.example.net/app" })).toThrow("provider"); + expect(() => parseCreateDestination({ label: "x", provider: "twitch", endpointUrl: "https://live.example.net/app" })).toThrow("endpointUrl"); + expect(() => parseCreateDestination({ label: "x", provider: "twitch", endpointUrl: "rtmp://user:pw@live.example.net/app" })).toThrow("endpointUrl"); + expect(() => parseUpdateDestination({})).toThrow("no recognized fields"); + expect(() => parseUpdateDestination({ lastPlatformState: "exploded" })).toThrow("lastPlatformState"); + }); + + it("delete is blocked (409) while an enabled output references the destination", async () => { + const s = store(); + const repo = repository({ transaction: s.transaction }); + const destination = await repo.create(WS, parseCreateDestination({ label: "D", provider: "twitch", endpointUrl: "rtmps://live.example.net/app" }), ACTOR); + s.outputs.push({ id: randomUUID(), workspace_id: WS, virtual_channel_id: randomUUID(), label: "O", profile: "720p30", + mode: "passthrough", enabled: true, destination_id: destination.id, credential_handle_id: null, + created_at: new Date(), updated_at: new Date() }); + await expect(repo.remove(WS, destination.id, ACTOR)).rejects.toThrow(ConflictException); + await expect(repo.remove(WS, destination.id, ACTOR)).rejects.toThrow(/referenced by 1 enabled/); + expect(s.destinations).toHaveLength(1); + }); + + it("delete proceeds once the referencing output is disabled: dependents disabled, credential revoked, row removed", async () => { + const s = store(); + const repo = repository({ transaction: s.transaction }); + const credential = randomUUID(); + s.credentials.push({ id: credential, workspace_id: WS, ciphertext: Buffer.alloc(8), key_version: 1, revoked_at: null }); + const destination = await repo.create(WS, parseCreateDestination({ label: "D", provider: "twitch", endpointUrl: "rtmps://live.example.net/app", credentialId: credential }), ACTOR); + s.outputs.push({ id: randomUUID(), workspace_id: WS, virtual_channel_id: randomUUID(), label: "O", profile: "720p30", + mode: "passthrough", enabled: false, destination_id: destination.id, credential_handle_id: null, + created_at: new Date(), updated_at: new Date() }); + const result = await repo.remove(WS, destination.id, ACTOR); + expect(result).toEqual({ disabledOutputs: 0, revokedCredential: true }); + expect(s.destinations).toHaveLength(0); + expect(s.credentials[0]?.revoked_at).not.toBeNull(); + await expect(repo.get(WS, destination.id)).rejects.toThrow(NotFoundException); + }); + + it("credential revoke disables dependents (outputs referencing a destination on that credential)", async () => { + const s = store(); + const repo = repository({ transaction: s.transaction }); + const credential = randomUUID(); + s.credentials.push({ id: credential, workspace_id: WS, ciphertext: Buffer.alloc(8), key_version: 1, revoked_at: null }); + const destination = await repo.create(WS, parseCreateDestination({ label: "D", provider: "peertube", endpointUrl: "rtmp://live.example.net/app", credentialId: credential }), ACTOR); + const channel = randomUUID(); + s.outputs.push({ id: randomUUID(), workspace_id: WS, virtual_channel_id: channel, label: "A", profile: "720p30", + mode: "passthrough", enabled: true, destination_id: destination.id, credential_handle_id: null, created_at: new Date(), updated_at: new Date() }); + s.outputs.push({ id: randomUUID(), workspace_id: WS, virtual_channel_id: channel, label: "B", profile: "720p30", + mode: "passthrough", enabled: true, destination_id: destination.id, credential_handle_id: null, created_at: new Date(), updated_at: new Date() }); + const disabled = await repo.disableDependentsOfCredential(WS, credential); + expect(disabled).toBe(2); + expect(s.outputs.every(o => !o.enabled)).toBe(true); + }); + + it("refuses a credential from another workspace or a revoked one on create", async () => { + const s = store(); + const repo = repository({ transaction: s.transaction }); + const foreign = randomUUID(); + s.credentials.push({ id: foreign, workspace_id: randomUUID(), ciphertext: Buffer.alloc(8), key_version: 1, revoked_at: null }); + await expect(repo.create(WS, parseCreateDestination({ label: "D", provider: "twitch", endpointUrl: "rtmps://live.example.net/app", credentialId: foreign }), ACTOR)) + .rejects.toThrow(/credential in this workspace/); + const revoked = randomUUID(); + s.credentials.push({ id: revoked, workspace_id: WS, ciphertext: Buffer.alloc(8), key_version: 1, revoked_at: new Date() }); + await expect(repo.create(WS, parseCreateDestination({ label: "D", provider: "twitch", endpointUrl: "rtmps://live.example.net/app", credentialId: revoked }), ACTOR)) + .rejects.toThrow(/revoked/); + }); +}); + +describe("go-live wiring (StreamOutput -> destination -> vault credential)", () => { + const CHANNEL = randomUUID(); + + function vaultFixture(): { vault: Vault; secrets: Map } { + const real = new Vault(); + const secrets = new Map(); + const vault = { + async seal(workspaceId: string, id: string, plaintext: Buffer) { return real.seal(workspaceId, id, plaintext); }, + async open(workspaceId: string, id: string, _sealed: { ciphertext: Buffer; keyVersion: number }) { + return Buffer.from(secrets.get(`${workspaceId}:${id}`) ?? ""); + }, + } as unknown as Vault; + return { vault, secrets }; + } + + it("resolves enabled destination-referenced outputs through the destination and its credential", async () => { + const s = store(); + const { vault, secrets } = vaultFixture(); + const repo = repository({ transaction: s.transaction, encryption: vault }); + const credential = randomUUID(); + s.credentials.push({ id: credential, workspace_id: WS, ciphertext: Buffer.alloc(8), key_version: 1, revoked_at: null }); + const destination = await repo.create(WS, parseCreateDestination({ label: "D", provider: "twitch", endpointUrl: "rtmps://live.example.net/app", credentialId: credential }), ACTOR); + s.outputs.push({ id: randomUUID(), workspace_id: WS, virtual_channel_id: CHANNEL, label: "A", profile: "1080p30", + mode: "normalize", enabled: true, destination_id: destination.id, credential_handle_id: null, created_at: new Date(), updated_at: new Date() }); + s.outputs.push({ id: randomUUID(), workspace_id: WS, virtual_channel_id: CHANNEL, label: "B", profile: "720p30", + mode: "passthrough", enabled: false, destination_id: destination.id, credential_handle_id: null, created_at: new Date(), updated_at: new Date() }); + const goLive = new GoLiveService(repo, { transaction: s.transaction }); + const targets = await goLive.resolveOutputs(WS, CHANNEL); + expect(targets).toHaveLength(1); + expect(targets[0]).toMatchObject({ endpointUrl: "rtmps://live.example.net/app", throughDestination: true, credentialHandleId: credential }); + expect(targets[0]?.output.label).toBe("A"); + // Redeem path: vault.open with tenant+handle AAD, raw key surfaced only here. + secrets.set(`${WS}:${destination.id}`, JSON.stringify({ stream_key: "live_123" })); + const redeemed = await goLive.redeemDestinationCredential(WS, destination.id); + expect(redeemed).toEqual({ endpointUrl: "rtmps://live.example.net/app", streamKey: "live_123", provider: "twitch" }); + }); + + it("fails closed when the destination credential is revoked or the destination has none", async () => { + const s = store(); + const { vault } = vaultFixture(); + const repo = repository({ transaction: s.transaction, encryption: vault }); + const credential = randomUUID(); + s.credentials.push({ id: credential, workspace_id: WS, ciphertext: Buffer.alloc(8), key_version: 1, revoked_at: new Date() }); + // The repository (correctly) refuses to create a destination on a revoked + // credential, so this destination was revoked after creation. + const revokedDestination = { id: randomUUID(), workspace_id: WS, label: "R", provider: "twitch", + endpoint_url: "rtmps://live.example.net/app", credential_id: credential, last_platform_state: "unknown", + created_at: new Date(), updated_at: new Date() }; + s.destinations.push(revokedDestination); + s.outputs.push({ id: randomUUID(), workspace_id: WS, virtual_channel_id: CHANNEL, label: "A", profile: "720p30", + mode: "passthrough", enabled: true, destination_id: revokedDestination.id, credential_handle_id: null, created_at: new Date(), updated_at: new Date() }); + const goLive = new GoLiveService(repo, { transaction: s.transaction }); + await expect(goLive.resolveOutputs(WS, CHANNEL)).rejects.toThrow(/revoked/); + await expect(goLive.redeemDestinationCredential(WS, revokedDestination.id)).rejects.toThrow(/revoked/); + }); + + it("resolves the program source through provider resolvers and refuses non-live sources", async () => { + const s = store(); + const repo = repository({ transaction: s.transaction }); + const seen: string[] = []; + const goLive = new GoLiveService(repo, { + transaction: s.transaction, + sourceResolvers: new Map([ + ["twitch", async (request) => { seen.push(request.channelId ?? ""); return { playbackHlsUrl: "https://usher.example.com/a.m3u8", live: true }; }], + ["offline-provider", async () => ({ playbackHlsUrl: null, live: false })], + ]), + }); + const resolved = await goLive.resolveProgramSource(WS, { provider: "twitch", channelId: "someone" }); + expect(resolved).toEqual({ playbackHlsUrl: "https://usher.example.com/a.m3u8", live: true, provider: "twitch" }); + expect(seen).toEqual(["someone"]); + await expect(goLive.resolveProgramSource(WS, { provider: "offline-provider", channelId: "x" })).rejects.toThrow(/not live/); + await expect(goLive.resolveProgramSource(WS, { provider: "nope", channelId: "x" })).rejects.toThrow(NotFoundException); + }); + + it("builds a RUNNING DesiredSpec for the take/start path from outputs + resolved source", async () => { + const s = store(); + const repo = repository({ transaction: s.transaction }); + const destination = await repo.create(WS, parseCreateDestination({ label: "D", provider: "twitch", endpointUrl: "rtmps://live.example.net/app" }), ACTOR); + s.credentials.push({ id: "cred-go-live", workspace_id: WS, ciphertext: Buffer.alloc(8), key_version: 1, revoked_at: null }); + s.destinations.forEach(d => { if (d.id === destination.id) d.credential_id = "cred-go-live"; }); + s.outputs.push({ id: randomUUID(), workspace_id: WS, virtual_channel_id: CHANNEL, label: "A", profile: "1080p60", + mode: "normalize", enabled: true, destination_id: destination.id, credential_handle_id: null, created_at: new Date(), updated_at: new Date() }); + const goLive = new GoLiveService(repo, { + transaction: s.transaction, + sourceResolvers: new Map([["twitch", async () => ({ playbackHlsUrl: "https://usher.example.com/a.m3u8", live: true })]]), + }); + const spec = await goLive.desiredSpecForGoLive(WS, CHANNEL, { provider: "twitch", channelId: "someone" }); + expect(spec).toEqual({ + desiredStatus: "RUNNING", + source: { kind: "hls", url: "https://usher.example.com/a.m3u8" }, + output: { endpointUrl: "rtmps://live.example.net/app", credentialHandleId: "cred-go-live" }, + profile: "1080p60", + mode: "normalize", + }); + // The spec is exactly MediaControlService.setDesired's DesiredSpec shape. + await expect(goLive.desiredSpecForGoLive(WS, randomUUID(), { provider: "twitch" })).rejects.toThrow(/no enabled stream outputs/); + }); + + it("generic-hls resolver uses the external URL directly and probes liveness", async () => { + const requests: string[] = []; + const resolver = createGenericHlsResolver(async (input) => { requests.push(String(input)); return new Response("", { status: 200 }); }, 250); + await expect(resolver({ externalUrl: "https://cdn.example.com/live/index.m3u8" })) + .resolves.toEqual({ playbackHlsUrl: "https://cdn.example.com/live/index.m3u8", live: true }); + expect(requests).toEqual(["https://cdn.example.com/live/index.m3u8"]); + const failing = createGenericHlsResolver(async () => new Response("gone", { status: 404 }), 250); + await expect(failing({ externalUrl: "https://cdn.example.com/live/index.m3u8" })) + .resolves.toEqual({ playbackHlsUrl: "https://cdn.example.com/live/index.m3u8", live: false }); + await expect(resolver({})).rejects.toThrow("externalUrl"); + }); +}); diff --git a/apps/api/src/live/live-source-resolver.ts b/apps/api/src/live/live-source-resolver.ts new file mode 100644 index 0000000..c2f481d --- /dev/null +++ b/apps/api/src/live/live-source-resolver.ts @@ -0,0 +1,72 @@ +import type { LiveSourceResolution } from "@tactical-media-broadcast-hub/core"; +import type { ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk"; + +/** + * Source resolvers (WP6 go-live slice). Connector-level: each connector owns + * its provider behaviour; the app resolves by provider id through the + * connector registry and never switches on provider names. + */ +export interface LiveSourceRequest { + /** Provider channel identity (e.g. a Twitch login). */ + channelId?: string; + /** External URL naming the same channel (provider-specific shape). */ + externalUrl?: string; +} + +export type LiveSourceResolver = (request: LiveSourceRequest) => Promise; + +export class LiveSourceNotFoundError extends Error { + constructor(readonly provider: string) { super(`no live source resolver for provider "${provider}"`); } +} + +/** + * Registry-shaped resolver: resolves by provider id through the connector + * registry. `connection` supplies the credential context a connector needs + * (some connectors call Helix with a bound token; callers pass the connection + * associated with the destination/source being resolved). + */ +export function createLiveSourceResolver(registry: ConnectorRegistry): (provider: string, request: LiveSourceRequest, connectionId?: string) => Promise { + return async (provider, request, connectionId) => { + if (!registry.has(provider)) throw new LiveSourceNotFoundError(provider); + const connector = registry.get(provider); + if (connector.resolveLiveSource === undefined) throw new LiveSourceNotFoundError(provider); + // A minimal connection carries the credential ref; connectors that need no + // token (generic-hls) ignore it. + const connection = { + id: connectionId ?? `${provider}:live-source`, + workspaceId: "live-source", + provider, + channelId: request.channelId ?? request.externalUrl ?? "", + credentialId: { kind: "credential-ref" as const, id: connectionId ?? `${provider}:live-source` }, + fields: {}, + createdAt: new Date(), + }; + return connector.resolveLiveSource(connection, request); + }; +} + +/** + * generic-hls: the external URL is the playback URL. Liveness is probed with a + * bounded HEAD/GET of the playlist; 2xx/3xx and a 4xx/5xx without transport + * error are both definitive, only a transport failure reports unknown (live + * kept false and playbackUrl still returned so callers can poll). + */ +export function createGenericHlsResolver(fetchImpl: typeof fetch = fetch, timeoutMs = 5_000): LiveSourceResolver { + return async (request) => { + const url = request.externalUrl; + if (url === undefined || url.length === 0) throw new Error("generic-hls resolver requires externalUrl"); + let live = false; + try { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetchImpl(url, { method: "GET", signal: controller.signal, redirect: "follow" }); + live = response.ok; + } finally { clearTimeout(timer); } + } catch { live = false; } + return { playbackHlsUrl: url, live }; + }; +} + +/** Registers the generic-hls pseudo connector id used by destinations/sources with no dedicated connector. */ +export const GENERIC_HLS_PROVIDER = "generic-hls"; diff --git a/apps/api/src/live/live.module.test.ts b/apps/api/src/live/live.module.test.ts new file mode 100644 index 0000000..041fef6 --- /dev/null +++ b/apps/api/src/live/live.module.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; +import { LiveModule } from "./live.module.js"; +import { LiveDestinationRepository } from "./live-destination.repository.js"; +import { GoLiveService } from "./go-live.service.js"; +import { StreamOutputController } from "./stream-output.controller.js"; +import { LiveDestinationController } from "./live-destination.controller.js"; +import type { INestApplicationContext } from "@nestjs/common"; +import { NestFactory } from "@nestjs/core"; + +/** + * Module wiring smoke test: LiveModule is importable without a database (the + * registry registration is env-gated) and provides its services/controllers + * with the DI graph intact. + */ +describe("LiveModule wiring", () => { + it("provides the repository, go-live service and controllers", async () => { + const hadDatabaseUrl = process.env.DATABASE_URL; // registry registration is DB-gated; module must still construct + delete process.env.DATABASE_URL; + const app: INestApplicationContext = await NestFactory.createApplicationContext(LiveModule, { logger: false }); + try { + expect(app.get(LiveDestinationRepository)).toBeInstanceOf(LiveDestinationRepository); + const goLive = app.get(GoLiveService, { strict: false }); + expect(goLive).toBeInstanceOf(GoLiveService); + expect(app.get(LiveDestinationController)).toBeInstanceOf(LiveDestinationController); + expect(app.get(StreamOutputController)).toBeInstanceOf(StreamOutputController); + } finally { + await app.close(); + if (hadDatabaseUrl !== undefined) process.env.DATABASE_URL = hadDatabaseUrl; + } + }); +}); diff --git a/apps/api/src/live/live.module.ts b/apps/api/src/live/live.module.ts new file mode 100644 index 0000000..df724ca --- /dev/null +++ b/apps/api/src/live/live.module.ts @@ -0,0 +1,58 @@ +import { Module } from "@nestjs/common"; +import { withTenant } from "../db/client.js"; +import { LiveDestinationController } from "./live-destination.controller.js"; +import { LiveDestinationRepository, LIVE_DESTINATION_DEPENDENCIES, type TenantTransaction } from "./live-destination.repository.js"; +import { GoLiveService, GO_LIVE_DEPENDENCIES } from "./go-live.service.js"; +import { StreamOutputController } from "./stream-output.controller.js"; +import { createPublishingRegistry } from "@tactical-media-broadcast-hub/connector-runtime"; +import { pool } from "../db/client.js"; + +/** Registers connector-level live source resolvers from the connector registry. */ +function registerSourceResolvers(goLive: GoLiveService): void { + // Registry pattern: the provider set comes from the publishing registry — + // connector packages (twitch et al.) own provider behaviour, so every + // connector implementing resolveLiveSource registers under its own manifest + // id; generic-hls resolves through the built-in prober. No provider names + // branch here. + if (process.env.DATABASE_URL === undefined) return; + try { + const registry = createPublishingRegistry(pool); + for (const connector of registry.list()) { + const { id } = connector.manifest(); + if (connector.resolveLiveSource === undefined) continue; + const resolveLiveSource = connector.resolveLiveSource.bind(connector); + goLive.registerSourceResolver(id, async (request) => resolveLiveSource( + { + id: `${id}:live-source`, workspaceId: "live-source", provider: id, + channelId: request.channelId ?? request.externalUrl ?? "", + credentialId: { kind: "credential-ref", id: `${id}:live-source` }, + fields: {}, createdAt: new Date(), + }, + request, + )); + } + } catch { /* No connectors configured in this process: only generic-hls stays registered. */ } +} + +@Module({ + controllers: [LiveDestinationController, StreamOutputController], + providers: [ + { provide: "STREAM_OUTPUT_TRANSACTION", useValue: withTenant satisfies TenantTransaction }, + { + provide: LiveDestinationRepository, + useFactory: (dependencies?: unknown) => new LiveDestinationRepository(dependencies as never), + inject: [{ token: LIVE_DESTINATION_DEPENDENCIES, optional: true }], + }, + { + provide: GoLiveService, + useFactory: (destinations: LiveDestinationRepository, dependencies?: unknown) => { + const service = new GoLiveService(destinations, dependencies as never); + registerSourceResolvers(service); + return service; + }, + inject: [LiveDestinationRepository, { token: GO_LIVE_DEPENDENCIES, optional: true }], + }, + ], + exports: [LiveDestinationRepository, GoLiveService], +}) +export class LiveModule {} diff --git a/apps/api/src/live/stream-output.controller.ts b/apps/api/src/live/stream-output.controller.ts new file mode 100644 index 0000000..7cebb65 --- /dev/null +++ b/apps/api/src/live/stream-output.controller.ts @@ -0,0 +1,162 @@ +import { BadRequestException, Body, Controller, Delete, Get, Inject, Param, Patch, Post, Req, UseGuards } from "@nestjs/common"; +import type { Request } from "express"; +import type { PoolClient } from "pg"; +import { parseId, requireSession } from "../http/validation.js"; +import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import { streamOutputTarget } from "@tactical-media-broadcast-hub/core"; +import { LiveDestinationRepository } from "./live-destination.repository.js"; +import type { OutputSummary, TenantTransaction } from "./live-destination.repository.js"; + +/** Creates/edits stream outputs (the StreamOutput half of the WP9 boundary). */ +@Controller("channels/:channelId/outputs") +@UseGuards(RbacGuard) +export class StreamOutputController { + constructor( + @Inject(LiveDestinationRepository) private readonly destinations: LiveDestinationRepository, + @Inject("STREAM_OUTPUT_TRANSACTION") private readonly transaction: TenantTransaction, + ) {} + + @Post() + @RequirePermission("streamoutput.manage") + async create(@Req() request: Request & AuthenticatedRequest, @Param("channelId") channelId: string, @Body() body: unknown): Promise { + const session = requireSession(request); + const parsed = parseId(channelId, "channelId"); + const dto = parseCreateOutput(body); + return this.transaction(session.workspaceId, async (client) => { + await this.assertChannel(client, session.workspaceId, parsed); + // Type + API validation of the aggregate boundary: exactly one target. + const target = streamOutputTarget({ destinationId: dto.destinationId, credentialHandleId: dto.credentialHandleId }); + if (target === null) throw new BadRequestException("output must reference a destination or its own credential handle"); + if (target.kind === "destination") await this.destinations.get(session.workspaceId, target.destinationId); + if (target.kind === "handle" && !/^vh_[A-Za-z0-9_-]{16,128}$/.test(target.credentialHandleId)) { + throw new BadRequestException("credentialHandleId must be a vault handle reference"); + } + const result = await client.query( + `INSERT INTO stream_outputs (workspace_id, virtual_channel_id, label, profile, mode, enabled, destination_id, credential_handle_id) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8) RETURNING *`, + [session.workspaceId, parsed, dto.label, dto.profile, dto.mode, dto.enabled, dto.destinationId ?? null, dto.credentialHandleId ?? null], + ); + return mapOutput(result.rows[0] ?? {}); + }); + } + + @Get() + @RequirePermission("streamoutput.manage") + async list(@Req() request: Request & AuthenticatedRequest, @Param("channelId") channelId: string): Promise { + const session = requireSession(request); + return this.transaction(session.workspaceId, async (client) => { + await this.assertChannel(client, session.workspaceId, parseId(channelId, "channelId")); + const result = await client.query( + "SELECT * FROM stream_outputs WHERE workspace_id = $1 AND virtual_channel_id = $2 ORDER BY created_at", + [session.workspaceId, channelId], + ); + return result.rows.map(mapOutput); + }); + } + + @Patch(":id") + @RequirePermission("streamoutput.manage") + async update(@Req() request: Request & AuthenticatedRequest, @Param("channelId") channelId: string, @Param("id") id: string, @Body() body: unknown): Promise { + const session = requireSession(request); + const parsedChannel = parseId(channelId, "channelId"); + const parsedId = parseId(id); + const dto = parseUpdateOutput(body); + return this.transaction(session.workspaceId, async (client) => { + await this.assertChannel(client, session.workspaceId, parsedChannel); + // Re-check the exclusivity invariant whenever a target field changes. + if (dto.destinationId !== undefined || dto.credentialHandleId !== undefined) { + const current = await client.query<{ destination_id: string | null; credential_handle_id: string | null }>( + "SELECT destination_id, credential_handle_id FROM stream_outputs WHERE workspace_id=$1 AND virtual_channel_id=$2 AND id=$3", + [session.workspaceId, parsedChannel, parsedId], + ); + const row = current.rows[0]; + if (row === undefined) throw new BadRequestException("stream output not found"); + const nextDestination = dto.destinationId === undefined ? row.destination_id ?? undefined : dto.destinationId ?? undefined; + const nextHandle = dto.credentialHandleId === undefined ? row.credential_handle_id ?? undefined : dto.credentialHandleId ?? undefined; + try { streamOutputTarget({ destinationId: nextDestination, credentialHandleId: nextHandle }); } catch { + throw new BadRequestException("output cannot reference both a destination and its own credential handle"); + } + if (nextDestination !== undefined) await this.destinations.get(session.workspaceId, nextDestination); + } + const result = await client.query( + `UPDATE stream_outputs SET + label = COALESCE($3, label), enabled = COALESCE($4, enabled), + profile = COALESCE($5, profile), mode = COALESCE($6, mode), + destination_id = CASE WHEN $7 THEN $8 ELSE destination_id END, + credential_handle_id = CASE WHEN $9 THEN $10 ELSE credential_handle_id END, + updated_at = now() + WHERE workspace_id = $1 AND virtual_channel_id = $2 AND id = $11 RETURNING *`, + [session.workspaceId, parsedChannel, dto.label ?? null, dto.enabled ?? null, dto.profile ?? null, dto.mode ?? null, + dto.destinationId !== undefined, dto.destinationId ?? null, + dto.credentialHandleId !== undefined, dto.credentialHandleId ?? null, parsedId], + ); + if (result.rows[0] === undefined) throw new BadRequestException("stream output not found"); + return mapOutput(result.rows[0]); + }); + } + + @Delete(":id") + @RequirePermission("streamoutput.manage") + async remove(@Req() request: Request & AuthenticatedRequest, @Param("channelId") channelId: string, @Param("id") id: string): Promise<{ id: string }> { + const session = requireSession(request); + const parsedId = parseId(id); + await this.transaction(session.workspaceId, async (client) => { + await this.assertChannel(client, session.workspaceId, parseId(channelId, "channelId")); + await client.query("DELETE FROM stream_outputs WHERE workspace_id=$1 AND virtual_channel_id=$2 AND id=$3", + [session.workspaceId, channelId, parsedId]); + }); + return { id: parsedId }; + } + + private async assertChannel(client: PoolClient, workspaceId: string, channelId: string): Promise { + const found = await client.query("SELECT id FROM virtual_channels WHERE workspace_id=$1 AND id=$2", [workspaceId, channelId]); + if (found.rowCount === 0) throw new BadRequestException("channelId must reference a virtual channel in this workspace"); + } +} + +export function mapOutput(row: Record): OutputSummary { + return { + id: String(row.id), workspaceId: String(row.workspace_id), virtualChannelId: String(row.virtual_channel_id), + label: String(row.label), profile: String(row.profile), mode: String(row.mode), enabled: Boolean(row.enabled), + destinationId: row.destination_id === null ? null : String(row.destination_id), + credentialHandleId: row.credential_handle_id === null ? null : String(row.credential_handle_id), + createdAt: new Date(row.created_at as Date), updatedAt: new Date(row.updated_at as Date), + }; +} + +const LABEL = /^.{1,120}$/s; + +export function parseCreateOutput(value: unknown): { label: string; profile: string; mode: string; enabled: boolean; destinationId?: string; credentialHandleId?: string } { + if (typeof value !== "object" || value === null) throw new BadRequestException("body must be an object"); + const body = value as Record; + const label = typeof body.label === "string" ? body.label.trim() : ""; + if (!LABEL.test(label)) throw new BadRequestException("label must be 1-120 characters"); + const profile = body.profile === undefined ? "720p30" : String(body.profile); + if (!["720p30", "1080p30", "1080p60"].includes(profile)) throw new BadRequestException("profile must be 720p30, 1080p30 or 1080p60"); + const mode = body.mode === undefined ? "passthrough" : String(body.mode); + if (!["passthrough", "normalize"].includes(mode)) throw new BadRequestException("mode must be passthrough or normalize"); + const enabled = body.enabled === undefined ? true : Boolean(body.enabled); + const dto: { label: string; profile: string; mode: string; enabled: boolean; destinationId?: string; credentialHandleId?: string } = { label, profile, mode, enabled }; + if (body.destinationId !== undefined && body.destinationId !== null) dto.destinationId = String(body.destinationId); + if (body.credentialHandleId !== undefined && body.credentialHandleId !== null) dto.credentialHandleId = String(body.credentialHandleId); + return dto; +} + +export function parseUpdateOutput(value: unknown): { label?: string; profile?: string; mode?: string; enabled?: boolean; destinationId?: string | null; credentialHandleId?: string | null } { + if (typeof value !== "object" || value === null) throw new BadRequestException("body must be an object"); + const body = value as Record; + const dto: { label?: string; profile?: string; mode?: string; enabled?: boolean; destinationId?: string | null; credentialHandleId?: string | null } = {}; + if (body.label !== undefined) { + const label = typeof body.label === "string" ? body.label.trim() : ""; + if (!LABEL.test(label)) throw new BadRequestException("label must be 1-120 characters"); + dto.label = label; + } + if (body.profile !== undefined && !["720p30", "1080p30", "1080p60"].includes(String(body.profile))) throw new BadRequestException("profile must be 720p30, 1080p30 or 1080p60"); + if (body.profile !== undefined) dto.profile = String(body.profile); + if (body.mode !== undefined && !["passthrough", "normalize"].includes(String(body.mode))) throw new BadRequestException("mode must be passthrough or normalize"); + if (body.mode !== undefined) dto.mode = String(body.mode); + if (body.enabled !== undefined) dto.enabled = Boolean(body.enabled); + if (body.destinationId !== undefined) dto.destinationId = body.destinationId === null ? null : String(body.destinationId); + if (body.credentialHandleId !== undefined) dto.credentialHandleId = body.credentialHandleId === null ? null : String(body.credentialHandleId); + return dto; +} diff --git a/apps/api/src/main.ts b/apps/api/src/main.ts new file mode 100644 index 0000000..cb07e8e --- /dev/null +++ b/apps/api/src/main.ts @@ -0,0 +1,40 @@ +import "./telemetry/otel"; +import { NestFactory } from "@nestjs/core"; +import { Module } from "@nestjs/common"; +import { AppModule } from "./app.module"; +import { FeedsModule } from "./feeds/feeds.module"; +import { ArchiveModule } from "./archive/archive.module"; +import { logger } from "./telemetry/logger"; + +@Module({ imports: [AppModule, ArchiveModule, FeedsModule] }) +class RootModule {} + +/** Browser origins allowed to call the API (comma-separated). Defaults cover the dev web server on :3001. */ +function corsOrigins(): string[] { + const configured = process.env.CORS_ORIGINS; + if (configured !== undefined && configured.trim() !== "") return configured.split(",").map((o) => o.trim()); + return ["http://localhost:3001", "http://127.0.0.1:3001", "http://localhost:3000", "http://127.0.0.1:3000"]; +} + +async function bootstrap(): Promise { + const app = await NestFactory.create(RootModule, { + logger: ["error", "warn"], + abortOnError: false, + }); + // Sessions use bearer tokens; OAuth callbacks also bind the initiating browser with an HttpOnly cookie. + app.enableCors({ + origin: corsOrigins(), + credentials: true, + methods: ["GET", "POST", "PATCH", "DELETE"], + allowedHeaders: ["Authorization", "Content-Type"], + }); + const port = Number(process.env.PORT ?? 3000); + app.enableShutdownHooks(); + await app.listen(port); + logger.info({ port }, "api listening"); +} + +void bootstrap().catch((error: unknown) => { + console.error("api bootstrap failed", error); + process.exitCode = 1; +}); diff --git a/apps/api/src/media-control/capacity.test.ts b/apps/api/src/media-control/capacity.test.ts new file mode 100644 index 0000000..d784724 --- /dev/null +++ b/apps/api/src/media-control/capacity.test.ts @@ -0,0 +1,551 @@ +import { generateKeyPairSync, randomUUID } from "node:crypto"; +import type { PoolClient } from "pg"; +import { HttpException } from "@nestjs/common"; +import { beforeEach, describe, expect, it } from "vitest"; +import { + CapacityGate, CapacityQueueFullError, ChannelPendingCapacityError, DEFAULT_CHANNEL_QUOTA, FAILOVER_RESERVE, + capacityUnits, effectiveUnits, fairShareOrder, jobResourceCaps, pendingBoundFor, +} from "./capacity.js"; +import { MemoryRedis } from "../production/testing/memory-redis.js"; +import { MediaControlService, MEDIA_CONTROL_KEYS as K } from "./media-control.service.js"; +import { StubCredentialHandleProvider } from "./credential-handles.js"; +import { Ed25519ControlSigner } from "./wire.js"; +import type { RedisReply } from "./redis.js"; +import type { DesiredSpec } from "./spec.js"; +import type { DirectorReceipt, TenantTransaction } from "../production/journal.js"; + +/** + * Capacity acceptance: per-worker + * unit accounting, the PENDING_CAPACITY queue with fair share across + * workspaces, admission on the take/start path (over-quota write rejected with + * nothing persisted), the bounded pending queue (8 = 2 × the default quota 4 → + * 429 CAPACITY_QUEUE_FULL), concurrency at quota − 1, tenant isolation, and + * per-job resource-cap isolation at the accounting level. + * + * `MemoryRedis` does not implement the capacity Lua scripts, so the gate runs + * its read-modify-write fallback under the in-process per-worker lock — the + * same path a Redis fake without `eval` support takes. + */ + +const WORKER = "media-worker-a"; +const HANDLE = "vh_0123456789abcdefXYZ"; +const ACTOR = { userId: null, role: "system" } as const; + +function running(profile: DesiredSpec["profile"] = "720p30"): DesiredSpec { + return { + desiredStatus: "RUNNING", + source: { kind: "hls", url: "https://cdn.example.com/live/index.m3u8" }, + output: { endpointUrl: "rtmps://live.example.net/app", credentialHandleId: HANDLE }, + profile, mode: "passthrough", + }; +} +const stopped: DesiredSpec = { desiredStatus: "STOPPED", source: null, output: null, profile: "720p30", mode: "passthrough" }; + +/** Drives the gate's read-modify-write fallback: the capacity scripts are unsupported here. */ +class FakeRedis extends MemoryRedis { + override async eval(script: string, keys: string[], args: string[]): Promise { + if (script.includes("tostring(cur)") || script.includes("tostring(v)")) throw new Error("no script support"); + return super.eval(script, keys, args); + } +} + +/** Aggregate DB fake: the director path (production channels) starts/stops channels. */ +class ProductionDb { + state = { version: 0, receipts: [] as DirectorReceipt[] }; + applied: Array<{ ch: string; status: string }> = []; + failAfter = 0; + readonly transaction: TenantTransaction = async (_ws: string, fn: (client: PoolClient) => Promise): Promise => { + const before = structuredClone(this.state); + const query = async (sql: string, values: unknown[] = []): Promise<{ rows: unknown[]; rowCount: number }> => { + let rows: unknown[] = []; + if (sql.startsWith("SELECT id FROM virtual_channels")) rows = [{ id: values[1] }]; // this channel exists + else if (sql.startsWith("SELECT desired_version")) rows = [{ desired_version: String(this.state.version), desired_status: "STOPPED" }]; + else if (sql.includes("FROM audit_log")) { + const found = sql.includes("requestId") + ? this.state.receipts.find(r => r.requestId === values[2]) + : this.state.receipts.at(-1); + rows = found ? [{ diff: found }] : []; + } else if (sql.startsWith("UPDATE virtual_channels")) { + this.state.version = Number(values[2]); + this.applied.push({ ch: String(values[1]), status: String(values[3]) }); + } else if (sql.startsWith("SELECT audit_append")) { + this.state.receipts.push(values[6] as DirectorReceipt); + if (this.failAfter > 0) { this.failAfter -= 1; throw new Error("commit refused"); } + } + return { rows, rowCount: rows.length }; + }; + try { return await fn({ query } as unknown as PoolClient); } + catch (error) { this.state = before; throw error; } // PostgreSQL rolls back with the transaction + }; +} + +const WS = randomUUID(); +const CH = randomUUID(); + +async function catchError(promise: Promise): Promise { + const error: unknown = await promise.then(() => null, (e: unknown) => e); + expect(error).toBeInstanceOf(HttpException); + return error as HttpException; +} +function codeOf(error: HttpException): unknown { + const response = error.getResponse(); + return typeof response === "object" && response !== null ? (response as Record).code : response; +} + +describe("§4.4 unit accounting", () => { + it("charges each running channel its profile's units of its worker's capacity", () => { + expect(capacityUnits("720p30")).toBe(4); + expect(capacityUnits("1080p30")).toBe(6); + expect(capacityUnits("1080p60")).toBe(10); + }); + + it("reserves the 20 % failover reserve from advertised capacity", () => { + expect(effectiveUnits(100, FAILOVER_RESERVE)).toBe(80); + expect(effectiveUnits(25, FAILOVER_RESERVE)).toBe(20); // 4 × 720p30 channels + expect(effectiveUnits(7, FAILOVER_RESERVE)).toBe(5); // floor + }); + + it("scales per-job caps with reserved units so Σ caps fit the worker", () => { + expect(jobResourceCaps("1080p60")).toEqual({ capacityUnits: 10, cpuShares: 2560, memoryMiB: 5120, fdBudget: 256, egressMbps: 20 }); + // Two 720p30 jobs fit a 10-unit worker (2×4 = 8 ≤ 10) with disjoint budgets: + // a job hitting its own cap cannot take a sibling's budget. + const a = jobResourceCaps("720p30"), b = jobResourceCaps("720p30"); + expect(a.cpuShares + b.cpuShares).toBeLessThanOrEqual(10 * 256); + expect(a.memoryMiB + b.memoryMiB).toBeLessThanOrEqual(10 * 512); + expect(a.fdBudget).toBe(b.fdBudget); // per-job RLIMIT_NOFILE, not shared + }); +}); + +describe("fair share across workspaces", () => { + const entry = (ws: string, seq: number, units = 4) => ({ workspaceId: ws, seq, units }); + + it("interleaves workspaces so one tenant's queue cannot delay another", () => { + const order = fairShareOrder([entry("A", 1), entry("A", 3), entry("A", 5), entry("B", 2)], () => 1).map(i => i.workspaceId); + expect(order).toEqual(["A", "B", "A", "A"]); + }); + + it("a heavier tenant earns proportionally more service without starving the light one", () => { + // A (weight 2) is served, then B's ratio is lower, so B runs second. + const order = fairShareOrder([entry("A", 1), entry("A", 2), entry("B", 3), entry("A", 4), entry("A", 5)], ws => (ws === "A" ? 2 : 1)); + expect(order.map(i => i.workspaceId)).toEqual(["A", "B", "A", "A", "A"]); + expect(order.findIndex(i => i.workspaceId === "B")).toBe(1); + }); + + it("keeps FIFO order within a workspace (admission_requested_at)", () => { + expect(fairShareOrder([entry("A", 9), entry("A", 2), entry("A", 5)], () => 1).map(i => i.seq)).toEqual([2, 5, 9]); + }); +}); + +describe("CapacityGate admission", () => { + let redis: MemoryRedis; + let gate: CapacityGate; + + beforeEach(() => { + redis = new MemoryRedis(); + gate = new CapacityGate({ redis, now: () => Date.parse("2026-10-07T12:00:00.000Z") }); + }); + + const start = (channelId: string, spec: DesiredSpec = running(), key = channelId) => + gate.admit(WS, channelId, WORKER, spec, ACTOR, key, "desired-v1", null); + const queued = (promise: Promise) => catchError(promise).then(error => { + expect(error).toBeInstanceOf(ChannelPendingCapacityError); + expect(error.getStatus()).toBe(202); + expect(codeOf(error)).toBe("PENDING_CAPACITY"); + return error as ChannelPendingCapacityError; + }); + + it("4 placed channels; the 5th queues as PENDING_CAPACITY while running channels are unaffected", async () => { + await gate.advertiseWorker(WORKER, 25); // effective 20 → four 4-unit channels + const placed = Array.from({ length: 4 }, () => randomUUID()); + for (const channelId of placed) await expect(start(channelId)).resolves.toMatchObject({ placed: true, units: 4 }); + for (const channelId of placed) expect(await gate.channelOf(WS, channelId)).toMatchObject({ state: "PLACED", units: 4 }); + + const fifth = randomUUID(); + const error = await queued(start(fifth)); + expect((error.getResponse() as Record).position).toBe(1); + expect(await gate.channelOf(WS, fifth)).toMatchObject({ state: "PENDING_CAPACITY", units: 4 }); + // The four running channels keep their placements and reservations. + const snapshot = await gate.snapshot(WS, [...placed, fifth]); + expect(snapshot.placedChannels).toBe(4); + expect(snapshot.pendingChannels).toBe(1); + expect(snapshot.workers[0]).toMatchObject({ usedUnits: 16, queued: 1 }); + }); + + it("freeing capacity never places more than the quota", async () => { + await gate.advertiseWorker(WORKER, 25); + const placed = Array.from({ length: 4 }, () => randomUUID()); + for (const channelId of placed) await start(channelId); + const waiting = Array.from({ length: 5 }, () => randomUUID()); + for (const channelId of waiting) await queued(start(channelId)); + const all = [...placed, ...waiting]; + + for (const channelId of placed) { + await gate.release(WS, channelId); + const admitted = await gate.drain(WORKER); + expect(admitted.length).toBeLessThanOrEqual(1); + const snapshot = await gate.snapshot(WS, all); + expect(snapshot.placedChannels).toBeLessThanOrEqual(DEFAULT_CHANNEL_QUOTA); + expect(snapshot.placedChannels + snapshot.pendingChannels).toBe(9 - placed.indexOf(channelId) - 1); + expect(snapshot.workers[0]?.usedUnits).toBeLessThanOrEqual(20); + } + const final = await gate.snapshot(WS, all); + expect(final.placedChannels).toBe(DEFAULT_CHANNEL_QUOTA); // exactly the quota, never more + expect(final.pendingChannels).toBe(1); + }); + + it("drives a workspace to the pending bound and rejects the 9th with 429 CAPACITY_QUEUE_FULL, nothing persisted", async () => { + await gate.advertiseWorker(WORKER, 25); + const placed = Array.from({ length: 4 }, () => randomUUID()); + for (const channelId of placed) await start(channelId); + const waiting: string[] = []; + for (let i = 0; i < pendingBoundFor(DEFAULT_CHANNEL_QUOTA); i += 1) { + const channelId = randomUUID(); + waiting.push(channelId); + await queued(start(channelId)); + } + expect(pendingBoundFor(DEFAULT_CHANNEL_QUOTA)).toBe(8); // 2 × the default quota of 4 + + const ninth = randomUUID(); + const error = await catchError(start(ninth)); + expect(error).toBeInstanceOf(CapacityQueueFullError); + expect(error.getStatus()).toBe(429); + expect(codeOf(error)).toBe("CAPACITY_QUEUE_FULL"); + // Nothing persisted for the rejected start. + expect(await gate.channelOf(WS, ninth)).toBeNull(); + const snapshot = await gate.snapshot(WS, [ninth]); + expect(snapshot.placedChannels).toBe(0); + expect(snapshot.pendingChannels).toBe(0); + expect((await gate.snapshot(WS, [...placed, ...waiting])).pendingChannels).toBe(8); + }); + + it("two concurrent starts at quota − 1 admit exactly one", async () => { + await gate.advertiseWorker(WORKER, 25); // effective 20: 4 channels by capacity, quota 4 + for (let i = 0; i < DEFAULT_CHANNEL_QUOTA - 1; i += 1) await start(randomUUID()); + const a = randomUUID(), b = randomUUID(); + const results = await Promise.allSettled([start(a), start(b)]); + expect(results.filter(r => r.status === "fulfilled")).toHaveLength(1); + const rejected = results.filter(r => r.status === "rejected").map(r => (r as PromiseRejectedResult).reason); + expect(rejected).toHaveLength(1); + expect(rejected[0]).toBeInstanceOf(ChannelPendingCapacityError); + // One placed, one queued — both accounted, never two placements. + expect((await gate.snapshot(WS, [a, b])).placedChannels).toBe(1); + const states = await Promise.all([gate.channelOf(WS, a), gate.channelOf(WS, b)]); + expect(states.filter(c => c?.state === "PLACED")).toHaveLength(1); + expect(states.filter(c => c?.state === "PENDING_CAPACITY")).toHaveLength(1); + }); + + it("a tenant saturating its quota cannot delay another tenant's placement", async () => { + await gate.advertiseWorker(WORKER, 100); // effective 80: room for both tenants + const other = randomUUID(); + // Tenant A saturates its quota and fills its pending queue. + for (let i = 0; i < DEFAULT_CHANNEL_QUOTA; i += 1) await start(randomUUID()); + for (let i = 0; i < pendingBoundFor(DEFAULT_CHANNEL_QUOTA); i += 1) await queued(start(randomUUID())); + // Tenant B places immediately, ahead of A's queued channels. + const b1 = randomUUID(), b2 = randomUUID(); + await expect(gate.admit(other, b1, WORKER, running(), ACTOR, b1, "desired-v1", null)).resolves.toMatchObject({ placed: true }); + await expect(gate.admit(other, b2, WORKER, running(), ACTOR, b2, "desired-v1", null)).resolves.toMatchObject({ placed: true }); + expect((await gate.snapshot(other, [b1, b2])).placedChannels).toBe(2); + }); + + it("re-admitting under the same idempotency key is a replay counted once", async () => { + await gate.advertiseWorker(WORKER, 25); + const channelId = randomUUID(); + await start(channelId, running(), "same-key"); + await expect(start(channelId, running(), "same-key")).resolves.toMatchObject({ replay: true }); + expect((await gate.snapshot(WS, [channelId])).placedChannels).toBe(1); + }); + + it("a profile change beyond free worker capacity queues instead of overcommitting", async () => { + await gate.advertiseWorker(WORKER, 10); // effective 8: one 6-unit channel fits + const channelId = randomUUID(); + await start(channelId, running("1080p30")); + const second = randomUUID(); + await queued(start(second, running("1080p60"))); // 10 units > 2 free + expect(await gate.channelOf(WS, second)).toMatchObject({ state: "PENDING_CAPACITY" }); + expect(await gate.channelOf(WS, channelId)).toMatchObject({ state: "PLACED", units: 6 }); // unaffected + }); + + it("undo reverses reservations so a failed desired-state write leaks no capacity", async () => { + await gate.advertiseWorker(WORKER, 25); + const channelId = randomUUID(); + const admitted = await start(channelId); + await admitted.undo(); + expect(await gate.channelOf(WS, channelId)).toBeNull(); + expect((await gate.snapshot(WS, [channelId])).placedChannels).toBe(0); + const refill = randomUUID(); + await expect(start(refill)).resolves.toMatchObject({ placed: true }); // units are reusable + expect((await gate.snapshot(WS, [refill])).workers[0]).toMatchObject({ usedUnits: 4 }); + }); + + it("records per-job throttle/kill events without touching sibling reservations", async () => { + await gate.advertiseWorker(WORKER, 25); + const a = randomUUID(), b = randomUUID(); + await start(a); + await start(b); + await gate.recordResourceEvent(WS, a, "killed", "memory.max exceeded"); + expect(await gate.channelOf(WS, a)).toMatchObject({ state: "PLACED", units: 4, lastEvent: { kind: "killed", reason: "memory.max exceeded" } }); + // b's placement and budget are untouched: the kill is isolated to a's own job. + const sibling = await gate.channelOf(WS, b); + expect(sibling).toMatchObject({ state: "PLACED", units: 4 }); + expect(sibling?.lastEvent).toBeUndefined(); + const snapshot = await gate.snapshot(WS, [a, b]); + expect(snapshot.placedChannels).toBe(2); + expect(snapshot.workers[0]?.usedUnits).toBe(8); + expect(snapshot.channels.find(c => c.channelId === a)?.caps).toEqual(jobResourceCaps("720p30")); + }); + + it("without an advertised capacity nothing is accounted (opt-in per worker)", async () => { + const channelId = randomUUID(); + const admitted = await start(channelId); + expect(admitted.channel).toBeNull(); + expect(await gate.channelOf(WS, channelId)).toBeNull(); + }); +}); + +describe("admission wired into the start path (setDesired)", () => { + let redis: FakeRedis; + let db: ProductionDb; + let service: MediaControlService; + + beforeEach(async () => { + redis = new FakeRedis(); + db = new ProductionDb(); + service = new MediaControlService({ + redis, transaction: db.transaction, assignments: [], + signer: new Ed25519ControlSigner("test", generateKeyPairSync("ed25519").privateKey), + handles: new StubCredentialHandleProvider(), + now: () => Date.parse("2026-10-07T12:00:00.000Z"), commandTtlMs: 30_000, + capacity: { workerUnits: { [WORKER]: 100 } }, + }); + await service.advertiseWorkerCapacity(WORKER, 100); // effective 80: room for both tenants + }); + + const start = (channelId: string, spec: DesiredSpec = running(), key = channelId) => + service.setDesired(WS, channelId, spec, { idempotencyKey: key, actor: ACTOR }); + const assign = (channelId: string) => service.assign(WS, channelId, WORKER, 1); + const desiredKey = (channelId: string) => redis.data.get(K.desired(WS, channelId)) ?? null; + const versionKey = (channelId: string) => redis.data.get(K.version(WS, channelId)) ?? null; + + it("4 placed channels; the 5th stays PENDING_CAPACITY while running channels are unaffected", async () => { + const placed = [CH, ...Array.from({ length: 3 }, () => randomUUID())]; + for (const channelId of placed) { await assign(channelId); await expect(start(channelId)).resolves.toBeTruthy(); } + for (const channelId of placed) expect(JSON.parse(desiredKey(channelId) ?? "null")).toMatchObject({ desiredStatus: "RUNNING" }); + + const fifth = randomUUID(); + await assign(fifth); + const error = await catchError(start(fifth)); + expect(codeOf(error)).toBe("PENDING_CAPACITY"); + expect((await service.channelCapacity(WS, fifth))?.state).toBe("PENDING_CAPACITY"); + // Nothing persisted for the queued start... + expect(versionKey(fifth)).toBeNull(); + expect(desiredKey(fifth)).toBeNull(); + // ...and the four running channels keep their committed desired state. + for (const channelId of placed) expect(JSON.parse(desiredKey(channelId) ?? "null")).toMatchObject({ desiredStatus: "RUNNING" }); + const capacity = await service.capacityGate.snapshot(WS, [...placed, fifth]); + expect(capacity).toMatchObject({ placedChannels: 4, pendingChannels: 1 }); + expect(capacity.workers[0]).toMatchObject({ usedUnits: 16 }); + }); + + it("an over-quota desired-state write is rejected with nothing persisted", async () => { + const placed = [CH, ...Array.from({ length: 3 }, () => randomUUID())]; + for (const channelId of placed) { await assign(channelId); await start(channelId); } + const before = redis.published.length; + const fifth = randomUUID(); + await assign(fifth); + await expect(start(fifth)).rejects.toThrow(); + expect(versionKey(fifth)).toBeNull(); + expect(desiredKey(fifth)).toBeNull(); + expect(db.applied.filter(a => a.ch === fifth)).toEqual([]); + expect(redis.published.length).toBe(before); // no command issued + }); + + it("freeing worker capacity never places more than the quota", async () => { + const placed = [CH, ...Array.from({ length: 3 }, () => randomUUID())]; + for (const channelId of placed) { await assign(channelId); await start(channelId); } + const waiting = Array.from({ length: 4 }, () => randomUUID()); + for (const channelId of waiting) { await assign(channelId); await expect(start(channelId)).rejects.toThrow(); } + + await start(placed[0]!, stopped, `${placed[0]}:stop`); // stop → free 4 units → the queue drains one + const capacity = await service.capacityGate.snapshot(WS, [...placed, ...waiting]); + expect(capacity.placedChannels).toBe(DEFAULT_CHANNEL_QUOTA); + expect(capacity.pendingChannels).toBe(3); + expect(capacity.workers[0]?.usedUnits).toBe(16); + // The admitted channel's start was re-applied: a committed desired snapshot exists. + const waitingIds: readonly string[] = waiting; + const admitted = capacity.channels.find(c => c.state === "PLACED" && waitingIds.includes(c.channelId)); + expect(admitted).toBeDefined(); + const admittedId = admitted?.channelId; + expect(typeof admittedId).toBe("string"); + expect(JSON.parse(desiredKey(admittedId ?? "") ?? "null")).toMatchObject({ desiredStatus: "RUNNING" }); + // The stopped channel no longer holds capacity. + expect(await service.channelCapacity(WS, placed[0]!)).toBeNull(); + }); + + it("drives a workspace to the pending bound (8) and 429s the 9th with nothing persisted", async () => { + const placed = [CH, ...Array.from({ length: 3 }, () => randomUUID())]; + for (const channelId of placed) { await assign(channelId); await start(channelId); } + const waiting: string[] = []; + for (let i = 0; i < pendingBoundFor(DEFAULT_CHANNEL_QUOTA); i += 1) { + const channelId = randomUUID(); + waiting.push(channelId); + await assign(channelId); + await expect(start(channelId)).rejects.toThrow("PENDING_CAPACITY"); + } + const before = redis.published.length; + const ninth = randomUUID(); + await assign(ninth); + const error = await catchError(start(ninth)); + expect(error).toBeInstanceOf(CapacityQueueFullError); + expect(error.getStatus()).toBe(429); + expect(codeOf(error)).toBe("CAPACITY_QUEUE_FULL"); + // Nothing persisted: no Redis desired state, no capacity record, no DB write, no command. + expect(versionKey(ninth)).toBeNull(); + expect(desiredKey(ninth)).toBeNull(); + expect(await service.channelCapacity(WS, ninth)).toBeNull(); + expect(db.applied.filter(a => a.ch === ninth)).toEqual([]); + expect(redis.published.length).toBe(before); + expect((await service.capacityGate.snapshot(WS, [...placed, ...waiting])).pendingChannels).toBe(8); + }); + + it("two concurrent starts at quota − 1 admit exactly one", async () => { + const placed = [CH, ...Array.from({ length: DEFAULT_CHANNEL_QUOTA - 2 }, () => randomUUID())]; + for (const channelId of placed) { await assign(channelId); await start(channelId); } + const a = randomUUID(), b = randomUUID(); + await assign(a); await assign(b); + const results = await Promise.allSettled([start(a), start(b)]); + expect(results.filter(r => r.status === "fulfilled")).toHaveLength(1); + expect(results.filter(r => r.status === "rejected")).toHaveLength(1); + const capacity = await service.capacityGate.snapshot(WS, [...placed, a, b]); + expect(capacity.placedChannels).toBe(DEFAULT_CHANNEL_QUOTA); // never one more + expect(capacity.pendingChannels).toBe(1); + expect(capacity.workers[0]?.usedUnits).toBe(16); + }); + + it("a tenant saturating its quota cannot delay another tenant's placement", async () => { + const tenantB = randomUUID(); + const b1 = randomUUID(), b2 = randomUUID(); + await service.assign(tenantB, b1, WORKER, 1); + await service.assign(tenantB, b2, WORKER, 1); + // Tenant A: quota saturated and its pending queue full. + const placed = [CH, ...Array.from({ length: 3 }, () => randomUUID())]; + for (const channelId of placed) { await assign(channelId); await start(channelId); } + for (let i = 0; i < pendingBoundFor(DEFAULT_CHANNEL_QUOTA); i += 1) { + const channelId = randomUUID(); + await assign(channelId); + await expect(start(channelId)).rejects.toThrow("PENDING_CAPACITY"); + } + // Tenant B still places immediately. + await expect(service.setDesired(tenantB, b1, running(), { idempotencyKey: b1, actor: ACTOR })).resolves.toBeTruthy(); + await expect(service.setDesired(tenantB, b2, running(), { idempotencyKey: b2, actor: ACTOR })).resolves.toBeTruthy(); + expect((await service.capacityGate.snapshot(tenantB, [b1, b2])).placedChannels).toBe(2); + expect(await service.channelCapacity(tenantB, b1)).toMatchObject({ state: "PLACED" }); + expect(await service.channelCapacity(tenantB, b2)).toMatchObject({ state: "PLACED" }); + }); + + it("per-job cap throttle/kill isolation is visible in the accounting", async () => { + await assign(CH); + await start(CH); + await service.recordResourceEvent(WS, CH, "throttled", "cpu.weight reached"); + expect(await service.channelCapacity(WS, CH)).toMatchObject({ + state: "PLACED", units: 4, lastEvent: { kind: "throttled", reason: "cpu.weight reached" }, + }); + const status = await service.getStatus(WS); + expect(status.capacity?.channels[0]?.caps).toEqual(jobResourceCaps("720p30")); + }); + + it("a mid-transaction failure after admission leaves no capacity leak", async () => { + const other = randomUUID(); + await assign(other); + db.failAfter = 1; // audit_append throws → the whole transaction rolls back + await expect(start(other)).rejects.toThrow("commit refused"); + expect(await service.channelCapacity(WS, other)).toBeNull(); + const snapshot = await service.capacityGate.snapshot(WS, [other]); + expect(snapshot.placedChannels).toBe(0); + expect(versionKey(other)).toBeNull(); + // The leaked-free units are reusable by a later start. + await expect(start(other)).resolves.toBeTruthy(); + }); +}); + +describe("admission on the director (take/start) path", () => { + let redis: FakeRedis; + let db: ProductionDb; + let service: MediaControlService; + let channel: string; + + beforeEach(async () => { + redis = new FakeRedis(); + db = new ProductionDb(); + service = new MediaControlService({ + redis, transaction: db.transaction, assignments: [], + signer: new Ed25519ControlSigner("test", generateKeyPairSync("ed25519").privateKey), + handles: new StubCredentialHandleProvider(), + now: () => Date.parse("2026-10-07T12:00:00.000Z"), commandTtlMs: 30_000, + capacity: { workerUnits: { [WORKER]: 25 } }, + }); + channel = randomUUID(); + await service.advertiseWorkerCapacity(WORKER, 25); + }); + + const desired = (channelId: string, status: "RUNNING" | "STOPPED") => + service.director(WS, channelId, { type: "desired", spec: status === "RUNNING" ? running() : stopped }, + `${channelId}:${status}`, ACTOR); + + it("a start over quota queues as PENDING_CAPACITY with nothing persisted", async () => { + const placed = [channel, ...Array.from({ length: 3 }, () => randomUUID())]; + for (const channelId of placed) { await service.assign(WS, channelId, WORKER, 1); await expect(desired(channelId, "RUNNING")).resolves.toBeTruthy(); } + const fifth = randomUUID(); + await service.assign(WS, fifth, WORKER, 1); + const error = await catchError(desired(fifth, "RUNNING")); + expect(codeOf(error)).toBe("PENDING_CAPACITY"); + expect(db.state.version).toBe(4); // four committed writes; the fifth rolled back + expect(db.state.receipts).toHaveLength(4); + expect(db.applied.filter(a => a.ch === fifth)).toEqual([]); + expect(await service.channelCapacity(WS, fifth)).toMatchObject({ state: "PENDING_CAPACITY" }); + }); + + it("a stop frees capacity and the queue re-applies exactly one admitted start", async () => { + const placed = [channel, ...Array.from({ length: 3 }, () => randomUUID())]; + for (const channelId of placed) { await service.assign(WS, channelId, WORKER, 1); await desired(channelId, "RUNNING"); } + const waiting: string[] = []; + for (let i = 0; i < 2; i += 1) { + const channelId = randomUUID(); + waiting.push(channelId); + await service.assign(WS, channelId, WORKER, 1); + await expect(desired(channelId, "RUNNING")).rejects.toThrow("PENDING_CAPACITY"); + } + + await desired(placed[0]!, "STOPPED"); + const capacity = await service.capacityGate.snapshot(WS, [...placed, ...waiting]); + expect(capacity.placedChannels).toBe(DEFAULT_CHANNEL_QUOTA); // never more than the quota + expect(capacity.pendingChannels).toBe(1); + const waitingIds: readonly string[] = waiting; + const admitted = capacity.channels.find(c => c.state === "PLACED" && waitingIds.includes(c.channelId)); + expect(admitted).toBeDefined(); + expect(JSON.parse(redis.data.get(K.desired(WS, admitted!.channelId)) ?? "null")).toMatchObject({ desiredStatus: "RUNNING" }); + }); + + it("a mid-transaction failure after admission leaks nothing", async () => { + await service.assign(WS, channel, WORKER, 1); + db.failAfter = 1; + await expect(desired(channel, "RUNNING")).rejects.toThrow("commit refused"); + expect(await service.channelCapacity(WS, channel)).toBeNull(); + expect((await service.capacityGate.snapshot(WS, [channel])).placedChannels).toBe(0); + expect(db.state.receipts).toHaveLength(0); + }); +}); + +describe("error shape", () => { + it("PENDING_CAPACITY is 202, CAPACITY_QUEUE_FULL is 429, both carry their code", () => { + const pending = new ChannelPendingCapacityError({ workspaceId: WS, channelId: CH, workerId: WORKER, units: 4, + profile: "720p30", requestedAt: "now", position: 1, pendingChannels: 1, pendingBound: 8 }); + expect(pending).toBeInstanceOf(HttpException); + expect(pending.getStatus()).toBe(202); + expect((pending.getResponse() as Record).code).toBe("PENDING_CAPACITY"); + + const full = new CapacityQueueFullError({ workspaceId: WS, channelId: CH, workerId: WORKER, pendingChannels: 8, pendingBound: 8 }); + expect(full).toBeInstanceOf(HttpException); + expect(full.getStatus()).toBe(429); + expect((full.getResponse() as Record).code).toBe("CAPACITY_QUEUE_FULL"); + }); +}); diff --git a/apps/api/src/media-control/capacity.ts b/apps/api/src/media-control/capacity.ts new file mode 100644 index 0000000..4c21ce0 --- /dev/null +++ b/apps/api/src/media-control/capacity.ts @@ -0,0 +1,720 @@ +import { HttpException } from "@nestjs/common"; +import type { DirectorActor, DirectorOperation } from "../production/journal.js"; +import type { ControlRedis } from "./redis.js"; +import type { DesiredSpec } from "./spec.js"; +import { OUTPUT_PRESETS, type OutputPreset } from "./wire.js"; + +/** + * Capacity & admission for the media-control desired-state plane + * (WP9, proposal §4.4 capacity). + * + * Model + * - Every RUNNING channel consumes `PROFILE_UNITS[profile]` capacity units of + * the worker it is placed on. A worker commits only + * `effectiveUnits(totalUnits) = floor(totalUnits × (1 − failoverReserve))` + * (the 20 % failover reserve of §4.4). + * - A workspace places at most `channelQuota` channels (default 4). Channels + * waiting in `PENDING_CAPACITY` are exempt — the running-VC quota counts + * placed channels only — and every placement re-checks the quota. + * - A start that does not fit is queued as `PENDING_CAPACITY` (ordered by + * `admission_requested_at`) instead of overcommitting. The queue is bounded + * at `pendingBound` = 2 × the channel quota; a start beyond the bound is + * rejected `429 CAPACITY_QUEUE_FULL` and nothing is persisted. + * - The queue drains weighted-fair across workspaces, so a tenant saturating + * its own quota cannot delay another tenant's placement. + * + * Safety + * - Reservations use atomic check-and-increment counters (worker units, + * workspace placed count) so two concurrent starts at quota − 1 admit + * exactly one, including across API processes. Redis fakes without `eval` + * for the admission scripts fall back to read-modify-write, which is correct + * within one process because every mutation runs under the gate's per-worker + * in-process lock. + * - The per-worker queue index is mutated under that same lock; the counters + * remain the authority on how much is placed, so a lost index update can + * never over-place, only delay a drain. + * - All keys are control-plane owned (`mcp:cap:*`); workers never read them. + * Queued admissions persist the parsed, secret-free `DesiredSpec` + * (credential handles only, never stream keys). + * + * Out of scope for this slice (scheduler-owned): `virtual_channels` rows with + * `observed_status = 'PENDING_CAPACITY'`, queue rebuild on scheduler restart + * and best-fit worker selection. `advertiseWorker` is the hand-off point. + */ + +/** Capacity units per output preset: roughly linear in pixels × frames. */ +export const PROFILE_UNITS: Readonly> = { + "720p30": 4, + "1080p30": 6, + "1080p60": 10, +}; + +/** Default running-VC quota per workspace (§4.4). */ +export const DEFAULT_CHANNEL_QUOTA = 4; +/** The pending queue holds at most 2 × the channel quota (§4.4). */ +export function pendingBoundFor(channelQuota: number): number { return 2 * channelQuota; } +/** 20 % of advertised worker capacity is held back for failover (§4.4). */ +export const FAILOVER_RESERVE = 0.2; + +/** Per-job cgroup/RLIMIT budgets derived from a channel's capacity units. */ +export const PER_UNIT_CPU_SHARES = 256; +export const PER_UNIT_MEMORY_MIB = 512; +export const PER_UNIT_EGRESS_MBPS = 2; +export const PER_JOB_FD_BUDGET = 256; + +export function capacityUnits(profile: OutputPreset): number { + const units = PROFILE_UNITS[profile]; + if (!Number.isSafeInteger(units) || units <= 0) throw new Error(`no capacity units for profile ${profile}`); + return units; +} + +/** Capacity a worker may actually commit to placements. */ +export function effectiveUnits(totalUnits: number, failoverReserve: number): number { + const reserve = Math.min(Math.max(failoverReserve, 0), 0.9); + return Math.max(0, Math.floor(totalUnits * (1 - reserve))); +} + +export interface JobResourceCaps { + capacityUnits: number; + cpuShares: number; + memoryMiB: number; + fdBudget: number; + egressMbps: number; +} + +/** + * Per-job resource caps for one channel's media process tree. Caps scale with + * the units the job reserves, so Σ caps of placed channels always fits the + * worker's effective capacity: a job throttled or killed at its own cap is + * isolated from sibling channels (§4.4 cgroup + RLIMIT_NOFILE limits). + */ +export function jobResourceCaps(profile: OutputPreset): JobResourceCaps { + const units = capacityUnits(profile); + return { + capacityUnits: units, + cpuShares: units * PER_UNIT_CPU_SHARES, + memoryMiB: units * PER_UNIT_MEMORY_MIB, + fdBudget: PER_JOB_FD_BUDGET, + egressMbps: units * PER_UNIT_EGRESS_MBPS, + }; +} + +export const CAPACITY_KEYS = { + /** Advertised worker capacity: `{ workerId, totalUnits }`. */ + worker: (workerId: string) => `mcp:cap:worker:${workerId}`, + used: (workerId: string) => `mcp:cap:used:${workerId}`, + /** Bounded pending queue for a worker, a JSON array ordered by `seq`. */ + queue: (workerId: string) => `mcp:cap:pq:${workerId}`, + placedCount: (workspaceId: string) => `mcp:cap:nplaced:${workspaceId}`, + pendingCount: (workspaceId: string) => `mcp:cap:npend:${workspaceId}`, + channel: (workspaceId: string, channelId: string) => `mcp:cap:ch:${workspaceId}:${channelId}`, + seq: () => "mcp:cap:seq", +} as const; + +export type CapacityState = "PLACED" | "PENDING_CAPACITY"; + +/** Public view of one channel's capacity accounting. */ +export interface ChannelCapacity { + workspaceId: string; + channelId: string; + workerId: string; + units: number; + profile: OutputPreset; + state: CapacityState; + /** Admission time (kept on re-queue so queue order follows first request). */ + requestedAt: string; + seq: number; + /** 1-based position in the worker's fair-share drain order; null when placed. */ + position: number | null; + caps: JobResourceCaps; + lastEvent?: { kind: "throttled" | "killed"; reason: string; at: string }; +} + +/** Everything a queued start needs to be re-applied when its placement is admitted. */ +export interface AdmissionPayload { + spec: DesiredSpec; + actor: DirectorActor; + idempotencyKey: string; + path: "desired-v1" | "director"; + operation: DirectorOperation | null; +} + +/** A stored admission record (the secret-free spec is persisted with it). */ +export interface StoredAdmission extends ChannelCapacity, AdmissionPayload { + recordKey: string; +} + +export interface QueueEntry { workspaceId: string; channelId: string; workerId: string; seq: number; units: number; requestedAt: string } + +export interface AdmissionResult { + placed: true; + units: number; + /** Null when this worker has no advertised capacity, so nothing is accounted. */ + channel: ChannelCapacity | null; + /** True when the idempotency key replayed an already-admitted start. */ + replay: boolean; + /** + * Reverses exactly the reservations this admission made. Call it when the + * desired-state write fails after admission: nothing persisted, no capacity + * leaked. It never drains the queue (an explicit free does that). + */ + undo(): Promise; +} + +export interface ReleaseResult { + state: "STOPPED" | CapacityState; + workerId: string | null; + units: number; + /** Channels the freed capacity admitted from the pending queue. */ + placed: string[]; +} + +export interface CapacitySnapshot { + workspaceId: string; + channelQuota: number; + pendingBound: number; + placedChannels: number; + pendingChannels: number; + channels: ChannelCapacity[]; + workers: Array<{ workerId: string; totalUnits: number | null; effectiveUnits: number | null; usedUnits: number; queued: number }>; +} + +export class CapacityQueueFullError extends HttpException { + constructor(detail: { workspaceId: string; channelId: string; workerId: string; pendingChannels: number; pendingBound: number }) { + super({ code: "CAPACITY_QUEUE_FULL", message: "capacity queue is full", ...detail }, 429); + } +} +export class ChannelPendingCapacityError extends HttpException { + constructor(detail: { + workspaceId: string; channelId: string; workerId: string; units: number; profile: OutputPreset; + requestedAt: string; position: number; pendingChannels: number; pendingBound: number; + }) { + super({ code: "PENDING_CAPACITY", message: "channel queued as PENDING_CAPACITY", ...detail }, 202); + } +} +export class CapacityProfileDeltaError extends HttpException { + constructor(detail: { workspaceId: string; channelId: string; workerId: string; units: number }) { + super({ code: "CAPACITY_PROFILE_DELTA", message: "profile change exceeds free worker capacity", ...detail }, 409); + } +} + +export interface CapacityConfig { + /** Max placed (running) channels per workspace. */ + channelQuota: number; + /** Max channels waiting in PENDING_CAPACITY per workspace (≤ 2 × quota). */ + pendingBound: number; + /** Fraction of advertised worker capacity held back for failover. */ + failoverReserve: number; + /** Weighted fair share per workspace (default 1). */ + weights: Record; + /** Worker capacities seeded at bootstrap; the scheduler advertises these in production. */ + workerUnits: Record; +} + +export const DEFAULT_CAPACITY_CONFIG: CapacityConfig = { + channelQuota: DEFAULT_CHANNEL_QUOTA, + pendingBound: pendingBoundFor(DEFAULT_CHANNEL_QUOTA), + failoverReserve: FAILOVER_RESERVE, + weights: {}, + workerUnits: {}, +}; + +export interface CapacityGateDependencies { + redis: ControlRedis; + now(): number; + config?: Partial; + /** Re-apply a queued start once its placement is admitted (wired by MediaControlService). */ + applyPending?(admission: StoredAdmission): Promise; +} + +/** Atomic check-and-increment: `GET + delta ≤ limit` then `SET`, else no write. */ +const INCR_IF_AT_MOST_SCRIPT = ` +local cur = tonumber(redis.call('GET', KEYS[1]) or '0') + tonumber(ARGV[1]) +if cur > tonumber(ARGV[2]) then return 0 end +redis.call('SET', KEYS[1], tostring(cur)) +return 1`; + +/** Atomic adjust with a zero floor (counters never go negative). */ +const ADJUST_SCRIPT = ` +local v = tonumber(redis.call('GET', KEYS[1]) or '0') + tonumber(ARGV[1]) +if v < 0 then v = 0 end +redis.call('SET', KEYS[1], tostring(v)) +return v`; + +/** + * Weighted fair drain order: the workspace with the lowest served-units/weight + * ratio goes next, FIFO (by `seq`) within a workspace. With equal weights this + * is round-robin over workspaces, so one tenant's long queue never runs ahead + * of another tenant's single waiting channel. + */ +export function fairShareOrder( + items: readonly T[], weightOf: (workspaceId: string) => number, +): T[] { + const heads = new Map(); + for (const item of [...items].sort((a, b) => a.seq - b.seq)) { + const list = heads.get(item.workspaceId) ?? []; + list.push(item); + heads.set(item.workspaceId, list); + } + const served = new Map(); + const ordered: T[] = []; + while (ordered.length < items.length) { + let pick: { workspaceId: string; ratio: number; head: T } | null = null; + for (const [workspaceId, list] of heads) { + const head = list[0]; + if (head === undefined) continue; + const ratio = (served.get(workspaceId) ?? 0) / weightOf(workspaceId); + if (pick === null || ratio < pick.ratio - 1e-9 || + (Math.abs(ratio - pick.ratio) <= 1e-9 && head.seq < pick.head.seq)) { + pick = { workspaceId, ratio, head }; + } + } + if (pick === null) break; + const list = heads.get(pick.workspaceId); + const item = list?.shift(); + if (item === undefined) break; + served.set(pick.workspaceId, (served.get(pick.workspaceId) ?? 0) + item.units); + ordered.push(item); + } + return ordered; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function isOutputPreset(value: unknown): value is OutputPreset { + return typeof value === "string" && (OUTPUT_PRESETS as readonly string[]).includes(value); +} +function parseStored(raw: string | null): StoredAdmission | null { + if (raw === null || raw === "") return null; + let value: unknown; + try { value = JSON.parse(raw); } catch { return null; } + if (!isRecord(value)) return null; + const spec = value.spec; + if (!isRecord(spec)) return null; + const profile: unknown = spec.profile; + if (!isOutputPreset(profile)) return null; + const actor = value.actor; + if (!isRecord(actor) || (typeof actor.userId !== "string" && actor.userId !== null)) return null; + const { workspaceId, channelId, workerId, units, seq, requestedAt, recordKey, idempotencyKey, path, state } = value; + if (typeof workspaceId !== "string" || typeof channelId !== "string" || typeof workerId !== "string" || + typeof units !== "number" || !Number.isSafeInteger(units) || units <= 0 || + typeof seq !== "number" || !Number.isSafeInteger(seq) || + typeof requestedAt !== "string" || typeof recordKey !== "string" || + typeof idempotencyKey !== "string" || (path !== "desired-v1" && path !== "director") || + (state !== "PLACED" && state !== "PENDING_CAPACITY")) return null; + const lastEvent = value.lastEvent; + return { + workspaceId, channelId, workerId, units, profile, state, + requestedAt, seq, position: null, caps: jobResourceCaps(profile), + ...(isRecord(lastEvent) && (lastEvent.kind === "throttled" || lastEvent.kind === "killed") && + typeof lastEvent.reason === "string" && typeof lastEvent.at === "string" + ? { lastEvent: { kind: lastEvent.kind, reason: lastEvent.reason, at: lastEvent.at } } : {}), + spec: spec as unknown as DesiredSpec, + actor: actor as unknown as DirectorActor, + idempotencyKey, path, + operation: (value.operation ?? null) as DirectorOperation | null, + recordKey, + }; +} +function parseQueueEntry(raw: string): QueueEntry | null { + let value: unknown; + try { value = JSON.parse(raw); } catch { return null; } + if (!isRecord(value) || typeof value.workspaceId !== "string" || typeof value.channelId !== "string" || + typeof value.workerId !== "string" || !Number.isSafeInteger(value.seq) || !Number.isSafeInteger(value.units) || + typeof value.requestedAt !== "string") return null; + const { workspaceId, channelId, workerId, seq, units, requestedAt } = value; + return { workspaceId, channelId, workerId, seq: seq as number, units: units as number, requestedAt }; +} +function project(stored: StoredAdmission, position: number | null): ChannelCapacity { + const { spec: _spec, actor: _actor, idempotencyKey: _key, operation: _op, recordKey: _rk, ...view } = stored; + void _spec; void _actor; void _key; void _op; void _rk; + return { ...view, position }; +} +function admissionKey(path: StoredAdmission["path"], idempotencyKey: string): string { + return `${path}:${idempotencyKey}`; +} + +export class CapacityGate { + private readonly deps: CapacityGateDependencies; + readonly config: CapacityConfig; + private readonly locks = new Map>(); + private scripts = true; + + constructor(deps: CapacityGateDependencies) { + this.deps = deps; + const provided = deps.config ?? {}; + const quota = provided.channelQuota ?? DEFAULT_CAPACITY_CONFIG.channelQuota; + this.config = { + channelQuota: quota, + pendingBound: Math.min(provided.pendingBound ?? pendingBoundFor(quota), pendingBoundFor(quota)), + failoverReserve: provided.failoverReserve ?? DEFAULT_CAPACITY_CONFIG.failoverReserve, + weights: provided.weights ?? {}, + workerUnits: provided.workerUnits ?? {}, + }; + } + + /** The scheduler advertises worker capacity; placement stays within it. */ + async advertiseWorker(workerId: string, totalUnits: number): Promise { + if (!/^[A-Za-z0-9._-]{1,64}$/.test(workerId)) throw new Error("workerId is invalid"); + if (!Number.isSafeInteger(totalUnits) || totalUnits <= 0) throw new Error("totalUnits must be a positive integer"); + await this.deps.redis.set(CAPACITY_KEYS.worker(workerId), JSON.stringify({ workerId, totalUnits })); + } + + /** Seed configured worker capacities (idempotent; called at bootstrap). */ + async seed(): Promise { + for (const [workerId, units] of Object.entries(this.config.workerUnits)) await this.advertiseWorker(workerId, units); + } + + private weightOf(workspaceId: string): number { + const weight = this.config.weights[workspaceId] ?? 1; + return weight > 0 ? weight : 1; + } + + private async read(key: string): Promise { + const raw = await this.deps.redis.get(key); + return raw === null || raw === "" ? null : raw; + } + private async counter(key: string): Promise { + return Math.max(0, Number(await this.read(key) ?? 0)); + } + private async recordOf(workspaceId: string, channelId: string): Promise { + return parseStored(await this.read(CAPACITY_KEYS.channel(workspaceId, channelId))); + } + private async write(stored: StoredAdmission): Promise { + await this.deps.redis.set(CAPACITY_KEYS.channel(stored.workspaceId, stored.channelId), JSON.stringify(stored)); + } + private async forget(workspaceId: string, channelId: string): Promise { + await this.deps.redis.set(CAPACITY_KEYS.channel(workspaceId, channelId), ""); + } + private async nextSeq(): Promise { + return this.deps.redis.incr(CAPACITY_KEYS.seq()); + } + + /** Atomic `counter += delta` if it would not exceed `limit`; false leaves it untouched. */ + private async incrIfAtMost(key: string, delta: number, limit: number): Promise { + const redis = this.deps.redis; + if (this.scripts) { + try { + const applied = await redis.eval(INCR_IF_AT_MOST_SCRIPT, [key], [String(delta), String(limit)]); + if (typeof applied === "number") return applied === 1; + if (Array.isArray(applied) && applied.length === 0) { /* fall through */ } + else throw new Error("unexpected INCR_IF_AT_MOST reply"); + } catch { + this.scripts = false; // fake without the script: read-modify-write under the in-process lock + } + } + const current = await this.counter(key); + if (current + delta > limit) return false; + await redis.set(key, String(current + delta)); + return true; + } + + /** Atomic adjust with a zero floor. */ + private async adjust(key: string, delta: number): Promise { + const redis = this.deps.redis; + if (this.scripts) { + try { + const applied = await redis.eval(ADJUST_SCRIPT, [key], [String(delta)]); + if (typeof applied === "number") return; + throw new Error("unexpected ADJUST reply"); + } catch { + this.scripts = false; + } + } + const current = await this.counter(key); + await redis.set(key, String(Math.max(0, current + delta))); + } + + private async workerUnitsOf(workerId: string): Promise { + const raw = await this.read(CAPACITY_KEYS.worker(workerId)); + if (raw === null) return null; + try { + const value = JSON.parse(raw) as { totalUnits?: unknown }; + return typeof value.totalUnits === "number" && Number.isSafeInteger(value.totalUnits) && value.totalUnits > 0 + ? value.totalUnits : null; + } catch { return null; } + } + + private async withLock(key: string, fn: () => Promise): Promise { + const previous = this.locks.get(key) ?? Promise.resolve(); + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + this.locks.set(key, previous.then(() => gate)); + await previous; + try { return await fn(); } finally { release(); } + } + + // ---- queue index (per-worker sorted set of waiting channels) ---- + + /** + * The per-worker queue index: one JSON array of waiting channels under a + * single key, mutated only under the gate's lock for this worker. It is an + * ordering hint for the fair drain; the unit/quota counters — not this index + * — decide what may be placed, so a stale index can only delay a drain. + */ + private async queueEntries(workerId: string): Promise { + const raw = await this.read(CAPACITY_KEYS.queue(workerId)); + if (raw === null) return []; + let value: unknown; + try { value = JSON.parse(raw); } catch { return []; } + if (!Array.isArray(value)) return []; + return value.flatMap(item => { + const entry = parseQueueEntry(typeof item === "string" ? item : JSON.stringify(item)); + return entry === null || entry.workerId !== workerId ? [] : [entry]; + }).sort((a, b) => a.seq - b.seq); + } + private async writeQueue(workerId: string, entries: QueueEntry[]): Promise { + await this.deps.redis.set(CAPACITY_KEYS.queue(workerId), JSON.stringify(entries)); + } + private async indexAdd(entry: QueueEntry): Promise { + const entries = await this.queueEntries(entry.workerId); + if (!entries.some(item => item.channelId === entry.channelId)) entries.push(entry); + await this.writeQueue(entry.workerId, entries); + } + private async indexRemove(entry: QueueEntry): Promise { + const entries = await this.queueEntries(entry.workerId); + await this.writeQueue(entry.workerId, entries.filter(item => item.channelId !== entry.channelId)); + } + private entryOf(stored: StoredAdmission): QueueEntry { + return { workspaceId: stored.workspaceId, channelId: stored.channelId, workerId: stored.workerId, + seq: stored.seq, units: stored.units, requestedAt: stored.requestedAt }; + } + + private async pendingPositions(workerId: string): Promise> { + const order = fairShareOrder(await this.queueEntries(workerId), ws => this.weightOf(ws)); + return new Map(order.map((entry, index) => [entry.channelId, index + 1])); + } + + // ---- admission ---- + + /** + * Admit a start (RUNNING desired state). Either reserves capacity and + * resolves `placed`, or throws: `ChannelPendingCapacityError` (queued, the + * desired-state write must not proceed) or `CapacityQueueFullError` (429, + * nothing persisted). No reservation survives a failed desired-state write: + * call `undo()`. + */ + async admit( + workspaceId: string, channelId: string, workerId: string, spec: DesiredSpec, + actor: DirectorActor, idempotencyKey: string, path: StoredAdmission["path"], + operation: DirectorOperation | null, + ): Promise { + const units = capacityUnits(spec.profile); + const key = admissionKey(path, idempotencyKey); + const nowIso = () => new Date(this.deps.now()).toISOString(); + return this.withLock(CAPACITY_KEYS.queue(workerId), async () => { + const existing = await this.recordOf(workspaceId, channelId); + if (existing !== null && existing.recordKey === key) { + if (existing.state === "PLACED") { + return { placed: true, units: existing.units, channel: project(existing, null), replay: true, + undo: async () => undefined }; + } + const positions = await this.pendingPositions(workerId); + throw new ChannelPendingCapacityError({ workspaceId, channelId, workerId, units: existing.units, + profile: existing.profile, requestedAt: existing.requestedAt, + position: positions.get(channelId) ?? 1, + pendingChannels: await this.counter(CAPACITY_KEYS.pendingCount(workspaceId)), + pendingBound: this.config.pendingBound }); + } + // A re-request while queued re-queues in place (keeps its original order). + if (existing?.state === "PENDING_CAPACITY") await this.dequeue(existing); + + const totalUnits = await this.workerUnitsOf(workerId); + if (totalUnits === null) { + // Capacity is not yet advertised for this worker (the scheduler owns + // advertisement). Admission is opt-in per worker: without an + // advertisement there is no unit budget to enforce, so the bootstrap + // placement path keeps working and nothing is accounted. + return { placed: true, units, channel: null, replay: false, undo: async () => undefined }; + } + const limit = effectiveUnits(totalUnits, this.config.failoverReserve); + const base = { workspaceId, channelId, workerId, profile: spec.profile, + requestedAt: existing?.requestedAt ?? nowIso(), seq: existing?.seq ?? await this.nextSeq(), + spec, actor, idempotencyKey, path, operation, recordKey: key }; + + if (existing?.state === "PLACED") { + // In-place update of a running channel: only the unit delta is reserved, + // the channel is already counted against the workspace quota. + const delta = units - existing.units; + if (delta > 0 && await this.counter(CAPACITY_KEYS.used(workerId)) + delta > limit) { + throw new CapacityProfileDeltaError({ workspaceId, channelId, workerId, units }); + } + if (delta !== 0) await this.adjust(CAPACITY_KEYS.used(workerId), delta); + const stored: StoredAdmission = { ...base, units, state: "PLACED", position: null, caps: jobResourceCaps(spec.profile) }; + await this.write(stored); + return { placed: true, units, channel: project(stored, null), replay: false, + undo: delta === 0 ? async () => undefined : async () => { + await this.forget(workspaceId, channelId); + await this.adjust(CAPACITY_KEYS.used(workerId), -delta); + } }; + } + + // Fresh placement: worker capacity and the workspace running-VC quota are + // each reserved atomically; a lost second reservation undoes the first. + const workerOk = totalUnits !== null && await this.incrIfAtMost(CAPACITY_KEYS.used(workerId), units, limit); + if (workerOk) { + const quotaOk = await this.incrIfAtMost(CAPACITY_KEYS.placedCount(workspaceId), 1, this.config.channelQuota); + if (quotaOk) { + const stored: StoredAdmission = { ...base, units, state: "PLACED", position: null, caps: jobResourceCaps(spec.profile) }; + await this.write(stored); + return { placed: true, units, channel: project(stored, null), replay: false, + undo: async () => { + await this.forget(workspaceId, channelId); + await this.adjust(CAPACITY_KEYS.placedCount(workspaceId), -1); + await this.adjust(CAPACITY_KEYS.used(workerId), -units); + } }; + } + await this.adjust(CAPACITY_KEYS.used(workerId), -units); // quota full → queue instead of overcommit + } + + const pending = await this.counter(CAPACITY_KEYS.pendingCount(workspaceId)); + if (pending + 1 > this.config.pendingBound) { + throw new CapacityQueueFullError({ workspaceId, channelId, workerId, pendingChannels: pending, + pendingBound: this.config.pendingBound }); + } + const stored: StoredAdmission = { ...base, units, state: "PENDING_CAPACITY", position: null, caps: jobResourceCaps(spec.profile) }; + await this.write(stored); + await this.indexAdd(this.entryOf(stored)); + await this.adjust(CAPACITY_KEYS.pendingCount(workspaceId), 1); + const positions = await this.pendingPositions(workerId); + throw new ChannelPendingCapacityError({ workspaceId, channelId, workerId, units, profile: spec.profile, + requestedAt: stored.requestedAt, position: positions.get(channelId) ?? 1, + pendingChannels: pending + 1, pendingBound: this.config.pendingBound }); + }); + } + + /** Remove a queued admission (record, index, pending count). */ + private async dequeue(stored: StoredAdmission): Promise { + await this.forget(stored.workspaceId, stored.channelId); + await this.indexRemove(this.entryOf(stored)); + await this.adjust(CAPACITY_KEYS.pendingCount(stored.workspaceId), -1); + } + + /** + * A channel stopped (desired STOPPED) or was deleted: free its reservation + * only — no drain happens here, because the caller frees capacity while its + * own desired-state transaction is still open. Call `drain(workerId)` once + * that transaction commits; freeing capacity can then only place what fits + * the worker limit *and* each workspace's quota. + */ + async release(workspaceId: string, channelId: string): Promise { + const stored = await this.recordOf(workspaceId, channelId); + if (stored === null) return { state: "STOPPED", workerId: null, units: 0, placed: [] }; + if (stored.state === "PENDING_CAPACITY") { + await this.withLock(CAPACITY_KEYS.queue(stored.workerId), () => this.dequeue(stored)); + return { state: "PENDING_CAPACITY", workerId: stored.workerId, units: stored.units, placed: [] }; + } + await this.withLock(CAPACITY_KEYS.queue(stored.workerId), async () => { + await this.forget(workspaceId, channelId); + await this.adjust(CAPACITY_KEYS.placedCount(workspaceId), -1); + await this.adjust(CAPACITY_KEYS.used(stored.workerId), -stored.units); + }); + return { state: "PLACED", workerId: stored.workerId, units: stored.units, placed: [] }; + } + + /** + * Place queued channels into free worker capacity, weighted-fair across + * workspaces. Never places beyond the worker's effective units or beyond any + * workspace's channel quota (the placement re-check). Re-applies each + * admitted start through `applyPending`; a failed re-apply rolls back to + * queued at the original position, so capacity is never leaked. + */ + async drain(workerId: string): Promise { + const admitted = await this.withLock(CAPACITY_KEYS.queue(workerId), () => this.collectDrain(workerId)); + for (const admission of admitted) await this.applyAdmission(admission); + return admitted.map(admission => admission.channelId); + } + + private async collectDrain(workerId: string): Promise { + const entries = await this.queueEntries(workerId); + if (entries.length === 0) return []; + const totalUnits = await this.workerUnitsOf(workerId); + if (totalUnits === null) return []; + const limit = effectiveUnits(totalUnits, this.config.failoverReserve); + let used = await this.counter(CAPACITY_KEYS.used(workerId)); + const admitted: StoredAdmission[] = []; + for (const entry of fairShareOrder(entries, ws => this.weightOf(ws))) { + if (used + entry.units > limit) continue; // freeing capacity never over-places + const stored = await this.recordOf(entry.workspaceId, entry.channelId); + if (stored === null || stored.state !== "PENDING_CAPACITY" || stored.workerId !== workerId) { + await this.indexRemove(entry); // stale index entry + continue; + } + // Placement re-checks the running-VC quota; pending channels were exempt. + const quotaOk = await this.incrIfAtMost(CAPACITY_KEYS.placedCount(entry.workspaceId), 1, this.config.channelQuota); + if (!quotaOk) continue; + await this.adjust(CAPACITY_KEYS.used(workerId), entry.units); + used += entry.units; + await this.forget(stored.workspaceId, stored.channelId); + await this.indexRemove(entry); + await this.adjust(CAPACITY_KEYS.pendingCount(stored.workspaceId), -1); + const placed: StoredAdmission = { ...stored, state: "PLACED", position: null }; + await this.write(placed); + admitted.push(placed); + } + return admitted; + } + + private async applyAdmission(admission: StoredAdmission): Promise { + const apply = this.deps.applyPending; + if (apply === undefined) return; + try { await apply(admission); } catch { await this.rollbackAdmission(admission); } + } + + /** A failed re-apply puts the channel back in the queue at its original position. */ + private async rollbackAdmission(admission: StoredAdmission): Promise { + await this.withLock(CAPACITY_KEYS.queue(admission.workerId), async () => { + await this.adjust(CAPACITY_KEYS.used(admission.workerId), -admission.units); + await this.adjust(CAPACITY_KEYS.placedCount(admission.workspaceId), -1); + await this.write({ ...admission, state: "PENDING_CAPACITY", position: null }); + await this.indexAdd(this.entryOf(admission)); + await this.adjust(CAPACITY_KEYS.pendingCount(admission.workspaceId), 1); + }); + } + + /** + * Record a per-job resource event (a job throttled or killed at its own + * cgroup/fd/egress cap). Purely an accounting annotation: it never touches + * other channels' reservations, which is the isolation §4.4 requires. + */ + async recordResourceEvent(workspaceId: string, channelId: string, kind: "throttled" | "killed", reason: string): Promise { + const stored = await this.recordOf(workspaceId, channelId); + if (stored === null) return; + stored.lastEvent = { kind, reason, at: new Date(this.deps.now()).toISOString() }; + await this.write(stored); + } + + async channelOf(workspaceId: string, channelId: string): Promise { + const stored = await this.recordOf(workspaceId, channelId); + if (stored === null) return null; + const position = stored.state === "PENDING_CAPACITY" ? (await this.pendingPositions(stored.workerId)).get(channelId) ?? 1 : null; + return project(stored, position); + } + + /** Accounting view for status/tests. `channelIds` are the workspace's channels. */ + async snapshot(workspaceId: string, channelIds: readonly string[]): Promise { + const records = await Promise.all(channelIds.map(channelId => this.recordOf(workspaceId, channelId))); + const waiting = records.filter((record): record is StoredAdmission => record !== null && record.state === "PENDING_CAPACITY"); + const workers = new Map(); + for (const record of records) { + if (record === null) continue; + const totalUnits = await this.workerUnitsOf(record.workerId); + const entry = workers.get(record.workerId) ?? { totalUnits, effectiveUnits: totalUnits === null ? null : effectiveUnits(totalUnits, this.config.failoverReserve), + usedUnits: await this.counter(CAPACITY_KEYS.used(record.workerId)), queued: (await this.queueEntries(record.workerId)).length }; + workers.set(record.workerId, entry); + } + const positions = await (async () => { + const map = new Map(); + for (const workerId of workers.keys()) for (const [channelId, position] of await this.pendingPositions(workerId)) map.set(channelId, position); + return map; + })(); + return { + workspaceId, channelQuota: this.config.channelQuota, pendingBound: this.config.pendingBound, + placedChannels: records.filter(record => record?.state === "PLACED").length, + pendingChannels: waiting.length, + channels: records.flatMap(record => record === null ? [] : [project(record, positions.get(record.channelId) ?? null)]), + workers: [...workers.entries()].map(([workerId, entry]) => ({ workerId, ...entry })), + }; + } +} diff --git a/apps/api/src/media-control/control-s3.integration.test.ts b/apps/api/src/media-control/control-s3.integration.test.ts new file mode 100644 index 0000000..11aa350 --- /dev/null +++ b/apps/api/src/media-control/control-s3.integration.test.ts @@ -0,0 +1,76 @@ +import { generateKeyPairSync, randomUUID } from "node:crypto"; +import { join } from "node:path"; +import { Pool } from "pg"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { runMigrations } from "../db/migrate.js"; +import { withTenant } from "../db/client.js"; +import { MemoryRedis } from "../production/testing/memory-redis.js"; +import type { TenantTransaction } from "../production/journal.js"; +import { Ed25519ControlSigner } from "./wire.js"; +import { MediaControlService } from "./media-control.service.js"; +import { RespControlRedis } from "./redis.js"; +const owner = new Pool({ connectionString: process.env.SUPER_DATABASE_URL ?? process.env.DATABASE_URL }); +const workspaces: string[] = []; + +describe.skipIf(!process.env.DATABASE_URL)("S3 bus journal on PostgreSQL with real row locking/rollback", () => { + beforeAll(async () => { await runMigrations(join(process.cwd(), "migrations")); }); + afterAll(async () => { for (const ws of workspaces) await owner.query("DELETE FROM workspaces WHERE id=$1", [ws]); await owner.end(); }); + async function fixture(transaction: TenantTransaction = withTenant): Promise<{ ws: string; ch: string; a: string; b: string; redis: MemoryRedis; service: MediaControlService }> { + const ws = randomUUID(), ch = randomUUID(), a = randomUUID(), b = randomUUID(); workspaces.push(ws); + await owner.query("INSERT INTO workspaces(id,name,slug) VALUES($1,'S3 tests',$2)", [ws, `s3-${ws}`]); + await withTenant(ws, async client => { + await client.query("INSERT INTO virtual_channels(id,workspace_id,slug,name) VALUES($1,$2,'test','S3')", [ch, ws]); + await client.query("INSERT INTO production_scenes(id,workspace_id,virtual_channel_id,label,layout) VALUES($1,$3,$4,'A','single'),($2,$3,$4,'B','single')", [a, b, ws, ch]); + await client.query("INSERT INTO production_buses(workspace_id,virtual_channel_id,bus,scene_id) VALUES($1,$2,'preview',$3),($1,$2,'program',$4)", [ws, ch, a, b]); + }); + const redis = new MemoryRedis(), service = new MediaControlService({ redis, transaction, signer: new Ed25519ControlSigner("test", generateKeyPairSync("ed25519").privateKey), assignments: [] }); + await service.assign(ws, ch, "worker", 1); return { ws, ch, a, b, redis, service }; + } + async function state(ws: string, ch: string): Promise<{ version: string; program: string; receipts: string }> { + return withTenant(ws, async client => { + const result = await client.query<{ version: string; program: string; receipts: string }>(`SELECT c.desired_version::text AS version,b.scene_id AS program, + (SELECT count(*)::text FROM audit_log WHERE workspace_id=$1 AND target_type='production_command') AS receipts + FROM virtual_channels c JOIN production_buses b ON b.workspace_id=c.workspace_id AND b.virtual_channel_id=c.id AND b.bus='program' + WHERE c.workspace_id=$1 AND c.id=$2`, [ws, ch]); + const row = result.rows[0]; if (!row) throw new Error("missing channel"); return row; + }); + } + const actor = { userId: null, role: "system" }; + it("duplicate and concurrent duplicates commit one bus effect/version/audit row", async () => { + const f = await fixture(); const first = await f.service.director(f.ws, f.ch, { type: "cut" }, "same", actor); + const commands = await Promise.all(Array.from({ length: 16 }, () => f.service.director(f.ws, f.ch, { type: "cut" }, "same", actor))); + expect(commands.every(command => command.commandId === first.commandId)).toBe(true); + expect(await state(f.ws, f.ch)).toEqual({ version: "1", program: f.a, receipts: "1" }); + expect(f.redis.published.filter(p => p.channel.startsWith("cmd:"))).toHaveLength(1); + }); + it("crash before commit rolls back buses/version/journal; retry applies once", async () => { + let crash = true; + const transaction: TenantTransaction = (ws, fn) => withTenant(ws, async client => { const result = await fn(client); if (crash) { crash = false; throw new Error("crash before commit"); } return result; }); + const f = await fixture(transaction); + await expect(f.service.director(f.ws, f.ch, { type: "cut" }, "crash", actor)).rejects.toThrow("crash before commit"); + expect(await state(f.ws, f.ch)).toEqual({ version: "0", program: f.b, receipts: "0" }); expect(f.redis.published).toHaveLength(0); + await f.service.director(f.ws, f.ch, { type: "cut" }, "crash", actor); + expect(await state(f.ws, f.ch)).toEqual({ version: "1", program: f.a, receipts: "1" }); + }); +}); + +describe.skipIf(!process.env.REDIS_URL)("S3 atomic delivery Lua on live Redis", () => { + it("concurrent receipt redelivery returns one commandId and preserves desiredVersion", async () => { + const redis = new RespControlRedis(process.env.REDIS_URL ?? ""); + const ws = randomUUID(), ch = randomUUID(), scene = randomUUID(), requestId = randomUUID(); + const service = new MediaControlService({ redis, signer: new Ed25519ControlSigner("test", generateKeyPairSync("ed25519").privateKey), assignments: [] }); + await service.assign(ws, ch, "lua-worker", 1); + const receipt = { operation: { type: "cut" as const }, requestId, fingerprint: "test", state: { locked: false, override: null }, + snapshot: { workspaceId: ws, channelId: ch, workerId: "lua-worker", assignmentEpoch: 1, desiredVersion: 1, desiredStatus: "STOPPED" as const, + profile: "720p30" as const, mode: "normalize" as const, source: null, output: null, + production: { scenes: [{ id: scene, label: "Scene", layout: "single" as const, slots: [] }], sources: [], overlays: [], buses: { preview: scene, program: scene }, transition: { type: "cut" as const }, audio: { masterGainDb: 0 } } } }; + try { + const results = await Promise.all(Array.from({ length: 20 }, () => service.deliverReceipt(receipt))); + expect(new Set(results.map(result => result.commandId)).size).toBe(1); + expect(await redis.get(`mcp:ver:${ws}:${ch}`)).toBe("1"); expect(await redis.get(`mcp:director:${ws}:${ch}`)).not.toBeNull(); + } finally { + await redis.eval("return redis.call('DEL',unpack(KEYS))", [`mcp:asg:${ws}:${ch}`, `mcp:channels:${ws}`, `mcp:des:${ws}:${ch}`, `mcp:ver:${ws}:${ch}`, `mcp:director:${ws}:${ch}`, `mcp:delivery:${requestId}`], []); + service.onModuleDestroy(); + } + }); +}); diff --git a/apps/api/src/media-control/control-s3.test.ts b/apps/api/src/media-control/control-s3.test.ts new file mode 100644 index 0000000..861333c --- /dev/null +++ b/apps/api/src/media-control/control-s3.test.ts @@ -0,0 +1,141 @@ +import { generateKeyPairSync, randomUUID } from "node:crypto"; +import type { PoolClient } from "pg"; +import { describe, expect, it } from "vitest"; +import { DELIVER_RECEIPT_SCRIPT, MediaControlService, MEDIA_CONTROL_KEYS as K } from "./media-control.service.js"; +import { MemoryRedis } from "../production/testing/memory-redis.js"; +import { Ed25519ControlSigner } from "./wire.js"; +import type { DirectorReceipt, TenantTransaction } from "../production/journal.js"; +import { RealtimeService, type RealtimePeer } from "../realtime/realtime.service.js"; + +const WS: string = randomUUID(), CH: string = randomUUID(), A: string = randomUUID(), B: string = randomUUID(); +class AggregateDatabase { + state = { version: 0, preview: A, program: B, receipts: [] as DirectorReceipt[] }; + crash = false; + private tail: Promise = Promise.resolve(); + readonly transaction: TenantTransaction = async (_ws: string, fn: (c: PoolClient) => Promise): Promise => { + const previous = this.tail; + let release = (): void => undefined; + this.tail = new Promise(r => { release = r; }); + await previous; + const before = structuredClone(this.state); + const query = async (sql: string, values: unknown[] = []): Promise<{ rows: unknown[]; rowCount: number }> => { + let rows: unknown[] = []; + if (sql.startsWith("SELECT id FROM virtual_channels")) rows = [{ id: CH }]; + else if (sql.startsWith("SELECT desired_version")) rows = [{ desired_version: String(this.state.version), desired_status: "STOPPED" }]; + else if (sql.includes("FROM audit_log")) { + const found = sql.includes("requestId") ? this.state.receipts.find(r => r.requestId === values[2]) : this.state.receipts.at(-1); + rows = found ? [{ diff: found }] : []; + } else if (sql.startsWith("SELECT scene_id")) rows = [{ scene_id: this.state.preview }]; + else if (sql.startsWith("INSERT INTO production_buses")) { if (sql.includes("'program'")) this.state.program = String(values[2]); else this.state.preview = String(values[2]); } + else if (sql.startsWith("UPDATE virtual_channels")) this.state.version = Number(values[2]); + else if (sql.startsWith("SELECT audit_append")) { if (this.crash) { this.crash = false; throw new Error("crash mid-transaction"); } this.state.receipts.push(values[6] as DirectorReceipt); } + else if (sql.startsWith("SELECT id,label,layout")) rows = [A, B].map(id => ({ id, label: id === A ? "A" : "B", layout: "single" })); + else if (sql.startsWith("SELECT bus,scene_id")) rows = [{ bus: "preview", scene_id: this.state.preview }, { bus: "program", scene_id: this.state.program }]; + else if (sql.startsWith("SELECT id FROM production_scenes")) rows = [A, B].includes(String(values[2])) ? [{ id: values[2] }] : []; + return { rows, rowCount: sql.startsWith("UPDATE") ? 1 : rows.length }; + }; + try { return await fn({ query } as unknown as PoolClient); } + catch (error) { this.state = before; throw error; } + finally { release(); } + }; +} +async function fixture(): Promise<{ db: AggregateDatabase; redis: MemoryRedis; service: MediaControlService }> { + const db = new AggregateDatabase(), redis = new MemoryRedis(); + const service = new MediaControlService({ transaction: db.transaction, redis, signer: new Ed25519ControlSigner("test", generateKeyPairSync("ed25519").privateKey), assignments: [] }); + await service.assign(WS, CH, "fake-worker", 1); + return { db, redis, service }; +} +const ACTOR = { userId: randomUUID(), role: "Owner" }; + +describe("S3 bus idempotency and transactional crash matrix", () => { + it("duplicate CUT reuses the receipt, one version and one bus effect", async () => { + const { db, redis, service } = await fixture(); + const first = await service.director(WS, CH, { type: "cut" }, "same", ACTOR); + expect(await service.director(WS, CH, { type: "cut" }, "same", ACTOR)).toEqual(first); + expect(db.state).toMatchObject({ version: 1, program: A }); expect(db.state.receipts).toHaveLength(1); + expect(redis.published.filter(p => p.channel.startsWith("cmd:"))).toHaveLength(1); + expect(db.state.receipts[0]?.state.override?.mode).toBe("until_released"); + }); + it("concurrent duplicate requests on separate service instances commit once", async () => { + const { db, redis, service } = await fixture(); + const second = new MediaControlService({ transaction: db.transaction, redis, signer: new Ed25519ControlSigner("test", generateKeyPairSync("ed25519").privateKey), assignments: [] }); + const results = await Promise.all(Array.from({ length: 20 }, (_, i) => (i % 2 ? service : second).director(WS, CH, { type: "take" }, "parallel", ACTOR))); + expect(new Set(results.map(r => r.commandId)).size).toBe(1); + expect(db.state.version).toBe(1); expect(db.state.receipts).toHaveLength(1); + expect(redis.published.filter(p => p.channel.startsWith("cmd:"))).toHaveLength(1); + }); + it("crash mid-tx rolls back bus, version and outbox; retry applies once", async () => { + const { db, redis, service } = await fixture(); db.crash = true; + await expect(service.director(WS, CH, { type: "cut" }, "crash", ACTOR)).rejects.toThrow("mid-transaction"); + expect(db.state).toMatchObject({ version: 0, program: B, receipts: [] }); expect(redis.published).toHaveLength(0); + await service.director(WS, CH, { type: "cut" }, "crash", ACTOR); + expect(db.state.version).toBe(1); expect(db.state.receipts).toHaveLength(1); + }); + it("crash after commit before publish is recovered from durable receipt", async () => { + const { db, redis, service } = await fixture(); redis.failDelivery = true; + await expect(service.director(WS, CH, { type: "cut" }, "after", ACTOR)).rejects.toThrow("before delivery"); + expect(db.state.version).toBe(1); expect(redis.published).toHaveLength(0); + await service.director(WS, CH, { type: "cut" }, "after", ACTOR); + expect(db.state.version).toBe(1); expect(redis.published.filter(p => p.channel.startsWith("cmd:"))).toHaveLength(1); + }); + it("lost reply after publish is recovered without a second publish", async () => { + const { db, redis, service } = await fixture(); const evaluate = redis.eval.bind(redis); + let lose = true; + redis.eval = async (script, keys, args) => { const result = await evaluate(script, keys, args); if (script === DELIVER_RECEIPT_SCRIPT && lose) { lose = false; throw new Error("reply lost"); } return result; }; + await expect(service.director(WS, CH, { type: "cut" }, "lost", ACTOR)).rejects.toThrow("reply lost"); + await service.director(WS, CH, { type: "cut" }, "lost", ACTOR); + expect(db.state.version).toBe(1); expect(redis.published.filter(p => p.channel.startsWith("cmd:"))).toHaveLength(1); + }); + it("rejects key reuse for another operation and cross-channel scene selection", async () => { + const { db, service } = await fixture(); await service.director(WS, CH, { type: "lock", enabled: true }, "reuse", ACTOR); + await expect(service.director(WS, CH, { type: "lock", enabled: false }, "reuse", ACTOR)).rejects.toThrow("another operation"); + await expect(service.director(WS, CH, { type: "pst-select", sceneId: randomUUID() }, "foreign", ACTOR)).rejects.toThrow("this channel"); + expect(db.state.version).toBe(1); + }); + it("PST-select, CUT, lock and override each share the version fence", async () => { + const { db, service } = await fixture(); + for (const [key, operation] of [ + ["select", { type: "pst-select", sceneId: B }], ["cut", { type: "cut" }], + ["lock", { type: "lock", enabled: true }], ["override", { type: "override", mode: "five_minutes" }], + ["auto", { type: "override", mode: null }], + ] as const) await service.director(WS, CH, operation, key, ACTOR); + expect(db.state.version).toBe(5); expect(db.state.program).toBe(B); expect(db.state.receipts.at(-1)?.state).toEqual({ locked: true, override: null }); + }); + it("full desired writes on production channels share the bus transaction version fence", async () => { + const { db, service } = await fixture(); + await service.director(WS, CH, { type: "cut" }, "seed-production", ACTOR); + const initial = db.state.receipts[0]?.snapshot; + if (!initial?.production) throw new Error("missing production snapshot"); + const spec = { desiredStatus: "STOPPED" as const, source: null, output: null, profile: "720p30" as const, + mode: "normalize" as const, production: initial.production }; + const results = await Promise.all([ + service.setDesired(WS, CH, spec, { actor: ACTOR, idempotencyKey: "full-desired" }), + service.director(WS, CH, { type: "lock", enabled: true }, "lock-concurrent", ACTOR), + ]); + expect(results.map(result => result.desiredVersion).sort()).toEqual([2, 3]); expect(db.state.version).toBe(3); + const repeated = await service.setDesired(WS, CH, spec, { actor: ACTOR, idempotencyKey: "full-desired" }); + expect(repeated).toEqual(results[0]); expect(db.state.version).toBe(3); + }); + it("legacy full snapshots on a stored channel cannot race the first production bus command", async () => { + const { db, service } = await fixture(); + const results = await Promise.all([ + service.setDesired(WS, CH, { desiredStatus: "STOPPED", source: null, output: null, profile: "720p30", mode: "normalize" }), + service.director(WS, CH, { type: "cut" }, "first-cut", ACTOR), + ]); + expect(results.map(result => result.desiredVersion).sort()).toEqual([1, 2]); expect(db.state.version).toBe(2); + }); + it("signed command reaches observed version and a realtime viewer within 300 ms", async () => { + const { redis, service } = await fixture(); + redis.echo = command => { setTimeout(() => { void redis.set(K.observed(WS, CH), JSON.stringify({ workspaceId: WS, channelId: CH, workerId: "fake-worker", assignmentEpoch: 1, state: "STOPPED", realizedVersion: command.body.desiredVersion, updatedAt: new Date().toISOString(), reason: "fake worker echo" })); }, 10); }; + const gateway = new RealtimeService(service, { subscriptions: { subscribe: async () => () => undefined, close: () => undefined }, authorize: async () => true, channelExists: async () => true }); + const frames: unknown[] = []; const peer: RealtimePeer = { send: value => { frames.push(value); return true; }, close: () => undefined }; + const connection = gateway.attach(peer); connection.message({ type: "auth", workspaceId: WS, token: "operator" }); + connection.message({ type: "subscribe", channelId: CH }); + await new Promise(r => setTimeout(r, 5)); + const start = performance.now(); const issued = await service.director(WS, CH, { type: "cut" }, "latency", ACTOR); + while (!frames.some(value => typeof value === "object" && value && "kind" in value && value.kind === "observed" && "data" in value && value.data && typeof value.data === "object" && "realizedVersion" in value.data && value.data.realizedVersion === issued.desiredVersion) && performance.now() - start < 300) await new Promise(r => setTimeout(r, 5)); + expect(performance.now() - start).toBeLessThan(300); + expect((await service.getStatus(WS)).channels[0]?.observed?.realizedVersion).toBe(issued.desiredVersion); + gateway.onModuleDestroy(); + }); +}); diff --git a/apps/api/src/media-control/credential-handles.ts b/apps/api/src/media-control/credential-handles.ts new file mode 100644 index 0000000..f4380a8 --- /dev/null +++ b/apps/api/src/media-control/credential-handles.ts @@ -0,0 +1,18 @@ +/** + * Stream keys never transit the control API. Outputs reference an opaque vault + * handle; the worker's VaultHandleBroker redeems it at relay start. This slice + * ships a format-only stub; the real provider must prove the handle belongs to + * the workspace (and channel) before a command is signed. + */ +export interface CredentialHandleProvider { + assertUsable(workspaceId: string, channelId: string, credentialHandleId: string): Promise; +} +export const MEDIA_CREDENTIAL_HANDLES = Symbol("MEDIA_CREDENTIAL_HANDLES"); + +const HANDLE = /^vh_[A-Za-z0-9_-]{16,128}$/; + +export class StubCredentialHandleProvider implements CredentialHandleProvider { + async assertUsable(_workspaceId: string, _channelId: string, credentialHandleId: string): Promise { + if (!HANDLE.test(credentialHandleId)) throw new Error("credential handle is not a vault handle reference"); + } +} diff --git a/apps/api/src/media-control/endpoints.ts b/apps/api/src/media-control/endpoints.ts new file mode 100644 index 0000000..fc9d621 --- /dev/null +++ b/apps/api/src/media-control/endpoints.ts @@ -0,0 +1,40 @@ +import { BlockList, isIP } from "node:net"; + +/** + * Mirror of apps/media-worker/src/relay/endpoints.ts shape rules. The API + * rejects early what the worker would reject at verify time; the worker's + * DNS-pinning validator remains the final authority on resolved hosts. + */ +export type EndpointUse = "hls" | "output"; + +const blocked = new BlockList(); +for (const [subnet, prefix] of [ + ["0.0.0.0", 8], ["10.0.0.0", 8], ["100.64.0.0", 10], ["127.0.0.0", 8], + ["169.254.0.0", 16], ["172.16.0.0", 12], ["192.0.0.0", 24], ["192.168.0.0", 16], + ["192.0.2.0", 24], ["198.18.0.0", 15], ["198.51.100.0", 24], + ["203.0.113.0", 24], ["224.0.0.0", 4], ["240.0.0.0", 4], +] as const) blocked.addSubnet(subnet, prefix, "ipv4"); +for (const [subnet, prefix] of [ + ["::", 128], ["::1", 128], ["fc00::", 7], ["fe80::", 10], ["ff00::", 8], + ["2001::", 32], ["2001:db8::", 32], ["2002::", 16], ["64:ff9b::", 96], +] as const) blocked.addSubnet(subnet, prefix, "ipv6"); + +export function assertPublicAddress(address: string): void { + const normalized = address.replace(/^\[|\]$/g, "").toLowerCase(); + const family = isIP(normalized); + if (family === 0) throw new Error("unresolved endpoint address"); + if (normalized.startsWith("::ffff:")) throw new Error("mapped endpoint rejected"); + if (blocked.check(normalized, family === 4 ? "ipv4" : "ipv6")) throw new Error("private endpoint rejected"); +} + +export function validateEndpointShape(raw: string, use: EndpointUse): URL { + let url: URL; + try { url = new URL(raw); } catch { throw new Error("invalid endpoint URL"); } + const schemes = use === "hls" ? ["https:"] : ["rtmp:", "rtmps:"]; + if (!schemes.includes(url.protocol) || !url.hostname || url.username || url.password || url.search || url.hash) throw new Error("endpoint scheme or credentials rejected"); + if (isIP(url.hostname.replace(/^\[|\]$/g, ""))) assertPublicAddress(url.hostname); + // Stricter than the worker: loopback names never resolve to a public relay target. + const host = url.hostname.toLowerCase(); + if (host === "localhost" || host.endsWith(".localhost")) throw new Error("private endpoint rejected"); + return url; +} diff --git a/apps/api/src/media-control/media-control.controller.ts b/apps/api/src/media-control/media-control.controller.ts new file mode 100644 index 0000000..b21765f --- /dev/null +++ b/apps/api/src/media-control/media-control.controller.ts @@ -0,0 +1,77 @@ +import { Body, Controller, Get, Headers, Inject, Param, Post, Put, Req, UseGuards } from "@nestjs/common"; +import type { Request } from "express"; +import { parseId, requireSession } from "../http/validation.js"; +import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import { MediaControlService, type IssuedCommand, type MediaControlStatus } from "./media-control.service.js"; +import { PreviewService } from "../production/preview.js"; +import type { DirectorOperation } from "../production/journal.js"; +import { parseDesiredSpec } from "./spec.js"; + +@Controller("media-control") +@UseGuards(RbacGuard) +export class MediaControlController { + constructor(@Inject(MediaControlService) private readonly control: MediaControlService, + @Inject(PreviewService) private readonly preview: PreviewService) {} + + @Put("channels/:channelId/desired") + @RequirePermission("media.manage") + async setDesired( + @Req() request: Request & AuthenticatedRequest, + @Param("channelId") channelId: string, + @Body() body: unknown, + @Headers("idempotency-key") idempotencyKey?: string, + ): Promise { + const session = requireSession(request); + return this.control.setDesired(session.workspaceId, parseId(channelId, "channelId"), parseDesiredSpec(body), { + actor: { userId: session.userId, role: session.role }, ...(idempotencyKey === undefined ? {} : { idempotencyKey }), + }); + } + + private command(request: Request & AuthenticatedRequest, channelId: string, key: string | undefined, operation: DirectorOperation): Promise { + const session = requireSession(request); + return this.control.director(session.workspaceId, parseId(channelId, "channelId"), operation, key ?? "", + { userId: session.userId, role: session.role }); + } + + @Post("channels/:channelId/take") + @RequirePermission("director.take") + take(@Req() req: Request & AuthenticatedRequest, @Param("channelId") ch: string, @Headers("idempotency-key") key?: string): Promise { + return this.command(req, ch, key, { type: "take" }); + } + + @Post("channels/:channelId/cut") + @RequirePermission("director.take") + cut(@Req() req: Request & AuthenticatedRequest, @Param("channelId") ch: string, @Headers("idempotency-key") key?: string): Promise { + return this.command(req, ch, key, { type: "cut" }); + } + + @Post("channels/:channelId/pst-select") + @RequirePermission("director.take") + select(@Req() req: Request & AuthenticatedRequest, @Param("channelId") ch: string, @Body() body: { sceneId: string }, @Headers("idempotency-key") key?: string): Promise { + return this.command(req, ch, key, { type: "pst-select", sceneId: body?.sceneId }); + } + + @Post("channels/:channelId/lock") + @RequirePermission("director.lock_program") + lock(@Req() req: Request & AuthenticatedRequest, @Param("channelId") ch: string, @Body() body: { enabled: boolean }, @Headers("idempotency-key") key?: string): Promise { + return this.command(req, ch, key, { type: "lock", enabled: body?.enabled }); + } + + @Post("channels/:channelId/override") + @RequirePermission("director.override_auto") + override(@Req() req: Request & AuthenticatedRequest, @Param("channelId") ch: string, @Body() body: { mode: import("../production/journal.js").OverrideMode | null }, @Headers("idempotency-key") key?: string): Promise { + return this.command(req, ch, key, { type: "override", mode: body?.mode }); + } + + @Get("channels/:channelId/preview/:tap") + @RequirePermission("director.view") + previewUrls(@Req() req: Request & AuthenticatedRequest, @Param("channelId") ch: string, @Param("tap") tap: string): Promise> { + return this.preview.urls(requireSession(req).workspaceId, parseId(ch, "channelId"), tap === "program" ? tap : parseId(tap, "tap")); + } + + @Get("status") + @RequirePermission("media.view") + async status(@Req() request: Request & AuthenticatedRequest): Promise { + return this.control.getStatus(requireSession(request).workspaceId); + } +} diff --git a/apps/api/src/media-control/media-control.integration.test.ts b/apps/api/src/media-control/media-control.integration.test.ts new file mode 100644 index 0000000..4dd923a --- /dev/null +++ b/apps/api/src/media-control/media-control.integration.test.ts @@ -0,0 +1,76 @@ +import { generateKeyPairSync, randomUUID } from "node:crypto"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { verifyCommand } from "../../../media-worker/src/bus/envelope.js"; +import { RedisBus } from "../../../media-worker/src/bus/redis.js"; +import { StubCredentialHandleProvider } from "./credential-handles.js"; +import { MEDIA_CONTROL_KEYS as K, MediaControlService } from "./media-control.service.js"; +import { RespControlRedis } from "./redis.js"; +import { Ed25519ControlSigner } from "./wire.js"; + +const redisUrl = process.env.REDIS_URL; + +/** Real wire: the worker's own RedisBus subscribes and reports; the API publishes and reads back. */ +describe.skipIf(redisUrl === undefined)("media control plane (live Redis)", () => { + const workspaceId = randomUUID(); + const channelId = randomUUID(); + const workerId = `it-worker-${randomUUID().slice(0, 8)}`; + const { privateKey, publicKey } = generateKeyPairSync("ed25519"); + let redis: RespControlRedis; + let bus: RedisBus; + let service: MediaControlService; + const received: string[] = []; + + beforeAll(async () => { + redis = new RespControlRedis(redisUrl ?? ""); + service = new MediaControlService({ redis, signer: new Ed25519ControlSigner("control-v1", privateKey), + handles: new StubCredentialHandleProvider(), assignments: [] }); + bus = await RedisBus.connect(redisUrl ?? ""); + await bus.subscribe(workerId, [workspaceId], (message) => { received.push(message); }); + }); + afterAll(async () => { + await redis.eval("return redis.call('DEL', unpack(KEYS))", [K.assignment(workspaceId, channelId), K.channels(workspaceId), + K.version(workspaceId, channelId), K.desired(workspaceId, channelId), K.observed(workspaceId, channelId), K.heartbeat(workerId)], []); + bus.close(); + redis.close(); + }); + + it("Lua assignment CAS enforces forward-only epochs", async () => { + await service.assign(workspaceId, channelId, workerId, 1); + await expect(service.assign(workspaceId, channelId, workerId, 0)).rejects.toThrow("stale"); + await expect(service.assign(workspaceId, channelId, "other-worker", 1)).rejects.toThrow("stale"); + expect(await service.getAssignment(workspaceId, channelId)).toEqual({ workerId, assignmentEpoch: 1 }); + }); + + it("delivers a signed command the worker verifies, with a SETNX audit record", async () => { + const issued = await service.setDesired(workspaceId, channelId, { + desiredStatus: "RUNNING", source: { kind: "hls", url: "https://cdn.example.com/live.m3u8" }, + output: { endpointUrl: "rtmps://live.example.net/app", credentialHandleId: "vh_integration_handle_01" }, + profile: "720p30", mode: "passthrough", + }); + expect(issued.receivers).toBe(1); + for (let i = 0; i < 50 && received.length === 0; i += 1) await new Promise((r) => setTimeout(r, 20)); + const command = verifyCommand(JSON.parse(received[0] ?? "null"), new Map([["control-v1", publicKey]]), workerId); + expect(command.body).toMatchObject({ commandId: issued.commandId, desiredVersion: issued.desiredVersion, assignmentEpoch: 1 }); + expect(await redis.set(K.issued(issued.commandId), "x", { nx: true, pxMs: 1000 })).toBe(false); + const ttl = await redis.eval("return redis.call('PTTL', KEYS[1])", [K.issued(issued.commandId)], []); + expect(typeof ttl === "number" && ttl > 0 && ttl <= 30_000).toBe(true); + }); + + it("Lua commit CAS rejects an older version", async () => { + await redis.set(K.desired(workspaceId, channelId), JSON.stringify({ desiredVersion: 1_000 })); + await expect(service.setDesired(workspaceId, channelId, { desiredStatus: "STOPPED", source: null, output: null, + profile: "720p30", mode: "passthrough" })).rejects.toThrow("newer desired version"); + await redis.eval("return redis.call('DEL', KEYS[1])", [K.desired(workspaceId, channelId)], []); + }); + + it("reads observed state and liveness written by the worker's RedisBus", async () => { + const issued = await service.setDesired(workspaceId, channelId, { desiredStatus: "STOPPED", source: null, output: null, + profile: "720p30", mode: "passthrough" }); + await bus.heartbeat(workerId, [{ workspaceId, channelId, workerId, assignmentEpoch: 1, state: "STOPPED", + realizedVersion: issued.desiredVersion, updatedAt: new Date().toISOString(), reason: "signed desired state stopped" }]); + const status = await service.getStatus(workspaceId); + expect(status.workers).toEqual([expect.objectContaining({ workerId, alive: true })]); + expect(status.channels).toEqual([expect.objectContaining({ channelId, converged: true, workerAlive: true, + desired: { desiredVersion: issued.desiredVersion, desiredStatus: "STOPPED" } })]); + }); +}); diff --git a/apps/api/src/media-control/media-control.module.ts b/apps/api/src/media-control/media-control.module.ts new file mode 100644 index 0000000..8f23c43 --- /dev/null +++ b/apps/api/src/media-control/media-control.module.ts @@ -0,0 +1,8 @@ +import { Module } from "@nestjs/common"; +import { MediaControlController } from "./media-control.controller.js"; +import { MediaControlService } from "./media-control.service.js"; +import { PreviewService } from "../production/preview.js"; +import { PreviewController } from "../production/preview.controller.js"; + +@Module({ controllers: [MediaControlController, PreviewController], providers: [MediaControlService, PreviewService], exports: [MediaControlService] }) +export class MediaControlModule {} diff --git a/apps/api/src/media-control/media-control.service.ts b/apps/api/src/media-control/media-control.service.ts new file mode 100644 index 0000000..655565e --- /dev/null +++ b/apps/api/src/media-control/media-control.service.ts @@ -0,0 +1,640 @@ +import { randomUUID, createPrivateKey } from "node:crypto"; +import { withTenant, pool } from "../db/client.js"; +import { productionSnapshot } from "../production/snapshot.js"; +import { findReceipt, fingerprint, requestIdentity, type DirectorOperation, type DirectorActor, type DirectorState, type DirectorReceipt, type TenantTransaction } from "../production/journal.js"; +import { readFileSync } from "node:fs"; +import { + BadRequestException, ConflictException, Inject, Injectable, OnModuleDestroy, OnModuleInit, Optional, ServiceUnavailableException, +} from "@nestjs/common"; +import { isUuid } from "../http/validation.js"; +import { MEDIA_CREDENTIAL_HANDLES, StubCredentialHandleProvider, type CredentialHandleProvider } from "./credential-handles.js"; +import { RespControlRedis, type ControlRedis } from "./redis.js"; +import { parseDesiredSpec, type DesiredSpec } from "./spec.js"; +import { Ed25519ControlSigner, type CommandBody, type ControlCommandSigner, type DesiredChannel, type ObservedChannel, type OutputPreset } from "./wire.js"; +import { CapacityGate, DEFAULT_CAPACITY_CONFIG, type CapacityConfig, type CapacitySnapshot, type ChannelCapacity, type StoredAdmission } from "./capacity.js"; + +/** + * Key layout. Worker-owned (read-only here): `cmd:{ws}:{worker}` pub/sub, + * `obs:{ws}:{ch}` (EX 60), `hb:{worker}` (EX 15). The worker never reads a + * desired-state key — it learns desired state only from signed commands — so + * desired snapshots live under control-plane-owned `mcp:*` keys. + */ +export const MEDIA_CONTROL_KEYS = { + assignment: (ws: string, ch: string) => `mcp:asg:${ws}:${ch}`, + channels: (ws: string) => `mcp:channels:${ws}`, + version: (ws: string, ch: string) => `mcp:ver:${ws}:${ch}`, + desired: (ws: string, ch: string) => `mcp:des:${ws}:${ch}`, + issued: (commandId: string) => `mcp:cmdlog:${commandId}`, + command: (ws: string, workerId: string) => `cmd:${ws}:${workerId}`, + observed: (ws: string, ch: string) => `obs:${ws}:${ch}`, + heartbeat: (workerId: string) => `hb:${workerId}`, +} as const; + +/** Worker heartbeat TTL (apps/media-worker/src/bus/redis.ts). */ +export const HEARTBEAT_TTL_MS = 15_000; +const DEFAULT_COMMAND_TTL_MS = 30_000; +const WORKER_ID = /^[A-Za-z0-9._-]{1,64}$/; + +/** Assignment CAS: epochs only move forward; one epoch names exactly one worker. */ +export const ASSIGN_SCRIPT = ` +local cur = redis.call('GET', KEYS[1]) +if cur then + local obj = cjson.decode(cur) + local epoch = tonumber(ARGV[2]) + if obj.assignmentEpoch > epoch then return 0 end + if obj.assignmentEpoch == epoch and obj.workerId ~= ARGV[3] then return 0 end +end +redis.call('SET', KEYS[1], ARGV[1]) +redis.call('SADD', KEYS[2], ARGV[4]) +return 1`; + +/** Desired CAS: commit only if the assignment is unchanged and no newer version is stored. */ +export const COMMIT_DESIRED_SCRIPT = ` +if redis.call('GET', KEYS[2]) ~= ARGV[3] then return -1 end +local cur = redis.call('GET', KEYS[1]) +if cur then + local obj = cjson.decode(cur) + if tonumber(obj.desiredVersion) >= tonumber(ARGV[2]) then return 0 end +end +redis.call('SET', KEYS[1], ARGV[1]) +return 1`; + +export interface Assignment { workerId: string; assignmentEpoch: number } +export interface IssuedCommand { + commandId: string; + workerId: string; + assignmentEpoch: number; + desiredVersion: number; + issuedAt: string; + expiresAt: string; + /** Subscribers that received the publish; 0 means no worker is listening right now. */ + receivers: number; +} +export interface ChannelStatus { + channelId: string; + workerId: string; + assignmentEpoch: number; + desired: { desiredVersion: number; desiredStatus: DesiredChannel["desiredStatus"]; production?: DesiredChannel["production"]; director?: DirectorState } | null; + observed: ObservedChannel | null; + /** True when the worker realized the current desired version under the current assignment. */ + converged: boolean; + workerAlive: boolean; +} +export interface WorkerStatus { workerId: string; alive: boolean; lastHeartbeatAt: string | null } +export interface MediaControlStatus { channels: ChannelStatus[]; workers: WorkerStatus[]; capacity?: CapacitySnapshot } + +/** An admission decision paired with one desired-state write. */ +export interface CapacityDecision { + /** Reverses the reservations when the desired-state write fails: nothing persisted, no capacity leaked. */ + undo(): Promise; + /** Queued starts admitted because this write freed capacity. */ + placedFromQueue: string[]; +} + +export interface MediaControlDependencies { + redis: ControlRedis; + signer: ControlCommandSigner | null; + handles: CredentialHandleProvider; + now: () => number; + commandTtlMs: number; + /** Bootstrap assignments (mirrors the worker's MEDIA_DESIRED_CONFIG until a scheduler owns placement). */ + assignments: Array; + transaction: TenantTransaction; + /** Capacity & admission (§4.4). Defaults: quota 4, pending bound 8, 20 % failover reserve. */ + capacity?: Partial; +} +export const MEDIA_CONTROL_DEPENDENCIES = Symbol("MEDIA_CONTROL_DEPENDENCIES"); + +/** Advertised worker capacity (units) until the scheduler owns placement: `{ "media-worker-a": 40 }`. */ +function envWorkerUnits(): Record { + const raw = process.env.MEDIA_WORKER_UNITS; + if (!raw) return {}; + const value: unknown = JSON.parse(raw); + if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error("MEDIA_WORKER_UNITS must be a JSON object"); + const units: Record = {}; + for (const [workerId, item] of Object.entries(value as Record)) { + if (typeof item !== "number" || !Number.isSafeInteger(item) || item <= 0) throw new Error("invalid worker capacity"); + units[workerId] = item; + } + return units; +} + +function envSigner(): ControlCommandSigner | null { + const path = process.env.MEDIA_CONTROL_PRIVATE_KEY; + if (!path) return null; + return new Ed25519ControlSigner(process.env.MEDIA_CONTROL_KID ?? "control-v1", createPrivateKey(readFileSync(path))); +} + +function envAssignments(): MediaControlDependencies["assignments"] { + const raw = process.env.MEDIA_CONTROL_ASSIGNMENTS; + if (!raw) return []; + const value: unknown = JSON.parse(raw); + if (!Array.isArray(value)) throw new Error("MEDIA_CONTROL_ASSIGNMENTS must be a JSON array"); + return value.map((item: unknown) => { + if (typeof item !== "object" || item === null) throw new Error("invalid assignment"); + const { workspaceId, channelId, workerId, assignmentEpoch } = item as Record; + if (typeof workspaceId !== "string" || typeof channelId !== "string" || typeof workerId !== "string" || + typeof assignmentEpoch !== "number") throw new Error("invalid assignment"); + return { workspaceId, channelId, workerId, assignmentEpoch }; + }); +} + +function parseJson(raw: string | null, guard: (value: unknown) => value is T): T | null { + if (raw === null) return null; + try { + const value: unknown = JSON.parse(raw); + return guard(value) ? value : null; + } catch { return null; } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function isAssignment(value: unknown): value is Assignment { + return isRecord(value) && typeof value.workerId === "string" && Number.isSafeInteger(value.assignmentEpoch); +} +function isSnapshot(value: unknown): value is DesiredChannel { + return isRecord(value) && typeof value.workspaceId === "string" && typeof value.channelId === "string" && + typeof value.workerId === "string" && Number.isSafeInteger(value.assignmentEpoch) && + Number.isSafeInteger(value.desiredVersion) && (value.desiredStatus === "RUNNING" || value.desiredStatus === "STOPPED") && + "source" in value && "output" in value && typeof value.profile === "string" && typeof value.mode === "string"; +} +function isObserved(value: unknown): value is ObservedChannel { + return isRecord(value) && typeof value.workerId === "string" && typeof value.state === "string" && + Number.isSafeInteger(value.assignmentEpoch) && (value.realizedVersion === null || Number.isSafeInteger(value.realizedVersion)); +} +function isHeartbeat(value: unknown): value is { workerId: string; sentAt: string } { + return isRecord(value) && typeof value.workerId === "string" && typeof value.sentAt === "string"; +} + +/** Canonical assignment JSON — the commit script compares it byte-for-byte. */ +function assignmentJson(a: Assignment): string { + return JSON.stringify({ workerId: a.workerId, assignmentEpoch: a.assignmentEpoch }); +} + +@Injectable() +export class MediaControlService implements OnModuleDestroy, OnModuleInit { + private readonly deps: MediaControlDependencies; + private bootstrapped: Promise | null = null; + private recoveryTimer: ReturnType | null = null; + private recovering = false; + private readonly databaseConfigured: boolean; + + constructor( + @Optional() @Inject(MEDIA_CONTROL_DEPENDENCIES) dependencies?: Partial, + @Optional() @Inject(MEDIA_CREDENTIAL_HANDLES) handles?: CredentialHandleProvider, + ) { + this.databaseConfigured = dependencies?.transaction !== undefined || process.env.DATABASE_URL !== undefined; + this.deps = { + redis: dependencies?.redis ?? new RespControlRedis(process.env.REDIS_URL ?? + `redis://${process.env.REDIS_HOST ?? "localhost"}:${process.env.REDIS_PORT ?? "6379"}`), + signer: dependencies?.signer !== undefined ? dependencies.signer : envSigner(), + handles: dependencies?.handles ?? handles ?? new StubCredentialHandleProvider(), + now: dependencies?.now ?? Date.now, + commandTtlMs: dependencies?.commandTtlMs ?? DEFAULT_COMMAND_TTL_MS, + assignments: dependencies?.assignments ?? envAssignments(), + transaction: dependencies?.transaction ?? withTenant, + }; + this.capacityGate = new CapacityGate({ + redis: this.deps.redis, now: this.deps.now, + config: { ...DEFAULT_CAPACITY_CONFIG, workerUnits: envWorkerUnits(), ...dependencies?.capacity }, + applyPending: admission => this.reapplyAdmission(admission), + }); + } + + /** §4.4 capacity & admission gate; the scheduler advertises worker capacity here. */ + readonly capacityGate: CapacityGate; + async advertiseWorkerCapacity(workerId: string, totalUnits: number): Promise { + await this.capacityGate.advertiseWorker(workerId, totalUnits); + } + + onModuleInit(): void { + const recover = async (): Promise => { + if (this.recovering || !this.deps.signer) return; + this.recovering = true; + try { await this.flushOutbox(); } catch { /* DB/Redis outage: retry on the next tick. */ } + finally { this.recovering = false; } + }; + void recover(); + this.recoveryTimer = setInterval(() => { void recover(); }, 1000); + this.recoveryTimer.unref(); + } + onModuleDestroy(): void { + if (this.recoveryTimer) clearInterval(this.recoveryTimer); + this.deps.redis.close(); + } + + private bootstrap(): Promise { + this.bootstrapped ??= (async () => { + await this.capacityGate.seed(); + for (const a of this.deps.assignments) await this.assign(a.workspaceId, a.channelId, a.workerId, a.assignmentEpoch); + })().catch((error: unknown) => { this.bootstrapped = null; throw error; }); + return this.bootstrapped; + } + + /** Place a channel on a worker. Epochs are monotonic; the worker must be configured with the same epoch. */ + async assign(workspaceId: string, channelId: string, workerId: string, assignmentEpoch: number): Promise { + assertIds(workspaceId, channelId); + if (!WORKER_ID.test(workerId)) throw new BadRequestException("workerId is invalid"); + if (!Number.isSafeInteger(assignmentEpoch) || assignmentEpoch < 0) throw new BadRequestException("assignmentEpoch is invalid"); + const k = MEDIA_CONTROL_KEYS; + const applied = await this.deps.redis.eval(ASSIGN_SCRIPT, [k.assignment(workspaceId, channelId), k.channels(workspaceId)], + [assignmentJson({ workerId, assignmentEpoch }), String(assignmentEpoch), workerId, channelId]); + if (applied !== 1) throw new ConflictException("assignment epoch is stale or already bound to another worker"); + } + + async getAssignment(workspaceId: string, channelId: string): Promise { + return parseJson(await this.deps.redis.get(MEDIA_CONTROL_KEYS.assignment(workspaceId, channelId)), isAssignment); + } + + /** + * Commit a desired snapshot, then sign and publish it. Order matters: the + * snapshot is durably committed (CAS on assignment + version) before signing, + * and the commandId is recorded (SET NX, TTL = envelope lifetime) before publish. + */ + async setDesired(workspaceId: string, channelId: string, spec: DesiredSpec, + options?: { idempotencyKey?: string; actor: DirectorActor }): Promise { + assertIds(workspaceId, channelId); + const checked = parseDesiredSpec(spec); + const signer = this.deps.signer; + if (signer === null) throw new ServiceUnavailableException("media control signing key is not configured"); + await this.bootstrap(); + const existing = parseJson(await this.deps.redis.get(MEDIA_CONTROL_KEYS.desired(workspaceId, channelId)), isSnapshot); + // Once a channel uses production, full snapshot writes must join the same + // PostgreSQL fence as bus/overlay writes; a Redis-only INCR could collide. + const productionChannel = this.databaseConfigured && await this.deps.transaction(workspaceId, async client => { + const channel = await client.query("SELECT id FROM virtual_channels WHERE workspace_id=$1 AND id=$2", [workspaceId, channelId]); + return Boolean(channel.rowCount); + }); + if (checked.production || existing?.production || productionChannel) { + return this.director(workspaceId, channelId, { type: "desired", spec: checked }, options?.idempotencyKey ?? randomUUID(), + options?.actor ?? { userId: null, role: "system" }); + } + const assignment = await this.getAssignment(workspaceId, channelId); + if (assignment === null) throw new ConflictException("channel is not assigned to a media worker"); + if (checked.output !== null) { + try { await this.deps.handles.assertUsable(workspaceId, channelId, checked.output.credentialHandleId); } catch (error: unknown) { + throw new BadRequestException(`output.credentialHandleId: ${error instanceof Error ? error.message : "unusable"}`); + } + } + // The queued admission replays under this key when capacity frees up. + const idempotencyKey = options?.idempotencyKey ?? randomUUID(); + const admission = await this.admitForCapacity(workspaceId, channelId, assignment.workerId, checked, + idempotencyKey, "desired-v1", null, options?.actor); + const stopWorker = checked.desiredStatus === "STOPPED" ? assignment.workerId : null; + + const k = MEDIA_CONTROL_KEYS; + const desiredVersion = await this.deps.redis.incr(k.version(workspaceId, channelId)); + const snapshot: DesiredChannel = { + workspaceId, channelId, workerId: assignment.workerId, assignmentEpoch: assignment.assignmentEpoch, + desiredVersion, desiredStatus: checked.desiredStatus, source: checked.source, output: checked.output, + profile: checked.profile, mode: checked.mode, + ...(checked.production === undefined ? {} : { production: checked.production }), + }; + const committed = await this.deps.redis.eval(COMMIT_DESIRED_SCRIPT, + [k.desired(workspaceId, channelId), k.assignment(workspaceId, channelId)], + [JSON.stringify(snapshot), String(desiredVersion), assignmentJson(assignment)]); + if (committed !== 1) { + await admission.undo(); // nothing persisted: no desired state, no capacity reservation + throw new ConflictException(committed === -1 + ? "channel assignment changed; retry" + : "a newer desired version was committed concurrently"); + } + + const issued = await this.issue(signer, snapshot); + // The commit is durable, so the capacity reservation is kept even if the + // publish fails: the outbox recovery loop re-delivers the snapshot and the + // reservation is never taken twice for one desired version. + if (stopWorker !== null) await this.drainQueued(stopWorker); + return issued; + } + + /** + * §4.4 admission on the take/start path. A RUNNING start either reserves + * capacity or throws (PENDING_CAPACITY / 429 CAPACITY_QUEUE_FULL) before any + * desired state is written; STOPPED frees the reservation and offers the + * freed capacity to the pending queue. Idempotent by idempotency key. + */ + private async admitForCapacity( + workspaceId: string, channelId: string, workerId: string, spec: DesiredSpec, + idempotencyKey: string, path: "desired-v1" | "director", operation: DirectorOperation | null, + actor: DirectorActor = { userId: null, role: "system" }, + ): Promise { + if (spec.desiredStatus === "STOPPED") { + const released = await this.capacityGate.release(workspaceId, channelId); + return { undo: async () => undefined, placedFromQueue: released.placed }; + } + const admitted = await this.capacityGate.admit(workspaceId, channelId, workerId, spec, + actor, idempotencyKey, path, operation); + return { undo: admitted.undo, placedFromQueue: [] }; + } + + /** + * Re-deliver the committed snapshot under a fresh commandId (same version): + * the worker fence accepts an equal version with an unseen commandId. Used + * when a publish reached no subscriber or a worker restarted. + */ + async republish(workspaceId: string, channelId: string): Promise { + assertIds(workspaceId, channelId); + const signer = this.deps.signer; + if (signer === null) throw new ServiceUnavailableException("media control signing key is not configured"); + const [assignment, snapshot] = await Promise.all([ + this.getAssignment(workspaceId, channelId), + this.deps.redis.get(MEDIA_CONTROL_KEYS.desired(workspaceId, channelId)).then((raw) => parseJson(raw, isSnapshot)), + ]); + if (assignment === null || snapshot === null) throw new ConflictException("no committed desired state for channel"); + if (snapshot.workspaceId !== workspaceId || snapshot.channelId !== channelId) throw new ConflictException("committed snapshot is corrupt"); + if (snapshot.workerId !== assignment.workerId || snapshot.assignmentEpoch !== assignment.assignmentEpoch) { + throw new ConflictException("channel was reassigned since the last commit; set desired state again"); + } + return this.issue(signer, snapshot); + } + + private async issue(signer: ControlCommandSigner, snapshot: DesiredChannel): Promise { + const now = this.deps.now(); + const body: CommandBody = { + workspaceId: snapshot.workspaceId, channelId: snapshot.channelId, workerId: snapshot.workerId, + assignmentEpoch: snapshot.assignmentEpoch, commandId: randomUUID(), desiredVersion: snapshot.desiredVersion, + issuedAt: new Date(now).toISOString(), expiresAt: new Date(now + this.deps.commandTtlMs).toISOString(), snapshot, + }; + const signed = await signer.signCommittedSnapshot(body); + await this.recordIssued(body, signed.kid); + const receivers = await this.deps.redis.publish(MEDIA_CONTROL_KEYS.command(body.workspaceId, body.workerId), JSON.stringify(signed)); + return { + commandId: body.commandId, workerId: body.workerId, assignmentEpoch: body.assignmentEpoch, + desiredVersion: body.desiredVersion, issuedAt: body.issuedAt, expiresAt: body.expiresAt, receivers, + }; + } + + /** Durable anti-replay audit: a commandId is issued at most once, retained until its envelope expires. */ + async recordIssued(body: CommandBody, kid: string): Promise { + const ttl = Date.parse(body.expiresAt) - this.deps.now(); + if (!(ttl > 0)) throw new Error("command already expired"); + const fresh = await this.deps.redis.set(MEDIA_CONTROL_KEYS.issued(body.commandId), JSON.stringify({ + workspaceId: body.workspaceId, channelId: body.channelId, workerId: body.workerId, + assignmentEpoch: body.assignmentEpoch, desiredVersion: body.desiredVersion, kid, + issuedAt: body.issuedAt, expiresAt: body.expiresAt, + }), { nx: true, pxMs: ttl }); + if (!fresh) throw new ConflictException("commandId already issued (replay refused)"); + } + + /** The channel row serializes writers; the receipt is the transactional outbox. + * No external effects occur inside this PostgreSQL transaction. */ + async director(workspaceId: string, channelId: string, operation: DirectorOperation, idempotencyKey: string, + actor: DirectorActor): Promise { + assertIds(workspaceId, channelId); + validateOperation(operation); + if (!this.deps.signer) throw new ServiceUnavailableException("media control signing key is not configured"); + await this.bootstrap(); + const requestId = requestIdentity(workspaceId, channelId, idempotencyKey); + const expected = fingerprint(operation); + // Holder object: TS cannot track assignments made inside the closure below. + const capacityUndo: { undo: (() => Promise) | null; releasedFromStop: boolean; workerId: string } = + { undo: null, releasedFromStop: false, workerId: "" }; + let receipt: DirectorReceipt; + const transaction = this.deps.transaction(workspaceId, async client => { + const channel = await client.query<{ desired_version: string; desired_status: "RUNNING" | "STOPPED" }>( + "SELECT desired_version,desired_status FROM virtual_channels WHERE workspace_id=$1 AND id=$2 FOR UPDATE", [workspaceId, channelId]); + const row = channel.rows[0]; + if (!row) throw new ConflictException("unknown production channel"); + const duplicate = await findReceipt(client, workspaceId, channelId, requestId, expected); + if (duplicate) return duplicate; + const latest = await client.query<{ diff: DirectorReceipt }>( + "SELECT diff FROM audit_log WHERE workspace_id=$1 AND target_type='production_command' AND target_id=$2 ORDER BY seq DESC LIMIT 1", [workspaceId, channelId]); + const state: DirectorState = structuredClone(latest.rows[0]?.diff.state ?? { locked: false, override: null }); + if (operation.type === "lock") state.locked = operation.enabled; + if (operation.type === "override") state.override = operation.mode === null ? null : { + mode: operation.mode, expiresAt: operation.mode === "five_minutes" ? new Date(this.deps.now() + 300_000).toISOString() : null, + }; + if (operation.type === "take" || operation.type === "cut") { + const preview = await client.query<{ scene_id: string | null }>( + "SELECT scene_id FROM production_buses WHERE workspace_id=$1 AND virtual_channel_id=$2 AND bus='preview'", [workspaceId, channelId]); + const sceneId = preview.rows[0]?.scene_id; + if (!sceneId) throw new ConflictException("preview bus is empty"); + await client.query(`INSERT INTO production_buses(workspace_id,virtual_channel_id,bus,scene_id) VALUES($1,$2,'program',$3) + ON CONFLICT(workspace_id,virtual_channel_id,bus) DO UPDATE SET scene_id=EXCLUDED.scene_id,updated_at=now()`, [workspaceId, channelId, sceneId]); + state.override = { mode: "until_released", expiresAt: null }; + } + if (operation.type === "pst-select") { + const scene = await client.query("SELECT id FROM production_scenes WHERE workspace_id=$1 AND virtual_channel_id=$2 AND id=$3", [workspaceId, channelId, operation.sceneId]); + if (!scene.rowCount) throw new BadRequestException("scene must belong to this channel"); + await client.query(`INSERT INTO production_buses(workspace_id,virtual_channel_id,bus,scene_id) VALUES($1,$2,'preview',$3) + ON CONFLICT(workspace_id,virtual_channel_id,bus) DO UPDATE SET scene_id=EXCLUDED.scene_id,updated_at=now()`, [workspaceId, channelId, operation.sceneId]); + } + if (operation.type === "overlay") { + const updated = await client.query(`UPDATE overlay_slots s SET current_revision_id=r.id,updated_at=now() + FROM overlay_revisions r WHERE s.workspace_id=$1 AND s.virtual_channel_id=$2 AND s.id=$3 + AND r.workspace_id=s.workspace_id AND r.virtual_channel_id=s.virtual_channel_id AND r.overlay_slot_id=s.id + AND r.id=$4 AND r.render_state='rendered' AND (s.current_revision_id IS NULL OR r.created_at >= + (SELECT active_revision.created_at FROM overlay_revisions active_revision WHERE active_revision.workspace_id=s.workspace_id AND active_revision.id=s.current_revision_id))`, [workspaceId, channelId, operation.slotId, operation.revisionId]); + if (!updated.rowCount) throw new BadRequestException("rendered revision must belong to this overlay slot"); + } + const assignment = await this.getAssignment(workspaceId, channelId); + if (!assignment) throw new ConflictException("channel is not assigned to a media worker"); + const cached = parseJson(await this.deps.redis.get(MEDIA_CONTROL_KEYS.desired(workspaceId, channelId)), isSnapshot); + const previous = latest.rows[0]?.diff.snapshot ?? cached; + const spec = operation.type === "desired" ? parseDesiredSpec(operation.spec) : parseDesiredSpec({ + desiredStatus: row.desired_status, source: null, output: previous?.output ?? null, + profile: previous?.profile ?? "720p30", mode: previous?.mode ?? "normalize", + production: await productionSnapshot(client, workspaceId, channelId), + }); + if (spec.output) await this.deps.handles.assertUsable(workspaceId, channelId, spec.output.credentialHandleId); + // §4.4 admission is part of the transaction: an over-quota write rolls + // the whole thing back — no bus effect, no version bump, no audit row. + // A start queued as PENDING_CAPACITY throws here too, so nothing is + // persisted for a channel whose placement is not admitted. + if (this.databaseConfigured) { + const decision = await this.admitForCapacity(workspaceId, channelId, assignment.workerId, spec, + idempotencyKey, "director", operation, actor); + capacityUndo.undo = () => decision.undo(); + if (spec.desiredStatus === "STOPPED") { capacityUndo.releasedFromStop = true; capacityUndo.workerId = assignment.workerId; } + } + const floor = Math.max(Number(row.desired_version), previous?.desiredVersion ?? 0, + Number(await this.deps.redis.get(MEDIA_CONTROL_KEYS.version(workspaceId, channelId)) ?? 0)); + if (!Number.isSafeInteger(floor) || floor >= Number.MAX_SAFE_INTEGER) throw new ConflictException("desired version exhausted"); + const desiredVersion = floor + 1; + await client.query("UPDATE virtual_channels SET desired_version=$3,desired_status=$4,updated_at=now() WHERE workspace_id=$1 AND id=$2", [workspaceId, channelId, desiredVersion, spec.desiredStatus]); + const snapshot: DesiredChannel = { workspaceId, channelId, ...assignment, desiredVersion, ...spec }; + const committed: DirectorReceipt = { operation, requestId, fingerprint: expected, snapshot, state }; + // Serialize our appends with one another, preserving the tenant audit chain. + await client.query("SELECT pg_advisory_xact_lock(hashtextextended($1,0))", [`production-audit:${workspaceId}`]); + await client.query("SELECT audit_append($1,$2,$3,$4,$5,$6,$7)", + [workspaceId, actor.userId, actor.role, `director.${operation.type}`, "production_command", channelId, committed]); + return committed; + }); + try { + receipt = await transaction; + } catch (error: unknown) { + const undo = capacityUndo.undo; + if (undo !== null) await undo(); // rolled back: nothing persisted, no capacity leaked + throw error; + } + // The transaction committed, so the reservation stands; a stop offers its + // freed capacity to the pending queue (each re-apply is idempotent under + // the stored idempotency key). + if (capacityUndo.releasedFromStop) await this.drainQueued(capacityUndo.workerId); + return this.deliverReceipt(receipt); + } + + /** Offer freed capacity to the pending queue (each re-apply is idempotent). */ + private async drainQueued(workerId: string): Promise { + await this.capacityGate.drain(workerId).catch(() => undefined); + } + + /** + * Re-apply a queued start once its placement is admitted by a drain. Runs + * under the original idempotency key, so a retry of the original request + * replays the same receipt instead of double-starting. + */ + private async reapplyAdmission(admission: StoredAdmission): Promise { + if (admission.path === "desired-v1") { + await this.setDesired(admission.workspaceId, admission.channelId, admission.spec, { + idempotencyKey: admission.idempotencyKey, actor: admission.actor, + }); + return; + } + const operation = admission.operation; + if (operation === null) throw new Error("queued admission is missing its director operation"); + await this.director(admission.workspaceId, admission.channelId, operation, + admission.idempotencyKey, admission.actor); + } + + async deliverReceipt(receipt: DirectorReceipt): Promise { + const signer = this.deps.signer; + if (!signer) throw new ServiceUnavailableException("media control signing key is not configured"); + const snapshot = receipt.snapshot; + const key = `mcp:delivery:${receipt.requestId}`; + const prior = await this.deps.redis.get(key); + if (prior) return JSON.parse(prior) as IssuedCommand; + const assignment = await this.getAssignment(snapshot.workspaceId, snapshot.channelId); + if (!assignment || assignmentJson(assignment) !== assignmentJson(snapshot)) throw new ConflictException("outbox assignment changed"); + const now = this.deps.now(); + const body: CommandBody = { workspaceId: snapshot.workspaceId, channelId: snapshot.channelId, workerId: snapshot.workerId, + assignmentEpoch: snapshot.assignmentEpoch, desiredVersion: snapshot.desiredVersion, commandId: randomUUID(), + issuedAt: new Date(now).toISOString(), expiresAt: new Date(now + this.deps.commandTtlMs).toISOString(), snapshot }; + const signed = await signer.signCommittedSnapshot(body); + const result: IssuedCommand = { commandId: body.commandId, workerId: body.workerId, assignmentEpoch: body.assignmentEpoch, + desiredVersion: body.desiredVersion, issuedAt: body.issuedAt, expiresAt: body.expiresAt, receivers: 0 }; + const k = MEDIA_CONTROL_KEYS; + const response = await this.deps.redis.eval(DELIVER_RECEIPT_SCRIPT, + [key, k.desired(body.workspaceId, body.channelId), k.version(body.workspaceId, body.channelId), + k.assignment(body.workspaceId, body.channelId), k.command(body.workspaceId, body.workerId), k.issued(body.commandId), + `state:${body.workspaceId}:${body.channelId}:desired`, `mcp:director:${body.workspaceId}:${body.channelId}`], + [JSON.stringify(result), JSON.stringify(snapshot), String(snapshot.desiredVersion), assignmentJson(assignment), + JSON.stringify(signed), String(this.deps.commandTtlMs), JSON.stringify({ desiredVersion: snapshot.desiredVersion, state: receipt.state })]); + if (response === -1) throw new ConflictException("outbox assignment changed"); + if (typeof response !== "string") throw new Error("invalid delivery receipt"); + return JSON.parse(response) as IssuedCommand; + } + + /** Recovery also runs after process restart, independent of an HTTP retry. */ + async flushOutbox(): Promise { + const workspaces = await pool.query<{ id: string }>("SELECT id FROM workspaces"); + for (const workspace of workspaces.rows) { + const receipts = await this.deps.transaction(workspace.id, async client => { + const result = await client.query<{ diff: DirectorReceipt }>( + "SELECT diff FROM audit_log WHERE workspace_id=$1 AND target_type='production_command' ORDER BY seq", [workspace.id]); + return result.rows.map(row => row.diff); + }); + for (const receipt of receipts) await this.deliverReceipt(receipt).catch(() => undefined); + } + } + + async getStatus(workspaceId: string): Promise { + if (!isUuid(workspaceId)) throw new BadRequestException("workspaceId must be a UUID"); + await this.bootstrap(); + const k = MEDIA_CONTROL_KEYS; + const channelIds = (await this.deps.redis.smembers(k.channels(workspaceId))).filter(isUuid).sort(); + if (channelIds.length === 0) return { channels: [], workers: [] }; + const [assignments, desired, observed, director] = await Promise.all([ + this.deps.redis.mget(channelIds.map((ch) => k.assignment(workspaceId, ch))), + this.deps.redis.mget(channelIds.map((ch) => k.desired(workspaceId, ch))), + this.deps.redis.mget(channelIds.map((ch) => k.observed(workspaceId, ch))), + this.deps.redis.mget(channelIds.map((ch) => `mcp:director:${workspaceId}:${ch}`)), + ]); + const parsedAssignments = assignments.map((raw) => parseJson(raw, isAssignment)); + const workerIds = [...new Set(parsedAssignments.flatMap((a) => (a === null ? [] : [a.workerId])))].sort(); + const beats = await this.deps.redis.mget(workerIds.map((id) => k.heartbeat(id))); + const now = this.deps.now(); + // Heartbeats carry every workspace's channels for that worker; only liveness is surfaced here. + const workers: WorkerStatus[] = workerIds.map((workerId, i) => { + const hb = parseJson(beats[i] ?? null, isHeartbeat); + const sentAt = hb === null || hb.workerId !== workerId ? NaN : Date.parse(hb.sentAt); + const alive = Number.isFinite(sentAt) && now - sentAt <= HEARTBEAT_TTL_MS; + return { workerId, alive, lastHeartbeatAt: Number.isFinite(sentAt) ? new Date(sentAt).toISOString() : null }; + }); + const alive = new Map(workers.map((w) => [w.workerId, w.alive])); + + const channels: ChannelStatus[] = []; + channelIds.forEach((channelId, i) => { + const assignment = parsedAssignments[i]; + if (!assignment) return; + const want = parseJson(desired[i] ?? null, isSnapshot); + const seen = parseJson(observed[i] ?? null, isObserved); + // An observation from a previous placement says nothing about the current one. + const current = seen !== null && seen.workerId === assignment.workerId && seen.assignmentEpoch === assignment.assignmentEpoch ? seen : null; + channels.push({ + channelId, workerId: assignment.workerId, assignmentEpoch: assignment.assignmentEpoch, + desired: want === null ? null : { desiredVersion: want.desiredVersion, desiredStatus: want.desiredStatus, + ...(want.production ? { production: want.production } : {}), + ...(director[i] ? { director: JSON.parse(director[i] ?? "null") as DirectorState } : {}) }, + observed: current, + converged: want !== null && current !== null && current.realizedVersion === want.desiredVersion, + workerAlive: alive.get(assignment.workerId) ?? false, + }); + }); + // §4.4 capacity accounting: only channels of this workspace are reported. + const capacity = await this.capacityGate.snapshot(workspaceId, channelIds).catch(() => null); + return capacity === null ? { channels, workers } : { channels, workers, capacity }; + } + + /** Per-job resource event: a channel throttled/killed at its own cgroup/fd/egress cap (§4.4). */ + async recordResourceEvent(workspaceId: string, channelId: string, kind: "throttled" | "killed", reason: string): Promise { + assertIds(workspaceId, channelId); + await this.capacityGate.recordResourceEvent(workspaceId, channelId, kind, reason); + } + + /** Capacity accounting for one channel (null when the channel holds no reservation). */ + async channelCapacity(workspaceId: string, channelId: string): Promise { + assertIds(workspaceId, channelId); + return this.capacityGate.channelOf(workspaceId, channelId); + } +} + +function assertIds(workspaceId: string, channelId: string): void { + if (!isUuid(workspaceId)) throw new BadRequestException("workspaceId must be a UUID"); + if (!isUuid(channelId)) throw new BadRequestException("channelId must be a UUID"); +} + +/** Atomic Redis delivery: one publish for a durable receipt even if the client + * loses the response. Old receipts may be delivered but never replace newer state. */ +export const DELIVER_RECEIPT_SCRIPT = ` +local prior = redis.call('GET', KEYS[1]) +if prior then return prior end +if redis.call('GET', KEYS[4]) ~= ARGV[4] then return -1 end +local cur = redis.call('GET', KEYS[2]) +if not cur or tonumber(cjson.decode(cur).desiredVersion) < tonumber(ARGV[3]) then + redis.call('SET', KEYS[2], ARGV[2]) + redis.call('SET', KEYS[3], ARGV[3]) + redis.call('SET', KEYS[8], cjson.encode(cjson.decode(ARGV[7]).state)) +end +redis.call('SET', KEYS[6], ARGV[5], 'PX', ARGV[6]) +local result = cjson.decode(ARGV[1]) +result.receivers = redis.call('PUBLISH', KEYS[5], ARGV[5]) +local receipt = cjson.encode(result) +redis.call('SET', KEYS[1], receipt) +redis.call('PUBLISH', KEYS[7], ARGV[7]) +return receipt`; + +function validateOperation(operation: DirectorOperation): void { + if (!operation || typeof operation !== "object") throw new BadRequestException("invalid director operation"); + const fields: Record = { desired: ["type", "spec"], take: ["type"], cut: ["type"], "pst-select": ["type", "sceneId"], + lock: ["type", "enabled"], override: ["type", "mode"], overlay: ["type", "slotId", "revisionId"] }; + const allowed = fields[operation.type]; + if (!allowed || Object.keys(operation).some(k => !allowed.includes(k))) throw new BadRequestException("invalid director operation"); + if (operation.type === "desired") parseDesiredSpec(operation.spec); + if (operation.type === "pst-select" && !isUuid(operation.sceneId)) throw new BadRequestException("sceneId must be a UUID"); + if (operation.type === "lock" && typeof operation.enabled !== "boolean") throw new BadRequestException("enabled must be boolean"); + if (operation.type === "override" && operation.mode !== null && !["until_released", "five_minutes", "until_source_offline", "until_next_event"].includes(operation.mode)) throw new BadRequestException("invalid override mode"); + if (operation.type === "overlay" && (!isUuid(operation.slotId) || !isUuid(operation.revisionId))) throw new BadRequestException("invalid overlay identity"); +} diff --git a/apps/api/src/media-control/media-control.test.ts b/apps/api/src/media-control/media-control.test.ts new file mode 100644 index 0000000..95345fd --- /dev/null +++ b/apps/api/src/media-control/media-control.test.ts @@ -0,0 +1,417 @@ +import { createPublicKey, generateKeyPairSync, randomUUID, type KeyObject } from "node:crypto"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { HttpException } from "@nestjs/common"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +// The worker is the contract: verify with its own code, not a copy. +import { signCommand as workerSign, verifyCommand as workerVerify } from "../../../media-worker/src/bus/envelope.js"; +import { Ed25519ControlSigner as WorkerSigner } from "../../../media-worker/src/bus/control-signer.js"; +import { DurableFence } from "../../../media-worker/src/bus/fence.js"; +import { validateEndpointShape as workerValidate } from "../../../media-worker/src/relay/endpoints.js"; +import { StubCredentialHandleProvider } from "./credential-handles.js"; +import { validateEndpointShape } from "./endpoints.js"; +import { + ASSIGN_SCRIPT, COMMIT_DESIRED_SCRIPT, HEARTBEAT_TTL_MS, MEDIA_CONTROL_KEYS as K, MediaControlService, +} from "./media-control.service.js"; +import type { ControlRedis, RedisReply } from "./redis.js"; +import { parseDesiredSpec, type DesiredSpec } from "./spec.js"; +import { parseProductionSpec } from "./production.js"; +import { Ed25519ControlSigner, type CommandBody, type SignedCommand } from "./wire.js"; + +/** In-memory Redis with the two control-plane scripts emulated by identity. */ +class FakeRedis implements ControlRedis { + readonly kv = new Map(); + readonly sets = new Map>(); + readonly published: Array<{ channel: string; message: string }> = []; + subscribers = 1; + constructor(private readonly now: () => number) {} + async get(key: string): Promise { + const entry = this.kv.get(key); + if (!entry) return null; + if (entry.expiresAt !== null && entry.expiresAt <= this.now()) { this.kv.delete(key); return null; } + return entry.value; + } + async mget(keys: string[]): Promise> { return Promise.all(keys.map((key) => this.get(key))); } + async set(key: string, value: string, options: { nx?: boolean; pxMs?: number } = {}): Promise { + if (options.nx && (await this.get(key)) !== null) return false; + this.kv.set(key, { value, expiresAt: options.pxMs === undefined ? null : this.now() + options.pxMs }); + return true; + } + async incr(key: string): Promise { + const next = Number((await this.get(key)) ?? "0") + 1; + this.kv.set(key, { value: String(next), expiresAt: null }); + return next; + } + async sadd(key: string, member: string): Promise { + const set = this.sets.get(key) ?? new Set(); + set.add(member); + this.sets.set(key, set); + } + async smembers(key: string): Promise { return [...(this.sets.get(key) ?? [])]; } + async publish(channel: string, message: string): Promise { + this.published.push({ channel, message }); + return this.subscribers; + } + async eval(script: string, keys: string[], args: string[]): Promise { + const [k1 = "", k2 = ""] = keys; + const [a1 = "", a2 = "", a3 = "", a4 = ""] = args; + if (script === ASSIGN_SCRIPT) { + const cur = await this.get(k1); + if (cur !== null) { + const obj = JSON.parse(cur) as { workerId: string; assignmentEpoch: number }; + if (obj.assignmentEpoch > Number(a2)) return 0; + if (obj.assignmentEpoch === Number(a2) && obj.workerId !== a3) return 0; + } + await this.set(k1, a1); + await this.sadd(k2, a4); + return 1; + } + if (script === COMMIT_DESIRED_SCRIPT) { + if ((await this.get(k2)) !== a3) return -1; + const cur = await this.get(k1); + if (cur !== null && (JSON.parse(cur) as { desiredVersion: number }).desiredVersion >= Number(a2)) return 0; + await this.set(k1, a1); + return 1; + } + throw new Error("unknown script"); + } + close(): void {} +} + +const WS = randomUUID(); +const CH = randomUUID(); +const WORKER = "media-worker-a"; +const KID = "control-v1"; +const HANDLE = "vh_0123456789abcdefXYZ"; +const RUNNING: DesiredSpec = { + desiredStatus: "RUNNING", + source: { kind: "hls", url: "https://cdn.example.com/live/index.m3u8" }, + output: { endpointUrl: "rtmps://live.example.net/app", credentialHandleId: HANDLE }, + profile: "1080p30", + mode: "normalize", +}; + +describe("desired-state v1/v2 parser", () => { + it("round-trips a v1 single HLS source without adding production fields", () => { + expect(parseDesiredSpec(structuredClone(RUNNING))).toEqual(RUNNING); + expect(JSON.stringify(parseDesiredSpec(structuredClone(RUNNING)))).toBe(JSON.stringify(RUNNING)); + }); + + const sourceId = randomUUID(); + const sceneId = randomUUID(); + const playlist = { + sources: [{ id: sourceId, label: "Reel", kind: "playlist", assetIds: [randomUUID()], loop: true }], + scenes: [{ id: sceneId, label: "Main", layout: "single", slots: [ + { index: 0, sourceId, rect: null, fit: "contain", audioMode: "primary", gainDb: -6 }, + ] }], + buses: { preview: sceneId, program: sceneId }, overlays: [], + transition: { type: "cut" }, audio: { masterGainDb: 0 }, + }; + + it("accepts playlist sources and a single primary audio slot", () => { + expect(parseProductionSpec(playlist).sources[0]?.kind).toBe("playlist"); + }); + + it("rejects filtergraph payloads in scene and source labels", () => { + for (const bad of ["a;b", "x[0]", "a,b", "it's", "movie=/etc/passwd"]) { + expect(() => parseProductionSpec({ ...playlist, scenes: [{ ...playlist.scenes[0], label: bad }] })).toThrow(); + expect(() => parseProductionSpec({ ...playlist, sources: [{ ...playlist.sources[0], label: bad }] })).toThrow(); + } + }); + + it("rejects two primary slots and gains outside -24..0 in 3 dB steps", () => { + const scene = playlist.scenes[0]; + const slot = scene?.slots[0]; + if (!scene || !slot) throw new Error("fixture missing slot"); + expect(() => parseProductionSpec({ ...playlist, scenes: [{ ...scene, layout: "2x1", slots: [slot, { ...slot, index: 1 }] }] })).toThrow(/primary/); + for (const gainDb of [-25, -4, 1]) { + expect(() => parseProductionSpec({ ...playlist, scenes: [{ ...scene, slots: [{ ...slot, gainDb }] }] })).toThrow(); + } + }); +}); + +function keys(): { privateKey: KeyObject; publicKey: KeyObject } { return generateKeyPairSync("ed25519"); } + +function statusOf(error: unknown): number { + return error instanceof HttpException ? error.getStatus() : -1; +} +async function rejectsWith(promise: Promise, status: number): Promise { + const error: unknown = await promise.then(() => null, (e: unknown) => e); + expect(statusOf(error)).toBe(status); +} + +describe("media control plane", () => { + let now: number; + let redis: FakeRedis; + let pair: { privateKey: KeyObject; publicKey: KeyObject }; + let workerKeys: Map; + let service: MediaControlService; + let fenceDir: string; + + beforeEach(async () => { + now = Date.parse("2026-10-07T12:00:00.000Z"); + redis = new FakeRedis(() => now); + pair = keys(); + workerKeys = new Map([[KID, pair.publicKey]]); + service = new MediaControlService({ + redis, signer: new Ed25519ControlSigner(KID, pair.privateKey), handles: new StubCredentialHandleProvider(), + now: () => now, commandTtlMs: 30_000, assignments: [], + }); + await service.assign(WS, CH, WORKER, 3); + fenceDir = mkdtempSync(join(tmpdir(), "mcp-fence-")); + }); + afterEach(() => rmSync(fenceDir, { recursive: true, force: true })); + + function lastPublished(): unknown { return JSON.parse(redis.published.at(-1)?.message ?? "null"); } + + describe("envelope interop with the worker", () => { + it("published command verifies under the worker's verifyCommand", async () => { + const issued = await service.setDesired(WS, CH, RUNNING); + expect(redis.published.at(-1)?.channel).toBe(`cmd:${WS}:${WORKER}`); + const verified = workerVerify(lastPublished(), workerKeys, WORKER, now); + expect(verified.body).toMatchObject({ workspaceId: WS, channelId: CH, workerId: WORKER, assignmentEpoch: 3, + commandId: issued.commandId, desiredVersion: 1 }); + expect(verified.body.snapshot.output).toEqual({ endpointUrl: RUNNING.output?.endpointUrl, credentialHandleId: HANDLE }); + }); + + it("produces byte-identical signatures to the worker's own Ed25519ControlSigner", async () => { + await service.setDesired(WS, CH, RUNNING); + const ours = lastPublished() as SignedCommand; + const theirs = await new WorkerSigner(KID, pair.privateKey).signCommittedSnapshot(ours.body); + expect(theirs.signature).toBe(ours.signature); + expect(workerSign(ours.body, KID, pair.privateKey).signature).toBe(ours.signature); + }); + + it("signs canonically even when snapshot keys arrive in a different order", async () => { + const signer = new Ed25519ControlSigner(KID, pair.privateKey); + const snapshot = { mode: "passthrough" as const, profile: "720p30" as const, output: null, source: null, + desiredStatus: "STOPPED" as const, desiredVersion: 9, assignmentEpoch: 3, workerId: WORKER, channelId: CH, workspaceId: WS }; + const body: CommandBody = { workspaceId: WS, channelId: CH, workerId: WORKER, assignmentEpoch: 3, commandId: randomUUID(), + desiredVersion: 9, issuedAt: new Date(now).toISOString(), expiresAt: new Date(now + 10_000).toISOString(), snapshot }; + const signed = await signer.signCommittedSnapshot(body); + expect(() => workerVerify(JSON.parse(JSON.stringify(signed)), workerKeys, WORKER, now)).not.toThrow(); + }); + + it("worker rejects tampering, unknown kid, wrong worker, and expiry", async () => { + await service.setDesired(WS, CH, RUNNING); + const signed = lastPublished() as SignedCommand; + const tampered = structuredClone(signed); + if (tampered.body.snapshot.source) tampered.body.snapshot.source.url = "https://evil.example.com/x.m3u8"; + expect(() => workerVerify(tampered, workerKeys, WORKER, now)).toThrow("invalid signature"); + expect(() => workerVerify({ ...signed, kid: "control-v2" }, workerKeys, WORKER, now)).toThrow("invalid signature"); + expect(() => workerVerify(signed, new Map([[KID, keys().publicKey]]), WORKER, now)).toThrow("invalid signature"); + expect(() => workerVerify(signed, workerKeys, "media-worker-b", now)).toThrow("invalid command fields"); + expect(() => workerVerify(signed, workerKeys, WORKER, now + 30_000)).toThrow("expired command"); + }); + + it("refuses non-Ed25519 keys", () => { + const { privateKey } = generateKeyPairSync("ec", { namedCurve: "P-256" }); + expect(() => new Ed25519ControlSigner(KID, privateKey)).toThrow("Ed25519"); + expect(() => new Ed25519ControlSigner(KID, createPublicKey(pair.privateKey))).toThrow("Ed25519"); + }); + }); + + describe("replay", () => { + it("worker fence accepts a command once and reports replay on redelivery", async () => { + await service.setDesired(WS, CH, RUNNING); + const fence = new DurableFence(join(fenceDir, "fence.json")); + const command = workerVerify(lastPublished(), workerKeys, WORKER, now); + expect(fence.accept(command.body, now)).toBe("accepted"); + expect(fence.accept(command.body, now)).toBe("replay"); + // Durable: a restarted worker still refuses it. + expect(new DurableFence(join(fenceDir, "fence.json")).accept(command.body, now)).toBe("replay"); + }); + + it("control plane records each commandId once (SETNX) with TTL = envelope lifetime", async () => { + const issued = await service.setDesired(WS, CH, RUNNING); + const record = redis.kv.get(K.issued(issued.commandId)); + expect(record?.expiresAt).toBe(Date.parse(issued.expiresAt)); + expect(JSON.parse(record?.value ?? "{}")).toMatchObject({ workerId: WORKER, desiredVersion: 1, kid: KID }); + const body = (lastPublished() as SignedCommand).body; + await rejectsWith(service.recordIssued(body, KID), 409); + now = Date.parse(issued.expiresAt); + await expect(service.recordIssued(body, KID)).rejects.toThrow("already expired"); + }); + + it("every issued command gets a distinct commandId", async () => { + const a = await service.setDesired(WS, CH, RUNNING); + const b = await service.republish(WS, CH); + expect(a.commandId).not.toBe(b.commandId); + expect(b.desiredVersion).toBe(a.desiredVersion); + }); + }); + + describe("desiredVersion monotonicity", () => { + it("allocates strictly increasing versions and the worker fence accepts them in order", async () => { + const fence = new DurableFence(join(fenceDir, "fence.json")); + const versions: number[] = []; + for (const spec of [RUNNING, { ...RUNNING, desiredStatus: "STOPPED" as const }, RUNNING]) { + const issued = await service.setDesired(WS, CH, spec); + versions.push(issued.desiredVersion); + expect(fence.accept(workerVerify(lastPublished(), workerKeys, WORKER, now).body, now)).toBe("accepted"); + } + expect(versions).toEqual([1, 2, 3]); + }); + + it("an older command delivered late is stale at the worker", async () => { + const fence = new DurableFence(join(fenceDir, "fence.json")); + await service.setDesired(WS, CH, RUNNING); + const older = workerVerify(lastPublished(), workerKeys, WORKER, now).body; + await service.setDesired(WS, CH, RUNNING); + const newer = workerVerify(lastPublished(), workerKeys, WORKER, now).body; + expect(fence.accept(newer, now)).toBe("accepted"); + expect(fence.accept(older, now)).toBe("stale"); + }); + + it("republish re-delivers the committed version under a fresh commandId the fence accepts", async () => { + const fence = new DurableFence(join(fenceDir, "fence.json")); + await service.setDesired(WS, CH, RUNNING); + expect(fence.accept(workerVerify(lastPublished(), workerKeys, WORKER, now).body, now)).toBe("accepted"); + await service.republish(WS, CH); + expect(fence.accept(workerVerify(lastPublished(), workerKeys, WORKER, now).body, now)).toBe("accepted"); + }); + + it("commit CAS refuses to overwrite a newer committed version", async () => { + await service.setDesired(WS, CH, RUNNING); + // Simulate a concurrent writer that committed version 5 after we allocated version 2. + redis.kv.set(K.desired(WS, CH), { value: JSON.stringify({ desiredVersion: 5 }), expiresAt: null }); + const before = redis.published.length; + await rejectsWith(service.setDesired(WS, CH, RUNNING), 409); + expect(redis.published.length).toBe(before); + }); + + it("commit CAS refuses when the assignment changed underneath", async () => { + const incr = redis.incr.bind(redis); + // Reassign between the assignment read and the commit (version allocation sits in between). + redis.incr = async (key: string) => { await service.assign(WS, CH, "media-worker-b", 4); return incr(key); }; + await rejectsWith(service.setDesired(WS, CH, RUNNING), 409); + expect(redis.published).toHaveLength(0); + redis.incr = incr; + const issued = await service.setDesired(WS, CH, RUNNING); + expect(issued).toMatchObject({ workerId: "media-worker-b", assignmentEpoch: 4 }); + expect(redis.published.at(-1)?.channel).toBe(`cmd:${WS}:media-worker-b`); + }); + + it("assignment epochs only move forward and one epoch binds one worker", async () => { + await rejectsWith(service.assign(WS, CH, WORKER, 2), 409); + await rejectsWith(service.assign(WS, CH, "media-worker-b", 3), 409); + await expect(service.assign(WS, CH, WORKER, 3)).resolves.toBeUndefined(); + }); + + it("republish refuses after reassignment", async () => { + await service.setDesired(WS, CH, RUNNING); + await service.assign(WS, CH, "media-worker-b", 4); + await rejectsWith(service.republish(WS, CH), 409); + }); + }); + + describe("fail-closed preconditions", () => { + it("503 without a signing key, 409 for unassigned channels, 400 for bad handles", async () => { + const unsigned = new MediaControlService({ redis, signer: null, handles: new StubCredentialHandleProvider(), now: () => now, assignments: [] }); + await rejectsWith(unsigned.setDesired(WS, CH, RUNNING), 503); + await rejectsWith(service.setDesired(WS, randomUUID(), RUNNING), 409); + await rejectsWith(service.setDesired(WS, CH, { ...RUNNING, output: { endpointUrl: "rtmps://live.example.net/app", credentialHandleId: "sk_live_abc" } }), 400); + expect(redis.published).toHaveLength(0); + }); + + it("bootstraps assignments from configuration", async () => { + const ch = randomUUID(); + const boot = new MediaControlService({ redis, signer: new Ed25519ControlSigner(KID, pair.privateKey), + handles: new StubCredentialHandleProvider(), now: () => now, assignments: [{ workspaceId: WS, channelId: ch, workerId: WORKER, assignmentEpoch: 1 }] }); + await expect(boot.setDesired(WS, ch, RUNNING)).resolves.toMatchObject({ assignmentEpoch: 1, desiredVersion: 1 }); + }); + }); + + describe("endpoint and body validation", () => { + const cases: Array<[string, "hls" | "output"]> = [ + ["https://cdn.example.com/a.m3u8", "hls"], ["http://cdn.example.com/a.m3u8", "hls"], + ["rtmp://live.example.net/app", "output"], ["rtmps://live.example.net/app", "output"], ["srt://live.example.net", "output"], + ["rtmp://user:pw@live.example.net/app", "output"], ["rtmp://live.example.net/app?key=abc", "output"], + ["https://cdn.example.com/a.m3u8#frag", "hls"], ["https://10.1.2.3/a.m3u8", "hls"], ["https://127.0.0.1/a.m3u8", "hls"], + ["https://169.254.169.254/latest", "hls"], ["rtmp://192.168.1.10/app", "output"], ["rtmp://100.64.0.1/app", "output"], + ["https://[::1]/a.m3u8", "hls"], ["https://[fe80::1]/a.m3u8", "hls"], ["https://[fd00::1]/a.m3u8", "hls"], + ["https://[::ffff:8.8.8.8]/a.m3u8", "hls"], ["https://8.8.8.8/a.m3u8", "hls"], ["rtmp://[2606:4700::1]/app", "output"], + ["not a url", "hls"], + ]; + it.each(cases)("agrees with the worker on %s (%s)", (url, use) => { + const ours = (() => { try { validateEndpointShape(url, use); return "ok"; } catch (e) { return String(e); } })(); + const theirs = (() => { try { workerValidate(url, use); return "ok"; } catch (e) { return String(e); } })(); + expect(ours).toBe(theirs); + }); + + it("is stricter than the worker for loopback hostnames", () => { + expect(() => validateEndpointShape("https://localhost/a.m3u8", "hls")).toThrow("private"); + expect(() => validateEndpointShape("rtmp://relay.localhost/app", "output")).toThrow("private"); + }); + + it("never accepts secrets in the body", () => { + for (const body of [ + { ...RUNNING, streamKey: "live_123" }, + { ...RUNNING, output: { ...RUNNING.output, streamKey: "live_123" } }, + { ...RUNNING, output: { ...RUNNING.output, password: "pw" } }, + { ...RUNNING, source: { ...RUNNING.source, token: "t" } }, + ]) expect(() => parseDesiredSpec(body)).toThrow("secrets are never accepted"); + expect(() => parseDesiredSpec({ ...RUNNING, output: { endpointUrl: "rtmp://live.example.net/app?key=abc", credentialHandleId: HANDLE } })) + .toThrow("credentials rejected"); + }); + + it("rejects malformed bodies", () => { + expect(() => parseDesiredSpec({ ...RUNNING, workerId: "x" })).toThrow("not allowed"); + expect(() => parseDesiredSpec({ ...RUNNING, desiredVersion: 99 })).toThrow("not allowed"); + expect(() => parseDesiredSpec({ ...RUNNING, output: null })).toThrow("RUNNING requires"); + expect(() => parseDesiredSpec({ ...RUNNING, profile: "4k" })).toThrow("profile"); + expect(() => parseDesiredSpec({ ...RUNNING, source: { kind: "rtmp", url: "https://cdn.example.com/a" } })).toThrow("source.kind"); + expect(() => parseDesiredSpec({ ...RUNNING, source: { kind: "hls", url: "https://10.0.0.1/a.m3u8" } })).toThrow("private"); + expect(() => parseDesiredSpec([])).toThrow("object"); + }); + + it("accepts STOPPED with no endpoints and applies defaults", () => { + expect(parseDesiredSpec({ desiredStatus: "STOPPED" })).toEqual({ desiredStatus: "STOPPED", source: null, output: null, profile: "720p30", mode: "passthrough" }); + }); + }); + + describe("status", () => { + function observe(overrides: Record): void { + redis.kv.set(K.observed(WS, CH), { value: JSON.stringify({ workspaceId: WS, channelId: CH, workerId: WORKER, assignmentEpoch: 3, + state: "RUNNING", realizedVersion: 1, updatedAt: new Date(now).toISOString(), reason: "ok", ...overrides }), expiresAt: now + 60_000 }); + } + function beat(sentAt: number): void { + redis.kv.set(K.heartbeat(WORKER), { value: JSON.stringify({ workerId: WORKER, sentAt: new Date(sentAt).toISOString(), channels: [] }), expiresAt: now + 15_000 }); + } + + it("reports desired vs observed, convergence, and worker liveness", async () => { + await service.setDesired(WS, CH, RUNNING); + observe({}); + beat(now - 1_000); + const status = await service.getStatus(WS); + expect(status.workers).toEqual([{ workerId: WORKER, alive: true, lastHeartbeatAt: new Date(now - 1_000).toISOString() }]); + expect(status.channels).toEqual([expect.objectContaining({ channelId: CH, workerId: WORKER, assignmentEpoch: 3, + desired: { desiredVersion: 1, desiredStatus: "RUNNING" }, converged: true, workerAlive: true })]); + }); + + it("is not converged until the realized version catches up", async () => { + await service.setDesired(WS, CH, RUNNING); + await service.setDesired(WS, CH, RUNNING); + observe({ realizedVersion: 1 }); + expect((await service.getStatus(WS)).channels[0]?.converged).toBe(false); + }); + + it("ignores observations from a previous placement and stale heartbeats", async () => { + await service.setDesired(WS, CH, RUNNING); + observe({ assignmentEpoch: 2 }); + beat(now - HEARTBEAT_TTL_MS - 1); + const status = await service.getStatus(WS); + expect(status.channels[0]).toMatchObject({ observed: null, converged: false, workerAlive: false }); + expect(status.workers[0]?.alive).toBe(false); + }); + + it("is scoped to the caller's workspace", async () => { + const other = randomUUID(); + await service.assign(other, randomUUID(), "media-worker-z", 0); + const status = await service.getStatus(WS); + expect(status.channels.map((c) => c.channelId)).toEqual([CH]); + expect(status.workers.map((w) => w.workerId)).toEqual([WORKER]); + }); + }); +}); diff --git a/apps/api/src/media-control/production.ts b/apps/api/src/media-control/production.ts new file mode 100644 index 0000000..e8206e7 --- /dev/null +++ b/apps/api/src/media-control/production.ts @@ -0,0 +1,358 @@ +/** + * Desired-state v2: the production part of a channel snapshot + * (docs/design/streaming-architecture.md §2, §4, §6 S1). + * + * This file is mirrored byte-for-byte at apps/media-worker/src/production.ts + * (a test asserts it). It has no imports so both builds compile it as-is. + * + * Everything that can reach a filtergraph or argv is typed: ids are UUIDs, + * assets are sha256 content ids, geometry is integer basis points, gains are + * 3 dB steps, enums are closed. Labels are display text for operators and logs + * only; they are still held to a strict allowlist so a filtergraph payload + * (`;`, `[`, `,`, `'`, `movie=`) can never ride along in a snapshot. + */ + +export type SceneLayout = "single" | "pip" | "2x1" | "2x2" | "3x3" | "4x4" | "custom"; +export type SlotFit = "contain" | "cover" | "crop"; +export type AudioMode = "off" | "on" | "primary"; +export type OverlayKind = "lower_third" | "chyron" | "logo" | "clock"; +export type ClockFormat = "HH:MM" | "HH:MM:SS"; +export type FadeMs = 300 | 500 | 1000 | 2000; + +export const SCENE_LAYOUTS: readonly SceneLayout[] = ["single", "pip", "2x1", "2x2", "3x3", "4x4", "custom"]; +export const SLOT_FITS: readonly SlotFit[] = ["contain", "cover", "crop"]; +export const AUDIO_MODES: readonly AudioMode[] = ["off", "on", "primary"]; +export const OVERLAY_KINDS: readonly OverlayKind[] = ["lower_third", "chyron", "logo", "clock"]; +export const CLOCK_FORMATS: readonly ClockFormat[] = ["HH:MM", "HH:MM:SS"]; +export const FADE_MS: readonly FadeMs[] = [300, 500, 1000, 2000]; + +/** Slot capacity per layout. `custom` places up to 16 explicit rectangles. */ +export const LAYOUT_CAPACITY: Readonly> = { + single: 1, pip: 2, "2x1": 2, "2x2": 4, "3x3": 9, "4x4": 16, custom: 16, +}; + +/** Geometry is integer basis points of the output frame (0..10000), resolution independent. */ +export const GEOMETRY_SCALE = 10_000; +export const GAIN_DB_MIN = -24; +export const GAIN_DB_MAX = 0; +export const GAIN_DB_STEP = 3; +export const CHYRON_SPEED_MIN = 40; +export const CHYRON_SPEED_MAX = 400; +export const CHYRON_STRIP_MAX_PX = 16_384; +export const MAX_SOURCES = 32; +export const MAX_SCENES = 32; +export const MAX_OVERLAYS = 64; +export const MAX_PLAYLIST_ITEMS = 500; +export const MAX_OVERLAY_LAYER = 7; + +export interface Rect { x: number; y: number; w: number; h: number } +export interface Point { x: number; y: number } + +export interface HlsProductionSource { id: string; label: string; kind: "hls"; url: string } +/** Ordered, pre-normalized video asset ids (video_assets.id), fed by the worker's asset feeder. */ +export interface PlaylistProductionSource { id: string; label: string; kind: "playlist"; assetIds: string[]; loop: boolean } +export type ProductionSource = HlsProductionSource | PlaylistProductionSource; + +export interface SceneSlot { + index: number; + /** null = empty slot (renders the fallback). */ + sourceId: string | null; + /** Required for `custom` and `pip` layouts; null means the layout's own grid cell. */ + rect: Rect | null; + fit: SlotFit; + audioMode: AudioMode; + gainDb: number; +} + +export interface ProductionScene { id: string; label: string; layout: SceneLayout; slots: SceneSlot[] } + +export interface ProductionBuses { preview: string | null; program: string | null } + +export interface ProductionOverlay { + /** overlay_slots.id */ + id: string; + sceneId: string; + kind: OverlayKind; + enabled: boolean; + layer: number; + position: Point; + /** overlay_revisions.id; null only for the server-generated clock. */ + revisionId: string | null; + /** sha256 content id of the rendered PNG; null only for the clock. */ + assetId: string | null; + chyron: { speedPxPerSec: number; stripWidthPx: number } | null; + clock: { format: ClockFormat } | null; +} + +export type Transition = { type: "cut" } | { type: "fade"; ms: FadeMs }; + +export interface ProgramAudio { masterGainDb: number } + +export interface ProductionSpec { + scenes: ProductionScene[]; + buses: ProductionBuses; + sources: ProductionSource[]; + overlays: ProductionOverlay[]; + transition: Transition; + audio: ProgramAudio; +} + +export class ProductionSpecError extends Error { + constructor(message: string) { + super(message); + this.name = "ProductionSpecError"; + } +} + +const SECRETISH = /key|secret|token|password|passwd|credential(?!HandleId)|auth/i; +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +const SHA256 = /^[0-9a-f]{64}$/; +/** Letters, digits, space and `_ . -` only: no filtergraph or shell metacharacters, no `=`. */ +const LABEL = /^[\p{L}\p{N}][\p{L}\p{N} _.-]{0,63}$/u; + +function fail(message: string): never { throw new ProductionSpecError(message); } + +function record(value: unknown, field: string, allowed: readonly string[]): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${field} must be an object`); + for (const key of Object.keys(value)) { + if (allowed.includes(key)) continue; + if (SECRETISH.test(key)) fail(`${field}.${key}: secrets are never accepted; reference a vault credentialHandleId`); + fail(`${field}.${key} is not allowed`); + } + for (const key of allowed) if (!(key in value)) fail(`${field}.${key} is required`); + return value as Record; +} + +function list(value: unknown, field: string, max: number): unknown[] { + if (!Array.isArray(value)) fail(`${field} must be an array`); + if (value.length > max) fail(`${field} allows at most ${max} entries`); + return value; +} + +function uuid(value: unknown, field: string): string { + if (typeof value !== "string" || !UUID.test(value)) fail(`${field} must be a lowercase UUID`); + return value; +} + +function nullableUuid(value: unknown, field: string): string | null { + return value === null ? null : uuid(value, field); +} + +function label(value: unknown, field: string): string { + if (typeof value !== "string" || !LABEL.test(value)) { + fail(`${field} must be 1-64 letters, digits, spaces, '_', '.', '-'`); + } + return value; +} + +function int(value: unknown, field: string, min: number, max: number): number { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < min || value > max) { + fail(`${field} must be an integer in ${min}..${max}`); + } + return value; +} + +function bool(value: unknown, field: string): boolean { + if (typeof value !== "boolean") fail(`${field} must be a boolean`); + return value; +} + +function oneOf(value: unknown, field: string, options: readonly T[]): T { + const found = options.find((option) => option === value); + if (found === undefined) fail(`${field} must be one of ${options.join(", ")}`); + return found; +} + +export function gainDb(value: unknown, field: string): number { + const gain = int(value, field, GAIN_DB_MIN, GAIN_DB_MAX); + if (gain % GAIN_DB_STEP !== 0) fail(`${field} must be a multiple of ${GAIN_DB_STEP} dB`); + return gain === 0 ? 0 : gain; +} + +function rect(value: unknown, field: string): Rect { + const r = record(value, field, ["x", "y", "w", "h"]); + const x = int(r.x, `${field}.x`, 0, GEOMETRY_SCALE - 1); + const y = int(r.y, `${field}.y`, 0, GEOMETRY_SCALE - 1); + const w = int(r.w, `${field}.w`, 1, GEOMETRY_SCALE); + const h = int(r.h, `${field}.h`, 1, GEOMETRY_SCALE); + if (x + w > GEOMETRY_SCALE || y + h > GEOMETRY_SCALE) fail(`${field} must lie inside the frame`); + return { x, y, w, h }; +} + +function point(value: unknown, field: string): Point { + const p = record(value, field, ["x", "y"]); + return { x: int(p.x, `${field}.x`, 0, GEOMETRY_SCALE), y: int(p.y, `${field}.y`, 0, GEOMETRY_SCALE) }; +} + +function unique(ids: string[], field: string): void { + if (new Set(ids).size !== ids.length) fail(`${field} ids must be unique`); +} + +/** + * HLS URLs are shape-checked by the caller with its own endpoint validator + * (API and worker differ on loopback strictness); here only bounds are checked. + */ +function source(value: unknown, field: string): ProductionSource { + if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${field} must be an object`); + const kind = (value as Record).kind; + if (kind === "hls") { + const s = record(value, field, ["id", "label", "kind", "url"]); + if (typeof s.url !== "string" || s.url.trim() === "" || s.url.length > 2048) fail(`${field}.url must be a non-empty string`); + return { id: uuid(s.id, `${field}.id`), label: label(s.label, `${field}.label`), kind: "hls", url: s.url }; + } + if (kind === "playlist") { + const s = record(value, field, ["id", "label", "kind", "assetIds", "loop"]); + const assetIds = list(s.assetIds, `${field}.assetIds`, MAX_PLAYLIST_ITEMS).map((item, i) => uuid(item, `${field}.assetIds[${i}]`)); + if (assetIds.length === 0) fail(`${field}.assetIds must not be empty`); + return { id: uuid(s.id, `${field}.id`), label: label(s.label, `${field}.label`), kind: "playlist", assetIds, loop: bool(s.loop, `${field}.loop`) }; + } + return fail(`${field}.kind must be hls or playlist`); +} + +function slot(value: unknown, field: string, layout: SceneLayout, sources: ReadonlySet): SceneSlot { + const s = record(value, field, ["index", "sourceId", "rect", "fit", "audioMode", "gainDb"]); + const index = int(s.index, `${field}.index`, 0, LAYOUT_CAPACITY[layout] - 1); + const sourceId = nullableUuid(s.sourceId, `${field}.sourceId`); + if (sourceId !== null && !sources.has(sourceId)) fail(`${field}.sourceId must name a source of this channel`); + const needsRect = layout === "custom" || (layout === "pip" && index > 0); + const slotRect = s.rect === null ? null : rect(s.rect, `${field}.rect`); + if (needsRect && slotRect === null) fail(`${field}.rect is required for the ${layout} layout`); + if (!needsRect && slotRect !== null) fail(`${field}.rect is only allowed for custom layouts and PIP insets`); + return { + index, sourceId, rect: slotRect, + fit: oneOf(s.fit, `${field}.fit`, SLOT_FITS), + audioMode: oneOf(s.audioMode, `${field}.audioMode`, AUDIO_MODES), + gainDb: gainDb(s.gainDb, `${field}.gainDb`), + }; +} + +function scene(value: unknown, field: string, sources: ReadonlySet): ProductionScene { + const s = record(value, field, ["id", "label", "layout", "slots"]); + const layout = oneOf(s.layout, `${field}.layout`, SCENE_LAYOUTS); + const slots = list(s.slots, `${field}.slots`, LAYOUT_CAPACITY[layout]).map((item, i) => slot(item, `${field}.slots[${i}]`, layout, sources)); + if (new Set(slots.map((x) => x.index)).size !== slots.length) fail(`${field}.slots indexes must be unique`); + if (slots.filter((x) => x.audioMode === "primary").length > 1) fail(`${field}.slots: at most one primary audio slot per scene`); + return { id: uuid(s.id, `${field}.id`), label: label(s.label, `${field}.label`), layout, slots }; +} + +function overlay(value: unknown, field: string, scenes: ReadonlySet): ProductionOverlay { + const o = record(value, field, ["id", "sceneId", "kind", "enabled", "layer", "position", "revisionId", "assetId", "chyron", "clock"]); + const sceneId = uuid(o.sceneId, `${field}.sceneId`); + if (!scenes.has(sceneId)) fail(`${field}.sceneId must name a scene of this channel`); + const kind = oneOf(o.kind, `${field}.kind`, OVERLAY_KINDS); + const rendered = kind !== "clock"; + let revisionId: string | null = null; + let assetId: string | null = null; + if (rendered) { + revisionId = uuid(o.revisionId, `${field}.revisionId`); + if (typeof o.assetId !== "string" || !SHA256.test(o.assetId)) fail(`${field}.assetId must be a sha256 content id`); + assetId = o.assetId; + } else if (o.revisionId !== null || o.assetId !== null) fail(`${field}: a clock overlay has no rendered asset`); + + let chyron: ProductionOverlay["chyron"] = null; + if (kind === "chyron") { + const c = record(o.chyron, `${field}.chyron`, ["speedPxPerSec", "stripWidthPx"]); + chyron = { + speedPxPerSec: int(c.speedPxPerSec, `${field}.chyron.speedPxPerSec`, CHYRON_SPEED_MIN, CHYRON_SPEED_MAX), + stripWidthPx: int(c.stripWidthPx, `${field}.chyron.stripWidthPx`, 1, CHYRON_STRIP_MAX_PX), + }; + } else if (o.chyron !== null) fail(`${field}.chyron is only allowed for chyron overlays`); + + let clock: ProductionOverlay["clock"] = null; + if (kind === "clock") { + const c = record(o.clock, `${field}.clock`, ["format"]); + clock = { format: oneOf(c.format, `${field}.clock.format`, CLOCK_FORMATS) }; + } else if (o.clock !== null) fail(`${field}.clock is only allowed for clock overlays`); + + return { + id: uuid(o.id, `${field}.id`), sceneId, kind, + enabled: bool(o.enabled, `${field}.enabled`), + layer: int(o.layer, `${field}.layer`, 0, MAX_OVERLAY_LAYER), + position: point(o.position, `${field}.position`), + revisionId, assetId, chyron, clock, + }; +} + +function transition(value: unknown, field: string): Transition { + if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${field} must be an object`); + const type = (value as Record).type; + if (type === "cut") { + record(value, field, ["type"]); + return { type: "cut" }; + } + if (type === "fade") { + const t = record(value, field, ["type", "ms"]); + return { type: "fade", ms: oneOf(t.ms, `${field}.ms`, FADE_MS) }; + } + return fail(`${field}.type must be cut or fade`); +} + +/** Parse and normalize; throws ProductionSpecError with a field path on the first problem. */ +export function parseProductionSpec(value: unknown, field = "production"): ProductionSpec { + const p = record(value, field, ["scenes", "buses", "sources", "overlays", "transition", "audio"]); + const sources = list(p.sources, `${field}.sources`, MAX_SOURCES).map((item, i) => source(item, `${field}.sources[${i}]`)); + unique(sources.map((s) => s.id), `${field}.sources`); + const sourceIds = new Set(sources.map((s) => s.id)); + const scenes = list(p.scenes, `${field}.scenes`, MAX_SCENES).map((item, i) => scene(item, `${field}.scenes[${i}]`, sourceIds)); + unique(scenes.map((s) => s.id), `${field}.scenes`); + const sceneIds = new Set(scenes.map((s) => s.id)); + const overlays = list(p.overlays, `${field}.overlays`, MAX_OVERLAYS).map((item, i) => overlay(item, `${field}.overlays[${i}]`, sceneIds)); + unique(overlays.map((o) => o.id), `${field}.overlays`); + + const b = record(p.buses, `${field}.buses`, ["preview", "program"]); + const buses: ProductionBuses = { + preview: nullableUuid(b.preview, `${field}.buses.preview`), + program: nullableUuid(b.program, `${field}.buses.program`), + }; + for (const bus of ["preview", "program"] as const) { + const id = buses[bus]; + if (id !== null && !sceneIds.has(id)) fail(`${field}.buses.${bus} must name a scene of this channel`); + } + const a = record(p.audio, `${field}.audio`, ["masterGainDb"]); + return { + scenes, buses, sources, overlays, + transition: transition(p.transition, `${field}.transition`), + audio: { masterGainDb: gainDb(a.masterGainDb, `${field}.audio.masterGainDb`) }, + }; +} + +export function isProductionSpec(value: unknown): value is ProductionSpec { + try { + parseProductionSpec(value); + return true; + } catch { + return false; + } +} + +/** Every remote URL a production snapshot can make the worker fetch. */ +export function productionHlsUrls(spec: ProductionSpec): string[] { + return spec.sources.flatMap((s) => (s.kind === "hls" ? [s.url] : [])); +} + +/** Declared key order for signing: insertion order of the input can never drift the signed bytes. */ +export function canonicalProduction(spec: ProductionSpec): ProductionSpec { + return { + scenes: spec.scenes.map((s) => ({ + id: s.id, label: s.label, layout: s.layout, + slots: s.slots.map((x) => ({ + index: x.index, sourceId: x.sourceId, + rect: x.rect === null ? null : { x: x.rect.x, y: x.rect.y, w: x.rect.w, h: x.rect.h }, + fit: x.fit, audioMode: x.audioMode, gainDb: x.gainDb, + })), + })), + buses: { preview: spec.buses.preview, program: spec.buses.program }, + sources: spec.sources.map((s): ProductionSource => (s.kind === "hls" + ? { id: s.id, label: s.label, kind: "hls", url: s.url } + : { id: s.id, label: s.label, kind: "playlist", assetIds: [...s.assetIds], loop: s.loop })), + overlays: spec.overlays.map((o) => ({ + id: o.id, sceneId: o.sceneId, kind: o.kind, enabled: o.enabled, layer: o.layer, + position: { x: o.position.x, y: o.position.y }, + revisionId: o.revisionId, assetId: o.assetId, + chyron: o.chyron === null ? null : { speedPxPerSec: o.chyron.speedPxPerSec, stripWidthPx: o.chyron.stripWidthPx }, + clock: o.clock === null ? null : { format: o.clock.format }, + })), + transition: spec.transition.type === "cut" ? { type: "cut" } : { type: "fade", ms: spec.transition.ms }, + audio: { masterGainDb: spec.audio.masterGainDb }, + }; +} diff --git a/apps/api/src/media-control/redis.ts b/apps/api/src/media-control/redis.ts new file mode 100644 index 0000000..76ad959 --- /dev/null +++ b/apps/api/src/media-control/redis.ts @@ -0,0 +1,177 @@ +import net, { type Socket } from "node:net"; +import tls from "node:tls"; + +/** Minimal Redis surface the control plane needs; faked in unit tests. */ +export interface ControlRedis { + get(key: string): Promise; + mget(keys: string[]): Promise>; + /** SET key value PX ttl [NX]; resolves false when NX lost. */ + set(key: string, value: string, options?: { nx?: boolean; pxMs?: number }): Promise; + incr(key: string): Promise; + sadd(key: string, member: string): Promise; + smembers(key: string): Promise; + publish(channel: string, message: string): Promise; + eval(script: string, keys: string[], args: string[]): Promise; + close(): void; +} + +export type RedisReply = string | number | null | RedisReply[] | RedisError; +export class RedisError extends Error {} + +const MAX_BULK = 4_000_000; +const MAX_ARRAY = 100_000; + +export function encode(parts: string[]): Buffer { + return Buffer.from(`*${parts.length}\r\n${parts.map((part) => `$${Buffer.byteLength(part)}\r\n${part}\r\n`).join("")}`); +} + +export function parse(buffer: Buffer, offset = 0): { reply: RedisReply; end: number } | null { + if (offset >= buffer.length) return null; + const endLine = buffer.indexOf("\r\n", offset); + if (endLine < 0) return null; + const type = String.fromCharCode(buffer[offset] ?? 0); + const line = buffer.subarray(offset + 1, endLine).toString(); + let pos = endLine + 2; + if (type === "+") return { reply: line, end: pos }; + if (type === "-") return { reply: new RedisError(line), end: pos }; + if (type === ":") return { reply: Number(line), end: pos }; + const length = Number(line); + if (type === "$") { + if (length === -1) return { reply: null, end: pos }; + if (!Number.isInteger(length) || length < 0 || length > MAX_BULK) throw new Error("invalid Redis bulk length"); + if (buffer.length < pos + length + 2) return null; + return { reply: buffer.subarray(pos, pos + length).toString(), end: pos + length + 2 }; + } + if (type === "*") { + if (length === -1) return { reply: null, end: pos }; + if (!Number.isInteger(length) || length < 0 || length > MAX_ARRAY) throw new Error("invalid Redis array length"); + const replies: RedisReply[] = []; + for (let i = 0; i < length; i += 1) { + const item = parse(buffer, pos); + if (!item) return null; + replies.push(item.reply); + pos = item.end; + } + return { reply: replies, end: pos }; + } + throw new Error("invalid Redis reply"); +} + +class RedisConnection { + private buffer = Buffer.alloc(0); + private readonly pending: Array<{ resolve: (reply: RedisReply) => void; reject: (error: Error) => void }> = []; + closed = false; + + private constructor(private readonly socket: Socket) { + socket.on("data", (chunk: Buffer) => { + this.buffer = Buffer.concat([this.buffer, chunk]); + if (this.buffer.length > MAX_BULK * 2) { socket.destroy(new Error("Redis reply too large")); return; } + try { + for (let parsed = parse(this.buffer); parsed; parsed = parse(this.buffer)) { + this.buffer = this.buffer.subarray(parsed.end); + const waiter = this.pending.shift(); + if (parsed.reply instanceof RedisError) waiter?.reject(parsed.reply); + else waiter?.resolve(parsed.reply); + } + } catch (error: unknown) { + socket.destroy(error instanceof Error ? error : new Error("Redis protocol error")); + } + }); + const fail = (error: Error): void => { + this.closed = true; + for (const item of this.pending.splice(0)) item.reject(error); + }; + socket.on("error", fail); + socket.on("close", () => fail(new Error("Redis disconnected"))); + } + + static async connect(url: URL): Promise { + const port = Number(url.port || 6379); + const socket = url.protocol === "rediss:" + ? tls.connect({ host: url.hostname, port, servername: url.hostname, rejectUnauthorized: true }) + : net.connect({ host: url.hostname, port }); + await new Promise((resolve, reject) => { + socket.once(url.protocol === "rediss:" ? "secureConnect" : "connect", () => resolve()); + socket.once("error", reject); + }); + const connection = new RedisConnection(socket); + if (url.password) await connection.send(["AUTH", decodeURIComponent(url.username || "default"), decodeURIComponent(url.password)]); + const db = url.pathname.replace(/^\//, ""); + if (db !== "") await connection.send(["SELECT", db]); + return connection; + } + + send(parts: string[]): Promise { + if (this.closed) return Promise.reject(new Error("Redis disconnected")); + return new Promise((resolve, reject) => { + this.pending.push({ resolve, reject }); + this.socket.write(encode(parts)); + }); + } + + close(): void { this.closed = true; this.socket.destroy(); } +} + +function asString(reply: RedisReply): string | null { + return typeof reply === "string" ? reply : null; +} + +/** Lazily (re)connecting client: a dropped socket is replaced on the next call. */ +export class RespControlRedis implements ControlRedis { + private connection: Promise | null = null; + private readonly url: URL; + + constructor(rawUrl: string) { + this.url = new URL(rawUrl); + if (this.url.protocol !== "redis:" && this.url.protocol !== "rediss:") throw new Error("invalid Redis URL"); + } + + private async send(parts: string[]): Promise { + let current = this.connection; + if (current) { + const conn = await current.catch(() => null); + if (!conn || conn.closed) current = null; + } + if (!current) { + current = RedisConnection.connect(this.url); + this.connection = current; + current.catch(() => { if (this.connection === current) this.connection = null; }); + } + return (await current).send(parts); + } + + async get(key: string): Promise { return asString(await this.send(["GET", key])); } + async mget(keys: string[]): Promise> { + if (keys.length === 0) return []; + const reply = await this.send(["MGET", ...keys]); + return Array.isArray(reply) ? reply.map(asString) : keys.map(() => null); + } + async set(key: string, value: string, options: { nx?: boolean; pxMs?: number } = {}): Promise { + const parts = ["SET", key, value]; + if (options.pxMs !== undefined) parts.push("PX", String(Math.max(1, Math.ceil(options.pxMs)))); + if (options.nx) parts.push("NX"); + return (await this.send(parts)) === "OK"; + } + async incr(key: string): Promise { + const reply = await this.send(["INCR", key]); + if (typeof reply !== "number") throw new Error("unexpected INCR reply"); + return reply; + } + async sadd(key: string, member: string): Promise { await this.send(["SADD", key, member]); } + async smembers(key: string): Promise { + const reply = await this.send(["SMEMBERS", key]); + return Array.isArray(reply) ? reply.filter((item): item is string => typeof item === "string") : []; + } + async publish(channel: string, message: string): Promise { + const reply = await this.send(["PUBLISH", channel, message]); + return typeof reply === "number" ? reply : 0; + } + async eval(script: string, keys: string[], args: string[]): Promise { + return this.send(["EVAL", script, String(keys.length), ...keys, ...args]); + } + close(): void { + const current = this.connection; + this.connection = null; + void current?.then((conn) => conn.close(), () => undefined); + } +} diff --git a/apps/api/src/media-control/spec.ts b/apps/api/src/media-control/spec.ts new file mode 100644 index 0000000..a3e14b2 --- /dev/null +++ b/apps/api/src/media-control/spec.ts @@ -0,0 +1,88 @@ +import { BadRequestException } from "@nestjs/common"; +import { validateEndpointShape } from "./endpoints.js"; +import { parseProductionSpec, ProductionSpecError, type ProductionSpec } from "./production.js"; +import { hasRunnableProgram, OUTPUT_MODES, OUTPUT_PRESETS, type DesiredState, type OutputMode, type OutputPreset } from "./wire.js"; + +/** Operator-controlled part of a desired snapshot; identity/assignment/version are control-plane owned. */ +export interface DesiredSpec { + desiredStatus: DesiredState; + source: { kind: "hls"; url: string } | null; + output: { endpointUrl: string; credentialHandleId: string } | null; + profile: OutputPreset; + mode: OutputMode; + /** v2 production; omitted for v1 specs so they round-trip unchanged. Excludes `source`. */ + production?: ProductionSpec; +} + +const SPEC_KEYS = ["desiredStatus", "source", "output", "profile", "mode", "production"]; +const SECRETISH = /key|secret|token|password|passwd|credential(?!HandleId)|auth/i; + +function object(value: unknown, field: string, allowed: string[]): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) throw new BadRequestException(`${field} must be an object`); + for (const key of Object.keys(value)) { + if (allowed.includes(key)) continue; + if (SECRETISH.test(key)) throw new BadRequestException(`${field}.${key}: secrets are never accepted; reference a vault credentialHandleId`); + throw new BadRequestException(`${field}.${key} is not allowed`); + } + return value as Record; +} + +function nonEmpty(value: unknown, field: string, max = 2048): string { + if (typeof value !== "string" || value.trim() === "" || value.length > max) throw new BadRequestException(`${field} must be a non-empty string`); + return value; +} + +function endpoint(raw: string, use: "hls" | "output", field: string): string { + try { validateEndpointShape(raw, use); } catch (error: unknown) { + throw new BadRequestException(`${field}: ${error instanceof Error ? error.message : "invalid endpoint"}`); + } + return raw; +} + +function isPreset(value: unknown): value is OutputPreset { return OUTPUT_PRESETS.some((preset) => preset === value); } +function isMode(value: unknown): value is OutputMode { return OUTPUT_MODES.some((mode) => mode === value); } + +export function parseDesiredSpec(value: unknown): DesiredSpec { + const body = object(value, "body", SPEC_KEYS); + const desiredStatus = body.desiredStatus; + if (desiredStatus !== "RUNNING" && desiredStatus !== "STOPPED") throw new BadRequestException("desiredStatus must be RUNNING or STOPPED"); + const profile = body.profile ?? "720p30"; + if (!isPreset(profile)) throw new BadRequestException(`profile must be one of ${OUTPUT_PRESETS.join(", ")}`); + const mode = body.mode ?? "passthrough"; + if (!isMode(mode)) throw new BadRequestException(`mode must be one of ${OUTPUT_MODES.join(", ")}`); + + let source: DesiredSpec["source"] = null; + if (body.source !== undefined && body.source !== null) { + const s = object(body.source, "source", ["kind", "url"]); + if (s.kind !== "hls") throw new BadRequestException("source.kind must be hls"); + source = { kind: "hls", url: endpoint(nonEmpty(s.url, "source.url"), "hls", "source.url") }; + } + let output: DesiredSpec["output"] = null; + if (body.output !== undefined && body.output !== null) { + const o = object(body.output, "output", ["endpointUrl", "credentialHandleId"]); + output = { + endpointUrl: endpoint(nonEmpty(o.endpointUrl, "output.endpointUrl"), "output", "output.endpointUrl"), + credentialHandleId: nonEmpty(o.credentialHandleId, "output.credentialHandleId", 160), + }; + } + if (body.production === undefined || body.production === null) { + if (desiredStatus === "RUNNING" && !hasRunnableProgram({ source, output })) { + throw new BadRequestException("RUNNING requires both source and output"); + } + return { desiredStatus, source, output, profile, mode }; + } + + if (source !== null) throw new BadRequestException("source must be null with production; list sources in production.sources"); + let production: ProductionSpec; + try { production = parseProductionSpec(body.production); } catch (error: unknown) { + if (error instanceof ProductionSpecError) throw new BadRequestException(error.message); + throw error; + } + production.sources.forEach((s, i) => { + if (s.kind === "hls") endpoint(s.url, "hls", `production.sources[${i}].url`); + }); + if (desiredStatus === "RUNNING" && !hasRunnableProgram({ source, output, production })) { + throw new BadRequestException("RUNNING requires output and a program scene (production.buses.program)"); + } + return { desiredStatus, source, output, profile, mode, production }; +} diff --git a/apps/api/src/media-control/wire.ts b/apps/api/src/media-control/wire.ts new file mode 100644 index 0000000..577be15 --- /dev/null +++ b/apps/api/src/media-control/wire.ts @@ -0,0 +1,125 @@ +import { sign, type KeyObject } from "node:crypto"; +import { validateEndpointShape } from "./endpoints.js"; +import { canonicalProduction, parseProductionSpec, productionHlsUrls, type ProductionSpec } from "./production.js"; + +/** + * Local mirror of the media worker's control contract + * (apps/media-worker/src/types.ts + bus/envelope.ts + bus/control-signer.ts). + * Kept in this module so the API build never compiles worker sources; the + * interop tests sign here and verify with the worker's own verifier. + */ +export type DesiredState = "STOPPED" | "RUNNING"; +export type OutputPreset = "720p30" | "1080p30" | "1080p60"; +export type OutputMode = "passthrough" | "normalize"; +export type ChannelState = "STOPPED" | "STARTING" | "RUNNING" | "DEGRADED" | "RESTARTING" | "FAILED"; + +export const OUTPUT_PRESETS: readonly OutputPreset[] = ["720p30", "1080p30", "1080p60"]; +export const OUTPUT_MODES: readonly OutputMode[] = ["passthrough", "normalize"]; + +/** A complete, secret-free channel snapshot. The control plane owns this record. */ +export interface DesiredChannel { + workspaceId: string; + channelId: string; + workerId: string; + assignmentEpoch: number; + desiredVersion: number; + desiredStatus: DesiredState; + source: { kind: "hls"; url: string } | null; + output: { endpointUrl: string; credentialHandleId: string } | null; + profile: OutputPreset; + mode: OutputMode; + /** + * Desired-state v2 (production: scenes, buses, sources, overlays, + * transition, audio). Absent on v1 snapshots, which keep their exact + * signed bytes; when present, `source` is null. + */ + production?: ProductionSpec; +} + +export interface ObservedChannel { + workspaceId: string; + channelId: string; + workerId: string; + assignmentEpoch: number; + state: ChannelState; + realizedVersion: number | null; + updatedAt: string; + reason: string; +} + +export interface CommandBody { + workspaceId: string; + channelId: string; + workerId: string; + assignmentEpoch: number; + commandId: string; + desiredVersion: number; + issuedAt: string; + expiresAt: string; + snapshot: DesiredChannel; +} +export interface SignedCommand { kid: string; body: CommandBody; signature: string } + +/** Fixed field order is the wire-format contract for both signing and verifying. */ +export function commandBytes(body: CommandBody): Buffer { + return Buffer.from(JSON.stringify({ + workspaceId: body.workspaceId, channelId: body.channelId, workerId: body.workerId, + assignmentEpoch: body.assignmentEpoch, commandId: body.commandId, + desiredVersion: body.desiredVersion, issuedAt: body.issuedAt, + expiresAt: body.expiresAt, snapshot: canonicalSnapshot(body.snapshot), + })); +} + +/** Snapshot keys in the worker's declared order, so key insertion order can't drift the bytes. */ +export function canonicalSnapshot(s: DesiredChannel): DesiredChannel { + const canonical: DesiredChannel = { + workspaceId: s.workspaceId, channelId: s.channelId, workerId: s.workerId, + assignmentEpoch: s.assignmentEpoch, desiredVersion: s.desiredVersion, desiredStatus: s.desiredStatus, + source: s.source === null ? null : { kind: s.source.kind, url: s.source.url }, + output: s.output === null ? null : { endpointUrl: s.output.endpointUrl, credentialHandleId: s.output.credentialHandleId }, + profile: s.profile, mode: s.mode, + }; + // v1 snapshots carry no production key at all, so their signed bytes are unchanged. + if (s.production !== undefined) canonical.production = canonicalProduction(s.production); + return canonical; +} + +/** Every remote HLS URL a snapshot can make the worker fetch (v1 source or v2 production sources). */ +export function snapshotHlsUrls(s: DesiredChannel): string[] { + return [...(s.source === null ? [] : [s.source.url]), ...(s.production === undefined ? [] : productionHlsUrls(s.production))]; +} + +/** RUNNING needs an output plus something to put on program: the v1 source, or a v2 program scene. */ +export function hasRunnableProgram(s: Pick): boolean { + if (s.output === null) return false; + return s.production === undefined ? s.source !== null : s.production.buses.program !== null; +} + +/** Caller must commit desired state and re-authorize before signing. */ +export interface ControlCommandSigner { + signCommittedSnapshot(body: CommandBody): Promise; +} + +export class Ed25519ControlSigner implements ControlCommandSigner { + constructor(private readonly kid: string, private readonly privateKey: KeyObject) { + if (privateKey.type !== "private" || privateKey.asymmetricKeyType !== "ed25519") throw new Error("control signer requires an Ed25519 private key"); + } + async signCommittedSnapshot(body: CommandBody): Promise { + assertBodyMatchesSnapshot(body); + for (const url of snapshotHlsUrls(body.snapshot)) validateEndpointShape(url, "hls"); + if (body.snapshot.output) validateEndpointShape(body.snapshot.output.endpointUrl, "output"); + const canonical: CommandBody = { ...body, snapshot: canonicalSnapshot(body.snapshot) }; + return { kid: this.kid, body: canonical, signature: sign(null, commandBytes(canonical), this.privateKey).toString("base64") }; + } +} + +function assertBodyMatchesSnapshot(body: CommandBody): void { + const s = body.snapshot; + if (s.workspaceId !== body.workspaceId || s.channelId !== body.channelId || s.workerId !== body.workerId || + s.assignmentEpoch !== body.assignmentEpoch || s.desiredVersion !== body.desiredVersion) throw new Error("snapshot mismatch"); + if (s.production !== undefined) { + if (s.source !== null) throw new Error("production snapshots carry sources in production.sources"); + parseProductionSpec(s.production); + } + if (s.desiredStatus === "RUNNING" && !hasRunnableProgram(s)) throw new Error("running snapshot requires a program source and output"); +} diff --git a/apps/api/src/production/journal.ts b/apps/api/src/production/journal.ts new file mode 100644 index 0000000..021062d --- /dev/null +++ b/apps/api/src/production/journal.ts @@ -0,0 +1,46 @@ +import type { PoolClient } from "pg"; +import { BadRequestException, ConflictException } from "@nestjs/common"; +import { createHash } from "node:crypto"; +import type { DesiredChannel } from "../media-control/wire.js"; +import type { DesiredSpec } from "../media-control/spec.js"; + +export type OverrideMode = "until_released" | "five_minutes" | "until_source_offline" | "until_next_event"; +export type DirectorOperation = + | { type: "desired"; spec: DesiredSpec } + | { type: "take" | "cut" } + | { type: "pst-select"; sceneId: string } + | { type: "lock"; enabled: boolean } + | { type: "override"; mode: OverrideMode | null } + | { type: "overlay"; slotId: string; revisionId: string }; +export interface DirectorState { locked: boolean; override: { mode: OverrideMode; expiresAt: string | null } | null } +export interface DirectorReceipt { + operation: DirectorOperation; + requestId: string; + fingerprint: string; + snapshot: DesiredChannel; + state: DirectorState; +} +export interface DirectorActor { userId: string | null; role: string } +export type TenantTransaction = (workspaceId: string, fn: (client: PoolClient) => Promise) => Promise; + +export function requestIdentity(workspaceId: string, channelId: string, idempotencyKey: string): string { + if (!/^[\x21-\x7e]{1,128}$/.test(idempotencyKey)) throw new BadRequestException("Idempotency-Key must contain 1–128 printable characters"); + return createHash("sha256").update(JSON.stringify([workspaceId, channelId, idempotencyKey])).digest("hex"); +} +export function fingerprint(operation: DirectorOperation): string { + // Rebuild a closed shape: field insertion order cannot change request identity. + const value = operation.type === "pst-select" ? [operation.type, operation.sceneId] + : operation.type === "lock" ? [operation.type, operation.enabled] + : operation.type === "override" ? [operation.type, operation.mode] + : operation.type === "overlay" ? [operation.type, operation.slotId, operation.revisionId] + : operation.type === "desired" ? [operation.type, operation.spec] : [operation.type]; + return createHash("sha256").update(JSON.stringify(value)).digest("hex"); +} +export async function findReceipt(client: PoolClient, workspaceId: string, channelId: string, requestId: string, expected: string): Promise { + const result = await client.query<{ diff: DirectorReceipt }>( + `SELECT diff FROM audit_log WHERE workspace_id=$1 AND target_type='production_command' + AND target_id=$2 AND diff->>'requestId'=$3 ORDER BY seq DESC LIMIT 1`, [workspaceId, channelId, requestId]); + const receipt = result.rows[0]?.diff; + if (receipt && receipt.fingerprint !== expected) throw new ConflictException("Idempotency-Key was used with another operation"); + return receipt ?? null; +} diff --git a/apps/api/src/production/preview.controller.ts b/apps/api/src/production/preview.controller.ts new file mode 100644 index 0000000..48e934e --- /dev/null +++ b/apps/api/src/production/preview.controller.ts @@ -0,0 +1,33 @@ +import { Controller, ForbiddenException, Get, Inject, Param, Query, Req, Res } from "@nestjs/common"; +import type { Request, Response } from "express"; +import { parseId } from "../http/validation.js"; +import { PreviewService } from "./preview.js"; + +/** Capability URLs are minted by the authenticated media-control endpoint. + * Proxy plaintext through this origin; encrypted S3 objects are never redirected. */ +@Controller("media-control/preview") +export class PreviewController { + constructor(@Inject(PreviewService) private readonly preview: PreviewService) {} + @Get(":workspaceId/:channelId/:tap/:file") + async read(@Req() request: Request, @Res() response: Response, + @Param("workspaceId") ws: string, @Param("channelId") ch: string, @Param("tap") tap: string, @Param("file") file: string, + @Query("expires") expires: string, @Query("signature") signature: string): Promise { + const origin = request.headers.origin; + const ownOrigin = process.env.PUBLIC_ORIGIN ?? `${request.protocol}://${request.get("host")}`; + if (origin && origin !== ownOrigin) throw new ForbiddenException("cross-origin preview denied"); + parseId(ws, "workspaceId"); parseId(ch, "channelId"); + if (tap !== "program") parseId(tap, "tap"); + this.preview.verify(ws, ch, tap, file, expires, signature); + const object = await this.preview.read(ws, ch, tap, file); + response.setHeader("ETag", object.etag); + response.setHeader("Cache-Control", "private, max-age=1"); + response.setHeader("Cross-Origin-Resource-Policy", "same-origin"); + response.setHeader("X-Content-Type-Options", "nosniff"); + response.setHeader("Vary", "Origin"); + response.removeHeader("Access-Control-Allow-Origin"); + if (request.headers["if-none-match"]?.split(/,\s*/).some(tag => tag === object.etag || tag === `W/${object.etag}` || tag === "*")) { + response.status(304).end(); return; + } + response.type(object.contentType).send(object.body); + } +} diff --git a/apps/api/src/production/preview.test.ts b/apps/api/src/production/preview.test.ts new file mode 100644 index 0000000..04f5556 --- /dev/null +++ b/apps/api/src/production/preview.test.ts @@ -0,0 +1,54 @@ +import { randomUUID } from "node:crypto"; +import { describe, expect, it, vi } from "vitest"; +import { MemoryRedis } from "./testing/memory-redis.js"; +import { PreviewService } from "./preview.js"; +import { keyForHash, sha256, type StorageService } from "../storage/storage.js"; +const WS = randomUUID(), CH = randomUUID(), TAP = randomUUID(); +function fixture(): { preview: PreviewService; storage: StorageService; redis: MemoryRedis; seed: (file: string, body: string, type?: string) => void; advance: (ms: number) => void } { + const redis = new MemoryRedis(), objects = new Map(); let now = Date.now(); + const storage: StorageService = { putContent: async () => { throw new Error("unused"); }, getObject: vi.fn(async key => { await new Promise(r => setTimeout(r, 1)); return objects.get(key) ?? Buffer.alloc(0); }), hasContent: async () => false, presign: async () => { throw new Error("must not redirect to encrypted S3"); } }; + const preview = new PreviewService({ redis, storage, secret: "test", now: () => now, assertTap: async () => undefined }); + const seed = (file: string, body: string, type = "video/mp2t"): void => { + const buffer = Buffer.from(body), key = keyForHash(sha256(buffer), type); objects.set(key, buffer); + redis.data.set(`preview:${WS}:${CH}:${TAP}:${file}`, JSON.stringify({ key, contentType: type })); + }; + return { preview, storage, redis, seed, advance: ms => { now += ms; } }; +} +describe("same-origin short-lived preview capabilities and shared cache", () => { + it("single-flight concurrent requests fill once and reuse an ETag", async () => { + const f = fixture(); f.seed("thumbnail.jpg", "jpeg", "image/jpeg"); + const objects = await Promise.all(Array.from({ length: 20 }, () => f.preview.read(WS, CH, TAP, "thumbnail.jpg"))); + expect(f.storage.getObject).toHaveBeenCalledTimes(1); expect(new Set(objects.map(o => o.etag)).size).toBe(1); + expect(objects[0]?.body.toString()).toBe("jpeg"); + f.advance(1001); await f.preview.read(WS, CH, TAP, "thumbnail.jpg"); expect(f.storage.getObject).toHaveBeenCalledTimes(2); + }); + it("keeps at most six segments per tap and fills evicted segments", async () => { + const f = fixture(); for (let i = 0; i < 7; i++) { f.seed(`s${i}.ts`, `segment-${i}`); await f.preview.read(WS, CH, TAP, `s${i}.ts`); } + await f.preview.read(WS, CH, TAP, "s0.ts"); expect(f.storage.getObject).toHaveBeenCalledTimes(8); + }); + it("signs path-bound URLs that expire after 60 seconds", async () => { + const f = fixture(); const urls = await f.preview.urls(WS, CH, TAP); + const url = new URL(urls.thumbnail ?? "", "https://same-origin.example"); + const expiry = url.searchParams.get("expires") ?? "", signature = url.searchParams.get("signature") ?? ""; + expect(url.pathname.startsWith("/media-control/preview/")).toBe(true); + expect(() => f.preview.verify(WS, CH, TAP, "thumbnail.jpg", expiry, signature)).not.toThrow(); + expect(() => f.preview.verify(randomUUID(), CH, TAP, "thumbnail.jpg", expiry, signature)).toThrow(); + expect(() => f.preview.verify(WS, CH, TAP, "audio.m3u8", expiry, signature)).toThrow(); + f.advance(60_000); expect(() => f.preview.verify(WS, CH, TAP, "thumbnail.jpg", expiry, signature)).toThrow("expired"); + }); + it("rewrites HLS media and init URLs to signed same-origin paths", async () => { + const f = fixture(); f.seed("video.m3u8", '#EXTM3U\n#EXT-X-MAP:URI="init.mp4"\n#EXTINF:1,\ns1.m4s\n', "application/vnd.apple.mpegurl"); + const object = await f.preview.read(WS, CH, TAP, "video.m3u8"); + expect(object.body.toString()).toContain(`/media-control/preview/${WS}/${CH}/${TAP}/init.mp4?expires=`); + expect(object.body.toString()).toContain(`/media-control/preview/${WS}/${CH}/${TAP}/s1.m4s?expires=`); + expect(object.body.toString()).not.toContain("https://"); + }); + it.each(["https://evil.example/s.ts", "../escape.ts", "//evil/s.ts", "file:///etc/passwd"])("rejects external/traversal HLS URI %s", async uri => { + const f = fixture(); f.seed("video.m3u8", `#EXTM3U\n${uri}\n`, "application/vnd.apple.mpegurl"); + await expect(f.preview.read(WS, CH, TAP, "video.m3u8")).rejects.toThrow("invalid preview file"); + }); + it("failed fill does not poison the single-flight cache", async () => { + const f = fixture(); await expect(f.preview.read(WS, CH, TAP, "s.ts")).rejects.toThrow("unavailable"); + f.seed("s.ts", "fixed"); expect((await f.preview.read(WS, CH, TAP, "s.ts")).body.toString()).toBe("fixed"); + }); +}); diff --git a/apps/api/src/production/preview.ts b/apps/api/src/production/preview.ts new file mode 100644 index 0000000..3401542 --- /dev/null +++ b/apps/api/src/production/preview.ts @@ -0,0 +1,99 @@ +import { createHmac, timingSafeEqual } from "node:crypto"; +import { BadRequestException, ForbiddenException, Inject, Injectable, NotFoundException, OnModuleDestroy, Optional } from "@nestjs/common"; +import { withTenant } from "../db/client.js"; +import { RespControlRedis, type ControlRedis } from "../media-control/redis.js"; +import { S3Storage, s3ConfigFromEnv, sha256, type StorageService } from "../storage/storage.js"; + +export const PREVIEW_DEPENDENCIES = Symbol("PREVIEW_DEPENDENCIES"); +export const PREVIEW_TTL_SECONDS = 60; +interface PreviewManifest { key: string; contentType: string } +interface PreviewObject { body: Buffer; contentType: string; etag: string } +interface PreviewDependencies { + redis: ControlRedis; + storage: StorageService; + secret: string; + now: () => number; + assertTap: (workspaceId: string, channelId: string, tap: string) => Promise; +} +const FILE = /^(?:[A-Za-z0-9_-]+\.(?:m3u8|m4s|ts|aac|jpg|jpeg|mp4))$/; +const TYPES = new Set(["image/jpeg", "application/vnd.apple.mpegurl", "application/x-mpegURL", "video/mp2t", "video/mp4", "audio/mp4", "audio/aac"]); + +@Injectable() +export class PreviewService implements OnModuleDestroy { + private readonly deps: PreviewDependencies; + private readonly sources = new Map>(); + private readonly flights = new Map>(); + constructor(@Optional() @Inject(PREVIEW_DEPENDENCIES) dependencies?: Partial) { + const secret = dependencies?.secret ?? process.env.PREVIEW_SIGNING_SECRET ?? process.env.SESSION_SECRET; + if (!secret && process.env.NODE_ENV === "production") throw new Error("PREVIEW_SIGNING_SECRET is required"); + this.deps = { redis: dependencies?.redis ?? new RespControlRedis(process.env.REDIS_URL ?? "redis://localhost:6379"), + storage: dependencies?.storage ?? new S3Storage(s3ConfigFromEnv()), secret: secret ?? "dev-preview-secret", + now: dependencies?.now ?? Date.now, + assertTap: dependencies?.assertTap ?? (async (ws, ch, tap) => { + const result = await withTenant(ws, client => client.query(tap === "program" + ? "SELECT id FROM virtual_channels WHERE workspace_id=$1 AND id=$2" + : "SELECT id FROM virtual_channel_sources WHERE workspace_id=$1 AND virtual_channel_id=$2 AND id=$3", tap === "program" ? [ws, ch] : [ws, ch, tap])); + if (!result.rowCount) throw new NotFoundException("preview tap not found"); + }), + }; + } + onModuleDestroy(): void { this.deps.redis.close(); this.sources.clear(); } + async urls(ws: string, ch: string, tap: string): Promise> { + await this.deps.assertTap(ws, ch, tap); + return { thumbnail: this.url(ws, ch, tap, "thumbnail.jpg"), video: this.url(ws, ch, tap, "video.m3u8"), audio: this.url(ws, ch, tap, "audio.m3u8") }; + } + private mac(path: string, expires: number): string { return createHmac("sha256", this.deps.secret).update(`${path}\n${expires}`).digest("hex"); } + private path(ws: string, ch: string, tap: string, file: string): string { + return `/media-control/preview/${ws}/${ch}/${tap}/${file}`; + } + url(ws: string, ch: string, tap: string, file: string): string { + if (!FILE.test(file)) throw new BadRequestException("invalid preview file"); + const path = this.path(ws, ch, tap, file); + const expires = Math.floor(this.deps.now() / 1000) + PREVIEW_TTL_SECONDS; + return `${path}?expires=${expires}&signature=${this.mac(path, expires)}`; + } + verify(ws: string, ch: string, tap: string, file: string, expires: string, signature: string): void { + if (!FILE.test(file) || !/^\d{1,12}$/.test(expires) || !/^[0-9a-f]{64}$/.test(signature)) throw new ForbiddenException("invalid preview signature"); + const expiry = Number(expires), now = Math.floor(this.deps.now() / 1000); + const expected = this.mac(this.path(ws, ch, tap, file), expiry); + if (expiry <= now || expiry > now + PREVIEW_TTL_SECONDS || !timingSafeEqual(Buffer.from(signature), Buffer.from(expected))) throw new ForbiddenException("expired or invalid preview signature"); + } + async read(ws: string, ch: string, tap: string, file: string): Promise { + const source = `${ws}:${ch}:${tap}`, key = `${source}:${file}`; + const cache = this.sources.get(source); + const hit = cache?.get(file); + if (hit && hit.expiresAt > this.deps.now()) { cache?.delete(file); cache?.set(file, hit); return hit.object; } + const existing = this.flights.get(key); + if (existing) return existing; + if (this.flights.size >= 128) throw new BadRequestException("preview fill capacity exceeded"); + const flight = this.fill(ws, ch, tap, file).then(object => { + const entries = this.sources.get(source) ?? new Map(); + entries.delete(file); entries.set(file, { expiresAt: this.deps.now() + 1000, object }); + while (entries.size > 6) entries.delete(entries.keys().next().value ?? ""); + this.sources.delete(source); this.sources.set(source, entries); + while (this.sources.size > 128) this.sources.delete(this.sources.keys().next().value ?? ""); + return object; + }).finally(() => this.flights.delete(key)); + this.flights.set(key, flight); + return flight; + } + private async fill(ws: string, ch: string, tap: string, file: string): Promise { + if (!FILE.test(file)) throw new BadRequestException("invalid preview file"); + const raw = await this.deps.redis.get(`preview:${ws}:${ch}:${tap}:${file}`); + if (!raw) throw new NotFoundException("preview unavailable"); + const manifest = JSON.parse(raw) as PreviewManifest; + if (!/^sha256\/[0-9a-f]{64}\.[a-z0-9]+$/.test(manifest.key) || !TYPES.has(manifest.contentType)) throw new BadRequestException("invalid preview manifest"); + let body = await this.deps.storage.getObject(manifest.key); + if (body.length > 4_000_000) throw new BadRequestException("preview object too large"); + if (file.endsWith(".m3u8")) { + const rewrite = (uri: string): string => this.url(ws, ch, tap, uri); + const playlist = body.toString("utf8"); + if (!playlist.startsWith("#EXTM3U")) throw new BadRequestException("invalid preview playlist"); + body = Buffer.from(playlist.split(/\r?\n/).map(line => { + if (line.startsWith("#")) return line.replace(/URI="([^"]+)"/g, (_match: string, uri: string) => `URI="${rewrite(uri)}"`); + return line.trim() === "" ? line : rewrite(line.trim()); + }).join("\n")); + } + return { body, contentType: manifest.contentType, etag: `"${sha256(body)}"` }; + } +} diff --git a/apps/api/src/production/production.controller.ts b/apps/api/src/production/production.controller.ts new file mode 100644 index 0000000..e62b5da --- /dev/null +++ b/apps/api/src/production/production.controller.ts @@ -0,0 +1,18 @@ +import { Body, Controller, Headers, Inject, Param, Post, Req, UseGuards } from "@nestjs/common"; +import type { Request } from "express"; +import { parseId, requireSession } from "../http/validation.js"; +import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import type { IssuedCommand } from "../media-control/media-control.service.js"; +import { ProductionService } from "./production.service.js"; +@Controller("production") +@UseGuards(RbacGuard) +export class ProductionController { + constructor(@Inject(ProductionService) private readonly production: ProductionService) {} + @Post("overlays/:overlayId") + @RequirePermission("director.change_layout") + render(@Req() request: Request & AuthenticatedRequest, @Param("overlayId") overlayId: string, + @Body() body: unknown, @Headers("idempotency-key") key?: string): Promise { + const session = requireSession(request); + return this.production.submit(session.workspaceId, parseId(overlayId, "overlayId"), body, key ?? "", { userId: session.userId, role: session.role }); + } +} diff --git a/apps/api/src/production/production.module.ts b/apps/api/src/production/production.module.ts new file mode 100644 index 0000000..4bf670a --- /dev/null +++ b/apps/api/src/production/production.module.ts @@ -0,0 +1,6 @@ +import { Module } from "@nestjs/common"; +import { MediaControlModule } from "../media-control/media-control.module.js"; +import { ProductionService } from "./production.service.js"; +import { ProductionController } from "./production.controller.js"; +@Module({ imports: [MediaControlModule], controllers: [ProductionController], providers: [ProductionService], exports: [ProductionService] }) +export class ProductionModule {} diff --git a/apps/api/src/production/production.service.ts b/apps/api/src/production/production.service.ts new file mode 100644 index 0000000..ed56264 --- /dev/null +++ b/apps/api/src/production/production.service.ts @@ -0,0 +1,94 @@ +import { createHash, randomUUID } from "node:crypto"; +import { ConflictException, Inject, Injectable, NotFoundException, OnModuleDestroy, OnModuleInit, Optional } from "@nestjs/common"; +import { withTenant, pool } from "../db/client.js"; +import { MediaControlService, type IssuedCommand } from "../media-control/media-control.service.js"; +import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js"; +import { requestIdentity, type DirectorActor, type TenantTransaction } from "./journal.js"; +import { renderOverlay, validateRenderInput, type RenderInput } from "./render.js"; +export const PRODUCTION_DEPENDENCIES = Symbol("PRODUCTION_DEPENDENCIES"); +interface RevisionFields { input: RenderInput; actor: DirectorActor; requestId: string; fingerprint: string } +interface Revision { id: string; virtual_channel_id: string; overlay_slot_id: string; render_state: string; fields: RevisionFields } +interface Dependencies { transaction: TenantTransaction; storage: StorageService; render: (input: unknown) => Promise } + +@Injectable() +export class ProductionService implements OnModuleInit, OnModuleDestroy { + private readonly deps: Dependencies; + private readonly flights = new Map>(); + private timer: ReturnType | null = null; + private recovering = false; + constructor(@Inject(MediaControlService) private readonly control: MediaControlService, + @Optional() @Inject(PRODUCTION_DEPENDENCIES) dependencies?: Partial) { + this.deps = { transaction: dependencies?.transaction ?? withTenant, storage: dependencies?.storage ?? new S3Storage(s3ConfigFromEnv()), + render: dependencies?.render ?? renderOverlay }; + } + onModuleInit(): void { + this.timer = setInterval(() => { void this.recover(); }, 1000); this.timer.unref(); + void this.recover(); + } + onModuleDestroy(): void { if (this.timer) clearInterval(this.timer); } + async submit(ws: string, slotId: string, value: unknown, idempotencyKey: string, actor: DirectorActor): Promise { + const input = validateRenderInput(value); + const expected = createHash("sha256").update(JSON.stringify(input)).digest("hex"); + const revision = await this.deps.transaction(ws, async client => { + const slot = await client.query<{ virtual_channel_id: string; kind: string }>("SELECT virtual_channel_id,kind FROM overlay_slots WHERE workspace_id=$1 AND id=$2 FOR UPDATE", [ws, slotId]); + const row = slot.rows[0]; + if (!row) throw new NotFoundException("overlay slot not found"); + if (row.kind !== input.templateId) throw new ConflictException("template must match overlay kind"); + const requestId = requestIdentity(ws, slotId, idempotencyKey); + const found = await client.query("SELECT * FROM overlay_revisions WHERE workspace_id=$1 AND overlay_slot_id=$2 AND fields->>'requestId'=$3", [ws, slotId, requestId]); + const previous = found.rows[0]; + if (previous) { + if (previous.fields.fingerprint !== expected) throw new ConflictException("Idempotency-Key was used with different overlay fields"); + return previous; + } + const id = randomUUID(); + const fields: RevisionFields = { input, actor, requestId, fingerprint: expected }; + await client.query("INSERT INTO overlay_revisions(id,workspace_id,virtual_channel_id,overlay_slot_id,template_id,fields) VALUES($1,$2,$3,$4,$5,$6)", [id, ws, row.virtual_channel_id, slotId, input.templateId, fields]); + return { id, virtual_channel_id: row.virtual_channel_id, overlay_slot_id: slotId, render_state: "pending", fields }; + }); + return this.run(ws, revision); + } + private run(ws: string, revision: Revision): Promise { + const key = `${ws}:${revision.id}`; + const previous = this.flights.get(key); + if (previous) return previous; + const flight = (async () => { + if (revision.render_state !== "rendered") { + try { + const png = await this.deps.render(revision.fields.input); + const stored = await this.deps.storage.putContent(png, "image/png"); + await this.deps.transaction(ws, client => client.query("UPDATE overlay_revisions SET render_state='rendered',asset_id=$3,updated_at=now() WHERE workspace_id=$1 AND id=$2", [ws, revision.id, stored.hash])); + } catch (error) { + await this.deps.transaction(ws, client => client.query("UPDATE overlay_revisions SET render_state='failed',asset_id=NULL,updated_at=now() WHERE workspace_id=$1 AND id=$2 AND render_state<>'rendered'", [ws, revision.id])); + throw error; + } + } + return this.control.director(ws, revision.virtual_channel_id, { type: "overlay", slotId: revision.overlay_slot_id, revisionId: revision.id }, + `overlay-render:${revision.id}`, revision.fields.actor); + })().finally(() => this.flights.delete(key)); + this.flights.set(key, flight); return flight; + } + private async recover(): Promise { + if (this.recovering) return; + this.recovering = true; + try { + const workspaces = await pool.query<{ id: string }>("SELECT id FROM workspaces"); + for (const ws of workspaces.rows) { + const revisions = await this.deps.transaction(ws.id, async client => { + const result = await client.query(`SELECT r.* FROM overlay_revisions r WHERE r.workspace_id=$1 + AND r.render_state <> 'failed' AND r.fields ? 'input' AND r.fields ? 'actor' + AND NOT EXISTS(SELECT 1 FROM overlay_slots s JOIN overlay_revisions active_revision ON active_revision.id=s.current_revision_id + AND active_revision.workspace_id=s.workspace_id WHERE s.workspace_id=r.workspace_id AND s.id=r.overlay_slot_id AND active_revision.created_at > r.created_at) + AND NOT EXISTS(SELECT 1 FROM audit_log a WHERE a.workspace_id=$1 + AND a.target_type='production_command' AND a.diff->'operation'->>'revisionId'=r.id::text) ORDER BY r.created_at LIMIT 100`, [ws.id]); + return result.rows; + }); + for (const revision of revisions) { + // Rendered rows are retried too: a crash may happen between storing PNG and swapping the slot. + await this.run(ws.id, revision).catch(() => undefined); + } + } + } catch { /* Retry after transient database/storage failures. */ } + finally { this.recovering = false; } + } +} diff --git a/apps/api/src/production/production.test.ts b/apps/api/src/production/production.test.ts new file mode 100644 index 0000000..da9e9aa --- /dev/null +++ b/apps/api/src/production/production.test.ts @@ -0,0 +1,67 @@ +import { randomUUID } from "node:crypto"; +import type { PoolClient } from "pg"; +import { describe, expect, it, vi } from "vitest"; +import { MediaControlService, type IssuedCommand } from "../media-control/media-control.service.js"; +import { sha256, keyForHash, type StorageService } from "../storage/storage.js"; +import { ProductionService } from "./production.service.js"; +import type { TenantTransaction } from "./journal.js"; +const WS = randomUUID(), CH = randomUUID(), SLOT = randomUUID(); +const ACTOR = { userId: randomUUID(), role: "Owner" }; +const ISSUED: IssuedCommand = { commandId: randomUUID(), workerId: "worker", assignmentEpoch: 1, desiredVersion: 1, issuedAt: new Date().toISOString(), expiresAt: new Date(Date.now() + 30000).toISOString(), receivers: 1 }; +interface Revision { id: string; virtual_channel_id: string; overlay_slot_id: string; fields: unknown; render_state: string; asset_id?: string } +function fixture(render?: (input: unknown) => Promise): { production: ProductionService; rows: Revision[]; storage: StorageService; control: MediaControlService } { + const rows: Revision[] = []; let tail = Promise.resolve(); + const transaction: TenantTransaction = async (_ws: string, fn: (client: PoolClient) => Promise): Promise => { + const wait = tail; let unlock = (): void => undefined; tail = new Promise(resolve => { unlock = resolve; }); await wait; + const query = async (sql: string, args: unknown[]): Promise<{ rows: unknown[]; rowCount: number }> => { + if (sql.startsWith("SELECT virtual_channel_id")) return { rows: [{ virtual_channel_id: CH, kind: "lower_third" }], rowCount: 1 }; + if (sql.startsWith("SELECT * FROM overlay_revisions")) { + const found = rows.filter(r => r.fields && typeof r.fields === "object" && "requestId" in r.fields && r.fields.requestId === args[2]); + return { rows: found, rowCount: found.length }; + } + if (sql.startsWith("INSERT")) rows.push({ id: String(args[0]), virtual_channel_id: String(args[2]), overlay_slot_id: String(args[3]), fields: args[5], render_state: "pending" }); + if (sql.startsWith("UPDATE")) { + const row = rows.find(r => r.id === args[1]); if (!row) throw new Error("missing revision"); + row.render_state = sql.includes("render_state='rendered'") ? "rendered" : "failed"; if (args[2]) row.asset_id = String(args[2]); + } + return { rows: [], rowCount: 1 }; + }; + try { return await fn({ query } as unknown as PoolClient); } finally { unlock(); } + }; + const control = new MediaControlService({ signer: null, assignments: [] }); vi.spyOn(control, "director").mockResolvedValue(ISSUED); + const storage: StorageService = { putContent: vi.fn(async (buffer, contentType) => { const hash = sha256(buffer); return { hash, key: keyForHash(hash, contentType), sizeBytes: buffer.length, contentType, deduplicated: false }; }), getObject: async () => Buffer.alloc(0), hasContent: async () => false, presign: async () => "" }; + return { rows, control, storage, production: new ProductionService(control, { transaction, storage, ...(render ? { render } : {}) }) }; +} +describe("revision → render job → content-addressed storage → desired-state pipeline", () => { + it("stores a real alpha PNG and swaps only a revision/asset reference through media-control", async () => { + const f = fixture(); const input = { templateId: "lower_third", fields: { title: '', subtitle: 'Name; [movie=hostile]' } }; + await expect(f.production.submit(WS, SLOT, input, "render-1", ACTOR)).resolves.toEqual(ISSUED); + expect(f.rows).toHaveLength(1); expect(f.rows[0]?.render_state).toBe("rendered"); expect(f.rows[0]?.asset_id).toMatch(/^[0-9a-f]{64}$/); + expect(f.storage.putContent).toHaveBeenCalledWith(expect.any(Buffer), "image/png"); + expect(f.control.director).toHaveBeenCalledWith(WS, CH, { type: "overlay", slotId: SLOT, revisionId: f.rows[0]?.id }, `overlay-render:${f.rows[0]?.id}`, ACTOR); + await f.production.submit(WS, SLOT, input, "render-1", ACTOR); expect(f.storage.putContent).toHaveBeenCalledTimes(1); + await expect(f.production.submit(WS, SLOT, { ...input, fields: { title: "changed" } }, "render-1", ACTOR)).rejects.toThrow("different overlay fields"); + }); + it("concurrent duplicate overlay requests share a single render job/revision", async () => { + const renderer = vi.fn(async () => { await new Promise(r => setTimeout(r, 10)); return Buffer.from("test-png"); }); const f = fixture(renderer); + await Promise.all(Array.from({ length: 10 }, () => f.production.submit(WS, SLOT, { templateId: "lower_third", fields: { title: "hello" } }, "parallel", ACTOR))); + expect(renderer).toHaveBeenCalledTimes(1); expect(f.rows).toHaveLength(1); expect(f.control.director).toHaveBeenCalledTimes(1); + }); + it("render failure records failed revision; retry resumes the same revision", async () => { + let fail = true; const f = fixture(async () => { if (fail) { fail = false; throw new Error("render failed"); } return Buffer.from("png"); }); + const input = { templateId: "lower_third", fields: { title: "hello" } }; + await expect(f.production.submit(WS, SLOT, input, "retry", ACTOR)).rejects.toThrow("render failed"); expect(f.rows[0]?.render_state).toBe("failed"); expect(f.control.director).not.toHaveBeenCalled(); + await f.production.submit(WS, SLOT, input, "retry", ACTOR); expect(f.rows).toHaveLength(1); expect(f.rows[0]?.render_state).toBe("rendered"); + }); + it("crash after PNG persistence retries only the desired-state swap", async () => { + const render = vi.fn(async () => Buffer.from("png")), f = fixture(render); + vi.mocked(f.control.director).mockRejectedValueOnce(new Error("publish outage")); + const input = { templateId: "lower_third", fields: { title: "hello" } }; + await expect(f.production.submit(WS, SLOT, input, "resume", ACTOR)).rejects.toThrow("publish outage"); expect(f.rows[0]?.render_state).toBe("rendered"); + await f.production.submit(WS, SLOT, input, "resume", ACTOR); expect(render).toHaveBeenCalledTimes(1); expect(f.storage.putContent).toHaveBeenCalledTimes(1); + }); + it("rejects hostile uploaded SVG before creating a revision or invoking storage", async () => { + const f = fixture(); await expect(f.production.submit(WS, SLOT, { templateId: "logo", fields: { svg: ']>&x;' } }, "hostile", ACTOR)).rejects.toThrow("unsafe SVG"); + expect(f.rows).toHaveLength(0); expect(f.storage.putContent).not.toHaveBeenCalled(); expect(f.control.director).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/src/production/render.test.ts b/apps/api/src/production/render.test.ts new file mode 100644 index 0000000..5bd581a --- /dev/null +++ b/apps/api/src/production/render.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from "vitest"; +import { renderOverlay, validateRenderInput, overlaySvg } from "./render.js"; +const png = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); +describe("overlay render job injection gates (installed sharp, isolated worker)", () => { + it.each(["lower_third", "chyron"])("renders %s hostile text as escaped text into an actual PNG", async templateId => { + const input = { templateId, fields: { title: ' & $(touch /tmp/owned)', subtitle: 'evil; [movie=/etc/passwd]; "', style: "red" } }; + const svg = overlaySvg(validateRenderInput(input)).svg; + expect(svg).toContain("<script>"); expect(svg).not.toContain("', '', + '', + '', + '
html
', + '', + '', + '', + '', + '', '', '', + ])("rejects hostile SVG through the render job: %s", async svg => { + await expect(renderOverlay({ templateId: "logo", fields: { svg } })).rejects.toThrow(); + }); + it.each(["\u0000", "\n", "\u202e", "a".repeat(81)])("rejects invalid text %s before sharp", async title => { + await expect(renderOverlay({ templateId: "lower_third", fields: { title } })).rejects.toThrow("invalid overlay text"); + }); +}); diff --git a/apps/api/src/production/render.ts b/apps/api/src/production/render.ts new file mode 100644 index 0000000..19d09ba --- /dev/null +++ b/apps/api/src/production/render.ts @@ -0,0 +1,118 @@ +// eslint-disable-next-line no-restricted-syntax -- Node worker_threads is a runtime builtin, not a provider identifier. +import { Worker } from "node:worker_threads"; +import { createRequire } from "node:module"; +import { BadRequestException } from "@nestjs/common"; + +export type OverlayTemplate = "lower_third" | "chyron" | "logo"; +export interface OverlayFields { title?: string; subtitle?: string; style?: "dark" | "red"; svg?: string } +export interface RenderInput { templateId: OverlayTemplate; fields: OverlayFields } +function text(value: unknown, max: number): string { + if (typeof value !== "string" || value.length > max || !/^[\p{L}\p{N}\p{P}\p{Zs}\p{M}\p{S}]*$/u.test(value)) throw new BadRequestException("invalid overlay text"); + return value; +} +function escape(value: string): string { + return value.replace(/[&<>"']/g, c => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[c] ?? c); +} +export function validateRenderInput(value: unknown): RenderInput { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new BadRequestException("invalid overlay input"); + const input = value as Record; + if (Object.keys(input).some(k => !["templateId", "fields"].includes(k)) || !["lower_third", "chyron", "logo"].includes(String(input.templateId))) throw new BadRequestException("invalid overlay template"); + if (!input.fields || typeof input.fields !== "object" || Array.isArray(input.fields)) throw new BadRequestException("invalid overlay fields"); + const fields = input.fields as Record; + if (Object.keys(fields).some(k => !["title", "subtitle", "style", "svg"].includes(k))) throw new BadRequestException("unknown overlay field"); + if (input.templateId === "logo") { + if (Object.keys(fields).some(k => k !== "svg") || typeof fields.svg !== "string") throw new BadRequestException("logo requires SVG only"); + return { templateId: "logo", fields: { svg: sanitizeLogo(fields.svg) } }; + } + if (fields.svg !== undefined || (fields.style !== undefined && fields.style !== "dark" && fields.style !== "red")) throw new BadRequestException("invalid overlay style"); + return { templateId: input.templateId as "lower_third" | "chyron", fields: { title: text(fields.title, 80), + subtitle: text(fields.subtitle ?? "", 120), style: fields.style === "red" ? "red" : "dark" } }; +} + +/** Closed geometric SVG language. No declarations, entities, CSS, URLs, script, + * animation, image/font loading, foreignObject or namespaced attributes. */ +export function sanitizeLogo(svg: string): string { + if (svg.length > 64_000 || /]+)>/g)) { + if (svg.slice(offset, match.index).trim() !== "") throw new BadRequestException("SVG text is not allowed"); + offset = (match.index ?? 0) + match[0].length; + const token = match[1] ?? ""; + const head = /^(\/)?([A-Za-z]+)([\s\S]*)$/.exec(token); + if (!head || !tags.has(head[2] ?? "")) throw new BadRequestException("unsafe SVG element"); + const tag = head[2] ?? "", tail = head[3] ?? ""; + if (head[1]) { + if (tail.trim() || stack.pop() !== tag) throw new BadRequestException("invalid SVG nesting"); + continue; + } + if (stack.length === 0 && (tag !== "svg" || ++roots !== 1)) throw new BadRequestException("invalid SVG root"); + if (stack.length > 32 || (tag === "svg" && stack.length > 0)) throw new BadRequestException("invalid SVG nesting"); + const selfClosing = /\/\s*$/.test(tail); + const attributeText = tail.replace(/\/\s*$/, ""); + let position = 0; + const names = new Set(); + for (const attr of attributeText.matchAll(/\s+([A-Za-z][A-Za-z0-9-]*)\s*=\s*(?:"([^"<>]*)"|'([^'<>]*)')/g)) { + if (attributeText.slice(position, attr.index).trim()) throw new BadRequestException("invalid SVG attribute"); + position = (attr.index ?? 0) + attr[0].length; + const name = attr[1] ?? "", value = attr[2] ?? attr[3] ?? ""; + if (!attrs.has(name) || names.has(name)) throw new BadRequestException("unsafe SVG attribute"); + names.add(name); + if (name === "xmlns" ? value !== "http://www.w3.org/2000/svg" : !/^[A-Za-z0-9#.,() +\-]*$/.test(value)) throw new BadRequestException("unsafe SVG attribute value"); + if (tag === "svg" && (name === "width" || name === "height") && (!/^\d+$/.test(value) || Number(value) < 1 || Number(value) > 1920)) throw new BadRequestException("SVG dimensions exceeded"); + } + if (attributeText.slice(position).trim()) throw new BadRequestException("invalid SVG attributes"); + if (!selfClosing) stack.push(tag); + } + if (roots !== 1 || stack.length || svg.slice(offset).trim()) throw new BadRequestException("invalid SVG"); + return svg; +} + +export function overlaySvg(input: RenderInput): { svg: string; width: number; height: number } { + if (input.templateId === "logo") return { svg: sanitizeLogo(input.fields.svg ?? ""), width: 512, height: 512 }; + const width = input.templateId === "chyron" ? 4096 : 1920; + const height = input.templateId === "chyron" ? 128 : 240; + const fill = input.fields.style === "red" ? "#FF1610" : "#171717"; + return { width, height, svg: `${escape(input.fields.title ?? "")}${escape(input.fields.subtitle ?? "")}` }; +} + +function sharpPath(): string { + const require = createRequire(__filename); + if (process.env.SHARP_MODULE_PATH) return require.resolve(process.env.SHARP_MODULE_PATH); + try { return require.resolve("sharp"); } catch { + throw new Error("sharp not found — install it or point SHARP_MODULE_PATH at a sharp build"); + } +} +const RENDER_SCRIPT = `const {parentPort,workerData}=require('node:worker_threads');const sharp=require(workerData.modulePath);sharp.cache(false);sharp.concurrency(1);sharp(Buffer.from(workerData.svg),{limitInputPixels:16777216}).resize(workerData.width,workerData.height,{fit:'contain',background:{r:0,g:0,b:0,alpha:0}}).png().toBuffer().then(png=>parentPort.postMessage(png)).catch(()=>{throw new Error('PNG rasterization failed')})`; +let activeJobs = 0; + +/** Isolated job: closed SVG language, bounded pixels/output, V8 heap and wall time. + * The thread receives sanitized pixels/geometry only, with no credentials in its environment. */ +export async function renderOverlay(value: unknown): Promise { + const input = validateRenderInput(value); + const rendered = overlaySvg(input); + const modulePath = sharpPath(); + if (activeJobs >= 4) throw new BadRequestException("overlay render capacity exceeded"); + activeJobs += 1; + try { + return await new Promise((resolve, reject) => { + const worker = new Worker(RENDER_SCRIPT, { eval: true, workerData: { ...rendered, modulePath }, + resourceLimits: { maxOldGenerationSizeMb: 128, maxYoungGenerationSizeMb: 16, stackSizeMb: 4 }, + env: { PATH: process.env.PATH, HOME: process.env.HOME, LANG: "C.UTF-8" } }); + let settled = false; + const finish = (error: Error | null, body?: Buffer): void => { + if (settled) return; settled = true; clearTimeout(timer); void worker.terminate(); + if (error || !body) reject(error ?? new Error("overlay render failed")); else resolve(body); + }; + const timer = setTimeout(() => finish(new Error("overlay render timed out")), 5000); + worker.once("message", (message: unknown) => { + if (!(message instanceof Uint8Array) || message.byteLength > 8_000_000 || message.byteLength < 8) finish(new Error("invalid overlay render output")); + else finish(null, Buffer.from(message)); + }); + worker.once("error", error => finish(error)); + worker.once("exit", () => finish(new Error("overlay render exited before producing PNG"))); + }); + } finally { activeJobs -= 1; } +} diff --git a/apps/api/src/production/snapshot.ts b/apps/api/src/production/snapshot.ts new file mode 100644 index 0000000..177490d --- /dev/null +++ b/apps/api/src/production/snapshot.ts @@ -0,0 +1,33 @@ +import type { PoolClient } from "pg"; +import { parseProductionSpec, type ProductionSpec } from "../media-control/production.js"; + +/** Read the aggregate only while virtual_channels is locked by the writer. */ +export async function productionSnapshot(client: PoolClient, workspaceId: string, channelId: string): Promise { + const args = [workspaceId, channelId]; + const scenes = await client.query<{ id: string; label: string; layout: string }>("SELECT id,label,layout FROM production_scenes WHERE workspace_id=$1 AND virtual_channel_id=$2 ORDER BY id", args); + const slots = await client.query<{ scene_id: string; slot_index: number; source_id: string | null; x: number | null; y: number | null; w: number | null; h: number | null; fit: string; audio_mode: string; gain_db: number }>("SELECT * FROM scene_slots WHERE workspace_id=$1 AND virtual_channel_id=$2 ORDER BY slot_index", args); + const sources = await client.query<{ id: string; label: string; kind: string; hls_url: string | null; playlist_loop: boolean | null; asset_ids: string[] }>( + `SELECT s.*, COALESCE((SELECT array_agg(substring(v.storage_key from 'sha256/([0-9a-f]{64})') ORDER BY i.position) FROM playlist_items i + JOIN video_assets v ON v.id=i.video_asset_id AND v.workspace_id=i.workspace_id + WHERE i.workspace_id=s.workspace_id AND i.source_id=s.id), '{}') AS asset_ids + FROM virtual_channel_sources s WHERE workspace_id=$1 AND virtual_channel_id=$2 ORDER BY id`, args); + const buses = await client.query<{ bus: "preview" | "program"; scene_id: string | null }>("SELECT bus,scene_id FROM production_buses WHERE workspace_id=$1 AND virtual_channel_id=$2", args); + const overlays = await client.query<{ id: string; scene_id: string; kind: string; enabled: boolean; layer: number; pos_x: number; pos_y: number; current_revision_id: string | null; asset_id: string | null; chyron_speed_px_s: number | null; chyron_strip_width_px: number | null; clock_format: string | null }>( + `SELECT s.*, r.asset_id FROM overlay_slots s LEFT JOIN overlay_revisions r + ON r.workspace_id=s.workspace_id AND r.id=s.current_revision_id AND r.render_state='rendered' + WHERE s.workspace_id=$1 AND s.virtual_channel_id=$2 AND (s.kind='clock' OR r.asset_id IS NOT NULL) ORDER BY s.layer,s.id`, args); + return parseProductionSpec({ + sources: sources.rows.map(s => s.kind === "hls" ? { id: s.id, label: s.label, kind: s.kind, url: s.hls_url } + : { id: s.id, label: s.label, kind: s.kind, assetIds: s.asset_ids, loop: s.playlist_loop }), + scenes: scenes.rows.map(s => ({ ...s, slots: slots.rows.filter(x => x.scene_id === s.id).map(x => ({ + index: x.slot_index, sourceId: x.source_id, rect: x.x === null ? null : { x: x.x, y: x.y, w: x.w, h: x.h }, + fit: x.fit, audioMode: x.audio_mode, gainDb: x.gain_db, + })) })), + buses: { preview: buses.rows.find(b => b.bus === "preview")?.scene_id ?? null, program: buses.rows.find(b => b.bus === "program")?.scene_id ?? null }, + overlays: overlays.rows.map(o => ({ id: o.id, sceneId: o.scene_id, kind: o.kind, enabled: o.enabled, layer: o.layer, + position: { x: o.pos_x, y: o.pos_y }, revisionId: o.current_revision_id, assetId: o.asset_id, + chyron: o.kind === "chyron" ? { speedPxPerSec: o.chyron_speed_px_s, stripWidthPx: o.chyron_strip_width_px } : null, + clock: o.kind === "clock" ? { format: o.clock_format } : null })), + transition: { type: "cut" }, audio: { masterGainDb: 0 }, + }); +} diff --git a/apps/api/src/production/testing/memory-redis.ts b/apps/api/src/production/testing/memory-redis.ts new file mode 100644 index 0000000..b23ea56 --- /dev/null +++ b/apps/api/src/production/testing/memory-redis.ts @@ -0,0 +1,34 @@ +import { ASSIGN_SCRIPT, COMMIT_DESIRED_SCRIPT, DELIVER_RECEIPT_SCRIPT } from "../../media-control/media-control.service.js"; +import type { ControlRedis, RedisReply } from "../../media-control/redis.js"; +import type { SignedCommand } from "../../media-control/wire.js"; +export class MemoryRedis implements ControlRedis { + readonly data = new Map(); + readonly sets = new Map>(); + readonly published: Array<{ channel: string; message: string }> = []; + failDelivery = false; + echo: ((command: SignedCommand) => void) | null = null; + async get(k: string): Promise { return this.data.get(k) ?? null; } + async mget(ks: string[]): Promise> { return ks.map(k => this.data.get(k) ?? null); } + async set(k: string, v: string, options?: { nx?: boolean }): Promise { if (options?.nx && this.data.has(k)) return false; this.data.set(k, v); return true; } + async incr(k: string): Promise { const n = Number(this.data.get(k) ?? 0) + 1; this.data.set(k, String(n)); return n; } + async sadd(k: string, v: string): Promise { const s = this.sets.get(k) ?? new Set(); s.add(v); this.sets.set(k, s); } + async smembers(k: string): Promise { return [...(this.sets.get(k) ?? [])]; } + async publish(channel: string, message: string): Promise { this.published.push({ channel, message }); if (channel.startsWith("cmd:")) this.echo?.(JSON.parse(message) as SignedCommand); return 1; } + async eval(script: string, keys: string[], args: string[]): Promise { + const k = (i: number): string => keys[i] ?? "", a = (i: number): string => args[i] ?? ""; + if (script === ASSIGN_SCRIPT) { this.data.set(k(0), a(0)); await this.sadd(k(1), a(3)); return 1; } + if (script === COMMIT_DESIRED_SCRIPT) { this.data.set(k(0), a(0)); return 1; } + if (script !== DELIVER_RECEIPT_SCRIPT) throw new Error("unknown script"); + if (this.failDelivery) { this.failDelivery = false; throw new Error("crash before delivery"); } + const prior = this.data.get(k(0)); if (prior) return prior; + if (this.data.get(k(3)) !== a(3)) return -1; + const current = JSON.parse(this.data.get(k(1)) ?? "null") as { desiredVersion: number } | null; + if (!current || current.desiredVersion < Number(a(2))) { this.data.set(k(1), a(1)); this.data.set(k(2), a(2)); this.data.set(k(7), JSON.stringify((JSON.parse(a(6)) as { state: unknown }).state)); } + this.data.set(k(5), a(4)); + const result = JSON.parse(a(0)) as Record; result.receivers = 1; + const receipt = JSON.stringify(result); + this.data.set(k(0), receipt); + await this.publish(k(4), a(4)); await this.publish(k(6), a(6)); return receipt; + } + close(): void {} +} diff --git a/apps/api/src/publishing/approval-gate.integration.test.ts b/apps/api/src/publishing/approval-gate.integration.test.ts new file mode 100644 index 0000000..cf8811e --- /dev/null +++ b/apps/api/src/publishing/approval-gate.integration.test.ts @@ -0,0 +1,365 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { Pool, type PoolClient } from "pg"; +import { PublishingExecutionRepository, assertPublishingSchema } from "@tactical-media-broadcast-hub/publishing-execution"; + +/** + * chatgov P1 gates against a live PostgreSQL (chatgov-design.md §7 P1 row): + * - immediate: ungated — a draft with NO compiled snapshot posts as today + * - admin_only: blocks a non-admin approver, fires for an admin approve + * - quorum: blocks with INSUFFICIENT_QUORUM + PENDING_POLL (poll side = P2) + * - partial fan-out (D8): 2 destinations, one immediate fires, one quorum + * blocks — gate_decisions carries per-destination rows + * - E3 revocation post-approve blocks with RESET_ON_CHANGE; no snapshot + * fails closed with OUTSIDE_POLICY + * + * Owns its own database (CHATGOV_IT_DATABASE_URL + CHATGOV_IT_SUPER_DATABASE_URL, + * P0 spine pattern) so it never races the shared tenancy suites. Skips clean. + */ +const appUrl = process.env.CHATGOV_IT_DATABASE_URL; +const ownerUrl = process.env.CHATGOV_IT_SUPER_DATABASE_URL; +const dbAvailable = appUrl !== undefined && ownerUrl !== undefined; + +type Modules = { + publishing: typeof import("./publishing.service.js"); + gate: typeof import("./approval-gate.service.js"); + chatgovGate: typeof import("../chatgov/gate.service.js"); + policyService: typeof import("../chatgov/policy.service.js"); + policy: typeof import("../chatgov/policy.js"); + dispatchGate: typeof import("../../../scheduler/src/activities/dispatch-gate.js"); + db: typeof import("../db/client.js"); +}; + +describe.skipIf(!dbAvailable)("chatgov P1 gates (live DB)", () => { + let m: Modules; + let ownerPool: Pool; + const suffix = `${Date.now()}${Math.floor(Math.random() * 1e6)}`; + let wsA: string; + let ownerId: string; + let adminId: string; + let approverId: string; + let connImmediate: string; + let connAdminOnly: string; + let connQuorum: string; + + async function asOwner(fn: (client: PoolClient) => Promise): Promise { + const client = await ownerPool.connect(); + try { + return await fn(client); + } finally { + client.release(); + } + } + + async function inTenant(wsId: string, fn: (client: PoolClient) => Promise): Promise { + const client = await ownerPool.connect(); + try { + await client.query("BEGIN"); + await client.query("SELECT set_config('app.workspace_id', $1, true)", [wsId]); + const result = await fn(client); + await client.query("COMMIT"); + return result; + } catch (error) { + await client.query("ROLLBACK"); + throw error; + } finally { + client.release(); + } + } + + /** state='passed'/'failed' rows must also satisfy the closed_at CHECK. */ + async function seedPoll(draftId: string, state: "open" | "passed", createdBy: string): Promise { + await inTenant(wsA, async (c) => { + await c.query( + `INSERT INTO polls (workspace_id, subject_kind, draft_id, question, rule, policy_version, closes_at, closed_at, close_reason, state, created_by) + VALUES ($1, 'draft', $2, 'ship it?', '{"version":1,"mode":"quorum","arms":[{"kind":"admins","x":2}],"join":"all"}'::jsonb, 'pv-x', + now() + interval '1 hour', + CASE WHEN $3::text = 'open'::text THEN NULL::timestamptz ELSE now() END, + CASE WHEN $3::text = 'open'::text THEN NULL::text ELSE 'timebox'::text END, + $3::text, $4)`, + [wsA, draftId, state, createdBy], + ); + }); + } + + beforeAll(async () => { + process.env.DATABASE_URL = appUrl; + process.env.SUPER_DATABASE_URL = ownerUrl; + m = { + publishing: await import("./publishing.service.js"), + gate: await import("./approval-gate.service.js"), + chatgovGate: await import("../chatgov/gate.service.js"), + policyService: await import("../chatgov/policy.service.js"), + policy: await import("../chatgov/policy.js"), + dispatchGate: await import("../../../scheduler/src/activities/dispatch-gate.js"), + db: await import("../db/client.js"), + }; + ownerPool = new Pool({ connectionString: ownerUrl }); + + await asOwner(async (c) => { + wsA = (await c.query("INSERT INTO workspaces (name, slug) VALUES ($1, $2) RETURNING id", ["cg-p1", `cg-p1-${suffix}`])).rows[0].id as string; + const user = async (name: string): Promise => + (await c.query( + "INSERT INTO users (external_id, username, email, display_name) VALUES ($1, $2, $3, $4) RETURNING id", + [`cgp1:${name}:${suffix}`, `cgp1-${name}-${suffix}`, `cgp1-${name}-${suffix}@example.com`, name], + )).rows[0].id as string; + ownerId = await user("owner"); + adminId = await user("admin"); + approverId = await user("approver"); + await c.query("INSERT INTO memberships (workspace_id, user_id, role, status) VALUES ($1, $2, 'Owner', 'active')", [wsA, ownerId]); + await c.query("INSERT INTO memberships (workspace_id, user_id, role, status) VALUES ($1, $2, 'Admin', 'active')", [wsA, adminId]); + await c.query("INSERT INTO memberships (workspace_id, user_id, role, status) VALUES ($1, $2, 'Approver', 'active')", [wsA, approverId]); + + const conn = async (name: string): Promise => + (await c.query("INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-social', $2) RETURNING id", [wsA, `p1-${name}`])).rows[0].id as string; + connImmediate = await conn("immediate"); + connAdminOnly = await conn("adminonly"); + connQuorum = await conn("quorum"); + }); + + // Compile policies the honest way — through PolicyService, exactly as the + // console does — so snapshots + versions are the production artifacts. + const policyService = new m.policyService.PolicyService(); + const immediateRule = await policyService.createRule(wsA, { scope: "account", accountId: connImmediate, policy: { mode: "immediate" } }, { userId: ownerId, role: "Owner" }); + const adminOnlyRule = await policyService.createRule(wsA, { scope: "account", accountId: connAdminOnly, policy: { mode: "admin_only", timebox: { duration: "PT2H", onExpire: "reject" } } }, { userId: ownerId, role: "Owner" }); + const quorumRule = await policyService.createRule(wsA, { scope: "account", accountId: connQuorum, policy: { mode: "quorum", arms: [{ kind: "admins", x: 2 }], timebox: { duration: "PT2H", onExpire: "reject" } } }, { userId: ownerId, role: "Owner" }); + expect(immediateRule.policy.mode).toBe("immediate"); + expect(adminOnlyRule.policy.mode).toBe("admin_only"); + expect(quorumRule.policy.mode).toBe("quorum"); + }, 60_000); + + afterAll(async () => { + if (ownerPool !== undefined) { + await asOwner(async (c) => { + await c.query("DELETE FROM workspaces WHERE slug = $1", [`cg-p1-${suffix}`]); + await c.query("DELETE FROM users WHERE external_id LIKE $1", [`cgp1:%:${suffix}`]); + }); + await ownerPool.end(); + } + await m?.db.pool.end(); + }); + + /** Service wired exactly like AppModule: real gate, no workflow start. */ + function makeService() { + return new m.publishing.PublishingService({ + tenantTransaction: m.db.withTenant, + workflowStarter: { startScheduledPublish: async () => undefined }, + repository: { + scheduleAttempt: async () => { throw new Error("not used"); }, + cancelAttempt: async () => undefined, + } as never, + assertReady: async () => undefined, + approvalGate: new m.gate.ApprovalGateService(new m.chatgovGate.GateService()), + }); + } + + async function seedDraft(targets: string[], state: "editing" | "in_review" = "editing"): Promise { + return asOwner(async (c) => { + await c.query("BEGIN"); + await c.query("SELECT set_config('app.workspace_id', $1, true)", [wsA]); + const post = { text: `p1 ${targets.join(",")}`, mediaAssetIds: [], visibility: "public", providerMetadata: { targetConnectionIds: targets } }; + const id = (await c.query( + `INSERT INTO drafts (workspace_id, author_user_id, post, state) VALUES ($1, $2, $3::jsonb, $4) RETURNING id`, + [wsA, ownerId, JSON.stringify(post), state], + )).rows[0].id as string; + await c.query("COMMIT"); + return id; + }); + } + + async function gateRows(draftId: string): Promise> { + return inTenant(wsA, async (c) => + (await c.query("SELECT gate, outcome, reason_codes, destination, policy_version FROM gate_decisions WHERE draft_id = $1 ORDER BY created_at, id", [draftId])).rows as never, + ); + } + + // --------------------------------------------------------------------- + it("immediate: fires for any approver and pins the policy_version (E1 row, no block)", async () => { + const draftId = await seedDraft([connImmediate], "in_review"); + const service = makeService(); + const draft = await service.approve(wsA, { userId: approverId, role: "Approver" }, draftId); + expect(draft.state).toBe("approved"); + const rows = await gateRows(draftId); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ gate: "E1", outcome: "fired", reason_codes: [] }); + expect(rows[0]?.destination.accountId).toBe(connImmediate); + }); + + it("workspace with NO rules: a draft posts exactly as today (gate no-op, zero rows)", async () => { + // Fresh workspace, no posting_rules at all — the pre-chatgov posture. + const wsPlain = await asOwner(async (c) => (await c.query("INSERT INTO workspaces (name, slug) VALUES ($1, $2) RETURNING id", ["cg-p1-plain", `cg-p1-plain-${suffix}`])).rows[0].id as string); + try { + const plainConn = (await asOwner(async (c) => (await c.query("INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-social', $2) RETURNING id", [wsPlain, "plain"])).rows[0].id)) as string; + const plainUser = (await asOwner(async (c) => (await c.query("INSERT INTO users (external_id, username, email, display_name) VALUES ($1, $2, $3, 'p') RETURNING id", [`cgp1:plain:${suffix}`, `cgp1-plain-${suffix}`, `cgp1-plain-${suffix}@e.com`])).rows[0].id)) as string; + await asOwner(async (c) => c.query("INSERT INTO memberships (workspace_id, user_id, role, status) VALUES ($1, $2, 'Publisher', 'active')", [wsPlain, plainUser])); + const post = { text: "plain", mediaAssetIds: [], visibility: "public", providerMetadata: { connectionId: plainConn } }; + const draftId = await inTenant(wsPlain, async (c) => (await c.query("INSERT INTO drafts (workspace_id, author_user_id, post, state) VALUES ($1, $2, $3::jsonb, 'in_review') RETURNING id", [wsPlain, plainUser, JSON.stringify(post)])).rows[0].id as string); + const service = makeService(); + const draft = await service.approve(wsPlain, { userId: plainUser, role: "Publisher" }, draftId); + expect(draft.state).toBe("approved"); // unchanged behavior + const rows = await inTenant(wsPlain, async (c) => (await c.query("SELECT count(*)::int AS n FROM gate_decisions WHERE draft_id = $1", [draftId])).rows[0].n); + expect(rows).toBe(0); // no gate rows for an ungated workspace + } finally { + await asOwner(async (c) => { await c.query("DELETE FROM workspaces WHERE id = $1", [wsPlain]); await c.query("DELETE FROM users WHERE id = (SELECT id FROM users WHERE external_id = $1)", [`cgp1:plain:${suffix}`]); }); + } + }); + + it("admin_only blocks a non-admin approver, then fires on one admin approve", async () => { + const draftId = await seedDraft([connAdminOnly], "in_review"); + const service = makeService(); + + await expect(service.approve(wsA, { userId: approverId, role: "Approver" }, draftId)).rejects.toMatchObject({ response: { code: "APPROVAL_GATE_BLOCKED" } }); + let state = await inTenant(wsA, async (c) => (await c.query("SELECT state FROM drafts WHERE id = $1", [draftId])).rows[0].state); + expect(state).toBe("in_review"); // approval did not land + let rows = await gateRows(draftId); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ gate: "E1", outcome: "blocked", reason_codes: ["INSUFFICIENT_QUORUM", "INELIGIBLE_ACTOR"] }); + + const draft = await service.approve(wsA, { userId: adminId, role: "Admin" }, draftId); + expect(draft.state).toBe("approved"); + state = await inTenant(wsA, async (c) => (await c.query("SELECT state FROM drafts WHERE id = $1", [draftId])).rows[0].state); + expect(state).toBe("approved"); + rows = await gateRows(draftId); + expect(rows).toHaveLength(2); + expect(rows[1]).toMatchObject({ gate: "E1", outcome: "fired", reason_codes: [] }); + }); + + it("quorum blocks with INSUFFICIENT_QUORUM + PENDING_POLL (poll side = P2)", async () => { + const draftId = await seedDraft([connQuorum], "in_review"); + const service = makeService(); + await expect(service.approve(wsA, { userId: ownerId, role: "Owner" }, draftId)).rejects.toMatchObject({ response: { code: "APPROVAL_GATE_BLOCKED" } }); + const rows = await gateRows(draftId); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ gate: "E1", outcome: "blocked", reason_codes: ["INSUFFICIENT_QUORUM", "PENDING_POLL"] }); + // And once a poll row reaches 'passed' (the P2 close path), E1 fires: + await seedPoll(draftId, "passed", ownerId); + const draft = await service.approve(wsA, { userId: ownerId, role: "Owner" }, draftId); + expect(draft.state).toBe("approved"); + const after = await gateRows(draftId); + expect(after[after.length - 1]).toMatchObject({ gate: "E1", outcome: "fired", reason_codes: [] }); + }); + + it("partial fan-out (D8): immediate fires, quorum blocks — per-destination rows", async () => { + const draftId = await seedDraft([connImmediate, connQuorum], "in_review"); + const service = makeService(); + await expect(service.approve(wsA, { userId: ownerId, role: "Owner" }, draftId)).rejects.toMatchObject({ response: { code: "APPROVAL_GATE_BLOCKED" } }); + const state = await inTenant(wsA, async (c) => (await c.query("SELECT state FROM drafts WHERE id = $1", [draftId])).rows[0].state); + expect(state).toBe("in_review"); // blocked side holds the approval back + const rows = await gateRows(draftId); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ gate: "E1", outcome: "blocked", destination: { accountId: connQuorum }, reason_codes: ["INSUFFICIENT_QUORUM", "PENDING_POLL"] }); + // queue surfaces the blocked destination with its codes + const blocked = rows.filter((r) => r.outcome === "blocked"); + expect(blocked.map((r) => r.destination.accountId)).toEqual([connQuorum]); + }); + + it("blocked queue endpoint seam: GateService.listForDraft reads every E1/E3 row", async () => { + const draftId = await seedDraft([connQuorum], "in_review"); + const service = makeService(); + await expect(service.approve(wsA, { userId: ownerId, role: "Owner" }, draftId)).rejects.toBeTruthy(); + const decisions = await new m.chatgovGate.GateService().listForDraft(wsA, draftId); + expect(decisions.length).toBeGreaterThanOrEqual(1); + expect(decisions[0]).toMatchObject({ gate: "E1", outcome: "blocked", reasonCodes: ["INSUFFICIENT_QUORUM", "PENDING_POLL"] }); + }); + + // --------------------------------------------------------------------- + describe("E3 pre-dispatch re-check", () => { + let repository: PublishingExecutionRepository; + let gate: InstanceType<(typeof m)["dispatchGate"]["DispatchGateService"]>; + + beforeAll(async () => { + repository = new PublishingExecutionRepository(new Pool({ connectionString: appUrl })); + gate = new m.dispatchGate.DispatchGateService(new Pool({ connectionString: appUrl })); + // scheduleAttempt requires an active connector-oauth credential per connection. + await asOwner(async (c) => { + for (const connectionId of [connImmediate, connAdminOnly, connQuorum]) { + await c.query( + "INSERT INTO credentials (workspace_id, connection_id, kind, ciphertext) VALUES ($1, $2, 'connector-oauth', '\\x00') ON CONFLICT DO NOTHING", + [wsA, connectionId], + ); + } + }); + }); + afterAll(async () => { await repository["pool"]?.end?.(); }); + + async function approveAndSchedule(connectionId: string): Promise<{ draftId: string; idempotencyKey: string }> { + // scheduleAttempt's execution fence requires providerMetadata.connectionId + // to equal the target — single-destination post, exactly the §4 shape. + const draftId = await asOwner(async (c) => { + await c.query("BEGIN"); + await c.query("SELECT set_config('app.workspace_id', $1, true)", [wsA]); + const post = { text: `p1 e3 ${connectionId}`, mediaAssetIds: [], visibility: "public", providerMetadata: { connectionId } }; + const id = (await c.query( + `INSERT INTO drafts (workspace_id, author_user_id, post, state) VALUES ($1, $2, $3::jsonb, 'in_review') RETURNING id`, + [wsA, ownerId, JSON.stringify(post)], + )).rows[0].id as string; + await c.query("COMMIT"); + return id; + }); + const service = makeService(); + // quorum needs the passed-poll seam (P2 owns the real transition) + if (connectionId === connQuorum) await seedPoll(draftId, "passed", ownerId); + await service.approve(wsA, { userId: ownerId, role: "Owner" }, draftId); + const attempt = await inTenant(wsA, async (c) => { + const scheduled = await repository.scheduleAttempt(c, { + workspaceId: wsA, draftId, connectionId, scheduledByUserId: ownerId, + requiredPermission: "publication.publish", publishAt: new Date(Date.now() - 1000), + }); + return { idempotencyKey: scheduled.idempotencyKey, scheduleId: scheduled.scheduleId, generation: scheduled.generation }; + }); + return { draftId, idempotencyKey: attempt.idempotencyKey }; + } + + it("fires a live approved immediate destination", async () => { + const { draftId, idempotencyKey } = await approveAndSchedule(connImmediate); + const outcome = await gate.check({ workspaceId: wsA, draftId, idempotencyKey, connectionId: connImmediate, platform: "mock-social" }); + expect(outcome).toMatchObject({ outcome: "fired", reasonCodes: [] }); + }); + + it("revocation post-approve blocks with RESET_ON_CHANGE and never sends", async () => { + const { draftId, idempotencyKey } = await approveAndSchedule(connImmediate); + await inTenant(wsA, async (c) => { + await c.query("UPDATE drafts SET state = 'rejected', approved_hash = NULL, approved_revision = NULL, reviewer_user_id = NULL WHERE id = $1", [draftId]); + }); + const outcome = await gate.check({ workspaceId: wsA, draftId, idempotencyKey, connectionId: connImmediate, platform: "mock-social" }); + expect(outcome.outcome).toBe("blocked"); + expect(outcome.reasonCodes).toEqual(["RESET_ON_CHANGE"]); + const rows = await gateRows(draftId); + expect(rows.some((r) => r.gate === "E3" && r.reason_codes.includes("RESET_ON_CHANGE"))).toBe(true); + }); + + it("draft edit after approve blocks with RESET_ON_CHANGE (hash re-pinned)", async () => { + const { draftId, idempotencyKey } = await approveAndSchedule(connImmediate); + await inTenant(wsA, async (c) => { + await c.query("UPDATE drafts SET state = 'editing', approved_hash = NULL, approved_revision = NULL WHERE id = $1", [draftId]); + }); + const outcome = await gate.check({ workspaceId: wsA, draftId, idempotencyKey, connectionId: connImmediate, platform: "mock-social" }); + expect(outcome.reasonCodes).toEqual(["RESET_ON_CHANGE"]); + }); + + it("no snapshot fails closed with OUTSIDE_POLICY", async () => { + // A brand-new connection has no compiled snapshot (PolicyService only + // recompiles on rule changes): once rules exist, approving a post + // targeting it must fail closed. + const fresh = await asOwner(async (c) => + (await c.query("INSERT INTO connections (workspace_id, provider, channel_id) VALUES ($1, 'mock-social', $2) RETURNING id", [wsA, `p1-outside-${suffix}`])).rows[0].id as string + ); + const draftId = await seedDraft([fresh], "in_review"); + const service = makeService(); + await expect(service.approve(wsA, { userId: ownerId, role: "Owner" }, draftId)).rejects.toMatchObject({ response: { code: "APPROVAL_GATE_BLOCKED" } }); + const rows = await gateRows(draftId); + expect(rows[0]).toMatchObject({ gate: "E1", outcome: "blocked", reason_codes: ["OUTSIDE_POLICY"] }); + }); + + it("quorum destination still blocked at E3 while the poll is open", async () => { + // Approve through the poll seam, then REOPEN the poll (P2 failure / + // re-vote path): E3 must re-check the live poll state at dispatch time. + const { draftId, idempotencyKey } = await approveAndSchedule(connQuorum); + await inTenant(wsA, async (c) => { + await c.query("UPDATE polls SET state = 'open', closed_at = NULL, close_reason = NULL WHERE draft_id = $1", [draftId]); + }); + const outcome = await gate.check({ workspaceId: wsA, draftId, idempotencyKey, connectionId: connQuorum, platform: "mock-social" }); + expect(outcome.outcome).toBe("blocked"); + expect(outcome.reasonCodes).toEqual(["INSUFFICIENT_QUORUM", "PENDING_POLL"]); + }); + }); +}); diff --git a/apps/api/src/publishing/approval-gate.service.ts b/apps/api/src/publishing/approval-gate.service.ts new file mode 100644 index 0000000..b18390b --- /dev/null +++ b/apps/api/src/publishing/approval-gate.service.ts @@ -0,0 +1,402 @@ +import { ConflictException, Injectable, Optional, Inject } from "@nestjs/common"; +import type { PoolClient } from "pg"; +import { scrub } from "../audit/audit.service.js"; +import { GateService, type DestinationRef, type GateDecisionRecord, type QuorumArmState } from "../chatgov/gate.service.js"; +import type { GateMode, PolicyRule, ReasonCode } from "../chatgov/policy.js"; +import { REASON_CODES } from "../chatgov/policy.js"; +import type { DraftPost } from "./publishing.service.js"; + +/** + * E1 approval gate (chatgov-design.md §4, P1): evaluated on the + * `review → approved` transition, PER DESTINATION (fan-out law D8). + * + * Mode semantics (§1d/§4): + * immediate — no gate. The E1 row is still recorded (it pins the + * policy_version E3 later re-checks) but nothing blocks. + * admin_only — the admin's approve IS the quorum ({kind:admins,x:1}). An + * approver who holds approval.grant but is not Owner/Admin + * does NOT satisfy the arm (INELIGIBLE_ACTOR) and the + * approval is refused until one admin approves. + * quorum — blocked until the linked chat poll passes (P2 owns the poll + * state machine). For P1 every console approve of a + * quorum-gated destination blocks with INSUFFICIENT_QUORUM + + * PENDING_POLL; once a polls row reaches state='passed' (the + * P2 close path) E1 fires. + * + * A destination with NO compiled snapshot posts as today: no gate, no row. + * A destination whose connection is not visible in the tenant (foreign or + * unknown id) fails closed with OUTSIDE_POLICY. + * + * Writes go to gate_decisions (P0 spine). Fired verdicts are written with the + * SAME client as the approval so the decision commits atomically with the + * transition and pins exactly the policy_version the approval landed under. + * Blocked verdicts cannot share that transaction — the approval rolls back — + * so they are recorded through GateService.record after the rollback, when no + * locks are held. (gate_decisions carries FOREIGN KEY (draft_id, workspace_id) + * REFERENCES drafts; an INSERT takes a KEY SHARE lock on the draft row, which + * the approval transaction's FOR UPDATE would deadlock against if the blocked + * rows were written before the throw.) + */ + +export const APPROVAL_GATE_ADMIN_ROLES: readonly string[] = ["Owner", "Admin"]; + +/** Thrown by the E1 hook to abort an approval; carries the per-destination verdicts. */ +export class ApprovalGateBlockedError extends ConflictException { + constructor(readonly verdicts: readonly E1Verdict[]) { + super({ + code: "APPROVAL_GATE_BLOCKED", + message: "approval blocked by posting policy for one or more destinations", + destinations: verdicts.map((verdict) => ({ + destination: verdict.destination, + mode: verdict.mode, + reasonCodes: verdict.reasonCodes, + })), + }); + } +} + +export interface E1Verdict { + destination: DestinationRef; + /** Effective mode at evaluation; "none" = no compiled snapshot (ungated). */ + mode: GateMode | "none"; + policyVersion: string | null; + outcome: "fired" | "blocked"; + reasonCodes: ReasonCode[]; + inputs: { + approvals?: number; + arms?: QuorumArmState[]; + pollState?: string; + policyMode?: GateMode | "none"; + [key: string]: unknown; + }; +} + +export interface E1Evaluation { + verdicts: E1Verdict[]; + blocked: E1Verdict[]; + fired: E1Verdict[]; +} + +const MISSING_TABLE = "42P01"; + +/** + * Pure E1 decision for ONE destination (unit-testable, no DB). + * `rule`/`policyVersion` are the destination's compiled snapshot; undefined + * means no snapshot row exists for the account (ungated, posts as today). + */ +export function evaluateE1Destination(input: { + destination: DestinationRef; + rule: PolicyRule | undefined; + policyVersion: string | undefined; + /** Approver holds an active Owner/Admin membership in the workspace. */ + adminApproval: boolean; + /** A polls row for this draft has reached state='passed' (P2 close path). */ + pollPassed: boolean; + pollState: string; +}): E1Verdict { + const { destination, adminApproval, pollPassed, pollState } = input; + if (input.rule === undefined || input.policyVersion === undefined) { + // No compiled snapshot: post as today — no gate, no reason codes. + return { + destination, + mode: "none", + policyVersion: null, + outcome: "fired", + reasonCodes: [], + inputs: { approvals: adminApproval ? 1 : 0, policyMode: "none" }, + }; + } + const rule = input.rule; + const policyVersion = input.policyVersion; + switch (rule.mode) { + case "immediate": + return { + destination, + mode: "immediate", + policyVersion, + outcome: "fired", + reasonCodes: [], + inputs: { approvals: adminApproval ? 1 : 0, policyMode: "immediate" }, + }; + case "admin_only": { + // §1d: admin_only = quorum:[{kind:admins,x:1}] with no participant arm — + // one admin approve satisfies it (§4 "admin_only mode = same path with + // a 1-arm poll or an E1-only console approval"). + const arms: QuorumArmState[] = [{ kind: "admins", got: adminApproval ? 1 : 0, needed: 1 }]; + return adminApproval + ? { destination, mode: "admin_only", policyVersion, outcome: "fired", reasonCodes: [], inputs: { approvals: 1, arms, policyMode: "admin_only" } } + : { + destination, + mode: "admin_only", + policyVersion, + outcome: "blocked", + reasonCodes: ["INSUFFICIENT_QUORUM", "INELIGIBLE_ACTOR"], + inputs: { approvals: 0, arms, policyMode: "admin_only" }, + }; + } + case "quorum": { + const arms: QuorumArmState[] = (rule.arms ?? []).map((arm) => ({ + kind: arm.kind, + got: pollPassed ? arm.x : 0, + needed: arm.x, + })); + return pollPassed + ? { destination, mode: "quorum", policyVersion, outcome: "fired", reasonCodes: [], inputs: { approvals: 1, arms, pollState: "passed", policyMode: "quorum" } } + : { + destination, + mode: "quorum", + policyVersion, + outcome: "blocked", + reasonCodes: ["INSUFFICIENT_QUORUM", "PENDING_POLL"], + inputs: { approvals: 0, arms, pollState, policyMode: "quorum" }, + }; + } + } +} + +/** Destination connection ids carried by a draft's post metadata (fan-out D8). */ +export function draftDestinationAccounts(post: DraftPost): string[] { + const metadata = post.providerMetadata ?? {}; + const targets = Array.isArray(metadata.targetConnectionIds) + ? metadata.targetConnectionIds.filter((id): id is string => typeof id === "string") + : typeof metadata.connectionId === "string" + ? [metadata.connectionId] + : []; + return [...new Set(targets)]; +} + +interface SnapshotRow { + account_id: string; + topic_binding_id: string | null; + effective: PolicyRule; + policy_version: string; +} + +@Injectable() +export class ApprovalGateService { + private readonly gates: GateService; + + constructor(@Optional() @Inject(GateService) gates?: GateService) { + this.gates = gates ?? new GateService(); + } + + /** Blocked-queue / draft gate timeline (GET /publishing/drafts/:id/gates). */ + async listForDraft(workspaceId: string, draftId: string): Promise { + return this.gates.listForDraft(workspaceId, draftId); + } + + /** + * Read + evaluate every destination of a draft. Runs INSIDE the approval + * transaction (consistent snapshot under the draft row lock) but performs + * no writes. Returns an empty evaluation when the draft has no destination + * metadata or chatgov is not deployed (missing tables) — both post as today. + */ + async evaluate( + client: PoolClient, + workspaceId: string, + draftId: string, + post: DraftPost, + approverUserId: string, + ): Promise { + const accountIds = draftDestinationAccounts(post); + if (accountIds.length === 0) return { verdicts: [], blocked: [], fired: [] }; + // Lazily detect chatgov deployment: a workspace that predates the spine + // (or a suite DB without migration 024) keeps today's ungated behavior. + let deployed = true; + try { + await client.query("SELECT 1 FROM posting_rules LIMIT 1"); + } catch (error) { + if (pgCode(error) === MISSING_TABLE) deployed = false; + else throw error; + } + if (!deployed) return { verdicts: [], blocked: [], fired: [] }; + + let connections: Map; + let snapshots: SnapshotRow[]; + let chatgovInUse: boolean; + let adminApproval: boolean; + let pollPassed: boolean; + let pollState: string; + try { + const connectionRows = await client.query<{ id: string; provider: string }>( + "SELECT id, provider FROM connections WHERE id = ANY($1::uuid[]) AND disconnected_at IS NULL", + [accountIds], + ); + connections = new Map(connectionRows.rows.map((row) => [String(row.id), String(row.provider)])); + const snapshotRows = await client.query( + "SELECT account_id, topic_binding_id, effective, policy_version FROM posting_policy_snapshots WHERE account_id = ANY($1::uuid[]) AND content_type = ''", + [accountIds], + ); + snapshots = snapshotRows.rows; + // A workspace with no live posting_rules has never opted into chatgov: + // every destination posts exactly as before (no gates, no rows). + const liveRules = await client.query<{ n: number }>( + "SELECT count(*)::int AS n FROM posting_rules WHERE superseded_by IS NULL", + ); + chatgovInUse = Number(liveRules.rows[0]?.n ?? 0) > 0; + const membership = await client.query<{ role: string }>( + "SELECT role FROM memberships WHERE workspace_id = $1 AND user_id = $2 AND status = 'active'", + [workspaceId, approverUserId], + ); + adminApproval = membership.rows.some((row) => APPROVAL_GATE_ADMIN_ROLES.includes(String(row.role))); + const poll = await client.query<{ state: string }>( + "SELECT state FROM polls WHERE draft_id = $1 ORDER BY created_at DESC LIMIT 1", + [draftId], + ); + pollState = String(poll.rows[0]?.state ?? "none"); + pollPassed = poll.rows.some((row) => String(row.state) === "passed"); + } catch (error) { + if (pgCode(error) === MISSING_TABLE) return { verdicts: [], blocked: [], fired: [] }; // chatgov not deployed + throw error; + } + + const topicBindingId = typeof post.providerMetadata?.topicBindingId === "string" ? post.providerMetadata.topicBindingId : null; + const verdicts: E1Verdict[] = accountIds.map((accountId) => { + const destination: DestinationRef = { + platform: connections.get(accountId) ?? "unknown", + accountId, + ...(topicBindingId === null ? {} : { topicBindingId }), + }; + if (!chatgovInUse) { + // No live rules anywhere in this workspace: it has not opted into + // governance, so every destination posts exactly as before. + return evaluateE1Destination({ destination, rule: undefined, policyVersion: undefined, adminApproval, pollPassed, pollState }); + } + if (!connections.has(accountId)) { + // Fail closed: the destination is not a visible connection of this + // tenant (§1d fallback:block posture — OUTSIDE_POLICY). + return { + destination, + mode: "none" as const, + policyVersion: null, + outcome: "blocked" as const, + reasonCodes: ["OUTSIDE_POLICY" as ReasonCode], + inputs: { policyMode: "none" as const, connectionVisible: false }, + }; + } + // Precedence per destination slot: a topic snapshot overrides the + // account root; the account root is the compiled default. + const accountSnapshots = snapshots.filter((row) => String(row.account_id) === accountId); + const snapshot = + (topicBindingId !== null ? accountSnapshots.find((row) => row.topic_binding_id === topicBindingId) : undefined) ?? + accountSnapshots.find((row) => row.topic_binding_id === null); + if (snapshot === undefined) { + // Rules exist but this destination has no compiled snapshot: the + // P0 hand-off posture — fail closed, never a silent ungated send. + return { + destination, + mode: "none", + policyVersion: null, + outcome: "blocked", + reasonCodes: ["OUTSIDE_POLICY"], + inputs: { policyMode: "none", connectionVisible: true }, + }; + } + return evaluateE1Destination({ + destination, + rule: snapshot?.effective, + policyVersion: snapshot?.policy_version, + adminApproval, + pollPassed, + pollState, + }); + }); + return { + verdicts, + blocked: verdicts.filter((verdict) => verdict.outcome === "blocked"), + fired: verdicts.filter((verdict) => verdict.outcome === "fired"), + }; + } + + /** + * Persist FIRED verdicts in the caller's transaction (the approval's own + * client): the decision row commits atomically with the `approved` + * transition and pins the exact policy_version the approval landed under. + */ + async recordFiredInTxn( + client: PoolClient, + workspaceId: string, + draftId: string, + actorUserId: string, + verdicts: readonly E1Verdict[], + ): Promise { + for (const verdict of verdicts) { + if (verdict.outcome !== "fired" || verdict.mode === "none") continue; // ungated: no gate, no row + await insertDecision(client, workspaceId, draftId, verdict, "user", actorUserId); + } + } + + /** + * Persist BLOCKED verdicts after the approval rolled back (no draft row + * lock is held, so the gate_decisions FK KEY SHARE cannot deadlock). + */ + async recordBlocked( + workspaceId: string, + draftId: string, + actorUserId: string, + verdicts: readonly E1Verdict[], + ): Promise { + for (const verdict of verdicts) { + if (verdict.outcome !== "blocked") continue; + await this.gates.record(workspaceId, { + draftId, + destination: verdict.destination, + gate: "E1", + policyVersion: verdict.policyVersion ?? "none", + outcome: "blocked", + inputs: verdict.inputs, + reasonCodes: verdict.reasonCodes, + actor: `user:${actorUserId}`, + }); + } + } +} + +/** Same row shape GateService.record writes (reason codes stay in-vocabulary). */ +async function insertDecision( + client: PoolClient, + workspaceId: string, + draftId: string, + verdict: E1Verdict, + actorKind: "user" | "system", + actorId: string, +): Promise { + const reasonCodes = verdict.reasonCodes.filter((code) => (REASON_CODES as readonly string[]).includes(code)); + await client.query( + `INSERT INTO gate_decisions + (workspace_id, draft_id, destination, gate, policy_version, outcome, inputs, reason_codes, actor) + VALUES ($1, $2, $3::jsonb, 'E1', $4, $5, $6::jsonb, $7, $8)`, + [ + workspaceId, + draftId, + JSON.stringify(verdict.destination), + verdict.policyVersion ?? "none", + verdict.outcome, + JSON.stringify(verdict.inputs), + reasonCodes, + `${actorKind}:${actorId}`, + ], + ); + await client.query("SELECT audit_append($1, NULL, 'sp_api', 'policy.gate', 'draft', $2, $3::jsonb)", [ + workspaceId, + draftId, + JSON.stringify( + scrub({ + gate: "E1", + destination: verdict.destination, + outcome: verdict.outcome, + reasonCodes, + policyVersion: verdict.policyVersion ?? "none", + inputs: verdict.inputs, + }) ?? {}, + ), + ]); +} + +function pgCode(error: unknown): string | undefined { + if (typeof error === "object" && error !== null && "code" in error) { + const code = (error as { code: unknown }).code; + return typeof code === "string" ? code : undefined; + } + return undefined; +} diff --git a/apps/api/src/publishing/approval-gate.test.ts b/apps/api/src/publishing/approval-gate.test.ts new file mode 100644 index 0000000..89a8b2e --- /dev/null +++ b/apps/api/src/publishing/approval-gate.test.ts @@ -0,0 +1,106 @@ +import { ConflictException } from "@nestjs/common"; +import { describe, expect, it } from "vitest"; +import { + ApprovalGateBlockedError, + APPROVAL_GATE_ADMIN_ROLES, + draftDestinationAccounts, + evaluateE1Destination, +} from "./approval-gate.service.js"; +import { policyHash, type PolicyRule } from "../chatgov/policy.js"; + +/** + * E1 gate semantics (chatgov-design.md §4) — pure decision table, no DB. + */ + +const destination = { platform: "telegram", accountId: "40000000-0000-4000-8000-000000000004" }; + +function rule(mode: PolicyRule["mode"], extra: Partial = {}): PolicyRule { + return { + version: 1, + mode, + join: "all", + allowVeto: false, + minDwellSeconds: 0, + anonymity: "named", + allowSelfApproval: false, + resetOnChange: true, + dryRun: false, + ...(mode === "quorum" ? { arms: [{ kind: "admins", x: 2 }], timebox: { duration: "PT2H", onExpire: "reject" } } : {}), + ...(mode === "admin_only" ? { timebox: { duration: "PT2H", onExpire: "reject" } } : {}), + ...extra, + } as PolicyRule; +} + +describe("E1 destination evaluation (chatgov §4 modes)", () => { + it("no snapshot: ungated — posts as today, no gate row, no reason codes", () => { + const verdict = evaluateE1Destination({ destination, rule: undefined, policyVersion: undefined, adminApproval: false, pollPassed: false, pollState: "none" }); + expect(verdict).toMatchObject({ mode: "none", outcome: "fired", reasonCodes: [], policyVersion: null }); + }); + + it("immediate: fires with no gate and pins the policy_version", () => { + const compiled = rule("immediate"); + const verdict = evaluateE1Destination({ destination, rule: compiled, policyVersion: policyHash(compiled), adminApproval: false, pollPassed: false, pollState: "none" }); + expect(verdict).toMatchObject({ mode: "immediate", outcome: "fired", reasonCodes: [] }); + expect(verdict.policyVersion).toBe(policyHash(compiled)); + }); + + it("admin_only: one admin approve IS the quorum (fires)", () => { + const compiled = rule("admin_only"); + const verdict = evaluateE1Destination({ destination, rule: compiled, policyVersion: policyHash(compiled), adminApproval: true, pollPassed: false, pollState: "none" }); + expect(verdict).toMatchObject({ mode: "admin_only", outcome: "fired", reasonCodes: [] }); + expect(verdict.inputs.arms).toEqual([{ kind: "admins", got: 1, needed: 1 }]); + }); + + it("admin_only: non-admin approver blocks (INELIGIBLE_ACTOR + INSUFFICIENT_QUORUM)", () => { + const compiled = rule("admin_only"); + const verdict = evaluateE1Destination({ destination, rule: compiled, policyVersion: policyHash(compiled), adminApproval: false, pollPassed: false, pollState: "none" }); + expect(verdict.outcome).toBe("blocked"); + expect(verdict.reasonCodes).toEqual(["INSUFFICIENT_QUORUM", "INELIGIBLE_ACTOR"]); + expect(verdict.inputs.arms).toEqual([{ kind: "admins", got: 0, needed: 1 }]); + }); + + it("quorum: blocks with INSUFFICIENT_QUORUM + PENDING_POLL until the poll passes (P2)", () => { + const compiled = rule("quorum"); + const verdict = evaluateE1Destination({ destination, rule: compiled, policyVersion: policyHash(compiled), adminApproval: true, pollPassed: false, pollState: "open" }); + expect(verdict.outcome).toBe("blocked"); + expect(verdict.reasonCodes).toEqual(["INSUFFICIENT_QUORUM", "PENDING_POLL"]); + expect(verdict.inputs.pollState).toBe("open"); + expect(verdict.inputs.arms).toEqual([{ kind: "admins", got: 0, needed: 2 }]); + }); + + it("quorum: fires when the linked poll has passed", () => { + const compiled = rule("quorum"); + const verdict = evaluateE1Destination({ destination, rule: compiled, policyVersion: policyHash(compiled), adminApproval: true, pollPassed: true, pollState: "passed" }); + expect(verdict).toMatchObject({ mode: "quorum", outcome: "fired", reasonCodes: [] }); + expect(verdict.inputs.arms).toEqual([{ kind: "admins", got: 2, needed: 2 }]); + }); +}); + +describe("draft destination fan-out (D8)", () => { + it("reads targetConnectionIds, falling back to connectionId", () => { + const a = "40000000-0000-4000-8000-000000000004"; + const b = "40000000-0000-4000-8000-000000000005"; + expect(draftDestinationAccounts({ text: "x", mediaAssetIds: [], visibility: "public", providerMetadata: { targetConnectionIds: [a, b] } })).toEqual([a, b]); + expect(draftDestinationAccounts({ text: "x", mediaAssetIds: [], visibility: "public", providerMetadata: { connectionId: a } })).toEqual([a]); + expect(draftDestinationAccounts({ text: "x", mediaAssetIds: [], visibility: "public", providerMetadata: {} })).toEqual([]); + }); + + it("admin roles are exactly Owner/Admin (§1a admins arm)", () => { + expect(APPROVAL_GATE_ADMIN_ROLES).toEqual(["Owner", "Admin"]); + }); +}); + +describe("ApprovalGateBlockedError", () => { + it("carries per-destination codes as a Conflict the controller maps to 409", () => { + const compiled = rule("quorum"); + const error = new ApprovalGateBlockedError([ + evaluateE1Destination({ destination, rule: compiled, policyVersion: policyHash(compiled), adminApproval: false, pollPassed: false, pollState: "open" }), + ]); + expect(error).toBeInstanceOf(ConflictException); + expect(error.getResponse()).toMatchObject({ code: "APPROVAL_GATE_BLOCKED" }); + expect((error.getResponse() as { destinations: unknown[] }).destinations[0]).toMatchObject({ + mode: "quorum", + reasonCodes: ["INSUFFICIENT_QUORUM", "PENDING_POLL"], + }); + }); +}); diff --git a/apps/api/src/publishing/poll-approval.ts b/apps/api/src/publishing/poll-approval.ts new file mode 100644 index 0000000..1e219d0 --- /dev/null +++ b/apps/api/src/publishing/poll-approval.ts @@ -0,0 +1,45 @@ +import { ConflictException } from '@nestjs/common'; +import { canonicalPostHash } from '@tactical-media-broadcast-hub/publishing-execution'; +import type { PoolClient } from 'pg'; +import { ApprovalGateService } from './approval-gate.service.js'; +import type { DraftPost } from './publishing.service.js'; + +/** E2's transactional publishing seam. A System audit actor delegates to a + * live Owner/Admin approval principal so the existing dispatch authorization + * continues to validate a real, revocable membership; no permission bypass. */ +export async function transitionPollDraft(client: PoolClient, input: { + workspaceId: string; draftId: string; pollId: string; passed: boolean; + text?: string; reason: string; escalate?: boolean; +}): Promise { + const current = (await client.query('SELECT * FROM drafts WHERE id = $1 FOR UPDATE', [input.draftId])).rows[0]; + if (!current || !['editing', 'in_review'].includes(String(current.state))) throw new ConflictException('poll subject is no longer reviewable'); + if (input.escalate) return; + if (input.passed && input.text !== undefined) { + // Separate writes are necessary: the revision trigger voids approval + // whenever content changes. Approve only the resulting revision/hash. + await client.query("UPDATE drafts SET post = jsonb_set(post, '{text}', to_jsonb($2::text)), updated_at = now() WHERE id = $1", [input.draftId, input.text]); + } + const draft = (await client.query('SELECT * FROM drafts WHERE id = $1', [input.draftId])).rows[0]!; + let reviewer: string | null = null; + if (input.passed) { + reviewer = (await client.query(`SELECT m.user_id FROM memberships m + JOIN role_permissions rp ON rp.role = m.role AND rp.permission = 'approval.grant' + WHERE m.workspace_id = $1 AND m.status = 'active' AND m.role IN ('Owner', 'Admin') + ORDER BY CASE m.role WHEN 'Owner' THEN 0 ELSE 1 END, m.user_id LIMIT 1`, [input.workspaceId])).rows[0]?.user_id as string | undefined ?? null; + if (!reviewer) throw new ConflictException('poll approval needs a live Owner/Admin authorizing principal'); + const gate = new ApprovalGateService(); + const evaluation = await gate.evaluate(client, input.workspaceId, input.draftId, draft.post as DraftPost, reviewer); + if (evaluation.blocked.length) throw new ConflictException({ code: 'OUTSIDE_POLICY', message: 'poll passed but E1 still blocks the draft', verdicts: evaluation.blocked }); + await gate.recordFiredInTxn(client, input.workspaceId, input.draftId, reviewer, evaluation.verdicts); + } + await client.query(`UPDATE drafts SET state = $2, reviewer_user_id = $3, + approved_revision = CASE WHEN $2 = 'approved' THEN revision ELSE NULL END, + approved_hash = $4, approved_at = CASE WHEN $2 = 'approved' THEN now() ELSE NULL END, + review_reason = $5, updated_at = now() WHERE id = $1`, + [input.draftId, input.passed ? 'approved' : 'rejected', reviewer, + input.passed ? canonicalPostHash(draft.post as DraftPost) : null, input.reason]); + await client.query("SELECT audit_append($1, NULL, 'System', $2, 'draft', $3, $4)", [input.workspaceId, + input.passed ? 'draft.approved' : 'draft.rejected', input.draftId, + { actor: `system:chat-poll-gate:${input.pollId}`, pollId: input.pollId, + authorizingUserId: reviewer, from: current.state, to: input.passed ? 'approved' : 'rejected', reason: input.reason }]); +} diff --git a/apps/api/src/publishing/publishing.controller.ts b/apps/api/src/publishing/publishing.controller.ts new file mode 100644 index 0000000..127b0ee --- /dev/null +++ b/apps/api/src/publishing/publishing.controller.ts @@ -0,0 +1,300 @@ +import { + BadRequestException, + Body, + Controller, + Delete, + Get, + HttpCode, + Inject, + Param, + Patch, + Post, + Query, + Req, + UseGuards, +} from "@nestjs/common"; +import type { Request } from "express"; +import { RbacGuard, RequireGroupAccess, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import { ApprovalGateService } from "./approval-gate.service.js"; +import type { GateDecisionRecord } from "../chatgov/gate.service.js"; +import { + PublishingService, + type Actor, + type DraftPost, + type DraftRecord, + type DraftState, + type ScheduledPublicationRecord, +} from "./publishing.service.js"; + +export interface CreateDraftDto { + post: DraftPost; +} + +export interface RejectDraftDto { + reason?: string; +} + +export interface ScheduleDraftDto { + connectionId: string; + publishAt: string; +} + +@Controller("publishing/drafts") +@UseGuards(RbacGuard) +export class PublishingController { + constructor( + @Inject(PublishingService) private readonly publishing: PublishingService, + @Inject(ApprovalGateService) private readonly gates: ApprovalGateService, + ) {} + + @Get() + @RequirePermission("draft.view") + async list( + @Req() request: Request & AuthenticatedRequest, + @Query("state") state: string | undefined, + @Query("limit") limit: string | undefined, + ): Promise { + const session = requireSession(request); + return this.publishing.listDrafts(session.workspaceId, { + ...(state === undefined || state === "" ? {} : { state: parseDraftState(state) }), + limit: parseListLimit(limit), + }); + } + + /** + * Blocked queue surfacing (chatgov-design.md §5): per-destination gate + * decisions for one draft — outcome, reason codes, quorum gap, pinned + * policy_version. Reads the P0 gate_decisions spine via GateService. + */ + @Get(":draftId/gates") + @RequirePermission("draft.view") + async gatesForDraft( + @Req() request: Request & AuthenticatedRequest, + @Param("draftId") draftId: string, + ): Promise { + const session = requireSession(request); + return this.gates.listForDraft(session.workspaceId, parseId(draftId)); + } + + @Post() + @RequirePermission("draft.create") + async create( + @Req() request: Request & AuthenticatedRequest, + @Body() body: unknown, + ): Promise { + const dto = parseCreateDraft(body); + const session = requireSession(request); + return this.publishing.createDraft(session.workspaceId, actorFrom(session), dto.post); + } + + @Patch(":draftId") + @RequirePermission("draft.edit") + async update( + @Req() request: Request & AuthenticatedRequest, + @Param("draftId") draftId: string, + @Body() body: unknown, + ): Promise { + const dto = parseCreateDraft(body); + const session = requireSession(request); + return this.publishing.updateDraft(session.workspaceId, actorFrom(session), parseId(draftId), dto.post); + } + + @Post(":draftId/submit") + @RequirePermission("approval.request") + async submit( + @Req() request: Request & AuthenticatedRequest, + @Param("draftId") draftId: string, + ): Promise { + const session = requireSession(request); + return this.publishing.submitForReview(session.workspaceId, actorFrom(session), parseId(draftId)); + } + + @Post(":draftId/approve") + @RequirePermission("approval.grant") + async approve( + @Req() request: Request & AuthenticatedRequest, + @Param("draftId") draftId: string, + ): Promise { + const session = requireSession(request); + return this.publishing.approve(session.workspaceId, actorFrom(session), parseId(draftId)); + } + + @Post(":draftId/reject") + @RequirePermission("approval.grant") + async reject( + @Req() request: Request & AuthenticatedRequest, + @Param("draftId") draftId: string, + @Body() body: unknown, + ): Promise { + const session = requireSession(request); + const dto = parseRejectDraft(body); + return this.publishing.reject(session.workspaceId, actorFrom(session), parseId(draftId), dto.reason); + } + + @Post(":draftId/schedule") + @HttpCode(202) + @RequirePermission("publication.schedule") + @RequireGroupAccess("body") + async schedule( + @Req() request: Request & AuthenticatedRequest, + @Param("draftId") draftId: string, + @Body() body: unknown, + ): Promise { + const session = requireSession(request); + const dto = parseScheduleDraft(body); + return this.publishing.schedule( + session.workspaceId, + actorFrom(session), + parseId(draftId), + dto.connectionId, + new Date(dto.publishAt), + ); + } + + @Post(":draftId/publish") + @HttpCode(202) + @RequirePermission("publication.publish") + @RequireGroupAccess("body") + async publishNow(@Req() request: Request & AuthenticatedRequest, @Param("draftId") draftId: string, @Body() body: unknown): Promise { + const session = requireSession(request); + const input = requireObject(body, "body"); + return this.publishing.schedule(session.workspaceId, actorFrom(session), parseId(draftId), parseId(requireString(input.connectionId, "connectionId")), new Date(), "publication.publish"); + } + + @Post(":draftId/retry") + @HttpCode(202) + @RequirePermission("publication.schedule") + @RequireGroupAccess("body") + async retry(@Req() request: Request & AuthenticatedRequest, @Param("draftId") draftId: string, @Body() body: unknown): Promise { + return this.resume(request, draftId, body, "failed"); + } + + @Post(":draftId/reauthorize") + @HttpCode(202) + @RequirePermission("publication.schedule") + @RequireGroupAccess("body") + async reauthorize(@Req() request: Request & AuthenticatedRequest, @Param("draftId") draftId: string, @Body() body: unknown): Promise { + return this.resume(request, draftId, body, "needs_reauthorization"); + } + + private async resume(request: Request & AuthenticatedRequest, draftId: string, body: unknown, state: "failed" | "needs_reauthorization"): Promise { + const session = requireSession(request); + const dto = parseScheduleDraft(body); + const input = requireObject(body, "body"); + if (typeof input.expectedGeneration !== "string" || !/^[1-9][0-9]*$/.test(input.expectedGeneration)) throw new BadRequestException("expectedGeneration must be a positive decimal string"); + return this.publishing.schedule(session.workspaceId, actorFrom(session), parseId(draftId), dto.connectionId, new Date(dto.publishAt), "publication.schedule", { state, generation: input.expectedGeneration }); + } + + @Delete(":draftId/schedule") + @RequirePermission("publication.schedule") + async cancelSchedule( + @Req() request: Request & AuthenticatedRequest, + @Param("draftId") draftId: string, + ): Promise { + const session = requireSession(request); + return this.publishing.cancelSchedule(session.workspaceId, actorFrom(session), parseId(draftId)); + } +} + +function requireSession(request: AuthenticatedRequest): NonNullable { + if (request.session === undefined) throw new Error("unreachable: RBAC guard did not attach a session"); + return request.session; +} + +function actorFrom(session: NonNullable): Actor { + return { userId: session.userId, role: session.role }; +} + +function parseCreateDraft(value: unknown): CreateDraftDto { + const body = requireObject(value, "body"); + const post = requireObject(body.post, "post"); + const text = requireString(post.text, "post.text"); + const visibility = requireVisibility(post.visibility); + if (!Array.isArray(post.mediaAssetIds) || !post.mediaAssetIds.every((id) => typeof id === "string")) { + throw new BadRequestException("post.mediaAssetIds must be an array of strings"); + } + const providerMetadata = requireObject(post.providerMetadata, "post.providerMetadata"); + if (post.mediaAssetIds.length > 0) throw new BadRequestException("publishing currently supports text only"); + if (["idempotencyKey", "draftId", "mediaIds"].some((key) => key in providerMetadata)) throw new BadRequestException("post metadata contains reserved execution keys"); + if (Buffer.byteLength(JSON.stringify(providerMetadata)) > 16_384) throw new BadRequestException("post metadata is too large"); + if (post.scheduledAt !== undefined) requireTimestamp(post.scheduledAt, "post.scheduledAt"); + return { + post: { + text, + mediaAssetIds: post.mediaAssetIds, + visibility, + providerMetadata, + ...(typeof post.scheduledAt === "string" ? { scheduledAt: post.scheduledAt } : {}), + }, + }; +} + +function parseRejectDraft(value: unknown): RejectDraftDto { + if (value === undefined || value === null) return {}; + const body = requireObject(value, "body"); + if (body.reason === undefined) return {}; + return { reason: requireString(body.reason, "reason") }; +} + +function parseScheduleDraft(value: unknown): ScheduleDraftDto { + const body = requireObject(value, "body"); + return { + connectionId: parseId(requireString(body.connectionId, "connectionId")), + publishAt: requireTimestamp(body.publishAt, "publishAt"), + }; +} + +function requireObject(value: unknown, field: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new BadRequestException(`${field} must be an object`); + } + return value as Record; +} + +function requireString(value: unknown, field: string): string { + if (typeof value !== "string" || value.trim() === "") { + throw new BadRequestException(`${field} must be a non-empty string`); + } + return value; +} + +function requireTimestamp(value: unknown, field: string): string { + const timestamp = requireString(value, field); + if (!Number.isFinite(Date.parse(timestamp))) throw new BadRequestException(`${field} must be an ISO timestamp`); + return timestamp; +} + +function parseId(value: string): string { + if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(value)) { + throw new BadRequestException("id must be a UUID"); + } + return value; +} + +const DRAFT_STATES: readonly DraftState[] = ["editing", "in_review", "approved", "rejected", "scheduled", "published"]; + +function parseDraftState(value: string): DraftState { + const state = DRAFT_STATES.find((s) => s === value); + if (state === undefined) throw new BadRequestException("state is invalid"); + return state; +} + +function parseListLimit(value: string | undefined): number { + if (value === undefined || value === "") return 50; + const n = Number(value); + if (!Number.isInteger(n) || n < 1) throw new BadRequestException("limit must be a positive integer"); + return Math.min(n, 200); +} + +function requireVisibility(value: unknown): DraftPost["visibility"] { + if ( + value !== "public" && + value !== "unlisted" && + value !== "private" && + value !== "followers" && + value !== "provider_default" + ) { + throw new BadRequestException("post.visibility is invalid"); + } + return value; +} diff --git a/apps/api/src/publishing/publishing.integration.test.ts b/apps/api/src/publishing/publishing.integration.test.ts new file mode 100644 index 0000000..4f9fa8d --- /dev/null +++ b/apps/api/src/publishing/publishing.integration.test.ts @@ -0,0 +1,53 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { Pool, type PoolClient } from "pg"; +import { PublishingExecutionRepository, canonicalPostHash, assertPublishingSchema } from "@tactical-media-broadcast-hub/publishing-execution"; +import { PgConnectionStore } from "../connections/connection-store.js"; +import { PublishingService } from "./publishing.service.js"; +const appUrl = process.env.PUBLISHING_IT_DATABASE_URL; const ownerUrl = process.env.PUBLISHING_IT_SUPER_DATABASE_URL; +describe.skipIf(!appUrl || !ownerUrl)("API publishing consumer (live local DB)", () => { + let pool: Pool; let owner: Pool; let workspaceId: string; let userId: string; let connectionId: string; let service: PublishingService; let store: PgConnectionStore; + beforeAll(async () => { + pool = new Pool({ connectionString: appUrl }); owner = new Pool({ connectionString: ownerUrl }); + workspaceId = String((await owner.query("INSERT INTO workspaces(name,slug) VALUES('API Test',$1) RETURNING id", [`api-sol-${Date.now()}`])).rows[0].id); + userId = String((await owner.query("INSERT INTO users(external_id,username,display_name) VALUES($1,$1,'API Test') RETURNING id", [`api-sol-${Date.now()}`])).rows[0].id); + await owner.query("INSERT INTO memberships(workspace_id,user_id,role,status) VALUES($1,$2,'Owner','active')", [workspaceId, userId]); + connectionId = String((await owner.query("INSERT INTO connections(workspace_id,provider,channel_id,fields) VALUES($1,'mastodon','channel','{\"instanceUrl\":\"https://social.test\"}') RETURNING id", [workspaceId])).rows[0].id); + await owner.query("INSERT INTO credentials(workspace_id,connection_id,kind,ciphertext) VALUES($1,$2,'connector-oauth','\\x00')", [workspaceId, connectionId]); + const tenantTransaction = async (workspace: string, fn: (client: PoolClient) => Promise): Promise => { + const client = await pool.connect(); try { await client.query("BEGIN"); await client.query("SELECT set_config('app.workspace_id',$1,true)", [workspace]); const result = await fn(client); await client.query("COMMIT"); return result; } catch (error) { await client.query("ROLLBACK"); throw error; } finally { client.release(); } + }; + store = new PgConnectionStore(tenantTransaction); + service = new PublishingService({ repository: new PublishingExecutionRepository(pool), assertReady: () => assertPublishingSchema(pool), tenantTransaction, workflowStarter: { startScheduledPublish: async () => undefined } }); + }); + afterAll(async () => { if (workspaceId) await owner.query("DELETE FROM workspaces WHERE id=$1", [workspaceId]); if (userId) await owner.query("DELETE FROM users WHERE id=$1", [userId]); await pool?.end(); await owner?.end(); }); + it("pins approval and journals scheduling atomically; identical schedule reuses the key and cancellation retains journal", async () => { + const actor = { userId, role: "Owner" }; const post = { text: "API integration", mediaAssetIds: [], visibility: "public" as const, providerMetadata: { connectionId } }; + const draft = await service.createDraft(workspaceId, actor, post); + await expect(service.schedule(workspaceId, actor, draft.id, connectionId, new Date())).rejects.toThrow(); + await service.submitForReview(workspaceId, actor, draft.id); await service.approve(workspaceId, actor, draft.id); + const approved = (await owner.query("SELECT approved_hash,approved_revision,reviewer_user_id FROM drafts WHERE id=$1", [draft.id])).rows[0]; + expect(approved).toMatchObject({ approved_hash: canonicalPostHash(post), approved_revision: "1", reviewer_user_id: userId }); + const at = new Date(Date.now()+60_000); const schedule = await service.schedule(workspaceId, actor, draft.id, connectionId, at); + const repeated = await service.schedule(workspaceId, actor, draft.id, connectionId, at); expect(repeated.idempotencyKey).toBe(schedule.idempotencyKey); + const journal = (await owner.query("SELECT execution_version,scheduled_by_user_id,credential_generation,generation FROM scheduled_publications WHERE id=$1", [schedule.id])).rows[0]; + expect(journal).toMatchObject({ execution_version: 1, scheduled_by_user_id: userId, credential_generation: "1", generation: "1" }); + await service.cancelSchedule(workspaceId, actor, draft.id); + expect((await owner.query("SELECT status FROM publishing_attempts WHERE idempotency_key=$1", [schedule.idempotencyKey])).rows[0]?.status).toBe("abandoned"); + expect((await owner.query("SELECT state,generation FROM scheduled_publications WHERE id=$1", [schedule.id])).rows[0]).toMatchObject({ state: "cancelled", generation: "2" }); + }); + it("connection removal closes an undispatched live lease, retains journal and leaves intent explicitly resumable", async () => { + const actor = { userId, role: "Owner" }; const post = { text: "undispatched removal", mediaAssetIds: [], visibility: "public" as const, providerMetadata: { connectionId } }; + const draft = await service.createDraft(workspaceId, actor, post); await service.submitForReview(workspaceId, actor, draft.id); await service.approve(workspaceId, actor, draft.id); + const schedule = await service.schedule(workspaceId, actor, draft.id, connectionId, new Date(Date.now()-1000)); + const repository = new PublishingExecutionRepository(pool); + const ref = { workspaceId, scheduleId: schedule.id, generation: schedule.generation, idempotencyKey: schedule.idempotencyKey }; + expect((await repository.claim(ref)).kind).toBe("claimed"); + const epoch = (await owner.query("SELECT authz_epoch FROM workspaces WHERE id=$1", [workspaceId])).rows[0].authz_epoch as string; + await store.remove({ ...actor, workspaceId, authzEpoch: Number(epoch), issuedAt: 0, expiresAt: Math.floor(Date.now()/1000)+300 }, connectionId, async () => undefined); + expect((await owner.query("SELECT status,owner_token,lease_until FROM publishing_attempts WHERE idempotency_key=$1", [schedule.idempotencyKey])).rows[0]).toMatchObject({ status: "abandoned", owner_token: null, lease_until: null }); + expect((await owner.query("SELECT state FROM drafts WHERE id=$1", [draft.id])).rows[0]?.state).toBe("approved"); + expect((await owner.query("SELECT state FROM scheduled_publications WHERE id=$1", [schedule.id])).rows[0]?.state).toBe("needs_reauthorization"); + expect((await owner.query("SELECT disconnected_at FROM connections WHERE id=$1", [connectionId])).rows[0]?.disconnected_at).toBeInstanceOf(Date); + expect((await repository.listDue()).some((item) => item.idempotencyKey === schedule.idempotencyKey)).toBe(false); + }); +}); diff --git a/apps/api/src/publishing/publishing.repository.integration.test.ts b/apps/api/src/publishing/publishing.repository.integration.test.ts new file mode 100644 index 0000000..92e9caa --- /dev/null +++ b/apps/api/src/publishing/publishing.repository.integration.test.ts @@ -0,0 +1,533 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { join } from "node:path"; +import { Pool, type PoolClient } from "pg"; + +/** + * Publishing execution spine (018 + PublishingExecutionRepository) against a + * live PostgreSQL, as sp_app (FORCE RLS binds). Uses its OWN database + * (PUBLISHING_IT_DATABASE_URL, plus PUBLISHING_IT_SUPER_DATABASE_URL for + * migrations and fixtures) for the same reason as invite-link.integration.test.ts: + * tenancy.integration.test.ts wipes every workspace in the shared DB and vitest + * runs files in parallel. Skipped when unset. + * + * Fixture rows (workspace, users, memberships, connections, credentials, + * approved drafts) are inserted by the owner; every pipeline action runs + * through the repository as sp_app. + */ +const appUrl = process.env.PUBLISHING_IT_DATABASE_URL; +const ownerUrl = process.env.PUBLISHING_IT_SUPER_DATABASE_URL; + +type Modules = { + repo: typeof import("./publishing.repository.js"); + db: typeof import("../db/client.js"); +}; + +describe.skipIf(appUrl === undefined || ownerUrl === undefined)("publishing execution repository (live DB)", () => { + let m: Modules; + let ownerPool: Pool; + let repo: InstanceType; + const suffix = `${Date.now()}${Math.floor(Math.random() * 1e6)}`; + let workspaceId: string; + let ownerUserId: string; + let adminUserId: string; + const userIds: string[] = []; + + async function asOwner(fn: (client: PoolClient) => Promise): Promise { + const client = await ownerPool.connect(); + try { + return await fn(client); + } finally { + client.release(); + } + } + + async function ownerQuery(sql: string, values: unknown[] = []): Promise>> { + return asOwner(async (c) => (await c.query(sql, values)).rows as Array>); + } + + async function makeUser(role: string): Promise { + const name = `pub${role.toLowerCase()}${suffix}${userIds.length}`; + const [user] = await ownerQuery( + "INSERT INTO users (external_id, username, display_name) VALUES ($1, $2, $3) RETURNING id", + [`local:${name}`, name, name], + ); + const id = String(user?.id); + userIds.push(id); + await ownerQuery("INSERT INTO memberships (workspace_id, user_id, role, status) VALUES ($1, $2, $3, 'active')", [ + workspaceId, + id, + role, + ]); + return id; + } + + /** A fresh connection + connector credential per test, so revocations never leak across tests. */ + async function makeConnection(): Promise<{ connectionId: string; credentialId: string }> { + const [conn] = await ownerQuery( + "INSERT INTO connections (workspace_id, provider, channel_id, fields) VALUES ($1, 'mastodon', $2, '{\"instance\":\"https://example.test\"}') RETURNING id", + [workspaceId, `chan-${Math.random()}`], + ); + const [cred] = await ownerQuery( + "INSERT INTO credentials (workspace_id, connection_id, kind, ciphertext) VALUES ($1, $2, 'connector-oauth', '\\x00'::bytea) RETURNING id", + [workspaceId, conn?.id], + ); + return { connectionId: String(conn?.id), credentialId: String(cred?.id) }; + } + + async function makeApprovedDraft(connectionId: string, text = "hello fediverse"): Promise { + const post = { + text, + mediaAssetIds: [], + visibility: "public", + scheduledAt: "2030-01-01T00:00:00.000Z", + providerMetadata: { connectionId, idempotencyKey: "user-supplied", draftId: "user-supplied" }, + }; + const [draft] = await ownerQuery( + `INSERT INTO drafts (workspace_id, author_user_id, post, state, reviewer_user_id, approved_revision, approved_hash, approved_at) + VALUES ($1, $2, $3, 'approved', $2, 1, $4, now()) RETURNING id`, + [workspaceId, ownerUserId, post, m.repo.canonicalPostHash(post)], + ); + return String(draft?.id); + } + + async function schedule( + principal = ownerUserId, + ): Promise<{ ref: import("./publishing.repository.js").AttemptRef; draftId: string; connectionId: string; credentialId: string; attemptId: string }> { + const { connectionId, credentialId } = await makeConnection(); + const draftId = await makeApprovedDraft(connectionId); + const scheduled = await m.db.withTenant(workspaceId, (client) => + repo.scheduleAttempt(client, { + workspaceId, + draftId, + connectionId, + scheduledByUserId: principal, + requiredPermission: "publication.publish", + publishAt: new Date(Date.now() - 1000), + }), + ); + return { + ref: { workspaceId, scheduleId: scheduled.scheduleId, generation: scheduled.generation, idempotencyKey: scheduled.idempotencyKey }, + draftId, + connectionId, + credentialId, + attemptId: scheduled.attemptId, + }; + } + + async function states(draftId: string): Promise<{ draft: string; schedule: string; scheduleCode: unknown; attempts: Array> }> { + const [d] = await ownerQuery("SELECT state FROM drafts WHERE id = $1", [draftId]); + const [s] = await ownerQuery("SELECT state, error_code FROM scheduled_publications WHERE draft_id = $1", [draftId]); + const attempts = await ownerQuery( + "SELECT attempt_no::text, status, error_code, claim_epoch::text, dispatch_count FROM publishing_attempts WHERE draft_id = $1 ORDER BY attempt_no", + [draftId], + ); + return { draft: String(d?.state), schedule: String(s?.state), scheduleCode: s?.error_code ?? null, attempts }; + } + + async function expireLease(attemptId: string): Promise { + await ownerQuery("UPDATE publishing_attempts SET lease_until = now() - interval '1 second' WHERE id = $1", [attemptId]); + } + + function sendSpy(remoteId = `109${Math.floor(Math.random() * 1e9)}`): { calls: number; send: () => Promise<{ remoteId: string }> } { + const spy = { + calls: 0, + send: async () => { + spy.calls += 1; + return { remoteId }; + }, + }; + return spy; + } + + function claimed(outcome: Awaited>) { + if (outcome.kind !== "claimed") throw new Error(`expected claimed, got ${JSON.stringify(outcome)}`); + return outcome.job; + } + + beforeAll(async () => { + // The pool reads DATABASE_URL at import time: point it at the isolated DB first. + process.env.DATABASE_URL = appUrl; + process.env.SUPER_DATABASE_URL = ownerUrl; + const { runMigrations } = await import("../db/migrate.js"); + await runMigrations(join(process.cwd(), "migrations")); + m = { + repo: await import("./publishing.repository.js"), + db: await import("../db/client.js"), + }; + ownerPool = new Pool({ connectionString: ownerUrl }); + repo = new m.repo.PublishingExecutionRepository(m.db.pool); + + const [ws] = await ownerQuery("INSERT INTO workspaces (name, slug) VALUES ('Publishing IT', $1) RETURNING id", [ + `publishing-it-${suffix}`, + ]); + workspaceId = String(ws?.id); + ownerUserId = await makeUser("Owner"); + adminUserId = await makeUser("Admin"); + }); + + afterAll(async () => { + if (ownerPool !== undefined) { + await asOwner(async (c) => { + await c.query("DELETE FROM workspaces WHERE id = $1", [workspaceId]); + await c.query("DELETE FROM users WHERE id = ANY($1::uuid[])", [userIds]); + }); + await ownerPool.end(); + } + await m?.db.pool.end(); + }); + + it("scheduleAttempt snapshots principal, approval proof and the exact credential generation", async () => { + const { ref, draftId, credentialId } = await schedule(); + expect(ref.idempotencyKey).toBe(`wp3:${workspaceId}:${draftId}:a1:publish`); + const [row] = await ownerQuery( + `SELECT sp.state, sp.execution_version, sp.scheduled_by_user_id, sp.required_permission, sp.credential_id, + sp.credential_generation::text, a.status, a.post_hash = d.approved_hash AS hash_pinned + FROM scheduled_publications sp + JOIN publishing_attempts a ON a.schedule_id = sp.id + JOIN drafts d ON d.id = sp.draft_id + WHERE sp.id = $1`, + [ref.scheduleId], + ); + expect(row).toMatchObject({ + state: "pending", + execution_version: 1, + scheduled_by_user_id: ownerUserId, + required_permission: "publication.publish", + credential_id: credentialId, + credential_generation: "1", + status: "ready", + hash_pinned: true, + }); + const due = await repo.listDue(100); + expect(due.map((d) => d.idempotencyKey)).toContain(ref.idempotencyKey); + await expect(repo.resolveWorkspace(ref.idempotencyKey)).resolves.toBe(workspaceId); + }); + + it("concurrent claims: exactly one worker wins, the rest wait on the live lease", async () => { + const { ref, draftId } = await schedule(); + const workers = Array.from({ length: 5 }, () => new m.repo.PublishingExecutionRepository(m.db.pool)); + const outcomes = await Promise.all(workers.map((w) => w.claim(ref))); + const kinds = outcomes.map((o) => o.kind).sort(); + expect(kinds).toEqual(["claimed", "wait", "wait", "wait", "wait"]); + for (const o of outcomes) if (o.kind === "wait") expect(o.reason).toBe("busy"); + + const winner = outcomes.find((o) => o.kind === "claimed"); + if (winner?.kind !== "claimed") throw new Error("no winner"); + expect(winner.job.lease.claimEpoch).toBe("1"); + expect(winner.job.connection.provider).toBe("mastodon"); + // Pipeline owns timing and the reserved execution metadata. + expect(winner.job.post).not.toHaveProperty("scheduledAt"); + expect(winner.job.post.providerMetadata).toMatchObject({ draftId, idempotencyKey: ref.idempotencyKey }); + const s = await states(draftId); + expect(s).toMatchObject({ draft: "scheduled", schedule: "in_flight" }); + expect(s.attempts).toEqual([expect.objectContaining({ status: "in_progress", claim_epoch: "1" })]); + }); + + it("credential revoked before the due claim pauses durably with zero sends; restoring does not resume", async () => { + const { ref, draftId, credentialId } = await schedule(); + await ownerQuery("UPDATE credentials SET revoked_at = now() WHERE id = $1", [credentialId]); + await expect(repo.claim(ref)).resolves.toEqual({ kind: "paused", code: "CREDENTIAL_CHANGED" }); + const s = await states(draftId); + expect(s).toMatchObject({ draft: "approved", schedule: "needs_reauthorization", scheduleCode: "CREDENTIAL_CHANGED" }); + expect(s.attempts).toEqual([expect.objectContaining({ status: "abandoned", dispatch_count: 0 })]); + + // Reconnecting rotates generation again: still paused, never auto-resumed. + await ownerQuery("UPDATE credentials SET revoked_at = NULL WHERE id = $1", [credentialId]); + const [cred] = await ownerQuery("SELECT generation::text FROM credentials WHERE id = $1", [credentialId]); + expect(cred?.generation).toBe("3"); + await expect(repo.claim(ref)).resolves.toMatchObject({ kind: "superseded" }); + expect((await states(draftId)).schedule).toBe("needs_reauthorization"); + }); + + it("credential rotated after claim: the dispatch gate refuses before any send", async () => { + const { ref, draftId, credentialId } = await schedule(); + const job = claimed(await repo.claim(ref)); + await ownerQuery("UPDATE credentials SET ciphertext = '\\x01'::bytea, rotated_at = now() WHERE id = $1", [credentialId]); + const spy = sendSpy(); + await expect(repo.withDispatchGate(job.lease, {}, spy.send)).resolves.toEqual({ kind: "paused", code: "CREDENTIAL_CHANGED" }); + expect(spy.calls).toBe(0); + expect(await states(draftId)).toMatchObject({ draft: "approved", schedule: "needs_reauthorization" }); + }); + + it("recorded principal demoted: claim pauses with AUTHZ_REVOKED", async () => { + const { ref, draftId } = await schedule(adminUserId); + await ownerQuery("UPDATE memberships SET role = 'Editor' WHERE workspace_id = $1 AND user_id = $2", [workspaceId, adminUserId]); + try { + await expect(repo.claim(ref)).resolves.toEqual({ kind: "paused", code: "AUTHZ_REVOKED" }); + expect((await states(draftId)).scheduleCode).toBe("AUTHZ_REVOKED"); + } finally { + await ownerQuery("UPDATE memberships SET role = 'Admin' WHERE workspace_id = $1 AND user_id = $2", [workspaceId, adminUserId]); + } + }); + + it("an unrelated epoch bump re-checks and continues still-authorized work", async () => { + const { ref } = await schedule(); + const extra = await makeUser("Analyst"); + expect(extra).toBeTruthy(); + const job = claimed(await repo.claim(ref)); + const spy = sendSpy(); + await expect(repo.withDispatchGate(job.lease, {}, spy.send)).resolves.toMatchObject({ kind: "sent" }); + expect(spy.calls).toBe(1); + }); + + it("receipts are idempotent on replay; finalization yields exactly one Publication", async () => { + const { ref, draftId } = await schedule(); + const job = claimed(await repo.claim(ref)); + const spy = sendSpy("10987654321"); + const gate = await repo.withDispatchGate(job.lease, {}, spy.send); + if (gate.kind !== "sent") throw new Error(`gate ${gate.kind}`); + + const target = { workspaceId, attemptId: job.lease.attemptId }; + const first = await repo.recordReceipt(target, { remoteId: gate.value.remoteId, remoteUrl: "https://example.test/@a/10987654321" }); + const replay = await repo.recordReceipt(target, { remoteId: gate.value.remoteId }); + expect(first.inserted).toBe(true); + expect(replay).toMatchObject({ inserted: false, receipt: { id: first.receipt.id } }); + await expect(repo.recordReceipt(target, { remoteId: "different" })).rejects.toMatchObject({ code: "RECEIPT_CONFLICT" }); + + const done = await repo.finalize(job.lease); + if (done.kind !== "done") throw new Error("finalize stale"); + expect(done.publication).toMatchObject({ remoteId: "10987654321", draftId, idempotencyKey: ref.idempotencyKey }); + + // Replays: no new send, same publication, finalize refuses the spent lease. + await expect(repo.claim(ref)).resolves.toEqual({ kind: "done", publication: done.publication }); + await expect(repo.finalize(job.lease)).resolves.toEqual({ kind: "stale" }); + await expect(repo.findPublication(workspaceId, ref.idempotencyKey)).resolves.toEqual(done.publication); + const pubs = await ownerQuery("SELECT count(*)::int AS n FROM publications WHERE idempotency_key = $1", [ref.idempotencyKey]); + expect(pubs[0]?.n).toBe(1); + expect(spy.calls).toBe(1); + const s = await states(draftId); + expect(s).toMatchObject({ draft: "published", schedule: "done" }); + expect(s.attempts).toEqual([expect.objectContaining({ status: "completed", dispatch_count: 1 })]); + expect((await repo.listDue(100)).map((d) => d.idempotencyKey)).not.toContain(ref.idempotencyKey); + + // Receipts are append-only for the runtime role. + await expect( + m.db.withTenant(workspaceId, (c) => c.query("UPDATE publishing_receipts SET remote_id = 'x' WHERE attempt_id = $1", [job.lease.attemptId])), + ).rejects.toMatchObject({ code: "42501" }); + }); + + it("lease expiry: takeover advances claim_epoch; the deposed owner records evidence but cannot complete", async () => { + const { ref, draftId } = await schedule(); + const a = claimed(await repo.claim(ref)); + expect(await repo.heartbeat(a.lease)).toBe(true); + const spy = sendSpy("555"); + const sent = await repo.withDispatchGate(a.lease, {}, spy.send); + expect(sent.kind).toBe("sent"); + + await expireLease(a.lease.attemptId); + expect(await repo.heartbeat(a.lease)).toBe(false); + expect((await repo.listDue(100)).map((d) => d.idempotencyKey)).toContain(ref.idempotencyKey); + + const b = claimed(await repo.claim(ref)); + expect(b.lease.claimEpoch).toBe(String(Number(a.lease.claimEpoch) + 1)); + expect(b.priorDispatch).toMatchObject({ count: 1 }); + + await expect(repo.withDispatchGate(a.lease, {}, spy.send)).resolves.toEqual({ kind: "lost" }); + await expect(repo.recordReceipt({ workspaceId, attemptId: a.lease.attemptId }, { remoteId: "555" })).resolves.toMatchObject({ inserted: true }); + await expect(repo.finalize(a.lease)).resolves.toEqual({ kind: "stale" }); + + // New owner adopts the evidence: no second send. + await expect(repo.withDispatchGate(b.lease, {}, spy.send)).resolves.toEqual({ kind: "finalize" }); + const done = await repo.finalize(b.lease); + expect(done).toMatchObject({ kind: "done", publication: { remoteId: "555" } }); + expect(spy.calls).toBe(1); + expect((await states(draftId)).draft).toBe("published"); + }); + + it("a recovery claim after a receipt goes straight to finalize", async () => { + const { ref } = await schedule(); + const a = claimed(await repo.claim(ref)); + await repo.withDispatchGate(a.lease, {}, sendSpy("777").send); + await repo.recordReceipt({ workspaceId, attemptId: a.lease.attemptId }, { remoteId: "777" }); + await expireLease(a.lease.attemptId); + const outcome = await repo.claim(ref); + if (outcome.kind !== "finalize") throw new Error(`expected finalize, got ${outcome.kind}`); + expect(outcome.receipt.remoteId).toBe("777"); + await expect(repo.finalize(outcome.job.lease)).resolves.toMatchObject({ kind: "done" }); + }); + + it("unsettled dispatch without a dedupe policy becomes unknown: never resent, content locked", async () => { + const { ref, draftId } = await schedule(); + const a = claimed(await repo.claim(ref)); + let calls = 0; + await expect( + repo.withDispatchGate(a.lease, {}, async () => { + calls += 1; + throw new Error("socket hang up"); + }), + ).rejects.toThrow("socket hang up"); + await expect( + ownerQuery("UPDATE drafts SET post = post || '{\"text\":\"edited\"}' WHERE id = $1", [draftId]), + ).rejects.toMatchObject({ code: "55000" }); + + await expireLease(a.lease.attemptId); + const b = claimed(await repo.claim(ref)); + await expect(repo.withDispatchGate(b.lease, {}, async () => { calls += 1; })).resolves.toEqual({ + kind: "unknown", + code: "PUBLISH_OUTCOME_UNKNOWN", + }); + expect(calls).toBe(1); + const s = await states(draftId); + expect(s).toMatchObject({ draft: "scheduled", schedule: "failed", scheduleCode: "PUBLISH_OUTCOME_UNKNOWN" }); + expect(s.attempts).toEqual([expect.objectContaining({ status: "unknown" })]); + await expect(repo.claim(ref)).resolves.toEqual({ kind: "unknown", code: "PUBLISH_OUTCOME_UNKNOWN" }); + await expect( + ownerQuery("UPDATE drafts SET post = post || '{\"text\":\"edited\"}' WHERE id = $1", [draftId]), + ).rejects.toMatchObject({ code: "55000" }); + }); + + it("verified dedupe policy permits one same-key resend inside the window", async () => { + const { ref } = await schedule(); + const a = claimed(await repo.claim(ref)); + await expect( + repo.withDispatchGate(a.lease, { allowRedispatch: true, dedupeWindowMs: 30 * 60_000 }, async () => { + throw new Error("timeout"); + }), + ).rejects.toThrow("timeout"); + await expireLease(a.lease.attemptId); + const b = claimed(await repo.claim(ref)); + const spy = sendSpy("888"); + const gate = await repo.withDispatchGate(b.lease, { allowRedispatch: true, dedupeWindowMs: 30 * 60_000 }, spy.send); + expect(gate.kind).toBe("sent"); + expect(b.post.providerMetadata.idempotencyKey).toBe(ref.idempotencyKey); + const [row] = await ownerQuery("SELECT dispatch_count FROM publishing_attempts WHERE id = $1", [b.lease.attemptId]); + expect(row?.dispatch_count).toBe(2); + }); + + it("definite non-delivery retries later under the same key", async () => { + const { ref, draftId } = await schedule(); + const a = claimed(await repo.claim(ref)); + await repo.withDispatchGate(a.lease, {}, async () => undefined); + const retry = await repo.retryLater(a.lease, "RATE_LIMITED", new Date(Date.now() + 60_000)); + expect(retry.kind).toBe("settled"); + expect(await states(draftId)).toMatchObject({ draft: "approved", schedule: "pending", scheduleCode: "RATE_LIMITED" }); + await expect(repo.claim(ref)).resolves.toMatchObject({ kind: "wait", reason: "not_due" }); + expect((await repo.listDue(100)).map((d) => d.idempotencyKey)).not.toContain(ref.idempotencyKey); + + await ownerQuery("UPDATE scheduled_publications SET next_attempt_at = now() - interval '1 second' WHERE id = $1", [ref.scheduleId]); + const b = claimed(await repo.claim(ref)); + expect(b.lease.idempotencyKey).toBe(ref.idempotencyKey); + expect(b.priorDispatch).toBeUndefined(); + }); + + it("generation fence: cancel + reschedule supersedes the old workflow without touching the new one", async () => { + const { ref, draftId, connectionId } = await schedule(); + await m.db.withTenant(workspaceId, (c) => + c.query("UPDATE scheduled_publications SET state = 'cancelled', error_code = 'CANCELLED' WHERE id = $1", [ref.scheduleId]), + ); + const next = await m.db.withTenant(workspaceId, (client) => + repo.scheduleAttempt(client, { + workspaceId, + draftId, + connectionId, + scheduledByUserId: ownerUserId, + requiredPermission: "publication.schedule", + publishAt: new Date(Date.now() - 1000), + }), + ); + expect(next).toMatchObject({ attemptNo: "2", generation: "2", idempotencyKey: `wp3:${workspaceId}:${draftId}:a2:publish` }); + + const spy = sendSpy(); + await expect(repo.claim(ref)).resolves.toEqual({ kind: "superseded", code: "SUPERSEDED" }); + expect(spy.calls).toBe(0); + const s = await states(draftId); + expect(s.schedule).toBe("pending"); + expect(s.attempts).toEqual([ + expect.objectContaining({ attempt_no: "1", status: "abandoned" }), + expect.objectContaining({ attempt_no: "2", status: "ready" }), + ]); + claimed(await repo.claim({ workspaceId, scheduleId: next.scheduleId, generation: next.generation, idempotencyKey: next.idempotencyKey })); + }); + + it("content edit before the due claim rotates the revision, voids approval and supersedes", async () => { + const { ref, draftId } = await schedule(); + await m.db.withTenant(workspaceId, (c) => + c.query("UPDATE drafts SET post = post || '{\"text\":\"changed\"}' WHERE id = $1", [draftId]), + ); + const [d] = await ownerQuery("SELECT revision::text, approved_revision, approved_hash, reviewer_user_id FROM drafts WHERE id = $1", [draftId]); + expect(d).toEqual({ revision: "2", approved_revision: null, approved_hash: null, reviewer_user_id: null }); + await expect(repo.claim(ref)).resolves.toEqual({ kind: "superseded", code: "SUPERSEDED" }); + expect(await states(draftId)).toMatchObject({ schedule: "cancelled", scheduleCode: "SUPERSEDED" }); + }); + + it("refuses a target differing from the approved snapshot and a forged approval hash", async () => { + const original = await makeConnection(); const other = await makeConnection(); + const draftId = await makeApprovedDraft(original.connectionId); + const input = { workspaceId, draftId, connectionId: other.connectionId, scheduledByUserId: ownerUserId, requiredPermission: "publication.publish" as const, publishAt: new Date() }; + await expect(m.db.withTenant(workspaceId, (c) => repo.scheduleAttempt(c, input))).rejects.toMatchObject({ code: "APPROVAL_MISMATCH" }); + await ownerQuery("UPDATE drafts SET approved_hash = 'forged' WHERE id = $1", [draftId]); + await expect(m.db.withTenant(workspaceId, (c) => repo.scheduleAttempt(c, { ...input, connectionId: original.connectionId }))).rejects.toMatchObject({ code: "APPROVAL_MISMATCH" }); + expect((await ownerQuery("SELECT * FROM publishing_attempts WHERE draft_id = $1", [draftId]))).toHaveLength(0); + }); + + it("keeps first dispatch and dedupe deadline across ambiguous and definite retries", async () => { + const { ref, attemptId } = await schedule(); + const first = claimed(await repo.claim(ref)); + await repo.withDispatchGate(first.lease, { allowRedispatch: true, dedupeWindowMs: 1_800_000 }, async () => undefined); + const [before] = await ownerQuery("SELECT first_dispatch_started_at, dispatch_started_at, provider_retry_deadline FROM publishing_attempts WHERE id = $1", [attemptId]); + await repo.retryDeduplicated(first.lease, "PUBLISH_AMBIGUOUS_DEDUPED", new Date(Date.now() - 1000)); + const second = claimed(await repo.claim(ref)); + expect(second.priorDispatch).toBeDefined(); expect(second.retryCount).toBe(1); + expect(second.firstDispatchStartedAt).toEqual(before?.first_dispatch_started_at); + expect(second.providerRetryDeadline).toEqual(before?.provider_retry_deadline); + await repo.withDispatchGate(second.lease, { allowRedispatch: true, dedupeWindowMs: 1_800_000 }, async () => undefined); + await repo.retryLater(second.lease, "RATE_LIMITED", new Date(Date.now() - 1000)); + const third = claimed(await repo.claim(ref)); + expect(third.priorDispatch).toBeUndefined(); expect(third.firstDispatchStartedAt).toEqual(before?.first_dispatch_started_at); + expect(third.providerRetryDeadline).toEqual(before?.provider_retry_deadline); + await expect(ownerQuery("UPDATE publishing_attempts SET first_dispatch_started_at = now() + interval '1 second' WHERE id = $1", [attemptId])).rejects.toMatchObject({ code: "23514" }); + await repo.fail(third.lease, "TEST_DONE"); + }); + + it("discovers late receipts on unknown work for evidence-only finalization", async () => { + const { ref, attemptId } = await schedule(); const first = claimed(await repo.claim(ref)); + await repo.withDispatchGate(first.lease, {}, async () => undefined); + await repo.markUnknown(first.lease); + expect((await repo.listDue()).some((item) => item.idempotencyKey === ref.idempotencyKey)).toBe(false); + await repo.recordReceipt({ workspaceId, attemptId }, { remoteId: "late-evidence" }); + expect((await repo.listDue()).find((item) => item.idempotencyKey === ref.idempotencyKey)).toMatchObject({ evidenceOnly: true }); + const recovery = await repo.claim(ref); + expect(recovery.kind).toBe("finalize"); + if (recovery.kind !== "finalize") throw new Error("receipt recovery missing"); + await expect(repo.finalize(recovery.job.lease)).resolves.toMatchObject({ kind: "done", publication: { remoteId: "late-evidence" } }); + expect((await ownerQuery("SELECT dispatch_count FROM publishing_attempts WHERE id = $1", [attemptId]))[0]?.dispatch_count).toBe(1); + }); + + it("pauses provider authentication rejection through owner CAS", async () => { + const { ref, draftId } = await schedule(); const first = claimed(await repo.claim(ref)); + await repo.pauseForReauthorization(first.lease); + expect((await states(draftId)).schedule).toBe("needs_reauthorization"); + await expect(repo.claim(ref)).resolves.toMatchObject({ kind: "superseded", code: "CREDENTIAL_REJECTED" }); + await expect(repo.pauseForReauthorization(first.lease)).resolves.toEqual({ kind: "stale" }); + }); + it("an expired owner records evidence but cannot finalize until a fresh claim", async () => { + const { ref, attemptId } = await schedule(); const first = claimed(await repo.claim(ref)); + await repo.withDispatchGate(first.lease, {}, async () => undefined); + await expireLease(attemptId); + await repo.recordReceipt({ workspaceId, attemptId }, { remoteId: "expired-owner-evidence" }); + await expect(repo.finalize(first.lease)).resolves.toEqual({ kind: "stale" }); + const recovery = await repo.claim(ref); + expect(recovery.kind).toBe("finalize"); + if (recovery.kind !== "finalize") throw new Error("fresh receipt claim missing"); + expect(recovery.job.lease.claimEpoch).not.toBe(first.lease.claimEpoch); + await expect(repo.finalize(recovery.job.lease)).resolves.toMatchObject({ kind: "done", publication: { remoteId: "expired-owner-evidence" } }); + }); + + it("journal is not deletable by the app; workspace purge cascades it", async () => { + const { attemptId } = await schedule(); + await expect( + m.db.withTenant(workspaceId, (c) => c.query("DELETE FROM publishing_attempts WHERE id = $1", [attemptId])), + ).rejects.toMatchObject({ code: "42501" }); + const receipts = await ownerQuery("SELECT count(*)::int AS n FROM publishing_receipts WHERE workspace_id = $1", [workspaceId]); + expect(receipts[0]?.n).toBeGreaterThan(0); + + await ownerQuery("DELETE FROM workspaces WHERE id = $1", [workspaceId]); + const left = await ownerQuery( + `SELECT (SELECT count(*) FROM publishing_receipts WHERE workspace_id = $1)::int + + (SELECT count(*) FROM publishing_attempts WHERE workspace_id = $1)::int AS n`, + [workspaceId], + ); + expect(left[0]?.n).toBe(0); + }); + +}); diff --git a/apps/api/src/publishing/publishing.repository.ts b/apps/api/src/publishing/publishing.repository.ts new file mode 100644 index 0000000..7fcb2d0 --- /dev/null +++ b/apps/api/src/publishing/publishing.repository.ts @@ -0,0 +1,2 @@ +// Compatibility export: the implementation is shared with the scheduler. +export * from "@tactical-media-broadcast-hub/publishing-execution"; diff --git a/apps/api/src/publishing/publishing.service.test.ts b/apps/api/src/publishing/publishing.service.test.ts new file mode 100644 index 0000000..62bb553 --- /dev/null +++ b/apps/api/src/publishing/publishing.service.test.ts @@ -0,0 +1,172 @@ +import { BadRequestException, ForbiddenException, NotFoundException } from "@nestjs/common"; +import type { PoolClient, QueryResult } from "pg"; +import { describe, expect, it, vi } from "vitest"; +import { + PublishingService, + type DraftPost, + type DraftState, + type PublishingDependencies, +} from "./publishing.service.js"; + +const workspaceId = "10000000-0000-4000-8000-000000000001"; +const userId = "20000000-0000-4000-8000-000000000002"; +const draftId = "30000000-0000-4000-8000-000000000003"; +const connectionId = "40000000-0000-4000-8000-000000000004"; +const actor = { userId, role: "Publisher" }; + +const post: DraftPost = { + text: "Ready to publish", + mediaAssetIds: [], + visibility: "public", + providerMetadata: {}, +}; + +interface FakeDatabase { + state: DraftState; + scheduled: boolean; +} + +function serviceWithDatabase(database: FakeDatabase, canUseConnection = true): PublishingService { + const now = new Date("2026-10-06T10:00:00.000Z"); + const row = (): Record => ({ + id: draftId, + workspace_id: workspaceId, + author_user_id: userId, + post, + state: database.state, + reviewer_user_id: null, + created_at: now, + updated_at: now, + }); + const result = (rows: Array>): QueryResult> => ({ + command: "", + rowCount: rows.length, + oid: 0, + fields: [], + rows, + }); + const client = { + async query(sql: string, values: readonly unknown[] = []): Promise>> { + if (sql.includes("FROM workspaces")) return result([{ id: workspaceId }]); + if (sql === "SELECT * FROM drafts WHERE id = $1 FOR UPDATE") return result([row()]); + if (sql === "SELECT * FROM scheduled_publications WHERE id = $1") return result([{ id: "schedule", workspace_id: workspaceId, draft_id: draftId, connection_id: connectionId, publish_at: now, generation: "1", state: "pending", idempotency_key: "key" }]); + if (sql.includes("SELECT audit_append")) return result([{ audit_append: "audit-id" }]); + if (sql.includes("FROM memberships m")) + return result(canUseConnection && values[2] === connectionId ? [{ "?column?": 1 }] : []); + if (sql.includes("SELECT id FROM drafts WHERE id = $1 AND state = 'approved' FOR UPDATE")) + return result(database.state === "approved" ? [{ id: draftId }] : []); + if (sql.includes("SELECT * FROM drafts WHERE id = $1 AND state IN ('approved', 'scheduled') FOR UPDATE")) + return result(database.state === "approved" || database.state === "scheduled" ? [row()] : []); + if (sql.includes("UPDATE drafts SET state = 'editing'")) { + if (database.state !== "scheduled") return result([]); + database.state = "editing"; + return result([row()]); + } + if (sql.includes("INSERT INTO scheduled_publications")) { + database.scheduled = true; + return result([{ + id: "50000000-0000-4000-8000-000000000005", + workspace_id: workspaceId, + draft_id: draftId, + connection_id: connectionId, + publish_at: now, + idempotency_key: "60000000-0000-4000-8000-000000000006", + state: "pending", + }]); + } + if (sql.includes("DELETE FROM scheduled_publications")) { + if (!database.scheduled) return result([]); + database.scheduled = false; + return result([{ id: "50000000-0000-4000-8000-000000000005" }]); + } + if (sql.includes("UPDATE drafts SET state = 'approved'")) { + if (database.state !== "scheduled") return result([]); + database.state = "approved"; + return result([row()]); + } + if (sql.includes("UPDATE drafts")) { + if (!sql.includes("state = $4")) return result([]); + const target = String(values[1]) as DraftState; + const expected = String(values[3]) as DraftState; + if (database.state !== expected) return result([]); + database.state = target; + return result([row()]); + } + if (sql.includes("SELECT state FROM drafts")) return result([{ state: database.state }]); + // RLS-scoped connection lookup: only this tenant's connection is visible. + if (sql.includes("FROM connections")) return result(values[0] === connectionId ? [{ provider: "mastodon" }] : []); + if (sql.includes("SELECT state FROM scheduled_publications")) { + return result(database.scheduled ? [{ state: "pending" }] : []); + } + throw new Error(`unexpected SQL in fake: ${sql}`); + }, + }; + const dependencies: PublishingDependencies = { + assertReady: async () => undefined, + repository: { + scheduleAttempt: vi.fn(async (_client, input) => { + if (database.state !== "approved") throw new BadRequestException("approval required"); + expect(input).toMatchObject({ workspaceId, draftId, connectionId, scheduledByUserId: userId, requiredPermission: "publication.schedule" }); + database.scheduled = true; + return { scheduleId: "schedule", attemptId: "attempt", attemptNo: "1", generation: "1", idempotencyKey: "key" }; + }), + cancelAttempt: vi.fn(async () => { database.scheduled = false; }), + }, + tenantTransaction: async (_tenant, fn) => fn(client as unknown as PoolClient), + workflowStarter: { startScheduledPublish: vi.fn(async () => undefined) }, + }; + return new PublishingService(dependencies); +} + +describe("publishing draft state machine", () => { + it("moves an editing draft through review and approval", async () => { + const database: FakeDatabase = { state: "editing", scheduled: false }; + const service = serviceWithDatabase(database); + await expect(service.submitForReview(workspaceId, actor, draftId)).resolves.toMatchObject({ state: "in_review" }); + await expect(service.approve(workspaceId, actor, draftId)).resolves.toMatchObject({ state: "approved" }); + expect(database.state).toBe("approved"); + }); + + it("rejects an illegal editing -> approved transition", async () => { + const service = serviceWithDatabase({ state: "editing", scheduled: false }); + await expect(service.approve(workspaceId, actor, draftId)).rejects.toBeInstanceOf(BadRequestException); + }); + + it("schedules an approved draft and cancels it back to approved", async () => { + const database: FakeDatabase = { state: "approved", scheduled: false }; + const service = serviceWithDatabase(database); + const scheduled = await service.schedule(workspaceId, actor, draftId, connectionId, new Date("2026-10-06T10:00:00Z")); + expect(scheduled.state).toBe("pending"); + expect(database).toEqual({ state: "approved", scheduled: true }); + + const cancelled = await service.cancelSchedule(workspaceId, actor, draftId); + expect(cancelled.state).toBe("approved"); + expect(database).toEqual({ state: "approved", scheduled: false }); + }); + + it("refuses to schedule onto a connection outside the tenant", async () => { + const database: FakeDatabase = { state: "approved", scheduled: false }; + const service = serviceWithDatabase(database); + const foreign = "40000000-0000-4000-8000-0000000000ff"; + await expect( + service.schedule(workspaceId, actor, draftId, foreign, new Date("2026-10-06T10:00:00Z")), + ).rejects.toBeInstanceOf(NotFoundException); + expect(database).toEqual({ state: "approved", scheduled: false }); + }); + + it("refuses an existing connection outside the actor's group scope", async () => { + const database: FakeDatabase = { state: "approved", scheduled: false }; + const service = serviceWithDatabase(database, false); + await expect(service.schedule(workspaceId, actor, draftId, connectionId, new Date("2026-10-06T10:00:00Z"))) + .rejects.toBeInstanceOf(ForbiddenException); + expect(database).toEqual({ state: "approved", scheduled: false }); + }); + + it("cancels a legacy pending schedule into editing for renewed approval", async () => { + const database: FakeDatabase = { state: "scheduled", scheduled: true }; + const service = serviceWithDatabase(database); + const draft = await service.cancelSchedule(workspaceId, actor, draftId); + expect(draft.state).toBe("editing"); + expect(database).toEqual({ state: "editing", scheduled: false }); + }); +}); diff --git a/apps/api/src/publishing/publishing.service.ts b/apps/api/src/publishing/publishing.service.ts new file mode 100644 index 0000000..26dc122 --- /dev/null +++ b/apps/api/src/publishing/publishing.service.ts @@ -0,0 +1,403 @@ +import { BadRequestException, ForbiddenException, Inject, Injectable, NotFoundException, Optional, ConflictException } from "@nestjs/common"; +import { isPublishingProvider } from "@tactical-media-broadcast-hub/connector-runtime"; +import { Client, Connection, WorkflowExecutionAlreadyStartedError } from "@temporalio/client"; +import { PublishingExecutionRepository, PublishingStateError, canonicalPostHash, assertPublishingSchema, type AttemptRef } from "@tactical-media-broadcast-hub/publishing-execution"; +import type { PoolClient } from "pg"; +import { markDraftPollsStale, autoOpenDraftPoll } from "../chatgov/poll.service.js"; +import { scrub } from "../audit/audit.service.js"; +import { ApprovalGateService, ApprovalGateBlockedError } from "./approval-gate.service.js"; +import { pool, withTenant } from "../db/client.js"; +import { userCanUseConnection } from "../groups/groups.service.js"; + +export type DraftState = "editing" | "in_review" | "approved" | "rejected" | "scheduled" | "published"; + +export interface DraftPost { + text: string; + mediaAssetIds: string[]; + visibility: "public" | "unlisted" | "private" | "followers" | "provider_default"; + scheduledAt?: string; + providerMetadata: Record; +} + +export interface DraftRecord { + id: string; + workspaceId: string; + authorUserId: string; + post: DraftPost; + state: DraftState; + reviewerUserId?: string; + createdAt: Date; + updatedAt: Date; +} + +export interface ScheduledPublicationRecord { + id: string; + workspaceId: string; + draftId: string; + connectionId: string; + publishAt: Date; + idempotencyKey: string; + generation: string; + state: "pending" | "in_flight" | "done" | "failed" | "needs_reauthorization" | "cancelled"; +} + +export interface Actor { + userId: string; + role: string; +} + +export interface WorkflowStarter { + startScheduledPublish(ref: AttemptRef, publishAt: Date): Promise; +} + +type TenantTransaction = (workspaceId: string, fn: (client: PoolClient) => Promise) => Promise; + +export interface PublishingDependencies { + tenantTransaction: TenantTransaction; + workflowStarter: WorkflowStarter; + repository: Pick; + assertReady: () => Promise; + /** + * E1 chatgov approval gate (chatgov-design.md §4). Optional: a plain + * `new PublishingService()` keeps today's ungated behavior; AppModule + * injects the real one. `null` disables it explicitly. + */ + approvalGate?: ApprovalGateService | null; +} + +export const PUBLISHING_DEPENDENCIES = Symbol("PUBLISHING_DEPENDENCIES"); + +/** The pending row is durable; a failed hint is repaired by discovery. */ +export class TemporalWorkflowStarter implements WorkflowStarter { + private connection?: Connection; + private connecting?: Promise; + private async client(): Promise { + this.connecting ??= Connection.connect({ address: process.env.TEMPORAL_ADDRESS ?? "localhost:7233" }) + .then((connection) => { this.connection = connection; return new Client({ connection, namespace: process.env.TEMPORAL_NAMESPACE ?? "default" }); }) + .catch((error: unknown) => { this.connecting = undefined; throw error; }); + return this.connecting; + } + async startScheduledPublish(ref: AttemptRef, publishAt: Date): Promise { + // Future work belongs to the due sweep; no long-lived API timer is needed. + if (publishAt.getTime() > Date.now()) return; + try { + const client = await this.client(); + await client.workflow.start("scheduledPublish", { + args: [ref], workflowId: `scheduled-publication-${ref.idempotencyKey}`, + taskQueue: process.env.TEMPORAL_TASK_QUEUE ?? "tmbh-publishing", + }); + } catch (error) { + if (error instanceof WorkflowExecutionAlreadyStartedError) return; + // Committed pending work will be retried by the worker sweep. + } + } + async onModuleDestroy(): Promise { await this.connecting?.catch(() => undefined); await this.connection?.close(); } +} + +const DEFAULT_DEPENDENCIES: PublishingDependencies = { + tenantTransaction: withTenant, + workflowStarter: new TemporalWorkflowStarter(), + repository: new PublishingExecutionRepository(pool), + assertReady: () => assertPublishingSchema(pool), + approvalGate: new ApprovalGateService(), +}; + +@Injectable() +export class PublishingService { + private readonly dependencies: PublishingDependencies; + + constructor( + @Optional() @Inject(PUBLISHING_DEPENDENCIES) dependencies?: Partial, + ) { + this.dependencies = { ...DEFAULT_DEPENDENCIES, ...dependencies }; + } + + async createDraft(workspaceId: string, actor: Actor, post: DraftPost): Promise { + await this.dependencies.assertReady(); + const draft = await this.dependencies.tenantTransaction(workspaceId, async (client) => { + await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [workspaceId]); + const result = await client.query( + `INSERT INTO drafts (workspace_id, author_user_id, post) + VALUES ($1, $2, $3) RETURNING *`, + [workspaceId, actor.userId, post], + ); + const draft = draftFromRow(requiredRow(result.rows[0], "created draft")); + await this.audit(client, workspaceId, actor, "draft.created", draft.id, { state: draft.state }); + return draft; + }); + return draft; + } + + async listDrafts( + workspaceId: string, + options: { state?: DraftState; limit: number }, + ): Promise { + return this.dependencies.tenantTransaction(workspaceId, async (client) => { + const result = await client.query( + `SELECT * FROM drafts + WHERE ($1::text IS NULL OR state = $1) + ORDER BY updated_at DESC + LIMIT $2`, + [options.state ?? null, options.limit], + ); + return result.rows.map((row: unknown) => draftFromRow(requiredRow(row, "draft"))); + }); + } + + async updateDraft( + workspaceId: string, + actor: Actor, + draftId: string, + post: DraftPost, + ): Promise { + await this.dependencies.assertReady(); + const draft = await this.dependencies.tenantTransaction(workspaceId, async (client) => { + await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [workspaceId]); + const result = await client.query( + `UPDATE drafts SET post = $2, updated_at = now() + WHERE id = $1 AND (state = 'editing' OR (state = 'in_review' AND EXISTS (SELECT 1 FROM polls WHERE draft_id = $1 AND state = 'open'))) RETURNING *`, + [draftId, post], + ); + if (result.rowCount !== 1) { + const current = await client.query("SELECT state FROM drafts WHERE id = $1", [draftId]); + if (current.rowCount === 0) throw new NotFoundException(`draft ${draftId} not found`); + throw new BadRequestException(`cannot edit draft in state ${String(current.rows[0]?.state)}`); + } + const draft = draftFromRow(requiredRow(result.rows[0], "updated draft")); + await markDraftPollsStale(client, workspaceId, draftId); + await this.audit(client, workspaceId, actor, "draft.updated", draftId, { state: draft.state }); + return draft; + }); + return draft; + } + + async submitForReview(workspaceId: string, actor: Actor, draftId: string): Promise { + return this.transition(workspaceId, actor, draftId, "editing", "in_review", "draft.submitted", { + from: "editing", + to: "in_review", + }); + } + + /** + * `review → approved` with the E1 chatgov gate (chatgov-design.md §4). + * + * E1 evaluates the destination's compiled posting_policy_snapshots PER + * DESTINATION (fan-out law D8): immediate fires (no gate), admin_only fires + * only for an Owner/Admin approver, quorum blocks with INSUFFICIENT_QUORUM + * + PENDING_POLL until the linked chat poll passes (P2 owns the poll state + * machine). Workspaces with no live posting_rules keep today's behavior + * exactly — the gate is a no-op for them; once rules exist, a destination + * with no compiled snapshot fails closed with OUTSIDE_POLICY (P0 hand-off). + * + * Fired decisions commit in the SAME transaction as the approval (the hook + * runs under the draft row lock, before the transition UPDATE). A blocked + * destination aborts that transaction (Conflict) and its verdicts are then + * recorded after the rollback — the blocked queue (gate_decisions) is the + * observable surface for them. + */ + async approve(workspaceId: string, actor: Actor, draftId: string): Promise { + const gate = this.dependencies.approvalGate ?? null; + try { + return await this.transition(workspaceId, actor, draftId, "in_review", "approved", "draft.approved", { + from: "in_review", + to: "approved", + }, actor.userId, gate === null ? undefined : async (client: PoolClient, draft: DraftRecord) => { + const evaluation = await gate.evaluate(client, workspaceId, draftId, draft.post, actor.userId); + if (evaluation.blocked.length > 0) throw new ApprovalGateBlockedError(evaluation.blocked); + await gate.recordFiredInTxn(client, workspaceId, draftId, actor.userId, evaluation.verdicts); + }); + } catch (error) { + if (error instanceof ApprovalGateBlockedError) { + // The approval rolled back; record the blocked verdicts with no draft + // lock held (gate_decisions' FK takes a KEY SHARE on the draft row, + // which would deadlock against the still-held FOR UPDATE). + await gate?.recordBlocked(workspaceId, draftId, actor.userId, error.verdicts).catch(() => undefined); + } + throw error; + } + } + + async reject(workspaceId: string, actor: Actor, draftId: string, reason?: string): Promise { + return this.transition(workspaceId, actor, draftId, "in_review", "rejected", "draft.rejected", { + from: "in_review", + to: "rejected", + ...(reason === undefined ? {} : { reason }), + }, actor.userId); + } + + async schedule( + workspaceId: string, + actor: Actor, + draftId: string, + connectionId: string, + publishAt: Date, + requiredPermission: "publication.schedule" | "publication.publish" = "publication.schedule", + expectedPrevious?: { state: "failed" | "needs_reauthorization"; generation: string }, + ): Promise { + if (!Number.isFinite(publishAt.getTime())) throw new BadRequestException("publishAt must be a valid timestamp"); + + await this.dependencies.assertReady(); + const scheduled = await this.dependencies.tenantTransaction(workspaceId, async (client) => { + await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [workspaceId]); + // FK checks bypass RLS, so a foreign tenant's connection id would satisfy + // the constraint; this RLS-scoped read is the actual tenancy check. + const connection = await client.query("SELECT provider FROM connections WHERE id = $1 AND disconnected_at IS NULL", [connectionId]); + if (connection.rowCount !== 1) throw new NotFoundException(`connection ${connectionId} not found`); + if (!isPublishingProvider(String(connection.rows[0]?.provider))) throw new BadRequestException("provider is not enabled for publishing"); + if (!(await userCanUseConnection(client, workspaceId, actor.userId, connectionId))) { + throw new ForbiddenException("connection is not shared with any of your groups"); + } + + let attempt; + try { + attempt = await this.dependencies.repository.scheduleAttempt(client, { + workspaceId, draftId, connectionId, scheduledByUserId: actor.userId, requiredPermission, publishAt, expectedPrevious, + }); + } catch (error) { + if (error instanceof PublishingStateError) throw new ConflictException({ code: error.code, message: error.message }); + throw error; + } + const result = await client.query("SELECT * FROM scheduled_publications WHERE id = $1", [attempt.scheduleId]); + const scheduled = scheduledFromRow(requiredRow(result.rows[0], "scheduled publication")); + await this.audit(client, workspaceId, actor, "publication.scheduled", draftId, { + connectionId, + publishAt: publishAt.toISOString(), + }); + return scheduled; + }); + await this.dependencies.workflowStarter + .startScheduledPublish({ workspaceId, scheduleId: scheduled.id, generation: scheduled.generation, idempotencyKey: scheduled.idempotencyKey }, publishAt) + .catch(() => undefined); + return scheduled; + } + + async cancelSchedule(workspaceId: string, actor: Actor, draftId: string): Promise { + await this.dependencies.assertReady(); + const draft = await this.dependencies.tenantTransaction(workspaceId, async (client) => { + await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [workspaceId]); + try { await this.dependencies.repository.cancelAttempt(client, workspaceId, draftId); } + catch (error) { + if (error instanceof PublishingStateError) throw new ConflictException({ code: error.code, message: error.message }); + throw error; + } + const changed = await client.query( + "SELECT * FROM drafts WHERE id = $1 AND state IN ('approved', 'scheduled') FOR UPDATE", + [draftId], + ); + if (changed.rowCount !== 1) await this.throwTransitionError(client, draftId, "approved", "editing"); + const oldState = String(changed.rows[0]?.state); + // Legacy pending schedules were marked scheduled before this claim gate + // existed. Reopen them for review instead of treating them as approved. + const reopened = oldState === "scheduled" + ? await client.query( + "UPDATE drafts SET state = 'editing', reviewer_user_id = NULL, updated_at = now() WHERE id = $1 AND state = 'scheduled' RETURNING *", + [draftId], + ) + : changed; + if (reopened.rowCount !== 1) throw new BadRequestException(`cannot reopen draft ${draftId}`); + const draft = draftFromRow(requiredRow(reopened.rows[0], "cancelled draft")); + await this.audit(client, workspaceId, actor, "publication.schedule_cancelled", draftId, { + from: oldState, + to: draft.state, + }); + return draft; + }); + return draft; + } + + private async transition( + workspaceId: string, + actor: Actor, + draftId: string, + from: DraftState, + to: DraftState, + action: string, + diff: Record, + reviewerUserId?: string, + /** E1 seam: runs inside the transition's transaction, under the draft row + * lock, before the UPDATE — may abort the whole transition by throwing. */ + afterLock?: (client: PoolClient, draft: DraftRecord) => Promise, + ): Promise { + await this.dependencies.assertReady(); + return this.dependencies.tenantTransaction(workspaceId, async (client) => { + await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [workspaceId]); + const current = await client.query("SELECT * FROM drafts WHERE id = $1 FOR UPDATE", [draftId]); + if (current.rowCount !== 1) throw new NotFoundException("draft not found"); + if (afterLock !== undefined) await afterLock(client, draftFromRow(current.rows[0])); + const hash = canonicalPostHash(current.rows[0].post as DraftPost); + const result = await client.query( + `UPDATE drafts + SET state = $2, reviewer_user_id = COALESCE($3, reviewer_user_id), updated_at = now(), + approved_revision = CASE WHEN $2 = 'approved' THEN revision ELSE NULL END, + approved_hash = CASE WHEN $2 = 'approved' THEN $5 ELSE NULL END, + approved_at = CASE WHEN $2 = 'approved' THEN now() ELSE NULL END + WHERE id = $1 AND state = $4 RETURNING *`, + [draftId, to, reviewerUserId ?? null, from, hash], + ); + if (result.rowCount !== 1) await this.throwTransitionError(client, draftId, from, to); + const draft = draftFromRow(requiredRow(result.rows[0], "transitioned draft")); + if (to === "in_review") await autoOpenDraftPoll(client, workspaceId, draftId, actor, draft.post); + await this.audit(client, workspaceId, actor, action, draftId, diff); + return draft; + }); + } + + private async throwTransitionError( + client: PoolClient, + draftId: string, + expected: DraftState, + target: DraftState, + ): Promise { + const current = await client.query("SELECT state FROM drafts WHERE id = $1", [draftId]); + if (current.rowCount === 0) throw new NotFoundException(`draft ${draftId} not found`); + throw new BadRequestException( + `illegal draft transition ${String(current.rows[0]?.state)} -> ${target}; expected ${expected}`, + ); + } + + private async audit( + client: PoolClient, + workspaceId: string, + actor: Actor, + action: string, + targetId: string, + diff: Record, + ): Promise { + await client.query("SELECT audit_append($1, $2, $3, $4, 'draft', $5, $6)", [ + workspaceId, actor.userId, actor.role, action, targetId, scrub(diff), + ]); + } +} + +function requiredRow(row: unknown, description: string): Record { + if (typeof row !== "object" || row === null) throw new Error(`database did not return ${description}`); + return row as Record; +} + +function draftFromRow(row: Record): DraftRecord { + return { + id: String(row.id), + workspaceId: String(row.workspace_id), + authorUserId: String(row.author_user_id), + post: row.post as DraftPost, + state: row.state as DraftState, + ...(row.reviewer_user_id === null || row.reviewer_user_id === undefined + ? {} + : { reviewerUserId: String(row.reviewer_user_id) }), + createdAt: new Date(String(row.created_at)), + updatedAt: new Date(String(row.updated_at)), + }; +} + +function scheduledFromRow(row: Record): ScheduledPublicationRecord { + return { + id: String(row.id), + workspaceId: String(row.workspace_id), + draftId: String(row.draft_id), + connectionId: String(row.connection_id), + publishAt: new Date(String(row.publish_at)), + idempotencyKey: String(row.idempotency_key), + generation: String(row.generation), + state: row.state as ScheduledPublicationRecord["state"], + }; +} diff --git a/apps/api/src/rbac/rbac.guard.ts b/apps/api/src/rbac/rbac.guard.ts new file mode 100644 index 0000000..6178f3b --- /dev/null +++ b/apps/api/src/rbac/rbac.guard.ts @@ -0,0 +1,83 @@ +import { BadRequestException, CanActivate, ExecutionContext, Inject, Injectable, SetMetadata, UnauthorizedException, ForbiddenException } from "@nestjs/common"; +import { Reflector } from "@nestjs/core"; +import { verifySession, epochValid, type SessionPayload } from "../auth/session.js"; +import { pool, withTenant } from "../db/client.js"; +import { userCanUseConnection } from "../groups/groups.service.js"; +import { isUuid } from "../http/validation.js"; + +export const PERMISSION_KEY = "required_permission"; +export const GROUP_ACCESS_KEY = "required_group_access"; + +export const RequirePermission = (permission: string): MethodDecorator => + SetMetadata(PERMISSION_KEY, permission) as MethodDecorator; + +/** + * Additionally gate a route on group scope for its `:connectionId` param: + * Owner/Admin pass; anyone else must belong to a group the connection is + * assigned to. Fails closed if the route has no `:connectionId`. + */ +export const RequireGroupAccess = (source: "params" | "body" = "params"): MethodDecorator => + SetMetadata(GROUP_ACCESS_KEY, source) as MethodDecorator; + +export interface AuthenticatedRequest { + session?: SessionPayload; +} + +/** + * RBAC guard: session signature -> workspace membership -> permission set. + * Epoch-fenced (finding #2): revocation bumps authz_epoch, session fails closed. + */ +@Injectable() +export class RbacGuard implements CanActivate { + constructor(@Inject(Reflector) private readonly reflector: Reflector) {} + + async canActivate(context: ExecutionContext): Promise { + const required = this.reflector.get(PERMISSION_KEY, context.getHandler()); + const http = context.switchToHttp(); + const req = http.getRequest<{ headers: Record; params?: Record; body?: unknown } & AuthenticatedRequest>(); + const auth = req.headers["authorization"]; + if (auth === undefined || !auth.startsWith("Bearer ")) throw new UnauthorizedException(); + const session = verifySession(auth.slice("Bearer ".length)); + if (session === null) throw new UnauthorizedException(); + + const ws = await pool.query("SELECT authz_epoch FROM workspaces WHERE id = $1", [session.workspaceId]); + // authz_epoch is bigint: node-postgres returns it as a string, so normalise + // before the strict comparison or every session reads as stale. + const rawEpoch = ws.rows[0]?.authz_epoch as string | number | undefined; + if (rawEpoch === undefined || !epochValid(session, Number(rawEpoch))) { + throw new UnauthorizedException("authorization epoch stale — re-authenticate"); + } + + if (required !== undefined) { + // memberships is FORCE RLS: the lookup must run inside the session's + // tenant context, otherwise sp_app sees zero rows and every check fails. + const allowed = await withTenant(session.workspaceId, (client) => + client.query( + `SELECT 1 FROM role_permissions rp + JOIN memberships m ON m.role = rp.role AND m.workspace_id = $1 AND m.user_id = $2 AND m.status = 'active' + WHERE rp.permission = $3`, + [session.workspaceId, session.userId, required], + ), + ); + if (allowed.rowCount === 0) throw new ForbiddenException(`missing permission: ${required}`); + } + + const groupSource = this.reflector.get<"params" | "body" | undefined>(GROUP_ACCESS_KEY, context.getHandler()); + if (groupSource !== undefined) { + const body = req.body; + const connectionId = groupSource === "body" + ? (typeof body === "object" && body !== null && "connectionId" in body ? body.connectionId : undefined) + : req.params?.connectionId; + if (connectionId === undefined || connectionId === "") { + throw new ForbiddenException("group access check requires a connectionId"); + } + if (typeof connectionId !== "string" || !isUuid(connectionId)) throw new BadRequestException("connectionId must be a UUID"); + const permitted = await withTenant(session.workspaceId, (client) => + userCanUseConnection(client, session.workspaceId, session.userId, connectionId), + ); + if (!permitted) throw new ForbiddenException("connection is not shared with any of your groups"); + } + req.session = session; + return true; + } +} diff --git a/apps/api/src/realtime/gateway.ts b/apps/api/src/realtime/gateway.ts new file mode 100644 index 0000000..de48d1b --- /dev/null +++ b/apps/api/src/realtime/gateway.ts @@ -0,0 +1,25 @@ +import type { PoolClient } from "pg"; +import { isUuid } from "../http/validation.js"; +import type { SessionPayload } from "../auth/session.js"; +export type SubscriptionDecision = { allow: true; workspaceId: string } | { allow: false; reason: string }; + +/** Takes an idle, checked-out client. Owns a tenant transaction so FORCE RLS + * membership checks work, and SET LOCAL cannot leak to the next pool borrower. */ +export async function authorizeSubscription(client: PoolClient, args: { token: string; workspaceId: string }, + verify: (token: string) => SessionPayload | null): Promise { + const session = verify(args.token); + if (!session || !isUuid(session.userId) || !isUuid(args.workspaceId) || !Number.isSafeInteger(session.authzEpoch)) return { allow: false, reason: "invalid session" }; + if (session.workspaceId !== args.workspaceId) return { allow: false, reason: "cross-workspace subscription denied" }; + await client.query("BEGIN"); + try { + await client.query("SELECT set_config('app.workspace_id', $1, true)", [args.workspaceId]); + const epoch = await client.query<{ authz_epoch: string | number }>("SELECT authz_epoch FROM workspaces WHERE id = $1", [args.workspaceId]); + const current = Number(epoch.rows[0]?.authz_epoch); + if (!Number.isSafeInteger(current) || session.authzEpoch !== current) return { allow: false, reason: "stale authorization epoch" }; + const permission = await client.query( + `SELECT 1 FROM role_permissions rp JOIN memberships m ON m.role=rp.role + WHERE m.workspace_id=$1 AND m.user_id=$2 AND m.status='active' AND rp.permission='stream.view'`, [args.workspaceId, session.userId]); + if (!permission.rowCount) return { allow: false, reason: "missing stream.view permission" }; + return { allow: true, workspaceId: args.workspaceId }; + } finally { await client.query("ROLLBACK"); } +} diff --git a/apps/api/src/realtime/realtime.module.ts b/apps/api/src/realtime/realtime.module.ts new file mode 100644 index 0000000..efc0cac --- /dev/null +++ b/apps/api/src/realtime/realtime.module.ts @@ -0,0 +1,27 @@ +import { Inject, Injectable, Module, OnApplicationBootstrap, OnModuleDestroy } from "@nestjs/common"; +import { HttpAdapterHost } from "@nestjs/core"; +import type { Server, IncomingMessage } from "node:http"; +import type { Duplex } from "node:stream"; +import { MediaControlModule } from "../media-control/media-control.module.js"; +import { RealtimeService } from "./realtime.service.js"; +import { RealtimeSocket } from "./websocket.js"; +@Injectable() +export class RealtimeTransport implements OnApplicationBootstrap, OnModuleDestroy { + private server: Server | null = null; + private readonly upgrade = (request: IncomingMessage, socket: Duplex, head: Buffer): void => { + if (request.url?.split("?")[0] !== "/realtime") return; + const origin = request.headers.origin; + const allowed = process.env.PUBLIC_ORIGIN ?? `http://${request.headers.host}`; + if (origin && origin !== allowed) { socket.end("HTTP/1.1 403 Forbidden\r\nConnection: close\r\n\r\n"); return; } + const peer = RealtimeSocket.accept(request, socket, head); if (!peer) return; + const handlers = this.realtime.attach(peer); peer.onMessage = handlers.message; peer.onClose = handlers.close; + }; + constructor(@Inject(HttpAdapterHost) private readonly adapter: HttpAdapterHost, @Inject(RealtimeService) private readonly realtime: RealtimeService) {} + onApplicationBootstrap(): void { + this.server = this.adapter.httpAdapter.getHttpServer() as Server; + this.server.on("upgrade", this.upgrade); + } + onModuleDestroy(): void { this.server?.off("upgrade", this.upgrade); } +} +@Module({ imports: [MediaControlModule], providers: [RealtimeService, RealtimeTransport], exports: [RealtimeService] }) +export class RealtimeModule {} diff --git a/apps/api/src/realtime/realtime.service.ts b/apps/api/src/realtime/realtime.service.ts new file mode 100644 index 0000000..c8b5b5b --- /dev/null +++ b/apps/api/src/realtime/realtime.service.ts @@ -0,0 +1,342 @@ +import { Inject, Injectable, OnModuleDestroy, Optional } from "@nestjs/common"; +import { pool, withTenant } from "../db/client.js"; +import { verifySession } from "../auth/session.js"; +import { isUuid } from "../http/validation.js"; +import { MediaControlService, type ChannelStatus } from "../media-control/media-control.service.js"; +import { authorizeChat, tokenWorkspace } from "../live-chat/auth.js"; +import { chatTopic, chatWindowKey, CHAT_WINDOW_SCRIPT } from "../live-chat/bridge.js"; +import { RespControlRedis } from "../media-control/redis.js"; +import type { LiveChatEventType } from "@tactical-media-broadcast-hub/core"; +import { authorizeSubscription } from "./gateway.js"; +import { RespRedisSubscriptions, type RedisSubscriptions } from "./subscribe.js"; +export const REALTIME_DEPENDENCIES = Symbol("REALTIME_DEPENDENCIES"); +export interface RealtimePeer { send(value: unknown): boolean; close(code?: number): void } +interface Dependencies { + subscriptions: RedisSubscriptions; + workspaceFromToken: (token: string) => string | null; + authorizeChat: (token: string, workspaceId: string, permission: string) => Promise; + chatLatestSeq: (workspaceId: string, liveId: string) => Promise; + chatWindow: (workspaceId: string, liveId: string) => Promise[]>; + authorize: (token: string, workspaceId: string) => Promise; + channelExists: (workspaceId: string, channelId: string) => Promise; + status: (workspaceId: string) => Promise<{ channels: ChannelStatus[] }>; +} +interface StateEvent { type: "state"; workspaceId: string; channelId: string; kind: "desired" | "observed"; seq: number; data: unknown } +/** S5: chat frames ride the same topic ring and seq counter (§3.3). */ +export interface ChatEvent { type: "chat"; workspaceId: string; channelId: string; seq: number; frame: Record } +type TopicEvent = StateEvent | ChatEvent; +interface Topic { + ws: string; ch: string; seq: number; ring: TopicEvent[]; listeners: Set; + unsubscribe: (() => void) | null; timer: ReturnType | null; idle: ReturnType | null; + polling: boolean; latest: Record; +} +interface ChatTopic { seq: number; pending: Map>; gapTimer: ReturnType | null; ws: string; liveId: string; ring: Record[]; listeners: Map; unsubscribe: (() => void) | null } +interface Viewer { canonical: boolean; chatIds: Map; channelIds: Map; token: string | null; ws: string | null; topics: Set; busy: Promise; authDeadline: number; queued: number; timer: ReturnType; closed: boolean } + +@Injectable() +export class RealtimeService implements OnModuleDestroy { + private readonly deps: Dependencies; + private readonly chatRedis = new RespControlRedis(process.env.REDIS_URL ?? "redis://localhost:6379"); + private readonly chatTopics = new Map(); + private readonly chatStarts = new Map>(); + private readonly topics = new Map(); + private readonly starts = new Map>(); + private readonly viewers = new Map(); + constructor(@Inject(MediaControlService) control: MediaControlService, + @Optional() @Inject(REALTIME_DEPENDENCIES) dependencies?: Partial) { + this.deps = { + workspaceFromToken: dependencies?.workspaceFromToken ?? tokenWorkspace, + authorizeChat: dependencies?.authorizeChat ?? (async (token, workspaceId, permission) => { + const client = await pool.connect(); + try { return Boolean(await authorizeChat(client, token, workspaceId, permission)); } finally { client.release(); } + }), + chatLatestSeq: dependencies?.chatLatestSeq ?? (async (workspaceId, liveId) => { + const result = await withTenant(workspaceId, client => client.query<{ seq: string | number }>("SELECT seq FROM live_chat_seq WHERE workspace_id=$1 AND channel_id=$2", [workspaceId, liveId])); + return Number(result.rows[0]?.seq ?? 0); + }), + chatWindow: dependencies?.chatWindow ?? (async (workspaceId, liveId) => { + const rows = await this.chatRedis.eval(CHAT_WINDOW_SCRIPT, [chatWindowKey(workspaceId, liveId)], []); + return Array.isArray(rows) ? rows.filter((row): row is string => typeof row === "string").map(row => JSON.parse(row) as Record) : []; + }), + subscriptions: dependencies?.subscriptions ?? new RespRedisSubscriptions(process.env.REDIS_URL ?? "redis://localhost:6379", () => { + for (const [peer, viewer] of this.viewers) peer.send(viewer.canonical ? { op: "resync", topic: "livechat" } : { type: "RESYNC_REQUIRED" }); + }), + authorize: dependencies?.authorize ?? (async (token, workspaceId) => { + const client = await pool.connect(); + try { return (await authorizeSubscription(client, { token, workspaceId }, verifySession)).allow; } + finally { client.release(); } + }), + channelExists: dependencies?.channelExists ?? (async (ws, ch) => { + const result = await withTenant(ws, client => client.query("SELECT id FROM virtual_channels WHERE workspace_id=$1 AND id=$2", [ws, ch])); + return Boolean(result.rowCount); + }), + status: dependencies?.status ?? (ws => control.getStatus(ws)), + }; + } + attach(peer: RealtimePeer): { message: (value: unknown) => void; close: () => void } { + const viewer: Viewer = { canonical: false, chatIds: new Map(), channelIds: new Map(), token: null, ws: null, topics: new Set(), busy: Promise.resolve(), authDeadline: Date.now() + 3000, queued: 0, + timer: setInterval(() => { void this.revalidate(peer); }, 1000), closed: false }; + viewer.timer.unref(); this.viewers.set(peer, viewer); + return { + message: value => { + // Bound upstream work too; a client cannot queue arbitrary authorization queries. + if (viewer.closed) return; + if (++viewer.queued > 32) { peer.close(1008); this.detach(peer); return; } + if (typeof value !== "object" || !value) { peer.close(1008); return; } + const message = value as Record; + viewer.busy = viewer.busy.then(() => this.message(peer, viewer, message)).finally(() => { viewer.queued -= 1; }).catch(() => { peer.close(1011); this.detach(peer); }); + }, close: () => this.detach(peer), + }; + } + private async check(token: string, ws: string, permission?: string): Promise { + let timer: ReturnType | undefined; + try { return await Promise.race([permission ? this.deps.authorizeChat(token, ws, permission) : this.deps.authorize(token, ws), new Promise(resolve => { timer = setTimeout(() => resolve(false), 1000); })]); } + catch { return false; } + finally { if (timer) clearTimeout(timer); } + } + private async revalidate(peer: RealtimePeer): Promise { + const viewer = this.viewers.get(peer); + if (!viewer || viewer.closed) return; + if (Date.now() >= viewer.authDeadline) { peer.close(1008); this.detach(peer); return; } + if (!viewer.token || !viewer.ws) return; + if (!await this.check(viewer.token, viewer.ws, viewer.canonical ? (viewer.chatIds.size ? "livechat.view" : "stream.view") : undefined) || + (viewer.channelIds.size > 0 && !await this.check(viewer.token, viewer.ws, "stream.view"))) { peer.close(1008); this.detach(peer); } + else viewer.authDeadline = Date.now() + 3000; + } + private async message(peer: RealtimePeer, viewer: Viewer, msg: Record): Promise { + if (viewer.closed) return; + if (msg.op !== undefined || viewer.canonical) { await this.canonicalMessage(peer, viewer, msg); return; } + if (viewer.token === null) { + if (msg.type !== "auth" || typeof msg.token !== "string" || msg.token.length > 8192 || typeof msg.workspaceId !== "string" || !await this.check(msg.token, msg.workspaceId)) { peer.close(1008); this.detach(peer); return; } + if (viewer.closed) return; + viewer.token = msg.token; viewer.ws = msg.workspaceId; viewer.authDeadline = Date.now() + 3000; + peer.send({ type: "auth_ack", workspaceId: viewer.ws }); return; + } + if (msg.type === "ack") return; + if (typeof msg.channelId !== "string" || !isUuid(msg.channelId) || !viewer.ws) { peer.close(1008); this.detach(peer); return; } + const key = `${viewer.ws}:${msg.channelId}`; + if (msg.type === "unsubscribe") { this.leave(peer, key); viewer.topics.delete(key); peer.send({ type: "unsubscribe_ack", channelId: msg.channelId }); return; } + if (msg.type !== "subscribe" || viewer.topics.size >= 32 || + (msg.lastSeq !== undefined && (!Number.isSafeInteger(msg.lastSeq) || Number(msg.lastSeq) < 0))) { peer.close(1008); this.detach(peer); return; } + // Recheck every subscription, including subscriptions on an already authenticated socket. + if (!await this.check(viewer.token, viewer.ws) || !await this.deps.channelExists(viewer.ws, msg.channelId)) { peer.send({ type: "subscribe_denied", channelId: msg.channelId }); return; } + if (viewer.closed) return; + const topic = await this.topic(viewer.ws, msg.channelId); + if (viewer.closed) { if (!topic.listeners.size) this.leave(peer, key); return; } + topic.listeners.add(peer); viewer.topics.add(key); viewer.authDeadline = Date.now() + 3000; + peer.send({ type: "subscribe_ack", channelId: msg.channelId, seq: topic.seq }); + if (msg.lastSeq !== undefined) { + const seq = Number(msg.lastSeq), oldest = topic.ring[0]?.seq ?? topic.seq + 1; + if (seq > topic.seq || seq < oldest - 1) peer.send({ type: "RESYNC_REQUIRED", channelId: msg.channelId }); + else for (const event of topic.ring) if (event.seq > seq) peer.send(event); + } else for (const kind of ["desired", "observed"] as const) { + const event = [...topic.ring].reverse().find((e): e is StateEvent => e.type === "state" && e.kind === kind); + if (event) peer.send(event); + } + } + private async canonicalMessage(peer: RealtimePeer, viewer: Viewer, msg: Record): Promise { + if (!viewer.token) { + const ws = typeof msg.token === "string" ? this.deps.workspaceFromToken(msg.token) : null; + if (msg.op !== "auth" || typeof msg.token !== "string" || msg.token.length > 8192 || !ws || + !await this.check(msg.token, ws, "stream.view")) { + peer.send({ op: "close", reason: "unauthorized" }); peer.close(1008); this.detach(peer); return; + } + if (viewer.closed) return; + viewer.canonical = true; viewer.token = msg.token; viewer.ws = ws; viewer.authDeadline = Date.now() + 3000; + peer.send({ op: "ready" }); return; + } + if (msg.op === "ping") { peer.send({ op: "pong" }); return; } + if (msg.op === "ack") return; + if (msg.op === "unsubscribe" && typeof msg.id === "string") { + const liveId = viewer.chatIds.get(msg.id); + if (liveId) { this.leaveChat(peer, viewer.ws ?? "", liveId); viewer.chatIds.delete(msg.id); } + const key = viewer.channelIds.get(msg.id); + if (key) { this.leave(peer, key); viewer.topics.delete(key); viewer.channelIds.delete(msg.id); } + return; + } + const liveId = msg.topic === "livechat" ? msg.liveId : msg.channelId; + if (msg.op !== "subscribe" || typeof msg.id !== "string" || msg.id.length === 0 || msg.id.length > 128 || + (msg.topic !== "livechat" && msg.topic !== "channel") || typeof liveId !== "string" || !isUuid(liveId) || !viewer.ws || + viewer.chatIds.size + viewer.topics.size >= 32 || viewer.chatIds.has(msg.id) || viewer.channelIds.has(msg.id) || + (msg.topic === "livechat" && [...viewer.chatIds.values()].includes(liveId)) || + (msg.topic === "channel" && viewer.topics.has(`${viewer.ws}:${liveId}`)) || + (msg.lastSeq !== undefined && (!Number.isSafeInteger(msg.lastSeq) || Number(msg.lastSeq) < 0))) { + peer.send({ op: "nack", id: msg.id, reason: "invalid subscription" }); return; + } + if (!await this.check(viewer.token, viewer.ws, msg.topic === "livechat" ? "livechat.view" : "stream.view") || + !await this.deps.channelExists(viewer.ws, liveId)) { + peer.send({ op: "nack", id: msg.id, reason: "subscription denied" }); return; + } + if (viewer.closed) return; + if (msg.topic === "channel") { + const topic = await this.topic(viewer.ws, liveId); + if (viewer.closed) return; + const key = `${viewer.ws}:${liveId}`; + topic.listeners.add(peer); viewer.topics.add(key); viewer.channelIds.set(msg.id, key); + peer.send({ op: "ack", id: msg.id, fromSeq: topic.seq }); + if (msg.lastSeq !== undefined) { + const seq = Number(msg.lastSeq), oldest = topic.ring[0]?.seq ?? topic.seq + 1; + if (seq > topic.seq || seq < oldest - 1) peer.send({ op: "resync", topic: "channel" }); + else for (const event of topic.ring) if (event.seq > seq) this.sendState(peer, event); + } else for (const event of topic.ring.slice(-2)) this.sendState(peer, event); + return; + } + const topic = await this.chatTopicFor(viewer.ws, liveId); + if (viewer.closed) { if (!topic.listeners.size) { topic.unsubscribe?.(); this.chatTopics.delete(chatTopic(topic.ws, liveId)); } return; } + topic.listeners.set(peer, msg.id); viewer.chatIds.set(msg.id, liveId); viewer.authDeadline = Date.now() + 3000; + const highest = topic.seq, oldest = Number(topic.ring[0]?.seq ?? highest + 1); + peer.send({ op: "ack", id: msg.id, fromSeq: highest }); + if (msg.lastSeq !== undefined) { + const seq = Number(msg.lastSeq); + const replay = topic.ring.filter(frame => Number(frame.seq) > seq); + const complete = replay.every((frame, i) => Number(frame.seq) === (i === 0 ? seq : Number(replay[i - 1]?.seq)) + 1); + if (seq > highest || seq < oldest - 1 || !complete) peer.send({ op: "resync", topic: "livechat" }); + else for (const frame of topic.ring) if (Number(frame.seq) > seq) this.sendChat(peer, frame); + } + } + private async chatTopicFor(ws: string, liveId: string): Promise { + const key = chatTopic(ws, liveId); + const pending = this.chatStarts.get(key); if (pending) return pending; + const existing = this.chatTopics.get(key); if (existing) return existing; + if (this.chatTopics.size + this.chatStarts.size >= 1024) throw new Error("chat topic capacity exceeded"); + const topic: ChatTopic = { seq: 0, pending: new Map(), gapTimer: null, ws, liveId, ring: [], listeners: new Map(), unsubscribe: null }; + const start = (async () => { + try { + topic.unsubscribe = await this.deps.subscriptions.subscribe(key, raw => { + try { this.receiveChat(topic, JSON.parse(raw) as Record); } catch { /* Ignore malformed frames. */ } + }); + const window = await this.deps.chatWindow(ws, liveId); + topic.ring = [...window, ...topic.ring].filter((frame, i, frames) => frames.findIndex(other => other.seq === frame.seq) === i) + .sort((a, b) => Number(a.seq) - Number(b.seq)).slice(-500); + topic.seq = topic.ring.length ? Number(topic.ring.at(-1)?.seq ?? 0) : await this.deps.chatLatestSeq(ws, liveId); + this.chatTopics.set(key, topic); return topic; + } catch (error) { topic.unsubscribe?.(); throw error; } + finally { this.chatStarts.delete(key); } + })(); + this.chatStarts.set(key, start); return start; + } + private receiveChat(topic: ChatTopic, frame: Record): void { + const seq = Number(frame.seq); + if (frame.kind !== "chat" || frame.liveStreamId !== topic.liveId || !Number.isSafeInteger(frame.seq) || seq <= 0 || + topic.ring.some(existing => existing.seq === frame.seq)) return; + // During the subscribe/snapshot handshake, merge notification races into the snapshot. + if (!topic.listeners.size) { topic.ring.push(frame); topic.ring = topic.ring.slice(-500); return; } + if (seq <= topic.seq) return; + topic.pending.set(seq, frame); + if (topic.pending.size > 500) { + for (const peer of topic.listeners.keys()) peer.send({ op: "resync", topic: "livechat" }); + topic.pending.clear(); return; + } + for (let next = topic.pending.get(topic.seq + 1); next; next = topic.pending.get(topic.seq + 1)) { + topic.pending.delete(++topic.seq); topic.ring.push(next); topic.ring = topic.ring.slice(-500); + for (const peer of topic.listeners.keys()) this.sendChat(peer, next); + } + if (topic.pending.size > 0 && !topic.gapTimer) { + topic.gapTimer = setTimeout(() => { + topic.gapTimer = null; + for (const peer of topic.listeners.keys()) peer.send({ op: "resync", topic: "livechat" }); + }, 1000); topic.gapTimer.unref(); + } else if (topic.pending.size === 0 && topic.gapTimer) { clearTimeout(topic.gapTimer); topic.gapTimer = null; } + } + private sendChat(peer: RealtimePeer, frame: Record): void { + const type: LiveChatEventType = frame.message !== undefined ? "chat.message" : frame.delivery !== undefined ? "chat.delivery" : frame.action !== undefined ? "chat.moderated" : "chat.gap"; + const viewer = this.viewers.get(peer); + if (!viewer || Date.now() >= viewer.authDeadline || !peer.send({ op: "msg", topic: "livechat", seq: frame.seq, type, payload: frame })) { + peer.send({ op: "resync", topic: "livechat" }); peer.close(1008); this.detach(peer); + } + } + private sendState(peer: RealtimePeer, event: TopicEvent): boolean { + return this.viewers.get(peer)?.canonical + ? peer.send({ op: "msg", topic: "channel", seq: event.seq, type: "stream.state", payload: event }) : peer.send(event); + } + private leaveChat(peer: RealtimePeer, ws: string, liveId: string): void { + const key = chatTopic(ws, liveId), topic = this.chatTopics.get(key); if (!topic) return; + topic.listeners.delete(peer); + if (!topic.listeners.size) { topic.unsubscribe?.(); if (topic.gapTimer) clearTimeout(topic.gapTimer); this.chatTopics.delete(key); } + } + private async topic(ws: string, ch: string): Promise { + const key = `${ws}:${ch}`; + let topic = this.topics.get(key); + if (topic) { + if (topic.idle) { clearTimeout(topic.idle); topic.idle = null; } + if (!topic.timer) await this.start(topic); + return topic; + } + if (this.topics.size >= 1024) throw new Error("realtime topic capacity exceeded"); + topic = { ws, ch, seq: 0, ring: [], listeners: new Set(), unsubscribe: null, timer: null, idle: null, polling: false, latest: {} }; + this.topics.set(key, topic); + try { await this.start(topic); } catch (error) { this.topics.delete(key); throw error; } + return topic; + } + private start(topic: Topic): Promise { + const key = `${topic.ws}:${topic.ch}`; + const pending = this.starts.get(key); + if (pending) return pending; + const starting = this.startTopic(topic).finally(() => this.starts.delete(key)); + this.starts.set(key, starting); return starting; + } + private async startTopic(topic: Topic): Promise { + topic.unsubscribe = await this.deps.subscriptions.subscribe(`state:${topic.ws}:${topic.ch}:desired`, () => { void this.poll(topic); }); + topic.timer = setInterval(() => { void this.poll(topic); }, 100); topic.timer.unref(); + await this.poll(topic); + } + private async poll(topic: Topic): Promise { + if (topic.polling) return; + topic.polling = true; + try { + const status = (await this.deps.status(topic.ws)).channels.find(ch => ch.channelId === topic.ch); + if (!status) return; + this.event(topic, "desired", status.desired); + this.event(topic, "observed", status.observed); + } catch { /* Expiring auth leases evict viewers if an outage also affects auth. */ } + finally { topic.polling = false; } + } + private event(topic: Topic, kind: "desired" | "observed", data: unknown): void { + const serialized = JSON.stringify(data); + if (topic.latest[kind] === serialized) return; + topic.latest[kind] = serialized; + const event: StateEvent = { type: "state", workspaceId: topic.ws, channelId: topic.ch, kind, seq: ++topic.seq, data }; + topic.ring.push(event); if (topic.ring.length > 256) topic.ring.shift(); + for (const peer of topic.listeners) { + const viewer = this.viewers.get(peer); + if (!viewer || Date.now() >= viewer.authDeadline || !this.sendState(peer, event)) { peer.close(1008); this.detach(peer); } + } + } + /** Fan a chat frame out to a channel's subscribers and into its resume ring. + * No topic yet (nobody is watching / hub has no channel row): dropped — chat + * history is the hub's store, not the gateway's. */ + publishChat(workspaceId: string, channelId: string, frame: Record): void { + const topic = this.topics.get(`${workspaceId}:${channelId}`); + if (!topic) return; + const event: ChatEvent = { type: "chat", workspaceId, channelId, seq: ++topic.seq, frame: { ...frame } }; + topic.ring.push(event); if (topic.ring.length > 256) topic.ring.shift(); + for (const peer of topic.listeners) { + const viewer = this.viewers.get(peer); + if (!viewer || Date.now() >= viewer.authDeadline || !this.sendState(peer, event)) { peer.close(1008); this.detach(peer); } + } + } + private leave(peer: RealtimePeer, key: string): void { + const topic = this.topics.get(key); if (!topic) return; + topic.listeners.delete(peer); + if (topic.listeners.size === 0) { + if (topic.idle) clearTimeout(topic.idle); + topic.unsubscribe?.(); topic.unsubscribe = null; + if (topic.timer) clearInterval(topic.timer); topic.timer = null; + topic.idle = setTimeout(() => this.topics.delete(key), 60_000); topic.idle.unref(); + } + } + private detach(peer: RealtimePeer): void { + const viewer = this.viewers.get(peer); if (!viewer) return; + viewer.closed = true; clearInterval(viewer.timer); + for (const liveId of viewer.chatIds.values()) this.leaveChat(peer, viewer.ws ?? "", liveId); + for (const key of viewer.topics) this.leave(peer, key); + this.viewers.delete(peer); + } + onModuleDestroy(): void { + for (const peer of this.viewers.keys()) { peer.close(1001); this.detach(peer); } + for (const topic of this.topics.values()) { if (topic.timer) clearInterval(topic.timer); if (topic.idle) clearTimeout(topic.idle); topic.unsubscribe?.(); } + this.topics.clear(); + for (const topic of this.chatTopics.values()) { topic.unsubscribe?.(); if (topic.gapTimer) clearTimeout(topic.gapTimer); } + this.chatTopics.clear(); this.chatRedis.close(); this.deps.subscriptions.close(); + } +} diff --git a/apps/api/src/realtime/realtime.test.ts b/apps/api/src/realtime/realtime.test.ts new file mode 100644 index 0000000..86f278e --- /dev/null +++ b/apps/api/src/realtime/realtime.test.ts @@ -0,0 +1,152 @@ +import { randomUUID } from "node:crypto"; +import { IncomingMessage } from "node:http"; +import { Socket } from "node:net"; +import { Duplex } from "node:stream"; +import type { PoolClient } from "pg"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { authorizeSubscription } from "./gateway.js"; +import { RealtimeService, type RealtimePeer } from "./realtime.service.js"; +import { RealtimeSocket } from "./websocket.js"; +import { MediaControlService, type ChannelStatus } from "../media-control/media-control.service.js"; +const WS = randomUUID(), USER = randomUUID(), CH = randomUUID(); +const session = { userId: USER, workspaceId: WS, authzEpoch: 7, role: "Operator", issuedAt: 1, expiresAt: 9999999999 }; +function tick(): Promise { return new Promise(resolve => setTimeout(resolve, 5)); } +function fixture(auth: () => Promise = async () => true): { service: RealtimeService; frames: unknown[]; peer: RealtimePeer; handlers: { message: (value: unknown) => void; close: () => void }; channel: ChannelStatus } { + const channel: ChannelStatus = { channelId: CH, workerId: "worker", assignmentEpoch: 1, desired: { desiredStatus: "STOPPED", desiredVersion: 1 }, observed: null, converged: false, workerAlive: true }; + const control = new MediaControlService({ signer: null, assignments: [] }); + const service = new RealtimeService(control, { authorize: auth, channelExists: async () => true, + status: async () => ({ channels: [channel] }), subscriptions: { subscribe: async () => () => undefined, close: () => undefined } }); + const frames: unknown[] = [], peer: RealtimePeer = { send: value => { frames.push(value); return true; }, close: vi.fn() }; + return { service, frames, peer, handlers: service.attach(peer), channel }; +} +afterEach(() => vi.useRealTimers()); +describe("realtime authorization port", () => { + it("normalizes bigint epochs and establishes transaction-local tenant context before RLS membership query", async () => { + const queries: string[] = []; + const client = { query: vi.fn(async (sql: string) => { + queries.push(sql); + if (sql.includes("FROM workspaces")) return { rows: [{ authz_epoch: "7" }], rowCount: 1 }; + if (sql.includes("FROM role_permissions")) { + expect(queries.some(q => q.includes("set_config"))).toBe(true); + return { rows: [{}], rowCount: 1 }; + } + return { rows: [], rowCount: 0 }; + }) } as unknown as PoolClient; + expect(await authorizeSubscription(client, { token: "valid", workspaceId: WS }, () => session)).toEqual({ allow: true, workspaceId: WS }); + expect(queries[0]).toBe("BEGIN"); expect(queries.at(-1)).toBe("ROLLBACK"); + }); + it.each(["8", "9007199254740993", "invalid"])("rejects stale/unsafe epoch %s", async epoch => { + const client = { query: async (sql: string) => ({ rows: sql.includes("FROM workspaces") ? [{ authz_epoch: epoch }] : [], rowCount: 0 }) } as unknown as PoolClient; + expect(await authorizeSubscription(client, { token: "valid", workspaceId: WS }, () => session)).toMatchObject({ allow: false }); + }); + it("rejects an inactive/non-member even when the token role claims Owner", async () => { + const client = { query: async (sql: string) => ({ rows: sql.includes("FROM workspaces") ? [{ authz_epoch: 7 }] : [], rowCount: 0 }) } as unknown as PoolClient; + expect(await authorizeSubscription(client, { token: "valid", workspaceId: WS }, () => ({ ...session, role: "Owner" }))).toMatchObject({ allow: false, reason: "missing stream.view permission" }); + }); + it("rejects cross-workspace subscriptions before database access", async () => { + const query = vi.fn(); const client = { query } as unknown as PoolClient; + expect(await authorizeSubscription(client, { token: "valid", workspaceId: randomUUID() }, () => session)).toMatchObject({ allow: false }); expect(query).not.toHaveBeenCalled(); + }); +}); +describe("per-channel realtime subscription lifecycle", () => { + it("authenticates in the first frame and checks each subscription before ack", async () => { + let allowed = true; const auth = vi.fn(async () => allowed); const f = fixture(auth); + f.handlers.message({ type: "auth", token: "operator", workspaceId: WS }); + f.handlers.message({ type: "subscribe", channelId: CH }); await tick(); + expect(f.frames).toContainEqual({ type: "subscribe_ack", channelId: CH, seq: 2 }); expect(auth).toHaveBeenCalledTimes(2); + expect(f.frames).toContainEqual(expect.objectContaining({ type: "state", kind: "observed", data: null })); + allowed = false; f.handlers.message({ type: "subscribe", channelId: randomUUID() }); await tick(); + expect(f.frames).toContainEqual(expect.objectContaining({ type: "subscribe_denied" })); f.service.onModuleDestroy(); + }); + it("epoch revocation closes and evicts subscriptions within five seconds", async () => { + vi.useFakeTimers(); let epoch = 7; + const client = { query: async (sql: string) => ({ rows: sql.includes("FROM workspaces") ? [{ authz_epoch: String(epoch) }] : [{}], rowCount: 1 }) } as unknown as PoolClient; + const f = fixture(async () => (await authorizeSubscription(client, { token: "operator", workspaceId: WS }, () => session)).allow); + f.handlers.message({ type: "auth", token: "operator", workspaceId: WS }); f.handlers.message({ type: "subscribe", channelId: CH }); + await vi.advanceTimersByTimeAsync(1); epoch += 1; + await vi.advanceTimersByTimeAsync(1100); expect(f.peer.close).toHaveBeenCalledWith(1008); + const count = f.frames.length; f.channel.desired = { desiredStatus: "STOPPED", desiredVersion: 2 }; + await vi.advanceTimersByTimeAsync(1000); expect(f.frames).toHaveLength(count); f.service.onModuleDestroy(); + }); + it("auth outages fail closed within five seconds", async () => { + vi.useFakeTimers(); let hang = false; const f = fixture(async () => hang ? new Promise(() => undefined) : true); + f.handlers.message({ type: "auth", token: "operator", workspaceId: WS }); f.handlers.message({ type: "subscribe", channelId: CH }); + await vi.advanceTimersByTimeAsync(1); hang = true; + await vi.advanceTimersByTimeAsync(2100); expect(f.peer.close).toHaveBeenCalled(); f.service.onModuleDestroy(); + }); + it("resumes buffered state events and requests resync for invalid cursors", async () => { + const f = fixture(); f.handlers.message({ type: "auth", token: "operator", workspaceId: WS }); + f.handlers.message({ type: "subscribe", channelId: CH, lastSeq: 1 }); await tick(); + expect(f.frames.filter(v => typeof v === "object" && v && "type" in v && v.type === "state")).toHaveLength(1); + f.handlers.message({ type: "subscribe", channelId: CH, lastSeq: 100 }); await tick(); + expect(f.frames).toContainEqual({ type: "RESYNC_REQUIRED", channelId: CH }); f.service.onModuleDestroy(); + }); + it("requires auth before subscribe and bounds incoming queued work", async () => { + const f = fixture(); f.handlers.message({ type: "subscribe", channelId: CH }); await tick(); expect(f.peer.close).toHaveBeenCalledWith(1008); f.service.onModuleDestroy(); + const slow = fixture(async () => new Promise(() => undefined)); + for (let i = 0; i < 40; i++) slow.handlers.message({ type: "auth", token: "operator", workspaceId: WS }); + expect(slow.peer.close).toHaveBeenCalled(); slow.service.onModuleDestroy(); + }); +}); +function masked(value: unknown): Buffer { + const body = Buffer.from(JSON.stringify(value)), mask = Buffer.from([1, 2, 3, 4]); + const header = Buffer.alloc(body.length < 126 ? 6 : 8); header[0] = 129; + if (body.length < 126) { header[1] = 128 | body.length; mask.copy(header, 2); } + else { header[1] = 128 | 126; header.writeUInt16BE(body.length, 2); mask.copy(header, 4); } + for (let i = 0; i < body.length; i++) body[i] = (body[i] ?? 0) ^ (mask[i % 4] ?? 0); + return Buffer.concat([header, body]); +} +describe("real WebSocket HTTP upgrade and wire frames", () => { + it("upgrades /realtime, accepts masked first-frame auth, then emits subscribe_ack and state", async () => { + const f = fixture(); f.handlers.close(); + const chunks: Buffer[] = []; + const socket = new Duplex({ read() {}, write(chunk: Buffer, _encoding, callback) { chunks.push(Buffer.from(chunk)); callback(); } }); + const request = new IncomingMessage(new Socket()); request.method = "GET"; request.url = "/realtime"; + request.headers = { host: "localhost", connection: "Upgrade", upgrade: "websocket", "sec-websocket-version": "13", "sec-websocket-key": "dGhlIHNhbXBsZSBub25jZQ==" }; + const peer = RealtimeSocket.accept(request, socket, Buffer.alloc(0)); expect(peer).not.toBeNull(); + if (!peer) throw new Error("handshake failed"); + const handlers = f.service.attach(peer); peer.onMessage = handlers.message; peer.onClose = handlers.close; + expect(Buffer.concat(chunks).toString()).toContain("101 Switching Protocols"); + const messages = Buffer.concat([masked({ type: "auth", workspaceId: WS, token: "operator" }), masked({ type: "subscribe", channelId: CH })]); + // Split in the middle of the header and payload, exercising stream framing. + socket.emit("data", messages.subarray(0, 3)); socket.emit("data", messages.subarray(3, 31)); socket.emit("data", messages.subarray(31)); + await tick(); + const wire = Buffer.concat(chunks).toString(); expect(wire).toContain("auth_ack"); expect(wire).toContain("subscribe_ack"); expect(wire).toContain('"kind":"observed"'); + f.service.onModuleDestroy(); socket.destroy(); request.destroy(); + }); +}); + +describe("S5 canonical chat socket lifecycle", () => { + it("revalidates device authorization and closes a revoked device within five seconds", async () => { + vi.useFakeTimers(); + const { issueDeviceToken } = await import("../live-chat/devices.js"), { authorizeChat } = await import("../live-chat/auth.js"); + const deviceId = randomUUID(); let epoch = 1; + const scopes = ["stream.view", "livechat.view"]; + const token = issueDeviceToken({ deviceId, workspaceId: WS, userId: USER, epoch: 1, scopes }); + const client = { query: async (sql: string) => ({ rows: sql.includes("FROM memberships") ? [{ role: "Operator", authz_epoch: 7 }] : sql.includes("FROM devices") ? [{ epoch, authz_epoch: 7, scopes }] : [] }) } as unknown as PoolClient; + const service = new RealtimeService(new MediaControlService({ signer: null, assignments: [] }), { subscriptions: { subscribe: async () => () => undefined, close: () => undefined }, + authorizeChat: async (token, ws, permission) => Boolean(await authorizeChat(client, token, ws, permission)), channelExists: async () => true, chatWindow: async () => [], chatLatestSeq: async () => 0 }); + const frames: unknown[] = [], peer: RealtimePeer = { send: frame => { frames.push(frame); return true; }, close: vi.fn() }, handlers = service.attach(peer); + try { + handlers.message({ op: "auth", token }); handlers.message({ op: "subscribe", id: "chat", topic: "livechat", liveId: CH }); + await vi.advanceTimersByTimeAsync(1); expect(frames).toContainEqual({ op: "ack", id: "chat", fromSeq: 0 }); + epoch += 1; await vi.advanceTimersByTimeAsync(1100); expect(peer.close).toHaveBeenCalledWith(1008); + } finally { service.onModuleDestroy(); } + }); + it("holds out-of-order notifications until the missing predecessor arrives", async () => { + const { issueSession } = await import("../auth/session.js"); + let receive: ((raw: string) => void) | undefined; + const service = new RealtimeService(new MediaControlService({ signer: null, assignments: [] }), { subscriptions: { subscribe: async (_topic, listener) => { receive = listener; return () => undefined; }, close: () => undefined }, + authorizeChat: async () => true, channelExists: async () => true, chatWindow: async () => [], chatLatestSeq: async () => 0 }); + const frames: Array> = [], peer: RealtimePeer = { send: frame => { frames.push(frame as Record); return true; }, close: vi.fn() }, handlers = service.attach(peer); + try { + handlers.message({ op: "auth", token: issueSession({ workspaceId: WS, userId: USER, role: "Operator", authzEpoch: 7 }) }); + handlers.message({ op: "subscribe", id: "chat", topic: "livechat", liveId: CH }); await tick(); + const frame = (seq: number) => JSON.stringify({ kind: "chat", liveStreamId: CH, channelId: CH, seq, provider: "mock-livestream", reason: "gap notice" }); + receive?.(frame(2)); expect(frames.filter(frame => frame.op === "msg")).toEqual([]); + receive?.(frame(1)); expect(frames.filter(frame => frame.op === "msg").map(frame => frame.seq)).toEqual([1,2]); + receive?.(frame(2)); expect(frames.filter(frame => frame.op === "msg")).toHaveLength(2); + handlers.message({ op: "ping" }); await tick(); expect(frames).toContainEqual({ op: "pong" }); + } finally { service.onModuleDestroy(); } + }); +}); diff --git a/apps/api/src/realtime/subscribe.test.ts b/apps/api/src/realtime/subscribe.test.ts new file mode 100644 index 0000000..99d4b38 --- /dev/null +++ b/apps/api/src/realtime/subscribe.test.ts @@ -0,0 +1,46 @@ +import { Duplex } from "node:stream"; +import type { Socket } from "node:net"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { encode, parse } from "../media-control/redis.js"; +import { RespRedisSubscriptions } from "./subscribe.js"; +class RedisSocket extends Duplex { + readonly commands: string[][] = []; + constructor() { super(); queueMicrotask(() => this.emit("connect")); } + override _read(): void { /* Incoming bytes are pushed by the RESP peer. */ } + override _write(bytes: Buffer, _encoding: BufferEncoding, callback: (error?: Error | null) => void): void { + const reply = parse(bytes)?.reply; + if (!Array.isArray(reply) || !reply.every((v): v is string => typeof v === "string")) { callback(new Error("invalid request")); return; } + this.commands.push(reply); + const name = reply[0]; + queueMicrotask(() => this.emit("data", name === "AUTH" || name === "SELECT" ? Buffer.from("+OK\r\n") : encode([name?.toLowerCase() ?? "", reply[1] ?? "", "1"]))); + callback(); + } + message(topic: string, message: string): void { + const frame = encode(["message", topic, message]); + this.emit("data", frame.subarray(0, 7)); this.emit("data", frame.subarray(7)); + } +} +afterEach(() => vi.useRealTimers()); +describe("shared Redis subscribe connection", () => { + it("AUTH/SELECT precede subscription ack, with ref-counted topics and fragmented RESP messages", async () => { + const socket = new RedisSocket(); const factory = vi.fn(() => socket as unknown as Socket); + const redis = new RespRedisSubscriptions("redis://user:password@localhost:6379/2", () => undefined, factory); + const first = vi.fn(), second = vi.fn(); + const unsubscribe1 = await redis.subscribe("state:workspace:channel:desired", first); + const unsubscribe2 = await redis.subscribe("state:workspace:channel:desired", second); + expect(factory).toHaveBeenCalledTimes(1); + expect(socket.commands).toEqual([["AUTH", "user", "password"], ["SELECT", "2"], ["SUBSCRIBE", "state:workspace:channel:desired"]]); + socket.message("state:workspace:channel:desired", "version 1"); expect(first).toHaveBeenCalledWith("version 1"); expect(second).toHaveBeenCalledWith("version 1"); + unsubscribe1(); expect(socket.commands.filter(c => c[0] === "UNSUBSCRIBE")).toHaveLength(0); + socket.message("state:workspace:channel:desired", "version 2"); expect(first).toHaveBeenCalledTimes(1); expect(second).toHaveBeenCalledTimes(2); + unsubscribe2(); await Promise.resolve(); expect(socket.commands.at(-1)).toEqual(["UNSUBSCRIBE", "state:workspace:channel:desired"]); redis.close(); + }); + it("reconnects and restores viewed topics, notifying consumers to resync", async () => { + vi.useFakeTimers(); const sockets: RedisSocket[] = [], notify = vi.fn(); + const redis = new RespRedisSubscriptions("redis://localhost", notify, () => { const socket = new RedisSocket(); sockets.push(socket); return socket as unknown as Socket; }); + const receive = vi.fn(); const subscribed = redis.subscribe("chat:workspace:live", receive); await vi.advanceTimersByTimeAsync(0); await subscribed; + sockets[0]?.destroy(); await vi.advanceTimersByTimeAsync(300); + expect(sockets).toHaveLength(2); expect(sockets[1]?.commands).toContainEqual(["SUBSCRIBE", "chat:workspace:live"]); expect(notify).toHaveBeenCalledTimes(1); + sockets[1]?.message("chat:workspace:live", "chat message"); expect(receive).toHaveBeenCalledWith("chat message"); redis.close(); + }); +}); diff --git a/apps/api/src/realtime/subscribe.ts b/apps/api/src/realtime/subscribe.ts new file mode 100644 index 0000000..ef1269c --- /dev/null +++ b/apps/api/src/realtime/subscribe.ts @@ -0,0 +1,104 @@ +import net, { type Socket } from "node:net"; +import tls from "node:tls"; +import { encode, parse, RedisError, type RedisReply } from "../media-control/redis.js"; +export interface RedisSubscriptions { + subscribe(topic: string, listener: (message: string) => void): Promise<() => void>; + close(): void; +} + +/** One shared subscribe connection, ref-counted topics, automatic reconnect. + * Redis command traffic must use a separate RespControlRedis connection. */ +export class RespRedisSubscriptions implements RedisSubscriptions { + private readonly listeners = new Map void>>(); + private socket: Socket | null = null; + private connecting: Promise | null = null; + private buffer = Buffer.alloc(0); + private readonly pending: Array<{ resolve: (reply: RedisReply) => void; reject: (error: Error) => void }> = []; + private retry: ReturnType | null = null; + private closed = false; + constructor(private readonly rawUrl: string, private readonly onReconnect: () => void = () => undefined, + private readonly socketFactory?: (url: URL) => Socket) {} + async subscribe(topic: string, listener: (message: string) => void): Promise<() => void> { + if (this.closed) throw new Error("subscriber closed"); + const existing = this.listeners.get(topic); + if (existing) { + existing.add(listener); + try { await this.connect(); } catch (error) { this.remove(topic, listener); throw error; } + } + else { + this.listeners.set(topic, new Set([listener])); + // connect() subscribes every registered topic on a new connection. + const connected = this.socket !== null && this.connecting === null; + try { await this.connect(); if (connected) await this.command(["SUBSCRIBE", topic]); } + catch (error) { this.remove(topic, listener); throw error; } + } + return () => this.remove(topic, listener); + } + private remove(topic: string, listener: (message: string) => void): void { + const group = this.listeners.get(topic); group?.delete(listener); + if (group?.size === 0) { + this.listeners.delete(topic); + if (this.socket) void this.command(["UNSUBSCRIBE", topic]).catch(() => undefined); + } + } + private connect(): Promise { + if (this.connecting) return this.connecting; + if (this.socket) return Promise.resolve(); + this.connecting = this.open().finally(() => { this.connecting = null; }); + return this.connecting; + } + private async open(): Promise { + const url = new URL(this.rawUrl); + if (!["redis:", "rediss:"].includes(url.protocol)) throw new Error("invalid Redis URL"); + const socket = this.socketFactory ? this.socketFactory(url) : url.protocol === "rediss:" ? tls.connect({ host: url.hostname, port: Number(url.port || 6379), servername: url.hostname, rejectUnauthorized: true }) + : net.connect({ host: url.hostname, port: Number(url.port || 6379) }); + this.socket = socket; this.buffer = Buffer.alloc(0); + socket.on("data", (chunk: Buffer) => { + this.buffer = Buffer.concat([this.buffer, chunk]); + if (this.buffer.length > 8_000_000) { socket.destroy(); return; } + try { + for (let parsed = parse(this.buffer); parsed; parsed = parse(this.buffer)) { + this.buffer = this.buffer.subarray(parsed.end); + const reply = parsed.reply; + if (Array.isArray(reply) && reply[0] === "message" && typeof reply[1] === "string" && typeof reply[2] === "string") { + for (const callback of this.listeners.get(reply[1]) ?? []) { try { callback(reply[2]); } catch { /* Isolate consumers. */ } } + } else { + const waiter = this.pending.shift(); + if (reply instanceof RedisError) waiter?.reject(reply); else waiter?.resolve(reply); + } + } + } catch { socket.destroy(); } + }); + socket.on("error", () => undefined); + socket.once("close", () => { + if (this.socket === socket) this.socket = null; + for (const waiter of this.pending.splice(0)) waiter.reject(new Error("Redis disconnected")); + if (!this.closed && !this.retry) { + this.retry = setTimeout(() => { this.retry = null; void this.connect().then(this.onReconnect).catch(() => undefined); }, 250); + this.retry.unref(); + } + }); + const timeout = setTimeout(() => socket.destroy(new Error("Redis connect timed out")), 2000); + await new Promise((resolve, reject) => { + socket.once(url.protocol === "rediss:" ? "secureConnect" : "connect", resolve); + socket.once("error", reject); + }).finally(() => clearTimeout(timeout)); + if (url.password) await this.command(["AUTH", decodeURIComponent(url.username || "default"), decodeURIComponent(url.password)]); + if (url.pathname.length > 1) await this.command(["SELECT", url.pathname.slice(1)]); + // Sequential ACKs ensure no topic is acknowledged to a viewer before Redis is listening. + for (const topic of this.listeners.keys()) await this.command(["SUBSCRIBE", topic]); + } + private command(parts: string[]): Promise { + const socket = this.socket; + if (!socket) return Promise.reject(new Error("Redis disconnected")); + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => socket.destroy(new Error("Redis subscription ack timed out")), 2000); + this.pending.push({ resolve: reply => { clearTimeout(timeout); resolve(reply); }, reject: error => { clearTimeout(timeout); reject(error); } }); + socket.write(encode(parts)); + }); + } + close(): void { + this.closed = true; if (this.retry) clearTimeout(this.retry); + this.listeners.clear(); this.socket?.destroy(); this.socket = null; + } +} diff --git a/apps/api/src/realtime/websocket.ts b/apps/api/src/realtime/websocket.ts new file mode 100644 index 0000000..5e4852b --- /dev/null +++ b/apps/api/src/realtime/websocket.ts @@ -0,0 +1,85 @@ +import { createHash } from "node:crypto"; +import type { IncomingMessage } from "node:http"; +import type { Duplex } from "node:stream"; + +/** RFC6455 transport without a new dependency; control frames, fragmentation, + * strict masking/UTF-8, and bounded buffering/backpressure. */ +export class RealtimeSocket { + private buffer = Buffer.alloc(0); + private fragments: Buffer[] = []; + private fragmentBytes = 0; + private closed = false; + onMessage: (value: unknown) => void = () => undefined; + onClose: () => void = () => undefined; + constructor(private readonly socket: Duplex) { + socket.on("data", (chunk: Buffer) => this.consume(chunk)); + socket.on("error", () => this.close()); + socket.on("close", () => { if (!this.closed) { this.closed = true; this.onClose(); } }); + } + static accept(request: IncomingMessage, socket: Duplex, head: Buffer): RealtimeSocket | null { + const key = request.headers["sec-websocket-key"]; + if (request.method !== "GET" || request.headers.upgrade?.toLowerCase() !== "websocket" || + !request.headers.connection?.toLowerCase().split(/\s*,\s*/).includes("upgrade") || request.headers["sec-websocket-version"] !== "13" || + typeof key !== "string" || !/^[A-Za-z0-9+/]{22}==$/.test(key)) { + socket.end("HTTP/1.1 400 Bad Request\r\nConnection: close\r\n\r\n"); return null; + } + const accept = createHash("sha1").update(`${key}258EAFA5-E914-47DA-95CA-C5AB0DC85B11`).digest("base64"); + socket.write(`HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ${accept}\r\n\r\n`); + const peer = new RealtimeSocket(socket); + if (head.length) queueMicrotask(() => peer.consume(head)); + return peer; + } + send(value: unknown): boolean { + const data = Buffer.from(JSON.stringify(value)); + if (this.closed || this.socket.writableLength > 256_000 || data.length > 256_000) { + const canonical = typeof value === "object" && value !== null && "op" in value; + this.frame(1, Buffer.from(JSON.stringify(canonical ? { op: "resync", topic: "topic" in value ? value.topic : "livechat" } : { type: "RESYNC_REQUIRED" }))); this.close(1013); return false; + } + this.frame(1, data); return true; + } + private frame(opcode: number, data: Buffer): void { + if (this.closed) return; + const header = Buffer.alloc(data.length < 126 ? 2 : data.length <= 65535 ? 4 : 10); + header[0] = 0x80 | opcode; + if (header.length === 2) header[1] = data.length; + else if (header.length === 4) { header[1] = 126; header.writeUInt16BE(data.length, 2); } + else { header[1] = 127; header.writeBigUInt64BE(BigInt(data.length), 2); } + this.socket.write(Buffer.concat([header, data])); + } + close(code = 1000): void { + if (this.closed) return; + const body = Buffer.alloc(2); body.writeUInt16BE(code); + this.frame(8, body); this.closed = true; this.onClose(); this.socket.end(); + const timer = setTimeout(() => this.socket.destroy(), 100); timer.unref(); + } + private consume(chunk: Buffer): void { + if (this.closed) return; + this.buffer = Buffer.concat([this.buffer, chunk]); + if (this.buffer.length > 128_000) { this.close(1009); return; } + while (this.buffer.length >= 2) { + const first = this.buffer[0] ?? 0, second = this.buffer[1] ?? 0; + const final = Boolean(first & 128), opcode = first & 15; + if (first & 112 || !(second & 128)) { this.close(1002); return; } + let length = second & 127, offset = 2; + if (length === 126) { if (this.buffer.length < 4) return; length = this.buffer.readUInt16BE(2); offset = 4; if (length < 126) { this.close(1002); return; } } + if (length === 127) { this.close(1009); return; } + if (length > 64_000 || (opcode >= 8 && (!final || length > 125))) { this.close(1009); return; } + if (this.buffer.length < offset + 4 + length) return; + const mask = this.buffer.subarray(offset, offset + 4); offset += 4; + const body = Buffer.from(this.buffer.subarray(offset, offset + length)); + this.buffer = this.buffer.subarray(offset + length); + for (let i = 0; i < body.length; i += 1) body[i] = (body[i] ?? 0) ^ (mask[i % 4] ?? 0); + if (opcode === 8) { this.close(); return; } + if (opcode === 9) { this.frame(10, body); continue; } + if (opcode === 10) continue; + if ((opcode !== 0 && opcode !== 1) || (opcode === 0 && !this.fragments.length) || (opcode === 1 && this.fragments.length)) { this.close(1002); return; } + this.fragments.push(body); this.fragmentBytes += body.length; + if (this.fragmentBytes > 64_000) { this.close(1009); return; } + if (final) { + const bytes = Buffer.concat(this.fragments); this.fragments = []; this.fragmentBytes = 0; + try { const value: unknown = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)); this.onMessage(value); } + catch { this.close(1007); return; } + } + } + } +} diff --git a/apps/api/src/settings/settings.controller.ts b/apps/api/src/settings/settings.controller.ts new file mode 100644 index 0000000..8229bcf --- /dev/null +++ b/apps/api/src/settings/settings.controller.ts @@ -0,0 +1,35 @@ +/** + * Instance settings API (admin only): + * + * GET /settings read the content-policy posture (Owner/Admin) + * PATCH /settings partial update + system audit event (Owner/Admin) + * + * Instance-level ⇒ the RBAC permission set (002) has no instance permission; + * the gate is the ROLE itself (task spec: Owner/Admin), checked in the + * controller after the standard RbacGuard session/epoch validation. RLS does + * not apply (system table, migration 028) — the role check IS the boundary. + */ +import { Body, Controller, ForbiddenException, Get, Patch, Req, UseGuards } from "@nestjs/common"; +import { requireSession } from "../http/validation.js"; +import { RbacGuard, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import { isInstanceAdmin, SettingsService, type InstanceSettingRecord } from "./settings.service.js"; + +@Controller("settings") +@UseGuards(RbacGuard) +export class SettingsController { + constructor(private readonly settings: SettingsService) {} + + @Get() + async get(@Req() request: AuthenticatedRequest): Promise { + const session = requireSession(request); + if (!isInstanceAdmin(session.role)) throw new ForbiddenException("instance settings are admin-only (Owner/Admin)"); + return this.settings.getSettings(); + } + + @Patch() + async patch(@Req() request: AuthenticatedRequest, @Body() body: unknown): Promise { + const session = requireSession(request); + if (!isInstanceAdmin(session.role)) throw new ForbiddenException("instance settings are admin-only (Owner/Admin)"); + return this.settings.updateSettings(body, { userId: session.userId, role: session.role }); + } +} diff --git a/apps/api/src/settings/settings.integration.test.ts b/apps/api/src/settings/settings.integration.test.ts new file mode 100644 index 0000000..9aaf100 --- /dev/null +++ b/apps/api/src/settings/settings.integration.test.ts @@ -0,0 +1,351 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { join } from "node:path"; +import { Pool, type PoolClient } from "pg"; + +/** + * Instance settings against a live PostgreSQL, as sp_app. + * Own scratch database (ADMIN_SETTINGS_IT_DATABASE_URL + ADMIN_SETTINGS_IT_ + * SUPER_DATABASE_URL) per the local-db-sol pattern: the shared-DB suites wipe + * every workspace and vitest runs files in parallel. Skipped when unset. + * + * Covers the task's list: seeded defaults, CRUD round-trip incl. partial + * updates and provenance (updated_at/updated_by), RBAC (non-admin 403 through + * the real controller; the real RbacGuard path), audit events on the SYSTEM + * chain (audit_append_system — an instance setting is not a tenant row), and + * the migration's CHECK constraints rejecting undecodable values. + */ +const appUrl = process.env.ADMIN_SETTINGS_IT_DATABASE_URL; +const ownerUrl = process.env.ADMIN_SETTINGS_IT_SUPER_DATABASE_URL; + +type Modules = { + service: typeof import("./settings.service.js"); + controller: typeof import("./settings.controller.js"); + guard: typeof import("../rbac/rbac.guard.js"); + session: typeof import("../auth/session.js"); + db: typeof import("../db/client.js"); +}; + +describe.skipIf(appUrl === undefined || ownerUrl === undefined)("instance settings (live DB)", () => { + let m: Modules; + let ownerPool: Pool; + let service: InstanceType; + let controller: InstanceType; + const suffix = `${Date.now()}${Math.floor(Math.random() * 1e6)}`; + let ws: string; + let adminUser: string; + let otherUser: string; + + async function asOwner(fn: (client: PoolClient) => Promise): Promise { + const client = await ownerPool.connect(); + try { + return await fn(client); + } finally { + client.release(); + } + } + + /** Reset to the seeded defaults (self-healing upsert: also restores deleted rows). */ + async function resetSettings(): Promise { + await asOwner(async (c) => { + await c.query( + `INSERT INTO instance_settings (key, value, updated_by) + VALUES ('youtube_full_harvest','false',NULL), ('youtube_cc_only_broadcast','true',NULL), + ('scrape_metadata_only','true',NULL), ('scrape_bounded_pilot','false',NULL), + ('intake_posture','metadata_only',NULL) + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_by = NULL, updated_at = now()`, + ); + }); + } + + beforeAll(async () => { + // The shared pool reads DATABASE_URL at import time: point it at this + // isolated DB first, then import the modules under test. + process.env.DATABASE_URL = appUrl; + process.env.SUPER_DATABASE_URL = ownerUrl; + const { runMigrations } = await import("../db/migrate.js"); + await runMigrations(join(process.cwd(), "migrations")); + m = { + service: await import("./settings.service.js"), + controller: await import("./settings.controller.js"), + guard: await import("../rbac/rbac.guard.js"), + session: await import("../auth/session.js"), + db: await import("../db/client.js"), + }; + ownerPool = new Pool({ connectionString: ownerUrl }); + + service = new m.service.SettingsService(); + controller = new m.controller.SettingsController(service); + + await asOwner(async (c) => { + ws = String((await c.query("INSERT INTO workspaces (name, slug) VALUES ($1, $2) RETURNING id", ["settings-it", `settings-it-${suffix}`])).rows[0].id); + const mkUser = async (tag: string): Promise => + String( + ( + await c.query("INSERT INTO users (external_id, username, email, display_name) VALUES ($1, $2, $3, $4) RETURNING id", [ + `settings:${tag}:${suffix}`, + `settings-${tag}-${suffix}`, + `settings-${tag}-${suffix}@example.com`, + tag, + ]) + ).rows[0].id, + ); + adminUser = await mkUser("admin"); + otherUser = await mkUser("member"); + // Give the users real memberships so the guard's role/permission + // lookups are exercised through the actual RBAC chain. + await c.query("SELECT set_config('app.workspace_id', $1, false)", [ws]); + await c.query("INSERT INTO memberships (workspace_id, user_id, role) VALUES ($1, $2, 'Owner'), ($1, $3, 'Publisher')", [ws, adminUser, otherUser]); + await c.query("SELECT set_config('app.workspace_id', '', false)"); + }); + }); + + afterAll(async () => { + if (ownerPool !== undefined) { + await asOwner(async (c) => { + await c.query("DELETE FROM workspaces WHERE id = $1", [ws]); + await c.query("DELETE FROM users WHERE external_id LIKE $1", [`settings:%:${suffix}`]); + }); + await ownerPool.end(); + } + await m?.db.pool.end(); + }); + + // ----------------------------------------------------------------------- + // Defaults + shape + // ----------------------------------------------------------------------- + + it("seeds the five keys at the SAFE defaults (decision record posture)", async () => { + // Read the seeded rows directly so this asserts the MIGRATION's seed, not + // service decoding. + const seeded = await asOwner(async (c) => + (await c.query<{ key: string; value: string; updated_by: string | null }>("SELECT key, value, updated_by FROM instance_settings")).rows, + ); + expect(seeded.map((r) => r.key).sort()).toEqual([...m.service.INSTANCE_SETTING_KEYS].sort()); + const byKey = new Map(seeded.map((r) => [r.key, r])); + expect(byKey.get("youtube_full_harvest")?.value).toBe("false"); + expect(byKey.get("youtube_cc_only_broadcast")?.value).toBe("true"); + expect(byKey.get("scrape_metadata_only")?.value).toBe("true"); + expect(byKey.get("scrape_bounded_pilot")?.value).toBe("false"); + expect(byKey.get("intake_posture")?.value).toBe("metadata_only"); + // Normalize to defaults for the rest of the suite (the scratch DB persists + // across runs; earlier runs may have left non-default values behind). + await resetSettings(); + const settings = await service.getSettings(); + expect(settings).toMatchObject({ + youtube_full_harvest: false, + youtube_cc_only_broadcast: true, + scrape_metadata_only: true, + scrape_bounded_pilot: false, + intake_posture: "metadata_only", + }); + // Seeded rows exist ⇒ nothing is pending, provenance present. + expect(settings.pending).toEqual([]); + expect(settings.updatedBy.youtube_full_harvest).toBeNull(); + }); + + it("typed getters mirror the record", async () => { + expect(await service.getYoutubeFullHarvest()).toBe(false); + expect(await service.getYoutubeCcOnlyBroadcast()).toBe(true); + expect(await service.getScrapeMetadataOnly()).toBe(true); + expect(await service.getScrapeBoundedPilot()).toBe(false); + expect(await service.getIntakePosture()).toBe("metadata_only"); + }); + + // ----------------------------------------------------------------------- + // CRUD (partial update + provenance) + // ----------------------------------------------------------------------- + + it("partial update flips one switch, leaves the others, records provenance", async () => { + await resetSettings(); + const actor = { userId: adminUser, role: "Owner" }; + const after = await service.updateSettings({ youtube_full_harvest: true }, actor); + expect(after.youtube_full_harvest).toBe(true); // written + expect(after.youtube_cc_only_broadcast).toBe(true); // untouched + expect(after.scrape_metadata_only).toBe(true); // untouched + expect(after.intake_posture).toBe("metadata_only"); // untouched + expect(after.updatedBy.youtube_full_harvest).toBe(adminUser); + expect(after.updatedBy.youtube_cc_only_broadcast).toBeNull(); + expect(new Date(after.updatedAt.youtube_full_harvest).getTime()).toBeGreaterThan(0); + // Typed getter sees the new value (the intake-side gate reads this). + expect(await service.getYoutubeFullHarvest()).toBe(true); + }); + + it("full update round-trips every key and re-updates overwrite", async () => { + await resetSettings(); + const actor = { userId: adminUser, role: "Admin" }; + const first = await service.updateSettings( + { youtube_full_harvest: true, youtube_cc_only_broadcast: false, scrape_metadata_only: false, scrape_bounded_pilot: true, intake_posture: "bounded_pilot" }, + actor, + ); + expect(first).toMatchObject({ + youtube_full_harvest: true, + youtube_cc_only_broadcast: false, + scrape_metadata_only: false, + scrape_bounded_pilot: true, + intake_posture: "bounded_pilot", + }); + // Re-update overwrites, does not duplicate rows (key PK). + const second = await service.updateSettings({ intake_posture: "metadata_only", scrape_bounded_pilot: false }, actor); + expect(second.intake_posture).toBe("metadata_only"); + expect(second.scrape_bounded_pilot).toBe(false); + expect(second.youtube_full_harvest).toBe(true); // still from the first write + const rows = await asOwner(async (c) => (await c.query<{ key: string }>("SELECT key FROM instance_settings ORDER BY key")).rows.map((r) => r.key)); + expect(rows.sort()).toEqual([...m.service.INSTANCE_SETTING_KEYS].sort()); + }); + + it("invalid patches change nothing", async () => { + await resetSettings(); + const before = await service.getSettings(); + for (const bad of [null, [], "x", {}, { nope: true }, { intake_posture: "everything" }, { scrape_bounded_pilot: "yes" }]) { + await expect(service.updateSettings(bad, { userId: adminUser, role: "Owner" })).rejects.toBeDefined(); + } + const after = await service.getSettings(); + expect({ ...after }).toEqual({ ...before, updatedAt: after.updatedAt, updatedBy: after.updatedBy, pending: after.pending }); + expect(await service.getIntakePosture()).toBe("metadata_only"); + }); + + // ----------------------------------------------------------------------- + // RBAC — non-admin is 403, through the controller AND the guard chain + // ----------------------------------------------------------------------- + + it("non-admin roles get 403 from GET and PATCH (controller gate)", async () => { + const req = (role: string) => ({ session: { userId: otherUser, workspaceId: ws, role, authzEpoch: 1, issuedAt: 0, expiresAt: 9e12 } }); + for (const role of ["Publisher", "Editor", "Approver", "Analyst", "Moderator", "BroadcastOperator"]) { + const errGet = await controller.get(req(role) as never).then( + () => null, + (e: unknown) => e, + ); + expect(errGet).toMatchObject({ status: 403 }); + const errPatch = await controller.patch(req(role) as never, { youtube_full_harvest: true }).then( + () => null, + (e: unknown) => e, + ); + expect(errPatch).toMatchObject({ status: 403 }); + } + // And nothing changed. + expect(await service.getYoutubeFullHarvest()).toBe(false); + }); + + it("non-admin is refused in the service layer too (no other call path widens posture)", async () => { + await expect(service.updateSettings({ youtube_full_harvest: true }, { userId: otherUser, role: "Publisher" })).rejects.toBeDefined(); + expect(await service.getYoutubeFullHarvest()).toBe(false); + }); + + it("Owner and Admin both pass the controller gate end-to-end", async () => { + await resetSettings(); + const req = (userId: string, role: string) => ({ session: { userId, workspaceId: ws, role, authzEpoch: 1, issuedAt: 0, expiresAt: 9e12 } }); + const asOwner = await controller.patch(req(adminUser, "Owner") as never, { scrape_bounded_pilot: true }); + expect(asOwner.scrape_bounded_pilot).toBe(true); + const asAdmin = await controller.patch(req(adminUser, "Admin") as never, { scrape_bounded_pilot: false }); + expect(asAdmin.scrape_bounded_pilot).toBe(false); + const read = await controller.get(req(adminUser, "Admin") as never); + expect(read.scrape_bounded_pilot).toBe(false); + }); + + it("the real RbacGuard + controller chain: valid session reaches GET, no session is 401", async () => { + const { Reflector } = await import("@nestjs/core"); + const epoch = await asOwner(async (c) => Number((await c.query("SELECT authz_epoch FROM workspaces WHERE id = $1", [ws])).rows[0].authz_epoch)); + const token = m.session.issueSession({ userId: adminUser, workspaceId: ws, role: "Owner", authzEpoch: epoch }); + const run = async (headers: Record): Promise => { + const reqObj = { headers, session: undefined as unknown }; + const ctx = { getHandler: () => controller.get, switchToHttp: () => ({ getRequest: () => reqObj }) } as never; + const guardOk = await new m.guard.RbacGuard(new Reflector()).canActivate(ctx); + if (guardOk) return controller.get(reqObj as never); + return null; + }; + // No bearer -> guard 401 before the controller is reached. + await expect(run({})).rejects.toMatchObject({ status: 401 }); + // Valid Owner session passes the guard (no permission metadata on the + // route) and the controller's role gate, returning the live settings. + const settings = (await run({ authorization: `Bearer ${token}` })) as import("./settings.service.js").InstanceSettingRecord; + expect(settings.intake_posture).toBe("metadata_only"); + }); + + // ----------------------------------------------------------------------- + // Audit events (system chain — instance rows are not tenant rows) + // ----------------------------------------------------------------------- + + it("every PATCH lands a system audit event with the before→after diff", async () => { + await resetSettings(); + await service.updateSettings({ youtube_full_harvest: true, intake_posture: "bounded_pilot" }, { userId: adminUser, role: "Owner" }); + + const events = await asOwner(async (c) => + ( + await c.query<{ action: string; target_type: string; target_id: string; actor_user_id: string; diff: Record; workspace_id: string | null }>( + "SELECT action, target_type, target_id, actor_user_id, diff, workspace_id FROM audit_log WHERE action = 'instance_settings.updated' ORDER BY seq", + ) + ).rows, + ); + expect(events.length).toBeGreaterThanOrEqual(1); + const last = events.at(-1); + expect(last).toBeDefined(); + if (last === undefined) return; + // System chain: workspace_id IS NULL, actor recorded, diff carries the change. + expect(last.workspace_id).toBeNull(); + expect(last.actor_user_id).toBe(adminUser); + expect(last.target_type).toBe("instance_settings"); + expect(last.target_id).toBe("instance"); + expect(last.diff).toMatchObject({ + youtube_full_harvest: { from: false, to: true }, + intake_posture: { from: "metadata_only", to: "bounded_pilot" }, + }); + }); + + it("a failed patch lands NO audit event", async () => { + await resetSettings(); + const before = await asOwner(async (c) => Number((await c.query("SELECT count(*) AS n FROM audit_log WHERE action = 'instance_settings.updated'")).rows[0].n)); + await expect(service.updateSettings({ intake_posture: "nonsense" }, { userId: adminUser, role: "Owner" })).rejects.toBeDefined(); + const after = await asOwner(async (c) => Number((await c.query("SELECT count(*) AS n FROM audit_log WHERE action = 'instance_settings.updated'")).rows[0].n)); + expect(after).toBe(before); + }); + + // ----------------------------------------------------------------------- + // Table-level guarantees (migration 028) + // ----------------------------------------------------------------------- + + it("instance_settings is a system table: no workspace_id, no RLS, sp_app-granted", async () => { + const meta = await asOwner(async (c) => + ( + await c.query<{ columns: number; policies: number; rls: string; grants: string[] }>( + `SELECT + (SELECT count(*) FROM information_schema.columns WHERE table_name = 'instance_settings' AND column_name = 'workspace_id')::int AS columns, + (SELECT count(*) FROM pg_policies WHERE tablename = 'instance_settings')::int AS policies, + c.relrowsecurity::text AS rls, + coalesce(array_agg(DISTINCT u.grantee) FILTER (WHERE u.grantee <> 'superposter'), '{}') AS grants + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN information_schema.role_table_grants u ON u.table_name = 'instance_settings' AND u.table_schema = 'public' + WHERE n.nspname = 'public' AND c.relname = 'instance_settings' + GROUP BY c.relrowsecurity`, + ) + ).rows.at(0), + ); + expect(meta).toBeDefined(); + if (meta === undefined) return; + expect(meta.columns).toBe(0); // no workspace_id column + expect(meta.policies).toBe(0); // no RLS policies + expect(meta.rls).toBe("false"); // RLS not even enabled + expect(meta.grants).toContain("sp_app"); // system table with sp_app grants + }); + + it("CHECK constraints reject unknown keys and undecodable values at the DB", async () => { + await expect(asOwner(async (c) => c.query("INSERT INTO instance_settings (key, value) VALUES ('allow_everything', 'true')"))).rejects.toThrow(); + await expect(asOwner(async (c) => c.query("INSERT INTO instance_settings (key, value) VALUES ('scrape_bounded_pilot', 'maybe')"))).rejects.toThrow(); + await expect(asOwner(async (c) => c.query("INSERT INTO instance_settings (key, value) VALUES ('intake_posture', 'wildwest')"))).rejects.toThrow(); + // A legit literal passes the CHECK (update the seeded row: key is the PK). + await asOwner(async (c) => c.query("UPDATE instance_settings SET value = 'community_submissions' WHERE key = 'intake_posture'")); + expect(await service.getIntakePosture()).toBe("community_submissions"); + // A MISSING row decodes to the safe default and is surfaced as pending. + await asOwner(async (c) => c.query("DELETE FROM instance_settings WHERE key = 'intake_posture'")); + const pending = await service.getSettings(); + expect(pending.intake_posture).toBe("metadata_only"); + expect(pending.pending).toContain("intake_posture"); + // Restore the seeded row for later groups. + await resetSettings(); + }); + + it("the migration is registered in schema_migrations as 028", async () => { + const applied = await asOwner(async (c) => (await c.query<{ name: string }>("SELECT name FROM schema_migrations WHERE name LIKE '028%'")).rows.map((r) => r.name)); + expect(applied).toContain("028_instance_settings.sql"); + }); +}); diff --git a/apps/api/src/settings/settings.module.ts b/apps/api/src/settings/settings.module.ts new file mode 100644 index 0000000..9219b88 --- /dev/null +++ b/apps/api/src/settings/settings.module.ts @@ -0,0 +1,16 @@ +/** + * Settings module wiring. The service talks to the system pool directly + * (withSystem): instance_settings is deliberately NOT tenant-scoped, so there + * is no withTenant context to establish — audit events go to the system chain + * via audit_append_system, which REJECTS a tenant context. + */ +import { Module } from "@nestjs/common"; +import { SettingsController } from "./settings.controller.js"; +import { SettingsService } from "./settings.service.js"; + +@Module({ + controllers: [SettingsController], + providers: [SettingsService], + exports: [SettingsService], +}) +export class SettingsModule {} diff --git a/apps/api/src/settings/settings.service.ts b/apps/api/src/settings/settings.service.ts new file mode 100644 index 0000000..9ff4baa --- /dev/null +++ b/apps/api/src/settings/settings.service.ts @@ -0,0 +1,229 @@ +/** + * Instance-level settings: the content-policy posture of THIS deployment. + * + * The firehose tiers are graded by legal/ToS exposure (decision record + * 2026-10-09-intake-posture.md). That exposure is + * per-instance — one egress IP, one legal entity — so the switches live in a + * SYSTEM table (migration 028): no workspace_id, no RLS, nothing per-tenant. + * Access control happens at the API seam: only Owner/Admin roles reach the + * mutating methods (controller role check), and every write lands a system + * audit-chain event (audit_append_system) so posture changes are provable. + * + * This module is the ONE place that interprets `instance_settings.value` + * strings. Key set is CHECK-constrained in 028 to exactly these five keys; + * values to exactly the literals decoded here, so an undecodable row cannot + * exist — unknown/missing rows fall back to the SAFE default and are surfaced + * as `pending` in GET /settings. + */ +/* eslint-disable no-restricted-syntax -- the setting keys NAME the platform each policy switch governs (youtube_full_harvest gates the yt-dlp path); this module is the policy boundary, same standing as intake-kinds.ts. */ + +import { BadRequestException, Injectable } from "@nestjs/common"; +import type { PoolClient } from "pg"; +import { withSystem } from "../db/client.js"; + +/** Overall intake posture (decision record tiers). */ +export const INTAKE_POSTURES = ["metadata_only", "community_submissions", "bounded_pilot"] as const; +export type IntakePosture = (typeof INTAKE_POSTURES)[number]; + +export function isIntakePosture(value: unknown): value is IntakePosture { + return typeof value === "string" && (INTAKE_POSTURES as readonly string[]).includes(value); +} + +/** The five instance settings keys (migration 028 CHECK). */ +export const INSTANCE_SETTING_KEYS = [ + "youtube_full_harvest", + "youtube_cc_only_broadcast", + "scrape_metadata_only", + "scrape_bounded_pilot", + "intake_posture", +] as const; +export type InstanceSettingKey = (typeof INSTANCE_SETTING_KEYS)[number]; + +/** Boolean-valued keys — decoded from 'true'/'false'. */ +export const BOOLEAN_SETTING_KEYS: readonly InstanceSettingKey[] = [ + "youtube_full_harvest", + "youtube_cc_only_broadcast", + "scrape_metadata_only", + "scrape_bounded_pilot", +]; + +/** Defaults = the safe posture (decision record 2026-10-09). */ +export const DEFAULT_INSTANCE_SETTINGS = { + youtube_full_harvest: false, // yt-dlp full video download: OFF + youtube_cc_only_broadcast: true, // broadcast playlists: CC-licensed only + scrape_metadata_only: true, // locked platforms: metadata only + scrape_bounded_pilot: false, // bounded read-only X pool: OFF + intake_posture: "metadata_only" as IntakePosture, +} satisfies Record; + +/** The decoded settings value object (what every consumer reads). */ +export interface InstanceSettings { + /** Enables yt-dlp video download for YouTube sources (ToS risk). */ + youtube_full_harvest: boolean; + /** CC-only filter for YouTube broadcast playlists (Creative Commons only). */ + youtube_cc_only_broadcast: boolean; + /** Locked platforms stay metadata-only (no authenticated crawling). */ + scrape_metadata_only: boolean; + /** Enables the bounded read-only X pool (decision record Tier 2). */ + scrape_bounded_pilot: boolean; + /** Overall intake posture. */ + intake_posture: IntakePosture; +} + +/** GET /settings shape: values + last-write provenance per key. */ +export interface InstanceSettingRecord extends InstanceSettings { + /** ISO timestamps of the last write per key (epoch for never-written keys). */ + updatedAt: Readonly>; + /** User ids of the last writer per key. */ + updatedBy: Readonly>; + /** Keys with no row yet (unset ⇒ default in force). Read-only information. */ + pending: readonly InstanceSettingKey[]; +} + +export interface SettingActor { + userId: string; + role: string; +} + +/** Roles allowed to change instance posture. Task spec: Owner/Admin. */ +export const INSTANCE_ADMIN_ROLES: readonly string[] = ["Owner", "Admin"]; + +export function isInstanceAdmin(role: string): boolean { + return INSTANCE_ADMIN_ROLES.includes(role); +} + +type RawRow = { key: string; value: string; updated_at: Date; updated_by: string | null }; + +/** System audit append, on the SAME transaction as the setting write. */ +async function auditSystemSettingChange( + client: PoolClient, + actor: SettingActor, + diff: Record, +): Promise { + await client.query("SELECT audit_append_system($1, $2, $3, $4, $5::jsonb)", [ + actor.userId, + "instance_settings.updated", + "instance_settings", + "instance", + JSON.stringify(diff), + ]); +} + +@Injectable() +export class SettingsService { + /** All five settings with metadata. Unknown rows cannot exist (028 CHECK). */ + async getSettings(): Promise { + return withSystem(async (client) => this.readAll(client)); + } + + async getYoutubeFullHarvest(): Promise { + return (await this.getSettings()).youtube_full_harvest; + } + + async getYoutubeCcOnlyBroadcast(): Promise { + return (await this.getSettings()).youtube_cc_only_broadcast; + } + + async getScrapeMetadataOnly(): Promise { + return (await this.getSettings()).scrape_metadata_only; + } + + async getScrapeBoundedPilot(): Promise { + return (await this.getSettings()).scrape_bounded_pilot; + } + + async getIntakePosture(): Promise { + return (await this.getSettings()).intake_posture; + } + + /** + * Partial update of instance settings. Validates shapes, writes every + * provided key in ONE transaction and lands a single system audit event + * carrying the before→after diff (posture changes are legal-exposure + * decisions; they must be provable after the fact). + */ + async updateSettings(patch: unknown, actor: SettingActor): Promise { + if (!isInstanceAdmin(actor.role)) { + // Defense in depth: the controller gates first, the service re-checks so + // no other call path can widen the posture. + throw new BadRequestException("only Owner/Admin may change instance settings"); + } + const input = this.validatePatch(patch); + return withSystem(async (client) => { + const before = await this.readAll(client); + const diff: Record = {}; + for (const key of INSTANCE_SETTING_KEYS) { + if (!(key in input)) continue; + const value = input[key]; + await client.query( + `INSERT INTO instance_settings (key, value, updated_by) VALUES ($1, $2, $3) + ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = now(), updated_by = EXCLUDED.updated_by`, + [key, String(value), actor.userId], + ); + diff[key] = { from: before[key], to: value }; + } + await auditSystemSettingChange(client, actor, diff); + return this.readAll(client); + }); + } + + /** + * Body-shape guard (shared `unknown` body convention): an object with any + * subset of the five keys; booleans must be actual booleans, posture must be + * a known enum value, unknown keys are a 400 (never silently dropped). + */ + private validatePatch(patch: unknown): Partial { + if (typeof patch !== "object" || patch === null || Array.isArray(patch)) { + throw new BadRequestException("body must be an object"); + } + const entries: Partial = {}; + let count = 0; + for (const [key, value] of Object.entries(patch as Record)) { + if (!(INSTANCE_SETTING_KEYS as readonly string[]).includes(key)) { + throw new BadRequestException(`unknown setting: ${key} (allowed: ${INSTANCE_SETTING_KEYS.join(", ")})`); + } + if (key === "intake_posture") { + if (!isIntakePosture(value)) throw new BadRequestException(`intake_posture must be one of: ${INTAKE_POSTURES.join(", ")}`); + entries.intake_posture = value; + } else if (typeof value === "boolean") { + entries[key as Exclude] = value; + } else { + throw new BadRequestException(`${key} must be a boolean`); + } + count += 1; + } + if (count === 0) throw new BadRequestException("body must set at least one setting"); + return entries; + } + + /** Read + decode every key; missing rows decode to the safe default. */ + private async readAll(client: PoolClient): Promise { + const result = await client.query("SELECT key, value, updated_at, updated_by FROM instance_settings"); + const rows = new Map(result.rows.map((row) => [row.key, row])); + const settings: InstanceSettings = { ...DEFAULT_INSTANCE_SETTINGS }; + const updatedAt = {} as Record; + const updatedBy = {} as Record; + const pending: InstanceSettingKey[] = []; + for (const key of INSTANCE_SETTING_KEYS) { + const row = rows.get(key); + if (row === undefined) { + pending.push(key); + updatedAt[key] = new Date(0).toISOString(); + updatedBy[key] = null; + continue; + } + if (key === "intake_posture") { + if (isIntakePosture(row.value)) settings.intake_posture = row.value; + } else if (row.value === "true" || row.value === "false") { + settings[key] = row.value === "true"; + } + // A row outside the decodable literals cannot exist (028 CHECK); if one + // ever does, the safe default above stays in force. + updatedAt[key] = row.updated_at.toISOString(); + updatedBy[key] = row.updated_by; + } + return { ...settings, updatedAt, updatedBy, pending }; + } +} + +/* eslint-enable no-restricted-syntax */ diff --git a/apps/api/src/settings/settings.test.ts b/apps/api/src/settings/settings.test.ts new file mode 100644 index 0000000..563a1ad --- /dev/null +++ b/apps/api/src/settings/settings.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, it } from "vitest"; +import { BadRequestException, ForbiddenException } from "@nestjs/common"; +import { + BOOLEAN_SETTING_KEYS, + DEFAULT_INSTANCE_SETTINGS, + INSTANCE_ADMIN_ROLES, + INSTANCE_SETTING_KEYS, + INTAKE_POSTURES, + isInstanceAdmin, + isIntakePosture, + SettingsService, +} from "./settings.service.js"; +import { SettingsController } from "./settings.controller.js"; + +/** + * Unit level: validation shapes, defaults, the role gate — no DB. The + * live-DB CRUD/RBAC/audit coverage is settings.integration.test.ts. + */ + +const service = new SettingsService(); +const admin = { userId: "00000000-0000-4000-8000-000000000001", role: "Owner" }; + +describe("instance settings — keys, defaults, roles", () => { + it("exposes exactly the five policy keys from the task spec", () => { + expect(INSTANCE_SETTING_KEYS).toEqual([ + "youtube_full_harvest", + "youtube_cc_only_broadcast", + "scrape_metadata_only", + "scrape_bounded_pilot", + "intake_posture", + ]); + expect(BOOLEAN_SETTING_KEYS).toHaveLength(4); + expect(INSTANCE_SETTING_KEYS).toHaveLength(5); + }); + + it("defaults are the SAFE posture (decision record 2026-10-09)", () => { + expect(DEFAULT_INSTANCE_SETTINGS.youtube_full_harvest).toBe(false); // yt-dlp OFF + expect(DEFAULT_INSTANCE_SETTINGS.youtube_cc_only_broadcast).toBe(true); // CC-only ON + expect(DEFAULT_INSTANCE_SETTINGS.scrape_metadata_only).toBe(true); // locked platforms: metadata + expect(DEFAULT_INSTANCE_SETTINGS.scrape_bounded_pilot).toBe(false); // no bounded X pool + expect(DEFAULT_INSTANCE_SETTINGS.intake_posture).toBe("metadata_only"); + }); + + it("only Owner/Admin are instance admins", () => { + expect(INSTANCE_ADMIN_ROLES).toEqual(["Owner", "Admin"]); + for (const role of ["Owner", "Admin"]) expect(isInstanceAdmin(role)).toBe(true); + for (const role of ["Publisher", "Editor", "Approver", "Analyst", "Moderator", "BroadcastOperator", ""]) { + expect(isInstanceAdmin(role)).toBe(false); + } + }); + + it("posture enum accepts exactly the three postures", () => { + for (const p of INTAKE_POSTURES) expect(isIntakePosture(p)).toBe(true); + for (const bad of ["metadata", "", "METADATA_ONLY", "full", undefined, null, 1]) { + expect(isIntakePosture(bad)).toBe(false); + } + }); +}); + +describe("instance settings — patch validation", () => { + it("a non-object body rejects before any query is issued", async () => { + await expect(service.updateSettings("x", admin)).rejects.toBeInstanceOf(BadRequestException); + }); + + it("rejects non-object bodies", async () => { + for (const bad of [undefined, null, "nope", 42, [], true]) { + await expect(service.updateSettings(bad, admin)).rejects.toBeInstanceOf(BadRequestException); + } + }); + + it("rejects unknown keys instead of silently dropping them", async () => { + await expect( + service.updateSettings({ youtube_full_harvest: true, allow_all_scraping: true }, admin), + ).rejects.toMatchObject({ status: 400, message: expect.stringMatching(/unknown setting/) }); + }); + + it("rejects non-boolean values for the boolean switches", async () => { + for (const bad of ["true", 1, null, undefined]) { + await expect(service.updateSettings({ scrape_bounded_pilot: bad }, admin)).rejects.toMatchObject({ status: 400 }); + } + }); + + it("rejects postures outside the enum", async () => { + for (const bad of ["full", "metadata-only", "", "METADATA_ONLY", true, null]) { + await expect(service.updateSettings({ intake_posture: bad }, admin)).rejects.toMatchObject({ status: 400 }); + } + }); + + it("rejects an empty patch", async () => { + await expect(service.updateSettings({}, admin)).rejects.toMatchObject({ status: 400, message: expect.stringMatching(/at least one/) }); + }); + + it("non-admin actors are refused in the service layer too (defense in depth)", async () => { + for (const role of ["Publisher", "Editor", "Approver", "Analyst", "Moderator", "BroadcastOperator"]) { + await expect(service.updateSettings({ intake_posture: "bounded_pilot" }, { userId: admin.userId, role })).rejects.toBeInstanceOf( + BadRequestException, + ); + } + }); +}); + +describe("instance settings — controller role gate", () => { + const controller = new SettingsController(service); + + const req = (role: string) => ({ session: { userId: admin.userId, workspaceId: "ws", role, authzEpoch: 1, issuedAt: 0, expiresAt: 1 } }); + + it("GET /settings is 403 for non-admin roles", async () => { + for (const role of ["Publisher", "Editor", "Approver", "Analyst", "Moderator", "StreamOperator", "VideoPublisher"]) { + await expect(controller.get(req(role) as never)).rejects.toBeInstanceOf(ForbiddenException); + } + }); + + it("PATCH /settings is 403 for non-admin roles", async () => { + for (const role of ["Publisher", "Editor", "Approver", "Analyst", "Moderator"]) { + await expect(controller.patch(req(role) as never, { intake_posture: "bounded_pilot" })).rejects.toBeInstanceOf( + ForbiddenException, + ); + } + }); + + it("Owner and Admin pass the controller gate (service then owns the rest)", async () => { + // A non-DB-backed service: getSettings would need the pool. Assert the + // gate itself by checking the failure is NOT Forbidden for admins. + for (const role of ["Owner", "Admin"]) { + const getErr = await controller.get(req(role) as never).then( + () => null, + (e: unknown) => e, + ); + expect(getErr).not.toBeInstanceOf(ForbiddenException); + const patchErr = await controller.patch(req(role) as never, {}).then( + () => null, + (e: unknown) => e, + ); + expect(patchErr).not.toBeInstanceOf(ForbiddenException); + } + }); +}); diff --git a/apps/api/src/storage/files-config.test.ts b/apps/api/src/storage/files-config.test.ts new file mode 100644 index 0000000..b289020 --- /dev/null +++ b/apps/api/src/storage/files-config.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from "vitest"; +import { filesIntegrationConfigFromEnv } from "./files-config.js"; + +// Synthetic non-secret account ID, used only for pure config tests; no network I/O. +const env = { + FILES_R2_ENDPOINT: `https://${"a".repeat(32)}.r2.cloudflarestorage.com`, + FILES_RESERVED_BUCKETS: "occupymedia-peertube-videos,occupymedia-peertube-playlists", +}; + +describe("files integration config", () => { + it("keeps payload ownership separate with no disk cache or pad implementation", () => { + const config = filesIntegrationConfigFromEnv(env); + expect(new Set(Object.values(config.buckets)).size).toBe(3); + expect(config).toMatchObject({ localPayloadCache: false, localMetadataAllowed: true, padsEnabled: false }); + expect(config.r2).toMatchObject({ region: "auto", forcePathStyle: true }); + }); + + it("rejects reuse of Matrix, Poster, or PeerTube buckets for files", () => { + for (const FILES_R2_BUCKET of ["occupymedia-matrix-media", "occupymedia-media-encrypted", "occupymedia-peertube-videos", "occupymedia-peertube-playlists"]) { + expect(() => filesIntegrationConfigFromEnv({ ...env, FILES_R2_BUCKET })).toThrow("separate"); + } + expect(() => filesIntegrationConfigFromEnv({ ...env, MATRIX_R2_BUCKET: "shared", S3_BUCKET: "shared" })).toThrow("separate"); + }); + + it("rejects absent/non-R2 endpoints and unsafe public URLs", () => { + expect(() => filesIntegrationConfigFromEnv({})).toThrow(); + for (const FILES_R2_ENDPOINT of ["http://localhost:9000", "https://s3.amazonaws.com", `${env.FILES_R2_ENDPOINT}/bucket`]) { + expect(() => filesIntegrationConfigFromEnv({ FILES_R2_ENDPOINT })).toThrow(); + } + for (const FILES_PUBLIC_URL of ["http://files.example.org", "https://user:password@files.example.org", "https://files.example.org/?token=x"]) { + expect(() => filesIntegrationConfigFromEnv({ ...env, FILES_PUBLIC_URL })).toThrow(); + } + expect(() => filesIntegrationConfigFromEnv({ ...env, FILES_R2_BUCKET: "" })).toThrow("bucket names"); + }); +}); diff --git a/apps/api/src/storage/files-config.ts b/apps/api/src/storage/files-config.ts new file mode 100644 index 0000000..6274f77 --- /dev/null +++ b/apps/api/src/storage/files-config.ts @@ -0,0 +1,54 @@ +/** Opt-in foundation config. Not loaded by API startup or the existing media worker. */ +export interface FilesIntegrationConfig { + payloadStore: "r2"; + localPayloadCache: false; + localMetadataAllowed: true; + r2: { endpoint: string; region: "auto"; forcePathStyle: true }; + buckets: { matrixMedia: string; filesBlobs: string; posterMedia: string }; + filesUrl: string; + officeUrl: string; + /** Disabled until a pad adapter can keep text/revision payloads in R2. */ + padsEnabled: false; +} + +type Environment = Readonly>; + +function httpsOrigin(value: string, name: string): string { + const url = new URL(value); + if (url.protocol !== "https:" || url.username || url.password || url.pathname !== "/" || url.search || url.hash) { + throw new Error(`${name} must be an HTTPS origin without credentials, path or query`); + } + return url.origin; +} + +export function filesIntegrationConfigFromEnv(env: Environment): FilesIntegrationConfig { + const endpoint = httpsOrigin(env.FILES_R2_ENDPOINT ?? "", "FILES_R2_ENDPOINT"); + if (!/^[a-f0-9]{32}\.r2\.cloudflarestorage\.com$/.test(new URL(endpoint).hostname)) { + throw new Error("FILES_R2_ENDPOINT must use the account R2 S3 endpoint"); + } + const buckets = { + matrixMedia: env.MATRIX_R2_BUCKET ?? "occupymedia-matrix-media", + filesBlobs: env.FILES_R2_BUCKET ?? "occupymedia-files-blobs", + posterMedia: env.S3_BUCKET ?? "occupymedia-media-encrypted", + }; + const names = Object.values(buckets); + if (names.some(name => !/^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$/.test(name))) { + throw new Error("Payload buckets must be valid R2 bucket names"); + } + // Other product buckets are supplied by deployment, not provider branching here. + const reserved = env.FILES_RESERVED_BUCKETS?.split(",").map(name => name.trim()); + if (!reserved?.length || reserved.some(name => !/^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$/.test(name))) { + throw new Error("FILES_RESERVED_BUCKETS must list other products' payload buckets"); + } + const exclusive = [...names, ...reserved]; + if (new Set(exclusive).size !== exclusive.length) { + throw new Error("Every product must have separate payload buckets"); + } + return { + payloadStore: "r2", localPayloadCache: false, localMetadataAllowed: true, + r2: { endpoint, region: "auto", forcePathStyle: true }, buckets, + filesUrl: httpsOrigin(env.FILES_PUBLIC_URL ?? "https://files.occupymedia.cc", "FILES_PUBLIC_URL"), + officeUrl: httpsOrigin(env.OFFICE_PUBLIC_URL ?? "https://office.occupymedia.cc", "OFFICE_PUBLIC_URL"), + padsEnabled: false, + }; +} diff --git a/apps/api/src/storage/storage.integration.test.ts b/apps/api/src/storage/storage.integration.test.ts new file mode 100644 index 0000000..b7ca992 --- /dev/null +++ b/apps/api/src/storage/storage.integration.test.ts @@ -0,0 +1,60 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { S3Storage } from "./storage.js"; + +/** + * Live roundtrip against any S3-compatible endpoint (S3_ENDPOINT env; + * dev = MinIO from infra compose). Skipped without env — the pure key-scheme + * suite in storage.test.ts always runs. + */ +const cfg = process.env.S3_ENDPOINT; + +describe.skipIf(cfg === undefined)("S3 storage (live endpoint)", () => { + let storage: S3Storage; + + beforeAll(() => { + storage = new S3Storage({ + endpoint: cfg, + region: process.env.S3_REGION ?? "us-east-1", + accessKeyId: process.env.S3_ACCESS_KEY ?? "tactical-media-broadcast-hub", + secretAccessKey: process.env.S3_SECRET_KEY ?? "tactical-media-broadcast-hub_dev", + bucket: process.env.S3_BUCKET ?? "tactical-media-broadcast-hub-media-test", + }); + }); + + afterAll(async () => { + // bucket intentionally retained for inspection; content is content-addressed test bytes + void storage; + }); + + it("put → get roundtrips bytes", async () => { + const payload = Buffer.from(`qa-roundtrip-${Date.now()}`); + const stored = await storage.putContent(payload, "image/png"); + expect(stored.deduplicated).toBe(false); + const back = await storage.getObject(stored.key); + expect(back.equals(payload)).toBe(true); + }); + + it("identical bytes deduplicate (second put is a no-op)", async () => { + const payload = Buffer.from("dedupe-me-identical"); + const first = await storage.putContent(payload, "video/mp4"); + const second = await storage.putContent(Buffer.from("dedupe-me-identical"), "video/mp4"); + expect(first.deduplicated).toBe(false); + expect(second.deduplicated).toBe(true); + expect(second.key).toBe(first.key); + expect(await storage.hasContent(second.hash, "video/mp4")).toBe(true); + }); + + it("same bytes, different content-type land on different keys", async () => { + const payload = Buffer.from("multi-format"); + const asPng = await storage.putContent(payload, "image/png"); + const asMp4 = await storage.putContent(payload, "video/mp4"); + expect(asPng.key).not.toBe(asMp4.key); + }); + + it("presigns a GET url", async () => { + const stored = await storage.putContent(Buffer.from("presign-me"), "image/png"); + const url = await storage.presign(stored.key, 60); + expect(url).toContain(encodeURIComponent(stored.key).split("%2F")[1] ?? stored.key.slice(7)); + expect(url.startsWith("http")).toBe(true); + }); +}); diff --git a/apps/api/src/storage/storage.test.ts b/apps/api/src/storage/storage.test.ts new file mode 100644 index 0000000..380c8cb --- /dev/null +++ b/apps/api/src/storage/storage.test.ts @@ -0,0 +1,118 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { createHash, randomBytes } from "node:crypto"; + +const s3Calls = vi.hoisted(() => ({ sent: [] as string[] })); + +vi.mock("@aws-sdk/client-s3", () => { + class FakeCommand { + constructor(readonly input: unknown, readonly marker: string) {} + } + class S3ServiceException extends Error { + readonly $metadata: { httpStatusCode?: number }; + constructor(message: string, status: number, name: string) { + super(message); + this.name = name; + this.$metadata = { httpStatusCode: status }; + } + } + return { + CreateBucketCommand: class extends FakeCommand { constructor(input: unknown) { super(input, "Create"); } }, + HeadObjectCommand: class extends FakeCommand { constructor(input: unknown) { super(input, "Head"); } }, + PutObjectCommand: class extends FakeCommand { constructor(input: unknown) { super(input, "Put"); } }, + GetObjectCommand: class extends FakeCommand { constructor(input: unknown) { super(input, "Get"); } }, + S3ServiceException, + S3Client: class { + constructor(readonly config: unknown) {} + async send(cmd: FakeCommand): Promise { + s3Calls.sent.push(cmd.marker); + if (cmd.marker === "Create") throw new S3ServiceException("Access Denied", 403, "AccessDenied"); + if (cmd.marker === "Head") throw new S3ServiceException("Not Found", 404, "NotFound"); + return {}; + } + }, + }; +}); + +import { S3Storage, extensionFor, keyForHash, open, seal, sha256 } from "./storage.js"; + +const TEST_KEK = createHash("sha256").update("test-kek").digest(); + +describe("storage key scheme (pure)", () => { + it("maps content types to extensions, unknown -> bin", () => { + expect(extensionFor("image/png")).toBe("png"); + expect(extensionFor("video/mp4")).toBe("mp4"); + expect(extensionFor("application/x-weird")).toBe("bin"); + }); + + it("keys are content-addressed and deterministic", () => { + const hash = sha256(Buffer.from("same bytes")); + expect(keyForHash(hash, "image/png")).toBe(`sha256/${hash}.png`); + expect(keyForHash(hash, "video/mp4")).toBe(`sha256/${hash}.mp4`); + expect(keyForHash(sha256(Buffer.from("same bytes")), "image/png")).toBe(keyForHash(hash, "image/png")); + expect(keyForHash(sha256(Buffer.from("other")), "image/png")).not.toBe(keyForHash(hash, "image/png")); + }); + + it("hash is stable across buffer instances", () => { + expect(sha256(Buffer.from("abc"))).toBe(sha256(Buffer.from("abc"))); + expect(sha256(Buffer.from("abc"))).toHaveLength(64); + }); +}); + +describe("client-side encryption (pure)", () => { + it("seal → open roundtrips arbitrary bytes", () => { + const plaintext = randomBytes(1024 * 64); + const sealed = seal(plaintext, TEST_KEK); + expect(open(sealed, TEST_KEK).equals(plaintext)).toBe(true); + }); + + it("ciphertext never contains plaintext (endpoint sees only sealed frame)", () => { + const plaintext = Buffer.from("TOPSECRET-media-bytes-123"); + const sealed = seal(plaintext, TEST_KEK); + expect(sealed.includes(plaintext)).toBe(false); + }); + + it("fresh CEK per seal → identical plaintext, different ciphertexts (dedup still by key)", () => { + const plaintext = Buffer.from("identical"); + const a = seal(plaintext, TEST_KEK); + const b = seal(plaintext, TEST_KEK); + expect(a.equals(b)).toBe(false); + expect(open(a, TEST_KEK).equals(open(b, TEST_KEK))).toBe(true); + }); + + it("tampered ciphertext fails GCM authentication", () => { + const sealed = seal(Buffer.from("authentic"), TEST_KEK); + const last = sealed.length - 1; + sealed[last] = (sealed[last] ?? 0) ^ 0xff; + expect(() => open(sealed, TEST_KEK)).toThrow(); + }); + + it("wrong KEK cannot open", () => { + const otherKek = createHash("sha256").update("other-kek").digest(); + const sealed = seal(Buffer.from("private"), TEST_KEK); + expect(() => open(sealed, otherKek)).toThrow(); + }); + + it("plaintext hash addressing + seal preserves dedup semantics", () => { + const a = Buffer.from("media"); + const b = Buffer.from("media"); + expect(sha256(a)).toBe(sha256(b)); + expect(keyForHash(sha256(a), "video/mp4")).toBe(keyForHash(sha256(b), "video/mp4")); + }); +}); + +describe("S3Storage with object-scoped credentials", () => { + beforeEach(() => { s3Calls.sent.length = 0; }); + + it("stores despite 403 on bucket create (pre-provisioned bucket, least-privilege token)", async () => { + const storage = new S3Storage({ + endpoint: "https://s3.example.invalid", + accessKeyId: "scoped-key", + secretAccessKey: "scoped-secret", + bucket: "pre-provisioned", + mediaKek: TEST_KEK, + }); + const stored = await storage.putContent(Buffer.from("scoped-token-probe"), "text/plain"); + expect(stored.deduplicated).toBe(false); + expect(s3Calls.sent).toEqual(["Create", "Head", "Put"]); + }); +}); diff --git a/apps/api/src/storage/storage.ts b/apps/api/src/storage/storage.ts new file mode 100644 index 0000000..2084774 --- /dev/null +++ b/apps/api/src/storage/storage.ts @@ -0,0 +1,224 @@ +import { createCipheriv, createDecipheriv, createHash, randomBytes } from "node:crypto"; +import { + CreateBucketCommand, + GetObjectCommand, + HeadObjectCommand, + PutObjectCommand, + S3Client, + S3ServiceException, +} from "@aws-sdk/client-s3"; +import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; + +/** + * S3-compatible object storage — the ONLY media I/O path (user directive: + * all archive/grabber content lives on object storage, ENCRYPTED — user + * directive 2026-10-06). Client-side envelope encryption: every object body + * is AES-256-GCM sealed with a random CEK; the CEK is wrapped by the media + * KEK (env S3_KEK). The endpoint never sees plaintext. Content addressing + * stays on the PLAINTEXT hash (dedup survives); getObject re-verifies the + * hash, so storage-level tampering is detected. SSE-at-rest headers are set + * additionally for endpoints that honor them (belt-and-braces). + */ + +export interface StoredObject { + key: string; + hash: string; + sizeBytes: number; + contentType: string; +} + +export interface StorageService { + putContent(buffer: Buffer, contentType: string): Promise; + getObject(key: string): Promise; + hasContent(hash: string, contentType: string): Promise; + presign(key: string, ttlSeconds: number): Promise; +} + +const EXTENSIONS: Record = { + "image/png": "png", + "image/jpeg": "jpg", + "image/webp": "webp", + "image/gif": "gif", + "video/mp4": "mp4", + "video/webm": "webm", + "video/quicktime": "mov", + "application/pdf": "pdf", +}; + +export function extensionFor(contentType: string): string { + return EXTENSIONS[contentType] ?? "bin"; +} + +export function keyForHash(hash: string, contentType: string): string { + return `sha256/${hash}.${extensionFor(contentType)}`; +} + +export function sha256(buffer: Buffer): string { + return createHash("sha256").update(buffer).digest("hex"); +} + +const FRAME_MAGIC = Buffer.from("SPENC1"); + +/** Seal plaintext under the KEK: returns [iv(12) | wrappedCEK | ciphertext+tag]. */ +export function seal(plaintext: Buffer, kek: Buffer): Buffer { + const cek = randomBytes(32); + const iv = randomBytes(12); + const sealedCek = aesGcmSeal(kek, iv, cek); + const cipher = createCipheriv("aes-256-gcm", cek, iv); + const body = Buffer.concat([cipher.update(plaintext), cipher.final(), cipher.getAuthTag()]); + return Buffer.concat([FRAME_MAGIC, iv, sealedCek, body]); +} + +/** Open a sealed frame; verifies GCM tags (CEK wrap + body) and returns plaintext. */ +export function open(sealed: Buffer, kek: Buffer): Buffer { + if (sealed.subarray(0, FRAME_MAGIC.length).equals(FRAME_MAGIC) === false) { + throw new Error("storage: not an encrypted frame (magic mismatch)"); + } + let offset = FRAME_MAGIC.length; + const iv = sealed.subarray(offset, offset + 12); + offset += 12; + const cekLen = 32 + 16; + const sealedCek = sealed.subarray(offset, offset + cekLen); + offset += cekLen; + const body = sealed.subarray(offset); + const cek = aesGcmOpen(kek, iv, sealedCek); + const tag = body.subarray(body.length - 16); + const ciphertext = body.subarray(0, body.length - 16); + const decipher = createDecipheriv("aes-256-gcm", cek, iv); + decipher.setAuthTag(tag); + return Buffer.concat([decipher.update(ciphertext), decipher.final()]); +} + +function aesGcmSeal(key: Buffer, iv: Buffer, plaintext: Buffer): Buffer { + const cipher = createCipheriv("aes-256-gcm", key, iv); + return Buffer.concat([cipher.update(plaintext), cipher.final(), cipher.getAuthTag()]); +} + +function aesGcmOpen(key: Buffer, iv: Buffer, sealed: Buffer): Buffer { + const tag = sealed.subarray(sealed.length - 16); + const decipher = createDecipheriv("aes-256-gcm", key, iv); + decipher.setAuthTag(tag); + return Buffer.concat([decipher.update(sealed.subarray(0, sealed.length - 16)), decipher.final()]); +} + +function loadMediaKek(): Buffer { + const env = process.env.S3_KEK; + if (env !== undefined && env !== "") { + const kek = Buffer.from(env, "base64"); + if (kek.length !== 32) throw new Error("S3_KEK must be 32 bytes base64"); + return kek; + } + if (process.env.NODE_ENV === "production") { + throw new Error("S3_KEK is required in production (media encryption is mandatory)"); + } + return createHash("sha256").update(`dev-media-kek:${process.env.HOSTNAME ?? "local"}`).digest(); +} + +export interface S3StorageConfig { + endpoint?: string; + region?: string; + accessKeyId?: string; + secretAccessKey?: string; + bucket?: string; + forcePathStyle?: boolean; + mediaKek?: Buffer; +} + +export function s3ConfigFromEnv(): S3StorageConfig { + return { + endpoint: process.env.S3_ENDPOINT, + region: process.env.S3_REGION ?? "us-east-1", + accessKeyId: process.env.S3_ACCESS_KEY, + secretAccessKey: process.env.S3_SECRET_KEY, + bucket: process.env.S3_BUCKET ?? "tactical-media-broadcast-hub-media", + forcePathStyle: process.env.S3_FORCE_PATH_STYLE !== "false", + }; +} + +export class S3Storage implements StorageService { + private readonly client: S3Client; + private readonly bucket: string; + private readonly kek: Buffer; + private bucketReady = false; + + constructor(config: S3StorageConfig) { + this.bucket = config.bucket ?? "tactical-media-broadcast-hub-media"; + this.kek = config.mediaKek ?? loadMediaKek(); + this.client = new S3Client({ + endpoint: config.endpoint, + region: config.region ?? "us-east-1", + credentials: + config.accessKeyId !== undefined && config.secretAccessKey !== undefined + ? { accessKeyId: config.accessKeyId, secretAccessKey: config.secretAccessKey } + : undefined, + forcePathStyle: config.forcePathStyle ?? true, + }); + } + + private async ensureBucket(): Promise { + if (this.bucketReady) return; + try { + await this.client.send(new CreateBucketCommand({ Bucket: this.bucket })); + } catch (err) { + const status = err instanceof S3ServiceException ? err.$metadata.httpStatusCode : undefined; + const name = err instanceof S3ServiceException ? err.name : ""; + // 403: least-privilege object tokens (e.g. R2 keys scoped to existing + // buckets) cannot create buckets. The bucket is pre-provisioned, so + // proceed — a genuinely missing bucket fails loudly on the real operation. + if (status !== 409 && name !== "BucketAlreadyOwnedByYou" && status !== 403) throw err; + } + this.bucketReady = true; + } + + private async headExists(key: string): Promise { + try { + await this.client.send(new HeadObjectCommand({ Bucket: this.bucket, Key: key })); + return true; + } catch (err) { + const status = err instanceof S3ServiceException ? err.$metadata.httpStatusCode : undefined; + const name = err instanceof S3ServiceException ? err.name : ""; + if (status === 404 || name === "NotFound") return false; + throw err; + } + } + + async putContent(buffer: Buffer, contentType: string): Promise { + await this.ensureBucket(); + const hash = sha256(buffer); + const key = keyForHash(hash, contentType); + if (await this.headExists(key)) { + return { key, hash, sizeBytes: buffer.byteLength, contentType, deduplicated: true }; + } + await this.client.send( + new PutObjectCommand({ + Bucket: this.bucket, + Key: key, + Body: seal(buffer, this.kek), + ContentType: contentType, + ServerSideEncryption: "AES256", + }), + ); + return { key, hash, sizeBytes: buffer.byteLength, contentType, deduplicated: false }; + } + + async getObject(key: string): Promise { + const res = await this.client.send(new GetObjectCommand({ Bucket: this.bucket, Key: key })); + if (res.Body === undefined) throw new Error(`empty body for ${key}`); + const stored = Buffer.from(await res.Body.transformToByteArray()); + const plaintext = open(stored, this.kek); + const hashFromKey = key.split("/")[1]?.split(".")[0]; + if (hashFromKey !== undefined && sha256(plaintext) !== hashFromKey) { + throw new Error(`storage: integrity failure for ${key} (plaintext hash mismatch)`); + } + return plaintext; + } + + async hasContent(hash: string, contentType: string): Promise { + await this.ensureBucket(); + return this.headExists(keyForHash(hash, contentType)); + } + + async presign(key: string, ttlSeconds: number): Promise { + return getSignedUrl(this.client, new GetObjectCommand({ Bucket: this.bucket, Key: key }), { expiresIn: ttlSeconds }); + } +} diff --git a/apps/api/src/team/invite-link.controller.ts b/apps/api/src/team/invite-link.controller.ts new file mode 100644 index 0000000..0b9584d --- /dev/null +++ b/apps/api/src/team/invite-link.controller.ts @@ -0,0 +1,95 @@ +import { Body, Controller, Delete, Get, HttpCode, Inject, Param, Post, Req, Res, UnauthorizedException, UseGuards } from "@nestjs/common"; +import type { Request, Response } from "express"; +import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import { + optionalEmail, + optionalString, + parseId, + requireObject, + requirePositiveInt, + requireSession, + requireString, +} from "../http/validation.js"; +import { verifySession } from "../auth/session.js"; +import { + InviteLinkService, + type InviteLink, + type InviteLinkCreateInput, + type InviteLinkInfo, +} from "./invite-link.service.js"; +import type { AuthResult } from "../auth/auth.service.js"; + +/** + * Admin surface for invite links, extending the `team` namespace: mint a + * multi-use link (token returned ONCE), list them (never the token), revoke. + */ +@Controller("team") +@UseGuards(RbacGuard) +export class InviteLinkController { + constructor(@Inject(InviteLinkService) private readonly links: InviteLinkService) {} + + @Post("invite-links") + @RequirePermission("workspace.members.manage") + async create(@Req() request: Request & AuthenticatedRequest, @Body() body: unknown): Promise { + const session = requireSession(request); + const dto = requireObject(body, "body"); + const input: InviteLinkCreateInput = { role: requireString(dto.role, "role"), expiresInHours: requirePositiveInt(dto.expiresInHours, "expiresInHours", 720) }; + const maxUses = dto.maxUses === undefined || dto.maxUses === null ? undefined : requirePositiveInt(dto.maxUses, "maxUses", 10_000); + return this.links.create(session.workspaceId, { userId: session.userId, role: session.role }, { ...input, ...(maxUses === undefined ? {} : { maxUses }) }); + } + + @Get("invite-links") + @RequirePermission("workspace.members.manage") + async list(@Req() request: Request & AuthenticatedRequest): Promise { + return this.links.list(requireSession(request).workspaceId); + } + + @Delete("invite-links/:id") + @RequirePermission("workspace.members.manage") + async revoke(@Req() request: Request & AuthenticatedRequest, @Param("id") id: string): Promise { + const session = requireSession(request); + return this.links.revoke(session.workspaceId, { userId: session.userId, role: session.role }, parseId(id)); + } +} + +/** + * Public claim surface. PATH-based routes keep the token out of the request + * body/log pairing issues while matching the SPA's route shape; there is NO + * RbacGuard here (the claimer may not be authenticated yet), so any presented + * Bearer session is verified manually, exactly like AuthController.reissue. + */ +@Controller("invites") +export class InviteClaimController { + constructor(@Inject(InviteLinkService) private readonly links: InviteLinkService) {} + + @Get(":token/info") + async info(@Param("token") token: string, @Res({ passthrough: true }) response: Response): Promise { + response.setHeader("Cache-Control", "no-store"); + return this.links.info(requireString(token, "token")); + } + + @Post(":token/claim") + @HttpCode(201) + async claim(@Param("token") token: string, @Req() request: Request, @Body() body: unknown): Promise { + const dto = requireObject(body, "body"); + const authorization = request.headers.authorization; + if (authorization !== undefined && !authorization.startsWith("Bearer ")) throw new UnauthorizedException(); + const session = authorization === undefined ? undefined : verifySession(authorization.slice("Bearer ".length)); + if (authorization !== undefined && session === null) throw new UnauthorizedException(); + // Shape-only extraction: username/password FORMAT validation lives in the + // service identity block, which runs after link validation so a bogus + // token only ever sees 404 here. + const username = optionalString(dto.username, "username"); + const password = optionalString(dto.password, "password"); + const displayName = optionalString(dto.displayName, "displayName"); + const email = optionalEmail(dto.email); + return this.links.claim({ + token: requireString(token, "token"), + ...(session === undefined ? {} : { session }), + ...(username === undefined ? {} : { username }), + ...(password === undefined ? {} : { password }), + ...(displayName === undefined ? {} : { displayName }), + ...(email === undefined ? {} : { email }), + }); + } +} diff --git a/apps/api/src/team/invite-link.integration.test.ts b/apps/api/src/team/invite-link.integration.test.ts new file mode 100644 index 0000000..4dd577b --- /dev/null +++ b/apps/api/src/team/invite-link.integration.test.ts @@ -0,0 +1,275 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { join } from "node:path"; +import { Pool, type PoolClient } from "pg"; + +/** + * Invite-link lifecycle against a live PostgreSQL, as sp_app (FORCE RLS binds). + * Uses its OWN database (INVITE_IT_DATABASE_URL, plus INVITE_IT_SUPER_DATABASE_URL + * for migrations/seeding) for the same reason as auth.integration.test.ts: + * tenancy.integration.test.ts wipes every workspace in the shared DB and vitest + * runs files in parallel. Skipped when unset. + */ +const appUrl = process.env.INVITE_IT_DATABASE_URL; +const ownerUrl = process.env.INVITE_IT_SUPER_DATABASE_URL; + +type Modules = { + links: typeof import("./invite-link.service.js"); + auth: typeof import("../auth/auth.service.js"); + session: typeof import("../auth/session.js"); + db: typeof import("../db/client.js"); +}; + +describe.skipIf(appUrl === undefined || ownerUrl === undefined)("invite links (live DB)", () => { + let m: Modules; + let ownerPool: Pool; + const suffix = `${Date.now()}${Math.floor(Math.random() * 1e6)}`; + const ownerUsername = `invowner${suffix}`; + let workspaceId: string; + let actor: { userId: string; role: string }; + + async function asOwner(fn: (client: PoolClient) => Promise): Promise { + const client = await ownerPool.connect(); + try { + return await fn(client); + } finally { + client.release(); + } + } + + beforeAll(async () => { + // The pool reads DATABASE_URL at import time: point it at the isolated DB first. + process.env.DATABASE_URL = appUrl; + process.env.SUPER_DATABASE_URL = ownerUrl; + const { runMigrations } = await import("../db/migrate.js"); + await runMigrations(join(process.cwd(), "migrations")); + m = { + links: await import("./invite-link.service.js"), + auth: await import("../auth/auth.service.js"), + session: await import("../auth/session.js"), + db: await import("../db/client.js"), + }; + ownerPool = new Pool({ connectionString: ownerUrl }); + + const auth = new m.auth.AuthService(); + const reg = await auth.register({ + workspaceName: "Invite IT", + username: ownerUsername, + password: "correct-horse-1", + displayName: "Owner", + email: `invowner-${suffix}@example.com`, + }); + workspaceId = reg.me.workspace.id; + const session = m.session.verifySession(reg.token); + if (session === null) throw new Error("session did not verify"); + actor = { userId: session.userId, role: session.role }; + }); + + afterAll(async () => { + if (ownerPool !== undefined) { + await asOwner(async (c) => { + await c.query("DELETE FROM workspaces WHERE id = $1", [workspaceId]); + await c.query("DELETE FROM users WHERE username = ANY($1::text[])", [ + [ownerUsername, ...claimedUsernames].filter((u): u is string => u !== null), + ]); + }); + await ownerPool.end(); + } + await m?.db.pool.end(); + }); + + /** Claimed usernames, for teardown. */ + const claimedUsernames: Array = []; + + /** Directly rewrite a link's state through the owner pool (inside the tenant context — RLS binds the owner too). */ + async function forceLinkState(linkId: string, sql: string, params: unknown[] = []): Promise { + await asOwner(async (c) => { + await c.query("BEGIN"); + await c.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]); + await c.query(sql, [linkId, ...params]); + await c.query("COMMIT"); + }); + } + + async function membershipRole(userId: string): Promise<{ role: string; status: string } | undefined> { + return asOwner(async (c) => { + await c.query("BEGIN"); + await c.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]); + const rows = ( + await c.query("SELECT role, status FROM memberships WHERE workspace_id = $1 AND user_id = $2", [workspaceId, userId]) + ).rows as Array<{ role: string; status: string }>; + await c.query("COMMIT"); + return rows[0]; + }); + } + + async function useCount(linkId: string): Promise { + return asOwner(async (c) => { + await c.query("BEGIN"); + await c.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]); + const rows = (await c.query("SELECT use_count FROM workspace_invite_links WHERE id = $1", [linkId])).rows as Array<{ use_count: number }>; + await c.query("COMMIT"); + return rows[0]?.use_count ?? -1; + }); + } + + async function auditActions(targetId: string): Promise { + return asOwner(async (c) => { + await c.query("BEGIN"); + await c.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]); + const rows = (await c.query("SELECT action, diff FROM audit_log WHERE target_id = $1 ORDER BY seq ASC", [targetId])).rows as Array<{ + action: string; + diff: Record | null; + }>; + await c.query("COMMIT"); + return rows.map((r) => r.action); + }); + } + + it("create -> info -> claim with a fresh username; the response is an AuthResult and the membership is active", async () => { + const links = new m.links.InviteLinkService(); + const created = await links.create(workspaceId, actor, { role: "Publisher", expiresInHours: 24, maxUses: 5 }); + expect(created.token).toMatch(/^[A-Za-z0-9_-]{40,}$/); + expect(created.url).toContain(`/join/${created.token}`); + expect(created.status).toBe("active"); + expect(created.useCount).toBe(0); + + const info = await links.info(created.token as string); + expect(info.workspaceName).toBe("Invite IT"); + expect(info.role).toBe("Publisher"); + expect(info.maxUses).toBe(5); + expect(info.usedCount).toBe(0); + await expect(links.info("no-such-token")).rejects.toMatchObject({ status: 404 }); + + const username = `joiner${suffix}`; + claimedUsernames.push(username); + const result = await links.claim({ token: created.token as string, username, password: "correct-horse-1" }); + expect(result.me.role).toBe("Publisher"); + expect(result.me.user.username).toBe(username); + expect(result.me.workspace.id).toBe(workspaceId); + expect(result.me.permissions).toContain("draft.create"); + const session = m.session.verifySession(result.token); + if (session === null) throw new Error("claim token did not verify"); + expect(session.workspaceId).toBe(workspaceId); + const membership = await membershipRole(session.userId); + expect(membership).toEqual({ role: "Publisher", status: "active" }); + expect(await useCount(created.id)).toBe(1); + expect(await auditActions(created.id)).toEqual(["invite_link.created"]); + expect(await auditActions(session.userId)).toEqual([]); // audit targets the membership id, not the user + }); + + it("double-claim by the same user is 409 and does not consume a use", async () => { + const links = new m.links.InviteLinkService(); + const created = await links.create(workspaceId, actor, { role: "Approver", expiresInHours: 24 }); + const username = `twice${suffix}`; + claimedUsernames.push(username); + const first = await links.claim({ token: created.token as string, username, password: "correct-horse-1", displayName: "Twice" }); + const userId = m.session.verifySession(first.token)?.userId; + if (userId === undefined) throw new Error("no session"); + await expect(links.claim({ token: created.token as string, username, password: "correct-horse-1" })).rejects.toMatchObject({ + status: 409, + }); + // Still 409 (not a 401) when the credentials are presented via Bearer instead. + const auth = new m.auth.AuthService(); + await expect(auth.login({ username, password: "correct-horse-1" })).resolves.toMatchObject({ me: { role: "Approver" } }); + expect(await useCount(created.id)).toBe(1); // unchanged: no partial consumption + }); + + it("an expired link is 410 on info and claim", async () => { + const links = new m.links.InviteLinkService(); + const created = await links.create(workspaceId, actor, { role: "Publisher", expiresInHours: 1 }); + await forceLinkState(created.id, "UPDATE workspace_invite_links SET expires_at = now() - interval '1 hour' WHERE id = $1"); + await expect(links.info(created.token as string)).rejects.toMatchObject({ status: 410 }); + const username = `late${suffix}`; + await expect(links.claim({ token: created.token as string, username, password: "correct-horse-1" })).rejects.toMatchObject({ + status: 410, + }); + // Rolled back: the whole claim transaction (user insert included) undid. + await expect(userByExternalId(username)).rejects.toThrow(/missing/); + expect((await links.list(workspaceId)).find((l) => l.id === created.id)?.status).toBe("expired"); + }); + + it("a link that exhausts max_uses is 410, and exhaustion is visible in list", async () => { + const links = new m.links.InviteLinkService(); + const created = await links.create(workspaceId, actor, { role: "Publisher", expiresInHours: 24, maxUses: 1 }); + const first = `single${suffix}`; + claimedUsernames.push(first); + await links.claim({ token: created.token as string, username: first, password: "correct-horse-1" }); + expect(await useCount(created.id)).toBe(1); + await expect(links.info(created.token as string)).rejects.toMatchObject({ status: 410 }); + const second = `second${suffix}`; + await expect(links.claim({ token: created.token as string, username: second, password: "correct-horse-1" })).rejects.toMatchObject({ + status: 410, + }); + const listed = (await links.list(workspaceId)).find((l) => l.id === created.id); + expect(listed?.status).toBe("exhausted"); + }); + + it("a revoked link is 410, revoking twice is 409, and an unknown id is 404", async () => { + const links = new m.links.InviteLinkService(); + const created = await links.create(workspaceId, actor, { role: "Publisher", expiresInHours: 24 }); + const revoked = await links.revoke(workspaceId, actor, created.id); + expect(revoked.status).toBe("revoked"); + expect(revoked.revokedAt).not.toBeNull(); + await expect(links.info(created.token as string)).rejects.toMatchObject({ status: 410 }); + const username = `postrev${suffix}`; + await expect(links.claim({ token: created.token as string, username, password: "correct-horse-1" })).rejects.toMatchObject({ + status: 410, + }); + await expect(links.revoke(workspaceId, actor, created.id)).rejects.toMatchObject({ status: 409 }); + await expect(links.revoke(workspaceId, actor, "00000000-0000-4000-8000-000000000000")).rejects.toMatchObject({ status: 404 }); + }); + + it("list returns derived statuses and NEVER the token; the audit diff never contains it either", async () => { + const links = new m.links.InviteLinkService(); + const active = await links.create(workspaceId, actor, { role: "Publisher", expiresInHours: 24 }); + const listed = await links.list(workspaceId); + expect(listed.length).toBeGreaterThan(0); + for (const link of listed) { + expect(link.token).toBeUndefined(); + expect(link.url).toBeUndefined(); + expect(["active", "expired", "revoked", "exhausted"]).toContain(link.status); + } + expect(listed.find((l) => l.id === active.id)?.status).toBe("active"); + + const auditRow = await asOwner(async (c) => { + await c.query("BEGIN"); + await c.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]); + const rows = ( + await c.query("SELECT diff FROM audit_log WHERE action = 'invite_link.created' AND target_id = $1", [active.id]) + ).rows as Array<{ diff: Record | null }>; + await c.query("COMMIT"); + return rows[0]?.diff ?? null; + }); + expect(auditRow).toEqual({ role: "Publisher", expiresAt: active.expiresAt, maxUses: null }); + expect(JSON.stringify(auditRow)).not.toContain(active.token as string); + }); + + it("an authenticated (Bearer) claim joins with the existing identity; unknown roles are refused at create", async () => { + const links = new m.links.InviteLinkService(); + const username = `bearer${suffix}`; + claimedUsernames.push(username); + // Register a separate workspace to prove the claim crosses tenants. + const auth = new m.auth.AuthService(); + const reg = await auth.register({ workspaceName: "Bearer Home", username, password: "correct-horse-1", displayName: "B" }); + const session = m.session.verifySession(reg.token); + if (session === null) throw new Error("no session"); + + const created = await links.create(workspaceId, actor, { role: "Approver", expiresInHours: 24 }); + const result = await links.claim({ token: created.token as string, session }); + expect(result.me.workspace.id).toBe(workspaceId); + expect(result.me.user.username).toBe(username); + expect(await membershipRole(session.userId)).toEqual({ role: created.role, status: "active" }); + + await expect(links.create(workspaceId, actor, { role: "NotARole", expiresInHours: 24 })).rejects.toMatchObject({ status: 400 }); + await expect(auth.login({ username, password: "correct-horse-1" })).resolves.toMatchObject({ me: { workspace: { id: reg.me.workspace.id } } }); + }); + + /** Resolve a claimed username to its user id via the owner pool (users has no RLS). */ + async function userByExternalId(username: string): Promise { + return asOwner(async (c) => { + const rows = (await c.query("SELECT id FROM users WHERE username = $1", [username])).rows as Array<{ id: string }>; + if (rows[0] === undefined) throw new Error(`user ${username} missing`); + return rows[0].id; + }); + } +}); diff --git a/apps/api/src/team/invite-link.service.ts b/apps/api/src/team/invite-link.service.ts new file mode 100644 index 0000000..41be9ad --- /dev/null +++ b/apps/api/src/team/invite-link.service.ts @@ -0,0 +1,407 @@ +import { randomBytes } from "node:crypto"; +import { + BadRequestException, + ConflictException, + GoneException, + Injectable, + NotFoundException, + UnauthorizedException, +} from "@nestjs/common"; +import type { PoolClient } from "pg"; +import { appendTenantAudit } from "../audit/audit.service.js"; +import { issueSessionInTenant, localExternalId, type AuthResult, type MeResponse } from "../auth/auth.service.js"; +import { MIN_PASSWORD_LENGTH, burnPasswordCheck, hashPassword, verifyPassword } from "../auth/password.js"; +import { verifySession, type SessionPayload } from "../auth/session.js"; +import { withSystem, withTenant } from "../db/client.js"; +import { requireString, requireUsername } from "../http/validation.js"; +import type { TeamActor } from "./team.service.js"; + +export type InviteLinkStatus = "active" | "expired" | "revoked" | "exhausted"; + +/** + * Admin-facing view of a link. `token`/`url` are returned by create ONLY — + * list/get never re-expose the secret, since anyone holding it can join. + */ +export interface InviteLink { + id: string; + role: string; + createdAt: string; // ISO + expiresAt: string; // ISO + maxUses: number | null; + useCount: number; + revokedAt: string | null; + status: InviteLinkStatus; // computed server-side, like MemberStatus + createdByUserId: string; + token?: string; // create response ONLY + url?: string; // create response ONLY: `${webOrigin}/join/` — shown once, never in list/audit +} + +export interface InviteLinkCreateInput { + role: string; + expiresInHours: number; // 1..720; server computes expires_at + maxUses?: number; // absent/null = unlimited until expiry +} + +/** Public, pre-auth landing-page metadata for a token (GET /invites/:token/info). */ +export interface InviteLinkInfo { + workspaceName: string; + role: string; + expiresAt: string; + maxUses: number | null; + usedCount: number; +} + +export interface InviteClaimInput { + token: string; + /** Present when the caller brought a valid Bearer session (verified manually — this route has no guard). */ + session?: SessionPayload | null; + /** Required when `session` is absent. */ + username?: string; + password?: string; + displayName?: string; + email?: string; +} + +const OWNER = "Owner"; + +interface InviteLinkRow { + id: string; + workspace_id: string; + role: string; + created_by: string; + created_at: Date; + expires_at: Date; + max_uses: number | null; + use_count: number; + revoked_at: Date | null; +} + +interface LinkClaimRow { + id: string; + workspace_id: string; + role: string; + expires_at: Date; + max_uses: number | null; + use_count: number; + revoked_at: Date | null; +} + +const LINK_COLUMNS = "id, workspace_id, role, created_by, created_at, expires_at, max_uses, use_count, revoked_at"; + +/** Same precedence as the claim guards: revoked beats expired beats exhausted. */ +function statusOf(row: Pick): InviteLinkStatus { + if (row.revoked_at !== null) return "revoked"; + if (new Date(row.expires_at).getTime() <= Date.now()) return "expired"; + if (row.max_uses !== null && row.use_count >= row.max_uses) return "exhausted"; + return "active"; +} + +function toIso(value: Date): string { + return new Date(value).toISOString(); +} + +/** Web origin the invite URL points at; the SPA owns the /join/ landing page. */ +function publicWebUrl(): string { + return (process.env.APP_URL ?? process.env.WEB_ORIGIN ?? "http://localhost:3001").replace(/\/+$/, ""); +} + +function isUniqueViolation(err: unknown): boolean { + return typeof err === "object" && err !== null && "code" in err && (err as { code?: string }).code === "23505"; +} + +@Injectable() +export class InviteLinkService { + async create(workspaceId: string, actor: TeamActor, input: InviteLinkCreateInput): Promise { + if (input.expiresInHours < 1 || input.expiresInHours > 720) throw new BadRequestException("expiresInHours must be between 1 and 720"); + if (input.maxUses !== undefined && (!Number.isInteger(input.maxUses) || input.maxUses < 1)) { + throw new BadRequestException("maxUses must be a positive integer"); + } + if (input.role === OWNER) throw new BadRequestException("invite links cannot grant the Owner role — invite members and promote them in Team"); + // 256 bits of entropy: tokens are unguessable and never stored hashed, so + // they are only ever shown once (create response) and never in list/audit. + const token = randomBytes(32).toString("base64url"); + const expiresAt = new Date(Date.now() + input.expiresInHours * 3_600_000); + + const row = await withTenant(workspaceId, async (client) => { + await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [workspaceId]); + await this.assertRoleExists(client, input.role); + const res = await client.query( + `INSERT INTO workspace_invite_links (workspace_id, role, token, created_by, expires_at, max_uses) + VALUES ($1, $2, $3, $4, $5, $6) RETURNING ${LINK_COLUMNS}`, + [workspaceId, input.role, token, actor.userId, expiresAt, input.maxUses ?? null], + ); + return res.rows[0] as InviteLinkRow; + }); + + await appendTenantAudit(workspaceId, { + actorUserId: actor.userId, + actorRole: actor.role, + action: "invite_link.created", + targetType: "invite_link", + targetId: row.id, + diff: { role: row.role, expiresAt: toIso(row.expires_at), maxUses: row.max_uses }, // never the token + }); + return { ...this.toLink(row), token, url: `${publicWebUrl()}/join/${token}` }; + } + + async list(workspaceId: string): Promise { + const res = await withTenant(workspaceId, (client) => + client.query(`SELECT ${LINK_COLUMNS} FROM workspace_invite_links WHERE workspace_id = $1 ORDER BY created_at DESC`, [workspaceId]), + ); + // The token column is deliberately not selected: listing is admin UI and + // must never re-expose a live secret. + return (res.rows as InviteLinkRow[]).map((row) => this.toLink(row)); + } + + async revoke(workspaceId: string, actor: TeamActor, linkId: string): Promise { + const outcome = await withTenant(workspaceId, async (client) => { + await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [workspaceId]); + const updated = await client.query( + `UPDATE workspace_invite_links SET revoked_at = now() + WHERE id = $1 AND workspace_id = $2 AND revoked_at IS NULL RETURNING ${LINK_COLUMNS}`, + [linkId, workspaceId], + ); + const row = updated.rows[0] as InviteLinkRow | undefined; + if (row !== undefined) return { row, changed: true }; + const existing = await client.query( + `SELECT ${LINK_COLUMNS} FROM workspace_invite_links WHERE id = $1 AND workspace_id = $2`, + [linkId, workspaceId], + ); + const found = existing.rows[0] as InviteLinkRow | undefined; + if (found === undefined) throw new NotFoundException(`invite link ${linkId} not found`); + return { row: found, changed: false }; + }); + if (!outcome.changed) throw new ConflictException("invite link already revoked"); + + await appendTenantAudit(workspaceId, { + actorUserId: actor.userId, + actorRole: actor.role, + action: "invite_link.revoked", + targetType: "invite_link", + targetId: linkId, + diff: { role: outcome.row.role, useCount: outcome.row.use_count }, // never the token + }); + return this.toLink(outcome.row); + } + + /** + * Pre-auth landing page data. `invite_link_for_claim` is the only way to see + * a link outside a tenant context; `workspaces` has no RLS, so the name join + * is safe under `withSystem`. Projection is name/role/counters only. + */ + async info(token: string): Promise { + const res = await withSystem((client) => + client.query( + `SELECT l.role, l.expires_at, l.max_uses, l.use_count, l.revoked_at, w.name AS workspace_name + FROM invite_link_for_claim($1) l JOIN workspaces w ON w.id = l.workspace_id`, + [token], + ), + ); + const row = res.rows[0] as + | { role: string; expires_at: Date; max_uses: number | null; use_count: number; revoked_at: Date | null; workspace_name: string } + | undefined; + if (row === undefined) throw new NotFoundException("invite link is invalid or has expired"); + const status = statusOf(row); + if (status === "revoked") throw new GoneException("invite link has been revoked"); + if (status === "expired") throw new GoneException("invite link has expired"); + if (status === "exhausted") throw new GoneException("invite link has no uses left"); + return { + workspaceName: row.workspace_name, + role: row.role, + expiresAt: toIso(row.expires_at), + maxUses: row.max_uses, + usedCount: row.use_count, + }; + } + + /** + * Register-or-sign-in + join in one system transaction, returning the same + * `AuthResult` shape as POST /auth/login so the SPA reuses its auth handler. + * + * Everything runs in ONE `withSystem` transaction (register's pattern): the + * claimer has no tenant context yet, so the link is resolved through the + * narrow SECURITY DEFINER lookup, then `set_config('app.workspace_id', …, true)` + * enters the target tenant for the RLS-bound membership insert, the atomic + * use consumption, and the session mint (which reads the post-bump epoch). + */ + async claim(input: InviteClaimInput): Promise { + const token = requireString(input.token, "token"); + const outcome = await withSystem(async (client) => { + // 1. Link resolution through the SECURITY DEFINER lookup (no tenant context exists yet). + const linkRes = await client.query( + "SELECT id, workspace_id, role, expires_at, max_uses, use_count, revoked_at FROM invite_link_for_claim($1)", + [token], + ); + const link = linkRes.rows[0] as LinkClaimRow | undefined; + if (link === undefined) throw new NotFoundException("invite link is invalid or has expired"); + + // 2. Validation (410 Gone for dead-but-real links) — BEFORE any credential + // work, so a made-up token only ever sees 404 and usernames stay unenumerable. + if (link.revoked_at !== null) throw new GoneException("invite link has been revoked"); + if (new Date(link.expires_at).getTime() <= Date.now()) throw new GoneException("invite link has expired"); + if (link.max_uses !== null && link.use_count >= link.max_uses) throw new GoneException("invite link has no uses left"); + await this.assertRoleAlive(client, link.role); + + // 3. Identity resolution (only when no Bearer session was presented). + let userId: string; + let username: string | null = null; + const session = input.session ?? undefined; + if (session !== undefined) { + userId = session.userId; + } else { + const name = requireUsername(input.username, "username"); + const password = requireString(input.password, "password"); + username = name; + const found = await client.query( + `SELECT u.id, p.password_hash FROM users u + LEFT JOIN user_passwords p ON p.user_id = u.id + WHERE u.external_id = $1`, + [localExternalId(name)], + ); + const row = found.rows[0] as { id: string; password_hash: string | null } | undefined; + if (row === undefined) { + // Register. Same anti-hijack rule as AuthService.register: an existing + // identity (including a password-less invited placeholder) can never + // be claimed by registering — see the branches below. + if (password.length < MIN_PASSWORD_LENGTH) { + throw new BadRequestException(`password must be at least ${MIN_PASSWORD_LENGTH} characters`); + } + const passwordHash = await hashPassword(password); + const displayName = (input.displayName ?? name).trim().slice(0, 100); + let created: { rows: Array<{ id: string }> }; + try { + created = await client.query( + `INSERT INTO users (external_id, username, email, display_name) VALUES ($1, $2, $3, $4) + ON CONFLICT (external_id) DO NOTHING + RETURNING id`, + [localExternalId(name), name, input.email ?? null, displayName], + ); + } catch (err) { + // Username collides with a differently-keyed account (e.g. an email + // placeholder that already took this username): surface 409, not a 500. + if (isUniqueViolation(err)) throw new ConflictException("an account with this username already exists — sign in instead"); + throw err; + } + const id = created.rows[0]?.id as string | undefined; + if (id === undefined) throw new ConflictException("an account with this username already exists — sign in instead"); + userId = id; + await client.query("INSERT INTO user_passwords (user_id, password_hash) VALUES ($1, $2)", [userId, passwordHash]); + } else if (row.password_hash === null) { + // Password-less invited placeholder: burning keeps claim timing flat. + await burnPasswordCheck(password); + throw new ConflictException("an account with this username already exists — sign in instead"); + } else if (!(await verifyPassword(password, row.password_hash))) { + throw new UnauthorizedException("invalid email or password"); + } else { + userId = row.id; + } + } + + // 4. Tenant entry: transaction-local, cannot leak past COMMIT/ROLLBACK. + await client.query("SELECT set_config('app.workspace_id', $1, true)", [link.workspace_id]); + await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [link.workspace_id]); + + // 5. Membership insert. Conflict (any existing row, including an 'invited' + // placeholder) => 409 and NO use consumed: a link must never be a + // privilege-upgrade vector for an existing membership. + const inserted = await client.query( + `INSERT INTO memberships (workspace_id, user_id, role, status) VALUES ($1, $2, $3, 'active') + ON CONFLICT (workspace_id, user_id) DO NOTHING + RETURNING id`, + [link.workspace_id, userId, link.role], + ); + const membershipId = inserted.rows[0]?.id as string | undefined; + if (membershipId === undefined) { + throw new ConflictException("you are already a member of this workspace — sign in instead"); + } + + // 6. Atomic consumption: one conditional statement, no SELECT-then-check + // TOCTOU. EvalPlanQual re-evaluates the guard on the locked row, so + // concurrent claimers cannot both spend the last use. + const consumed = await client.query( + `UPDATE workspace_invite_links SET use_count = use_count + 1 + WHERE id = $1 AND revoked_at IS NULL AND expires_at > now() + AND (max_uses IS NULL OR use_count < max_uses)`, + [link.id], + ); + if ((consumed.rowCount ?? 0) === 0) throw new GoneException("invite link is no longer active"); + + // 7. Session mint inside the same tx, reading the post-bump epoch and the + // fresh membership row (the membership insert bumped the workspace epoch). + const sessionToken = await issueSessionInTenant(client, link.workspace_id, userId); + return { + workspaceId: link.workspace_id, + userId, + membershipId, + linkId: link.id, + role: link.role, + token: sessionToken, + username, + }; + }); + + // 8. After commit (appendTenantAudit opens its own withTenant). + const me = await this.meFor(outcome.workspaceId, outcome.userId, outcome.token); + await appendTenantAudit(outcome.workspaceId, { + actorUserId: outcome.userId, + actorRole: outcome.role, + action: "membership.joined", + targetType: "membership", + targetId: outcome.membershipId, + diff: { linkId: outcome.linkId, role: outcome.role, username: outcome.username ?? me.user.username }, // never the token + }); + return { token: outcome.token, me }; + } + + private toLink(row: InviteLinkRow): InviteLink { + return { + id: row.id, + role: row.role, + createdAt: toIso(row.created_at), + expiresAt: toIso(row.expires_at), + maxUses: row.max_uses, + useCount: row.use_count, + revokedAt: row.revoked_at === null ? null : toIso(row.revoked_at), + status: statusOf(row), + createdByUserId: row.created_by, + }; + } + + /** AuthService.meFor's projection (it is private there), reused for the claim response. */ + private async meFor(workspaceId: string, userId: string, token: string): Promise { + return withTenant(workspaceId, async (client) => { + await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [workspaceId]); + const res = await client.query( + `SELECT m.role, u.username, u.email, u.display_name, w.name, w.slug + FROM memberships m + JOIN users u ON u.id = m.user_id + JOIN workspaces w ON w.id = m.workspace_id + WHERE m.workspace_id = $1 AND m.user_id = $2 AND m.status = 'active'`, + [workspaceId, userId], + ); + const row = res.rows[0] as + | { role: string; username: string; email: string | null; display_name: string; name: string; slug: string } + | undefined; + if (row === undefined) throw new UnauthorizedException("membership no longer active"); + const perms = await client.query("SELECT permission FROM role_permissions WHERE role = $1 ORDER BY permission", [row.role]); + return { + user: { id: userId, username: row.username, email: row.email, displayName: row.display_name }, + workspace: { id: workspaceId, name: row.name, slug: row.slug }, + role: row.role, + permissions: (perms.rows as Array<{ permission: string }>).map((p) => p.permission), + expiresAt: verifySession(token)?.expiresAt ?? 0, + }; + }); + } + + private async assertRoleExists(client: PoolClient, role: string): Promise { + const res = await client.query("SELECT 1 FROM role_permissions WHERE role = $1 LIMIT 1", [role]); + if (res.rowCount === 0) throw new BadRequestException(`unknown role: ${role}`); + } + + /** Claim-time role check: a removed role means the link is dead, not a bad request. */ + private async assertRoleAlive(client: PoolClient, role: string): Promise { + const res = await client.query("SELECT 1 FROM role_permissions WHERE role = $1 LIMIT 1", [role]); + if (res.rowCount === 0) throw new GoneException("invite link references a removed role"); + } +} diff --git a/apps/api/src/team/team.controller.ts b/apps/api/src/team/team.controller.ts new file mode 100644 index 0000000..d694ab0 --- /dev/null +++ b/apps/api/src/team/team.controller.ts @@ -0,0 +1,61 @@ +import { Body, Controller, Delete, Get, Inject, Param, Patch, Post, Req, UseGuards } from "@nestjs/common"; +import type { Request } from "express"; +import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import { parseId, requireEmail, requireObject, requireSession, requireString } from "../http/validation.js"; +import { TeamService, type Member, type MemberMutationResult } from "./team.service.js"; + +@Controller("team") +@UseGuards(RbacGuard) +export class TeamController { + constructor(@Inject(TeamService) private readonly team: TeamService) {} + + /** Every role holds workspace.view, so any member can see who else is in the workspace. */ + @Get("members") + @RequirePermission("workspace.view") + async list(@Req() request: Request & AuthenticatedRequest): Promise { + return this.team.list(requireSession(request).workspaceId); + } + + @Get("roles") + @RequirePermission("workspace.view") + async roles(): Promise { + return this.team.roles(); + } + + @Post("members/invite") + @RequirePermission("workspace.members.manage") + async invite(@Req() request: Request & AuthenticatedRequest, @Body() body: unknown): Promise { + const session = requireSession(request); + const dto = requireObject(body, "body"); + return this.team.invite( + session.workspaceId, + { userId: session.userId, role: session.role }, + requireEmail(dto.email), + requireString(dto.role, "role"), + ); + } + + @Patch("members/:id/role") + @RequirePermission("workspace.members.manage") + async changeRole( + @Req() request: Request & AuthenticatedRequest, + @Param("id") id: string, + @Body() body: unknown, + ): Promise { + const session = requireSession(request); + const dto = requireObject(body, "body"); + return this.team.changeRole( + session.workspaceId, + { userId: session.userId, role: session.role }, + parseId(id), + requireString(dto.role, "role"), + ); + } + + @Delete("members/:id") + @RequirePermission("workspace.members.manage") + async remove(@Req() request: Request & AuthenticatedRequest, @Param("id") id: string): Promise { + const session = requireSession(request); + return this.team.remove(session.workspaceId, { userId: session.userId, role: session.role }, parseId(id)); + } +} diff --git a/apps/api/src/team/team.service.ts b/apps/api/src/team/team.service.ts new file mode 100644 index 0000000..25236f4 --- /dev/null +++ b/apps/api/src/team/team.service.ts @@ -0,0 +1,214 @@ +import { BadRequestException, ConflictException, ForbiddenException, Injectable, NotFoundException } from "@nestjs/common"; +import type { PoolClient } from "pg"; +import { randomBytes } from "node:crypto"; +import { appendTenantAudit } from "../audit/audit.service.js"; +import { issueSessionInTenant, localExternalId } from "../auth/auth.service.js"; +import { pool, withTenant } from "../db/client.js"; + +export type MemberStatus = "active" | "invited"; + +export interface Member { + id: string; + userId: string; + username: string; + email: string | null; + displayName: string; + role: string; + status: MemberStatus; + createdAt: string; +} + +/** + * Every membership mutation bumps the workspace authz epoch (DB trigger), which + * invalidates ALL live sessions — including the caller's. Mutations therefore + * return a fresh token for the caller, minted inside the same transaction from + * the post-change membership row, so the acting admin is not signed out. + */ +export interface MemberMutationResult { + member: Member | null; + token: string | null; +} + +export interface TeamActor { + userId: string; + role: string; +} + +const OWNER = "Owner"; + +interface MemberRow { + id: string; + user_id: string; + username: string; + email: string | null; + display_name: string; + role: string; + status: MemberStatus; + created_at: Date; +} + +function memberFromRow(row: MemberRow): Member { + return { + id: row.id, + userId: row.user_id, + username: row.username, + email: row.email, + displayName: row.display_name, + role: row.role, + status: row.status, + createdAt: new Date(row.created_at).toISOString(), + }; +} + +const MEMBER_SELECT = `SELECT m.id, m.user_id, u.username, u.email, u.display_name, m.role, m.status, m.created_at + FROM memberships m JOIN users u ON u.id = m.user_id`; + +/** Placeholder username derived from the invite email's local-part, with a uniqueness fallback. */ +function usernameFromEmail(email: string): string { + return (email.split("@")[0] ?? "member").toLowerCase().replace(/[^a-z0-9_.-]/g, "").slice(0, 32) || "member"; +} + +@Injectable() +export class TeamService { + async list(workspaceId: string): Promise { + const res = await withTenant(workspaceId, (client) => client.query(`${MEMBER_SELECT} ORDER BY m.created_at ASC`)); + return (res.rows as MemberRow[]).map(memberFromRow); + } + + async roles(): Promise { + // role_permissions is global reference data (no RLS). + const res = await pool.query("SELECT DISTINCT role FROM role_permissions ORDER BY role"); + return (res.rows as Array<{ role: string }>).map((r) => r.role); + } + + async invite(workspaceId: string, actor: TeamActor, email: string, role: string): Promise { + const result = await withTenant(workspaceId, async (client) => { + await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [workspaceId]); + await this.assertRoleExists(client, role); + const actorRole = await this.actorRole(client, workspaceId, actor.userId); + if (role === OWNER && actorRole !== OWNER) throw new ForbiddenException("only an Owner can invite another Owner"); + + const existing = await client.query("SELECT id FROM users WHERE lower(email) = $1 ORDER BY created_at ASC LIMIT 1", [email]); + let userId = existing.rows[0]?.id as string | undefined; + if (userId === undefined) { + const base = usernameFromEmail(email); + const displayName = email.split("@")[0] ?? email; + for (let attempt = 0; userId === undefined && attempt < 3; attempt += 1) { + const username = attempt === 0 ? base : `${base}_${randomBytes(2).toString("hex")}`; + const created = await client.query( + `INSERT INTO users (external_id, username, email, display_name) VALUES ($1, $2, $3, $4) + ON CONFLICT (username) DO NOTHING + RETURNING id`, + [localExternalId(email), username, email, displayName], + ); + userId = created.rows[0]?.id as string | undefined; + } + if (userId === undefined) throw new ConflictException("could not derive a username for this invite"); + } + const inserted = await client.query( + `INSERT INTO memberships (workspace_id, user_id, role, status) VALUES ($1, $2, $3, 'invited') + ON CONFLICT (workspace_id, user_id) DO NOTHING RETURNING id`, + [workspaceId, userId, role], + ); + const membershipId = inserted.rows[0]?.id as string | undefined; + if (membershipId === undefined) throw new ConflictException(`${email} is already a member of this workspace`); + return { + member: await this.memberById(client, membershipId), + token: await issueSessionInTenant(client, workspaceId, actor.userId), + }; + }); + await appendTenantAudit(workspaceId, { + actorUserId: actor.userId, + actorRole: actor.role, + action: "membership.invited", + targetType: "membership", + targetId: result.member.id, + diff: { email, role }, + }); + return result; + } + + async changeRole(workspaceId: string, actor: TeamActor, membershipId: string, role: string): Promise { + const result = await withTenant(workspaceId, async (client) => { + await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [workspaceId]); + await this.assertRoleExists(client, role); + const target = await this.memberById(client, membershipId, true); + if (target.role === role) return { member: target, token: null, previous: role }; + const actorRole = await this.actorRole(client, workspaceId, actor.userId); + if ((role === OWNER || target.role === OWNER) && actorRole !== OWNER) { + throw new ForbiddenException("only an Owner can grant or revoke the Owner role"); + } + if (target.role === OWNER && target.status === "active") await this.assertNotLastOwner(client); + await client.query("UPDATE memberships SET role = $2 WHERE id = $1", [membershipId, role]); + return { + member: await this.memberById(client, membershipId), + token: await issueSessionInTenant(client, workspaceId, actor.userId), + previous: target.role, + }; + }); + if (result.previous !== role) { + await appendTenantAudit(workspaceId, { + actorUserId: actor.userId, + actorRole: actor.role, + action: "membership.role_changed", + targetType: "membership", + targetId: membershipId, + diff: { email: result.member.email, role: [result.previous, role] }, + }); + } + return { member: result.member, token: result.token }; + } + + async remove(workspaceId: string, actor: TeamActor, membershipId: string): Promise { + const removed = await withTenant(workspaceId, async (client) => { + await client.query("SELECT id FROM workspaces WHERE id = $1 FOR UPDATE", [workspaceId]); + const target = await this.memberById(client, membershipId, true); + const actorRole = await this.actorRole(client, workspaceId, actor.userId); + if (target.role === OWNER) { + if (actorRole !== OWNER) throw new ForbiddenException("only an Owner can remove an Owner"); + if (target.status === "active") await this.assertNotLastOwner(client); + } + await client.query("DELETE FROM memberships WHERE id = $1", [membershipId]); + // Removing yourself ends your access: no replacement token. + const token = target.userId === actor.userId ? null : await issueSessionInTenant(client, workspaceId, actor.userId); + return { target, token }; + }); + await appendTenantAudit(workspaceId, { + actorUserId: actor.userId, + actorRole: actor.role, + action: "membership.removed", + targetType: "membership", + targetId: membershipId, + diff: { email: removed.target.email, role: removed.target.role }, + }); + return { member: null, token: removed.token }; + } + + private async memberById(client: PoolClient, membershipId: string, lock = false): Promise { + const res = await client.query(`${MEMBER_SELECT} WHERE m.id = $1${lock ? " FOR UPDATE OF m" : ""}`, [membershipId]); + const row = res.rows[0] as MemberRow | undefined; + if (row === undefined) throw new NotFoundException(`member ${membershipId} not found`); + return memberFromRow(row); + } + + private async actorRole(client: PoolClient, workspaceId: string, userId: string): Promise { + const res = await client.query( + "SELECT role FROM memberships WHERE workspace_id = $1 AND user_id = $2 AND status = 'active'", + [workspaceId, userId], + ); + const role = res.rows[0]?.role as string | undefined; + if (role === undefined) throw new ForbiddenException("no active membership in this workspace"); + return role; + } + + private async assertRoleExists(client: PoolClient, role: string): Promise { + const res = await client.query("SELECT 1 FROM role_permissions WHERE role = $1 LIMIT 1", [role]); + if (res.rowCount === 0) throw new BadRequestException(`unknown role: ${role}`); + } + + private async assertNotLastOwner(client: PoolClient): Promise { + // Lock every Owner row so two concurrent demotions cannot both pass the check. + const res = await client.query(`SELECT id FROM memberships WHERE role = $1 AND status = 'active' FOR UPDATE`, [OWNER]); + if ((res.rowCount ?? 0) <= 1) throw new BadRequestException("a workspace must keep at least one active Owner"); + } +} diff --git a/apps/api/src/telemetry/logger.test.ts b/apps/api/src/telemetry/logger.test.ts new file mode 100644 index 0000000..be25ab8 --- /dev/null +++ b/apps/api/src/telemetry/logger.test.ts @@ -0,0 +1,15 @@ +import { expect, it } from "vitest"; +import { createApiLogger } from "./logger.js"; + +it("redacts bot/OAuth secrets and callback/request data before serializing API logs", () => { + const lines: string[] = []; + const log = createApiLogger({ write: (line) => { lines.push(line); } }); + log.info({ + token: "bot-value", client_secret: "client-value", access_token: "access-value", refresh_token: "refresh-value", code_verifier: "pkce-value", + data: { botToken: "nested-bot", access_token: "nested-access" }, + req: { headers: { authorization: "Bearer session-value", cookie: "cookie-value" }, body: { token: "body-token" }, query: { code: "code-value" }, url: "/connections/callback?code=url-code" }, + }, "safe event"); + const output = lines.join(""); + expect(output).toContain("safe event"); + for (const secret of ["bot-value", "client-value", "access-value", "refresh-value", "pkce-value", "nested-bot", "nested-access", "session-value", "cookie-value", "body-token", "code-value", "url-code"]) expect(output).not.toContain(secret); +}); diff --git a/apps/api/src/telemetry/logger.ts b/apps/api/src/telemetry/logger.ts new file mode 100644 index 0000000..4e9313e --- /dev/null +++ b/apps/api/src/telemetry/logger.ts @@ -0,0 +1,19 @@ +import pino, { type DestinationStream } from "pino"; + +export function createApiLogger(destination?: DestinationStream) { + const options = { + level: process.env.LOG_LEVEL ?? "info", + redact: { + paths: [ + "accessToken", "refreshToken", "apiKey", "streamKey", + "*.accessToken", "*.refreshToken", "*.apiKey", "*.streamKey", + "token", "botToken", "clientSecret", "client_secret", "access_token", "refresh_token", "code_verifier", + "*.token", "*.botToken", "*.clientSecret", "*.client_secret", "*.access_token", "*.refresh_token", "*.code_verifier", + "req.headers.authorization", "req.headers.cookie", "req.body", "req.query", "req.url", + ], + censor: "[REDACTED]", + }, + }; + return destination ? pino(options, destination) : pino(options); +} +export const logger = createApiLogger(); diff --git a/apps/api/src/telemetry/otel.ts b/apps/api/src/telemetry/otel.ts new file mode 100644 index 0000000..d632c2b --- /dev/null +++ b/apps/api/src/telemetry/otel.ts @@ -0,0 +1,41 @@ +import { logger } from "./logger"; + +interface OtelModule { + start(): void; +} + +export function startOtel(): void { + const endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT; + if (!endpoint) { + logger.info("otel: no OTEL_EXPORTER_OTLP_ENDPOINT — tracing disabled"); + return; + } + void (async () => { + try { + const { NodeSDK } = await import("@opentelemetry/sdk-node"); + const { getNodeAutoInstrumentations } = await import("@opentelemetry/auto-instrumentations-node"); + const { OTLPTraceExporter } = await import("@opentelemetry/exporter-trace-otlp-http"); + const sdk: OtelModule = new NodeSDK({ + traceExporter: new OTLPTraceExporter({ url: `${endpoint}/v1/traces` }), + instrumentations: [getNodeAutoInstrumentations({ + // Bot API paths contain credentials; OAuth callback query strings and + // invite-link paths contain join tokens. + "@opentelemetry/instrumentation-http": { + ignoreIncomingRequestHook: (request) => { + const url = request.url ?? ""; + return url.startsWith("/connections/callback") || url.startsWith("/invites/"); + }, + ignoreOutgoingRequestHook: (request) => /\/bot[^/]+\//.test(request.path ?? ""), + }, + "@opentelemetry/instrumentation-undici": { enabled: false }, + })], + }) as unknown as OtelModule; + sdk.start(); + logger.info({ endpoint }, "otel: tracing enabled"); + } catch (err) { + logger.warn({ err: String(err) }, "otel: failed to initialize — continuing without tracing"); + } + })(); +} + +startOtel(); diff --git a/apps/api/src/vault/vault.test.ts b/apps/api/src/vault/vault.test.ts new file mode 100644 index 0000000..9b94345 --- /dev/null +++ b/apps/api/src/vault/vault.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { Vault } from "./vault.js"; + +describe("vault envelope encryption", () => { + it("persistent envelopes survive restarts and reject cross-workspace/handle use and tampering", async () => { + const sealed = await new Vault().seal("ws-a", "vh-a", Buffer.from("persistent-token")); + expect(sealed.ciphertext.includes("persistent-token")).toBe(false); + expect((await new Vault().open("ws-a", "vh-a", sealed)).toString()).toBe("persistent-token"); + await expect(new Vault().open("ws-b", "vh-a", sealed)).rejects.toThrow(); + await expect(new Vault().open("ws-a", "vh-b", sealed)).rejects.toThrow(); + const tampered = Buffer.from(sealed.ciphertext); + tampered[tampered.length - 1] = (tampered[tampered.length - 1] ?? 0) ^ 1; + await expect(new Vault().open("ws-a", "vh-a", { ...sealed, ciphertext: tampered })).rejects.toThrow(); + }); + it("round-trips a secret", async () => { + const v = new Vault(); + const sealed = await v.put("ws-1", "cred-1", Buffer.from("access-token-abc")); + const opened = await v.get("ws-1", "cred-1", sealed); + expect(opened.toString()).toBe("access-token-abc"); + }); + + it("ciphertext is not plaintext", async () => { + const v = new Vault(); + const sealed = await v.put("ws-1", "cred-1", Buffer.from("access-token-abc")); + expect(sealed.ciphertext.includes("access-token-abc")).toBe(false); + }); + + it("rotation invalidates pre-rotation ciphertext (old DEK dropped)", async () => { + const v = new Vault(); + const before = await v.put("ws-1", "cred-1", Buffer.from("secret-1")); + const rotated = await v.rotate("ws-1", [{ credentialId: "cred-1", plaintext: Buffer.from("secret-1") }]); + const rewrapped = rotated.get("cred-1"); + expect(rewrapped).toBeDefined(); + if (rewrapped === undefined) return; + expect(rewrapped.keyVersion).toBeGreaterThan(before.keyVersion); + // the OLD sealed blob must no longer decrypt (its DEK is gone) + await expect(v.get("ws-1", "cred-1", before)).rejects.toThrow(); + // the NEW blob decrypts + const opened = await v.get("ws-1", "cred-1", rewrapped); + expect(opened.toString()).toBe("secret-1"); + }); +}); diff --git a/apps/api/src/vault/vault.ts b/apps/api/src/vault/vault.ts new file mode 100644 index 0000000..58447be --- /dev/null +++ b/apps/api/src/vault/vault.ts @@ -0,0 +1 @@ +export { Vault, vault, loadKek, type SealedCredential } from "@tactical-media-broadcast-hub/connector-runtime"; diff --git a/apps/api/src/video/ffprobe.test.ts b/apps/api/src/video/ffprobe.test.ts new file mode 100644 index 0000000..fd75412 --- /dev/null +++ b/apps/api/src/video/ffprobe.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { buildFfprobeArgs, parseFfprobeOutput } from "./ffprobe.js"; + +describe("ffprobe invocation", () => { + it("rejects containers without a video stream", () => { + expect(() => parseFfprobeOutput(JSON.stringify({ streams: [{ codec_type: "audio" }] }))).toThrow("no video stream"); + }); + it("keeps an injection-shaped input in one argv element", () => { + const input = "/videos/example.mp4; touch /tmp/should-not-exist"; + const args = buildFfprobeArgs(input); + + expect(args.at(-1)).toBe(input); + expect(args.at(-2)).toBe("-i"); + expect(args).not.toContain("sh"); + expect(args).not.toContain("-c"); + }); + + it("extracts video and audio metadata from JSON", () => { + const result = parseFfprobeOutput(JSON.stringify({ + streams: [ + { + codec_type: "video", + codec_name: "h264", + width: 1920, + height: 1080, + avg_frame_rate: "30000/1001", + }, + { codec_type: "audio", codec_name: "aac" }, + ], + format: { duration: "12.345" }, + })); + + expect(result).toEqual({ + width: 1920, + height: 1080, + durationMs: 12_345, + frameRate: 30000 / 1001, + codec: "h264", + audioCodec: "aac", + }); + }); +}); diff --git a/apps/api/src/video/ffprobe.ts b/apps/api/src/video/ffprobe.ts new file mode 100644 index 0000000..4cc6e16 --- /dev/null +++ b/apps/api/src/video/ffprobe.ts @@ -0,0 +1,159 @@ +import { spawn } from "node:child_process"; + +const FFPROBE_TIMEOUT_MS = 30_000; +const MAX_OUTPUT_BYTES = 10 * 1024 * 1024; + +export interface VideoProbeResult { + width?: number; + height?: number; + durationMs?: number; + frameRate?: number; + codec?: string; + audioCodec?: string; +} + +/** + * Keep the input as a discrete argv value. `-i` ensures even a path beginning + * with a dash is treated as input rather than another ffprobe option. + */ +export function buildFfprobeArgs(inputPath: string): string[] { + return [ + "-v", + "error", + "-print_format", + "json", + "-show_format", + "-show_streams", + "-i", + inputPath, + ]; +} + +export async function probeVideoFile( + inputPath: string, + executable = process.env.FFPROBE_PATH ?? "ffprobe", +): Promise { + if (inputPath.length === 0) throw new Error("ffprobe input path must not be empty"); + + const output = await new Promise((resolve, reject) => { + const child = spawn(executable, buildFfprobeArgs(inputPath), { + shell: false, + windowsHide: true, + stdio: ["ignore", "pipe", "pipe"], + }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + let outputBytes = 0; + let settled = false; + + const finish = (error: Error | undefined, value?: string): void => { + if (settled) return; + settled = true; + clearTimeout(timeout); + if (error === undefined) resolve(value ?? ""); + else reject(error); + }; + + const timeout = setTimeout(() => { + child.kill("SIGKILL"); + finish(new Error(`ffprobe timed out after ${FFPROBE_TIMEOUT_MS}ms`)); + }, FFPROBE_TIMEOUT_MS); + + child.stdout.on("data", (chunk: Buffer) => { + outputBytes += chunk.length; + if (outputBytes > MAX_OUTPUT_BYTES) { + child.kill("SIGKILL"); + finish(new Error("ffprobe output exceeded the 10 MiB limit")); + return; + } + stdout.push(chunk); + }); + child.stderr.on("data", (chunk: Buffer) => { + if (Buffer.concat(stderr).length < 16_384) stderr.push(chunk); + }); + child.once("error", (error) => finish(error)); + child.once("close", (code, signal) => { + if (code !== 0) { + const detail = Buffer.concat(stderr).toString("utf8").trim(); + finish( + new Error( + `ffprobe failed (${signal === null ? `exit ${String(code)}` : `signal ${signal}`})${ + detail === "" ? "" : `: ${detail}` + }`, + ), + ); + return; + } + finish(undefined, Buffer.concat(stdout).toString("utf8")); + }); + }); + + return parseFfprobeOutput(output); +} + +export function parseFfprobeOutput(output: string): VideoProbeResult { + let parsed: unknown; + try { + parsed = JSON.parse(output) as unknown; + } catch { + throw new Error("ffprobe returned invalid JSON"); + } + const root = asRecord(parsed); + if (root === undefined) throw new Error("ffprobe returned an invalid result"); + const streams = Array.isArray(root.streams) + ? root.streams.map(asRecord).filter((stream): stream is Record => stream !== undefined) + : []; + const format = asRecord(root.format); + const video = streams.find((stream) => stream.codec_type === "video"); + if (video === undefined) throw new Error("ffprobe found no video stream"); + const audio = streams.find((stream) => stream.codec_type === "audio"); + const durationSeconds = finiteNumber(format?.duration) ?? finiteNumber(video?.duration); + const frameRate = rationalNumber(video?.avg_frame_rate) ?? rationalNumber(video?.r_frame_rate); + + return { + ...optionalNumber("width", finiteInteger(video?.width)), + ...optionalNumber("height", finiteInteger(video?.height)), + ...optionalNumber( + "durationMs", + durationSeconds === undefined ? undefined : Math.max(0, Math.round(durationSeconds * 1000)), + ), + ...optionalNumber("frameRate", frameRate), + ...optionalString("codec", video?.codec_name), + ...optionalString("audioCodec", audio?.codec_name), + }; +} + +function asRecord(value: unknown): Record | undefined { + if (typeof value !== "object" || value === null || Array.isArray(value)) return undefined; + return value as Record; +} + +function finiteNumber(value: unknown): number | undefined { + const number = typeof value === "number" ? value : typeof value === "string" ? Number(value) : NaN; + return Number.isFinite(number) ? number : undefined; +} + +function finiteInteger(value: unknown): number | undefined { + const number = finiteNumber(value); + return number !== undefined && Number.isInteger(number) ? number : undefined; +} + +function rationalNumber(value: unknown): number | undefined { + if (typeof value !== "string") return finiteNumber(value); + const [numeratorText, denominatorText, extra] = value.split("/"); + if (extra !== undefined || numeratorText === undefined || denominatorText === undefined) { + return finiteNumber(value); + } + const numerator = Number(numeratorText); + const denominator = Number(denominatorText); + if (!Number.isFinite(numerator) || !Number.isFinite(denominator) || denominator === 0) return undefined; + return numerator / denominator; +} + +function optionalNumber(key: K, value: number | undefined): Partial> { + return value === undefined ? {} : { [key]: value } as Record; +} + +function optionalString(key: K, value: unknown): Partial> { + return typeof value !== "string" || value === "" ? {} : { [key]: value } as Record; +} diff --git a/apps/api/src/video/test-fixtures.ts b/apps/api/src/video/test-fixtures.ts new file mode 100644 index 0000000..63bdc2a --- /dev/null +++ b/apps/api/src/video/test-fixtures.ts @@ -0,0 +1,156 @@ +import { keyForHash, sha256, type StorageService } from "../storage/storage.js"; +import type { PoolClient, QueryResult } from "pg"; +import { vi } from "vitest"; +import { + VideoService, + type VideoServiceDependencies, +} from "./video.service.js"; + +export const workspaceId = "10000000-0000-4000-8000-000000000001"; +export const assetId = "20000000-0000-4000-8000-000000000002"; +export const jobId = "30000000-0000-4000-8000-000000000003"; + +export interface FakeVideoDatabase { + asset?: Record; + job?: Record; + transitions?: string[]; +} + +function result(rows: Array>): QueryResult> { + return { command: "", rowCount: rows.length, oid: 0, fields: [], rows }; +} + +export function serviceWithDatabase( + database: FakeVideoDatabase, + probe: VideoServiceDependencies["probe"], + storageOverride?: StorageService, +): { service: VideoService; appendAudit: VideoServiceDependencies["appendAudit"]; storage: StorageService; objects: Map } { + const objects = new Map(); + const storage: StorageService = storageOverride ?? { + putContent: vi.fn(async (bytes: Buffer, contentType: string) => { + const hash = sha256(bytes); + const key = keyForHash(hash, contentType); + const deduplicated = objects.has(key); + objects.set(key, Buffer.from(bytes)); + return { key, hash, sizeBytes: bytes.length, contentType, deduplicated }; + }), + getObject: vi.fn(async (key: string) => { + const bytes = objects.get(key); + if (bytes === undefined) throw new Error("Object not found"); + return Buffer.from(bytes); + }), + hasContent: async (hash, type) => objects.has(keyForHash(hash, type)), + presign: async () => { throw new Error("Encrypted source must be decrypted before probing"); }, + }; + database.transitions = []; + const client = { + async query(sql: string, values: readonly unknown[] = []): Promise>> { + if (sql.includes("INSERT INTO video_assets")) { + database.asset = { + id: assetId, + workspace_id: values[0], + storage_key: values[1], + mime_type: values[2], + size_bytes: values[3], + width: null, + height: null, + duration_ms: null, + frame_rate: null, + codec: null, + audio_codec: null, + thumbnail_key: null, + created_at: new Date("2026-10-07T00:00:00Z"), + }; + return result([database.asset]); + } + if (sql.includes("INSERT INTO video_processing_jobs")) { + database.job = { + id: jobId, + workspace_id: values[0], + video_asset_id: values[1], + state: "queued", + progress_percent: 0, + output_key: null, + error: null, + }; + database.transitions?.push("queued"); + return result([database.job]); + } + if (sql === "SELECT * FROM video_assets WHERE id = $1") { + return result(database.asset === undefined ? [] : [database.asset]); + } + if (sql.includes("SET state = 'probing'")) { + if (database.job?.state !== "queued") return result([]); + database.job.state = "probing"; + database.transitions?.push("probing"); + database.job.progress_percent = 10; + database.job.error = null; + return result([{ id: jobId }]); + } + if (sql.includes("SET state = 'failed'")) { + if (database.job !== undefined) { + database.job.state = "failed"; + database.transitions?.push("failed"); + database.job.error = values[1]; + } + return result([{ id: jobId }]); + } + if (sql.includes("UPDATE video_assets")) { + if (database.asset === undefined) return result([]); + database.asset.width = values[1]; + database.asset.height = values[2]; + database.asset.duration_ms = values[3]; + database.asset.frame_rate = values[4]; + database.asset.codec = values[5]; + database.asset.audio_codec = values[6]; + return result([database.asset]); + } + if (sql.includes("SET state = 'transcoding'")) { + if (database.job?.state !== "probing") return result([]); + database.job.state = "transcoding"; + database.transitions?.push("transcoding"); + database.job.progress_percent = 90; + return result([{ id: jobId }]); + } + if (sql.includes("SET state = 'complete'")) { + if (database.job?.state !== "transcoding") return result([]); + database.job.state = "complete"; + database.transitions?.push("complete"); + database.job.output_key = values[1]; + database.job.progress_percent = 100; + database.job.error = null; + return result([{ id: jobId }]); + } + if (sql.includes("SELECT state FROM video_processing_jobs")) { + return result(database.job === undefined ? [] : [{ state: database.job.state }]); + } + if (sql.includes("SELECT va.*")) { + if (database.asset === undefined) return result([]); + return result([{ + ...database.asset, + job_id: database.job?.id, + job_video_asset_id: database.job?.video_asset_id, + job_state: database.job?.state, + job_progress_percent: database.job?.progress_percent, + job_output_key: database.job?.output_key, + job_error: database.job?.error, + }]); + } + if (sql.includes("DELETE FROM video_assets")) { + if (database.asset === undefined) return result([]); + delete database.asset; + delete database.job; + return result([{ id: assetId }]); + } + throw new Error(`unexpected SQL in fake video database: ${sql}`); + }, + }; + const appendAudit = vi.fn(async () => "audit-id"); + const dependencies: VideoServiceDependencies = { + tenantTransaction: async (_tenant, fn) => fn(client as unknown as PoolClient), + appendAudit, + probe, + storage, + }; + return { service: new VideoService(dependencies), appendAudit, storage, objects }; +} diff --git a/apps/api/src/video/upload-limits.ts b/apps/api/src/video/upload-limits.ts new file mode 100644 index 0000000..5f79e64 --- /dev/null +++ b/apps/api/src/video/upload-limits.ts @@ -0,0 +1,2 @@ +// StorageService seals whole Buffers; bound multipart memory use per request. +export const MAX_VIDEO_UPLOAD_BYTES = 100 * 1024 * 1024; diff --git a/apps/api/src/video/video.controller.ts b/apps/api/src/video/video.controller.ts new file mode 100644 index 0000000..1a6b2f0 --- /dev/null +++ b/apps/api/src/video/video.controller.ts @@ -0,0 +1,103 @@ +import { + BadRequestException, + Body, + Controller, + Delete, + Get, + Inject, + Param, + Post, + Query, + Req, + UseGuards, + UseInterceptors, + UploadedFile, +} from "@nestjs/common"; +import { FileInterceptor } from "@nestjs/platform-express"; +import type { Request } from "express"; +import { parseId, parseLimit, requireObject, requireSession, requireString } from "../http/validation.js"; +import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import { + VideoService, + type VideoLibraryPage, + type VideoStatus, + type VideoUploadResult, +} from "./video.service.js"; +import { MAX_VIDEO_UPLOAD_BYTES } from "./upload-limits.js"; + +interface VideoFile { + buffer: Buffer; + mimetype: string; +} + +interface UploadVideoDto { + storageKey: string; + mimeType: string; + sizeBytes: number; +} + +@Controller("video") +@UseGuards(RbacGuard) +export class VideoController { + constructor(@Inject(VideoService) private readonly video: VideoService) {} + + @Post("upload") + @RequirePermission("media.manage") + @UseInterceptors(FileInterceptor("file", { + limits: { fileSize: MAX_VIDEO_UPLOAD_BYTES, files: 1, fields: 3, fieldSize: 1024, parts: 4 }, + })) + async upload( + @Req() request: Request & AuthenticatedRequest, + @Body() body: unknown, + @UploadedFile() file?: VideoFile, + ): Promise { + const session = requireSession(request); + if (file !== undefined) { + return this.video.uploadVideoBytes(session.workspaceId, file.buffer, file.mimetype); + } + if (request.is("multipart/form-data")) throw new BadRequestException("a video file is required (field: file)"); + const dto = parseUploadVideo(body); + return this.video.uploadVideo(session.workspaceId, dto.storageKey, dto.mimeType, dto.sizeBytes); + } + + @Get() + @RequirePermission("media.view") + async list( + @Req() request: Request & AuthenticatedRequest, + @Query("cursor") cursor: string | undefined, + @Query("limit") limit: string | undefined, + ): Promise { + return this.video.listVideos(requireSession(request).workspaceId, cursor, parseLimit(limit, 50, 200)); + } + + @Get(":id/status") + @RequirePermission("media.view") + async status( + @Req() request: Request & AuthenticatedRequest, + @Param("id") id: string, + ): Promise { + return this.video.getVideoStatus(parseId(id), requireSession(request).workspaceId); + } + + @Delete(":id") + @RequirePermission("media.manage") + async remove( + @Req() request: Request & AuthenticatedRequest, + @Param("id") id: string, + ): Promise { + await this.video.deleteVideo(parseId(id), requireSession(request).workspaceId); + } +} + +function parseUploadVideo(value: unknown): UploadVideoDto { + const body = requireObject(value, "body"); + const sizeBytes = body.sizeBytes; + if (typeof sizeBytes !== "number" || !Number.isSafeInteger(sizeBytes) || sizeBytes < 1) { + throw new BadRequestException("sizeBytes must be a positive safe integer"); + } + return { + storageKey: requireString(body.storageKey, "storageKey"), + mimeType: requireString(body.mimeType, "mimeType"), + sizeBytes, + }; +} diff --git a/apps/api/src/video/video.http.test.ts b/apps/api/src/video/video.http.test.ts new file mode 100644 index 0000000..cafa1ab --- /dev/null +++ b/apps/api/src/video/video.http.test.ts @@ -0,0 +1,124 @@ +import "reflect-metadata"; +import { readFile } from "node:fs/promises"; +import { Module, UnauthorizedException, ForbiddenException, type ExecutionContext, type INestApplication } from "@nestjs/common"; +import { NestFactory, Reflector } from "@nestjs/core"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { injectHttp } from "../connections/http-test-transport.js"; +import { RbacGuard, PERMISSION_KEY } from "../rbac/rbac.guard.js"; +import { VideoController } from "./video.controller.js"; +import { VideoService, type VideoStatus } from "./video.service.js"; +import { serviceWithDatabase, workspaceId, type FakeVideoDatabase } from "./test-fixtures.js"; + +@Module({}) +class TestModule {} + +describe("video multipart HTTP routes", () => { + let app: INestApplication | undefined; + let canManage = true; + + beforeEach(() => { + canManage = true; + vi.spyOn(RbacGuard.prototype, "canActivate").mockImplementation(async (context: ExecutionContext) => { + const request = context.switchToHttp().getRequest<{ headers: Record; session?: { workspaceId: string } }>(); + if (request.headers.authorization !== "Bearer test-session") throw new UnauthorizedException(); + if (!canManage && new Reflector().get(PERMISSION_KEY, context.getHandler()) === "media.manage") { + throw new ForbiddenException("missing permission: media.manage"); + } + request.session = { workspaceId }; + return true; + }); + }); + + afterEach(async () => { + await app?.close(); + app = undefined; + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + }); + + async function start(service: VideoService): Promise { + app = await NestFactory.create({ + module: TestModule, + controllers: [VideoController], + providers: [{ provide: VideoService, useValue: service }], + }, { logger: false }); + await app.init(); + } + + async function upload(form: FormData, authenticated = true): Promise { + const encoded = new Request("http://api.test/video/upload", { method: "POST", body: form }); + // ASCII fixture bytes preserve their exact value in the existing HTTP injector. + return injectHttp(app?.getHttpAdapter().getInstance(), encoded.url, { + method: "POST", + headers: { + ...(authenticated ? { authorization: "Bearer test-session" } : {}), + "content-type": encoded.headers.get("content-type") ?? "", + }, + body: Buffer.from(await encoded.arrayBuffer()).toString("utf8"), + }); + } + + function videoForm(): FormData { + const form = new FormData(); + form.append("file", new Blob(["video bytes from browser"], { type: "video/mp4" }), "example.mp4"); + return form; + } + + it("accepts FormData bytes and returns completed probe metadata through GET status", async () => { + const probe = vi.fn(async (path: string) => { + expect(await readFile(path, "utf8")).toBe("video bytes from browser"); + return { durationMs: 1000, width: 320, height: 180, frameRate: 25, codec: "h264" }; + }); + const { service } = serviceWithDatabase({}, probe); + await start(service); + const response = await upload(videoForm()); + expect(response.status).toBe(201); + const result = await response.json() as VideoStatus; + expect(result.processingJob.state).toBe("complete"); + expect(result.videoAsset).toMatchObject({ durationMs: 1000, width: 320, height: 180 }); + const status = await injectHttp(app?.getHttpAdapter().getInstance(), `http://api.test/video/${result.videoAsset.id}/status`, { + headers: { authorization: "Bearer test-session" }, + }); + expect(status.status).toBe(200); + expect(await status.json()).toEqual(result); + }); + + it("surfaces ffprobe failure through upload and status", async () => { + const { service } = serviceWithDatabase({}, async () => { throw new Error("ffprobe rejected media"); }); + await start(service); + const response = await upload(videoForm()); + expect(response.status).toBe(201); + expect((await response.json() as VideoStatus).processingJob).toMatchObject({ state: "failed", error: "ffprobe rejected media" }); + }); + + it("returns HTTP 503 with real unconfigured storage, without touching the DB", async () => { + vi.stubEnv("S3_ACCESS_KEY", ""); + vi.stubEnv("S3_SECRET_KEY", ""); + const transaction = vi.fn(); + await start(new VideoService({ tenantTransaction: transaction })); + const response = await upload(videoForm()); + expect(response.status).toBe(503); + expect(await response.json()).toMatchObject({ message: expect.stringContaining("Video storage is not configured") }); + expect(transaction).not.toHaveBeenCalled(); + }); + + it("rejects missing files, non-video MIME, multiple files, and unauthorized uploads", async () => { + const database: FakeVideoDatabase = {}; + const { service, storage } = serviceWithDatabase(database, async () => ({})); + await start(service); + const empty = new FormData(); + empty.append("mimeType", "video/mp4"); + expect((await upload(empty)).status).toBe(400); + const image = new FormData(); + image.append("file", new Blob(["image"], { type: "image/png" }), "example.png"); + expect((await upload(image)).status).toBe(400); + const multiple = videoForm(); + multiple.append("file", new Blob(["second"], { type: "video/mp4" }), "second.mp4"); + expect((await upload(multiple)).status).toBe(400); + expect((await upload(videoForm(), false)).status).toBe(401); + canManage = false; + expect((await upload(videoForm())).status).toBe(403); + expect(storage.putContent).not.toHaveBeenCalled(); + expect(database.asset).toBeUndefined(); + }); +}); diff --git a/apps/api/src/video/video.integration.test.ts b/apps/api/src/video/video.integration.test.ts new file mode 100644 index 0000000..37e8611 --- /dev/null +++ b/apps/api/src/video/video.integration.test.ts @@ -0,0 +1,129 @@ +import { randomUUID } from "node:crypto"; +import { join } from "node:path"; +import { Pool, type PoolClient } from "pg"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { runMigrations } from "../db/migrate.js"; +import { sha256, keyForHash, type StorageService } from "../storage/storage.js"; +import { VideoService, type VideoServiceDependencies } from "./video.service.js"; + +const dbAvailable = process.env.DATABASE_URL !== undefined; +const ownerPool = new Pool({ connectionString: process.env.SUPER_DATABASE_URL ?? process.env.DATABASE_URL }); +const appPool = new Pool({ connectionString: process.env.DATABASE_URL }); + +async function asTenant(workspaceId: string, fn: (client: PoolClient) => Promise): Promise { + const client = await appPool.connect(); + try { + await client.query("BEGIN"); + await client.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]); + const value = await fn(client); + await client.query("COMMIT"); + return value; + } catch (error) { + await client.query("ROLLBACK").catch(() => undefined); + throw error; + } finally { + client.release(); + } +} + +describe.skipIf(!dbAvailable)("video library (live DB)", () => { + let workspaceId: string; + let service: VideoService; + const objects = new Map(); + const storage: StorageService = { + putContent: async (bytes, contentType) => { + const hash = sha256(bytes); + const key = keyForHash(hash, contentType); + const deduplicated = objects.has(key); + objects.set(key, Buffer.from(bytes)); + return { key, hash, sizeBytes: bytes.length, contentType, deduplicated }; + }, + getObject: async (key) => { + const bytes = objects.get(key); + if (bytes === undefined) throw new Error("missing stored object"); + return Buffer.from(bytes); + }, + hasContent: async (hash, type) => objects.has(keyForHash(hash, type)), + presign: async () => "unused", + }; + + beforeAll(async () => { + await runMigrations(process.env.MIGRATIONS_DIR ?? join(process.cwd(), "migrations")); + const workspace = await ownerPool.query( + "INSERT INTO workspaces (name, slug) VALUES ('Video integration', $1) RETURNING id", + [`video-${randomUUID()}`], + ); + workspaceId = String(workspace.rows[0]?.id); + const dependencies: VideoServiceDependencies = { + tenantTransaction: asTenant, + appendAudit: async (tenantId, event) => asTenant(tenantId, async (client) => { + const audit = await client.query( + "SELECT audit_append($1, $2, $3, $4, $5, $6, $7) AS id", + [ + tenantId, + event.actorUserId, + event.actorRole, + event.action, + event.targetType, + event.targetId, + event.diff ?? null, + ], + ); + return String(audit.rows[0]?.id); + }), + probe: async () => ({ width: 640, height: 360, durationMs: 1000, frameRate: 30, codec: "h264" }), + storage, + }; + service = new VideoService(dependencies); + }); + + afterAll(async () => { + if (workspaceId !== undefined) await ownerPool.query("DELETE FROM workspaces WHERE id = $1", [workspaceId]); + await appPool.end(); + await ownerPool.end(); + }); + + it("uploads, pages, reads status, and deletes a tenant video", async () => { + const uploaded = await service.uploadVideoBytes(workspaceId, Buffer.from("integration video bytes"), "video/mp4"); + const page = await service.listVideos(workspaceId, undefined, 20); + expect(page.items).toHaveLength(1); + expect(page.items[0]?.videoAsset.id).toBe(uploaded.videoAsset.id); + expect(page.items[0]?.processingState).toBe("complete"); + + const status = await service.getVideoStatus(uploaded.videoAsset.id, workspaceId); + expect(status.processingJob.state).toBe("complete"); + expect(status.processingJob.processingNote).toContain("Transcode skipped"); + expect(status.videoAsset).toMatchObject({ width: 640, height: 360, durationMs: 1000, codec: "h264" }); + await expect(service.getVideoStatus(uploaded.videoAsset.id, randomUUID())).rejects.toThrow(/not found/); + + await service.deleteVideo(uploaded.videoAsset.id, workspaceId); + await expect(service.getVideoStatus(uploaded.videoAsset.id, workspaceId)).rejects.toThrow(/not found/); + expect(objects.has(uploaded.videoAsset.storageKey)).toBe(true); + const jobs = await asTenant(workspaceId, (client) => client.query("SELECT id FROM video_processing_jobs WHERE video_asset_id = $1", [uploaded.videoAsset.id])); + expect(jobs.rowCount).toBe(0); + }); + + it("persists probe failure and exposes the error through status", async () => { + const failedService = new VideoService({ + tenantTransaction: asTenant, + appendAudit: async () => "test-audit", + storage, + probe: async () => { throw new Error("ffprobe: invalid media"); }, + }); + const uploaded = await failedService.uploadVideoBytes(workspaceId, Buffer.from("invalid video"), "video/mp4"); + expect((await failedService.getVideoStatus(uploaded.videoAsset.id, workspaceId)).processingJob).toMatchObject({ state: "failed", error: "ffprobe: invalid media" }); + await failedService.deleteVideo(uploaded.videoAsset.id, workspaceId); + }); + + it("shares content-addressed bytes between assets and retains them when one asset is deleted", async () => { + const bytes = Buffer.from("same video twice"); + const first = await service.uploadVideoBytes(workspaceId, bytes, "video/mp4"); + const second = await service.uploadVideoBytes(workspaceId, bytes, "video/mp4"); + expect(second.videoAsset.id).not.toBe(first.videoAsset.id); + expect(second.videoAsset.storageKey).toBe(first.videoAsset.storageKey); + await service.deleteVideo(first.videoAsset.id, workspaceId); + expect((await service.getVideoStatus(second.videoAsset.id, workspaceId)).processingJob.state).toBe("complete"); + expect(await storage.getObject(second.videoAsset.storageKey)).toEqual(bytes); + await service.deleteVideo(second.videoAsset.id, workspaceId); + }); +}); diff --git a/apps/api/src/video/video.roundtrip.test.ts b/apps/api/src/video/video.roundtrip.test.ts new file mode 100644 index 0000000..00e08ab --- /dev/null +++ b/apps/api/src/video/video.roundtrip.test.ts @@ -0,0 +1,46 @@ +import { readFile } from "node:fs/promises"; +import { describe, expect, it } from "vitest"; +import { keyForHash, sha256, seal, open, type StorageService } from "../storage/storage.js"; +import { probeVideoFile } from "./ffprobe.js"; +import { serviceWithDatabase, workspaceId } from "./test-fixtures.js"; + +// Opt in with a generated/real MP4; the normal suite needs no ffmpeg installation. +const fixture = process.env.VIDEO_ROUNDTRIP_FIXTURE; + +describe.skipIf(fixture === undefined)("real ffprobe roundtrip (encrypted in-memory storage)", () => { + it("uploads actual MP4 bytes, decrypts the stored source, and persists real probe metadata", async () => { + if (fixture === undefined) throw new Error("VIDEO_ROUNDTRIP_FIXTURE is required"); + const bytes = await readFile(fixture); + const objects = new Map(); + const kek = Buffer.alloc(32, 7); + const storage: StorageService = { + putContent: async (source, contentType) => { + const hash = sha256(source); + const key = keyForHash(hash, contentType); + const deduplicated = objects.has(key); + if (!deduplicated) objects.set(key, seal(source, kek)); + return { key, hash, sizeBytes: source.length, contentType, deduplicated }; + }, + getObject: async (key) => { + const encrypted = objects.get(key); + if (encrypted === undefined) throw new Error("missing object"); + const source = open(encrypted, kek); + expect(key).toBe(keyForHash(sha256(source), "video/mp4")); + return source; + }, + hasContent: async (hash, contentType) => objects.has(keyForHash(hash, contentType)), + presign: async () => { throw new Error("must decrypt, not presign"); }, + }; + const { service } = serviceWithDatabase({}, probeVideoFile, storage); + const uploaded = await service.uploadVideoBytes(workspaceId, bytes, "video/mp4"); + const status = await service.getVideoStatus(uploaded.videoAsset.id, workspaceId); + expect(status.processingJob.state).toBe("complete"); + expect(status.videoAsset).toMatchObject({ durationMs: 1000, width: 320, height: 180, frameRate: 25 }); + expect(status.videoAsset.codec).toBeDefined(); + expect(objects.get(status.videoAsset.storageKey)?.equals(bytes)).toBe(false); + expect(await storage.getObject(status.videoAsset.storageKey)).toEqual(bytes); + const again = await storage.putContent(bytes, "video/mp4"); + expect(again).toMatchObject({ key: status.videoAsset.storageKey, deduplicated: true }); + console.info("Real ffprobe status:", JSON.stringify(status)); + }); +}); diff --git a/apps/api/src/video/video.service.test.ts b/apps/api/src/video/video.service.test.ts new file mode 100644 index 0000000..c76c637 --- /dev/null +++ b/apps/api/src/video/video.service.test.ts @@ -0,0 +1,108 @@ +import { readFile, stat } from "node:fs/promises"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { VideoService } from "./video.service.js"; +import { assetId, serviceWithDatabase, workspaceId, type FakeVideoDatabase } from "./test-fixtures.js"; +import { MAX_VIDEO_UPLOAD_BYTES } from "./upload-limits.js"; + +afterEach(() => vi.unstubAllEnvs()); + +describe("video byte ingest and processing-job state machine", () => { + it("stores bytes, probes the decrypted stored object, and completes through a no-op transcode", async () => { + const database: FakeVideoDatabase = {}; + const bytes = Buffer.from("actual video bytes"); + let inputPath = ""; + const probe = vi.fn(async (path: string) => { + inputPath = path; + expect(await readFile(path)).toEqual(bytes); + expect((await stat(path)).mode & 0o777).toBe(0o600); + expect(database.job?.state).toBe("probing"); + return { width: 1280, height: 720, durationMs: 42_000, frameRate: 30, codec: "h264", audioCodec: "aac" }; + }); + const { service, appendAudit, storage } = serviceWithDatabase(database, probe); + const uploaded = await service.uploadVideoBytes(workspaceId, bytes, "video/mp4"); + expect(storage.putContent).toHaveBeenCalledWith(bytes, "video/mp4"); + expect(uploaded.videoAsset.storageKey).toMatch(/^sha256\/[a-f0-9]{64}\.mp4$/); + expect(storage.getObject).toHaveBeenCalledWith(uploaded.videoAsset.storageKey); + expect(uploaded.videoAsset).toMatchObject({ sizeBytes: bytes.length, width: 1280, height: 720, durationMs: 42_000, codec: "h264" }); + expect(uploaded.processingJob).toMatchObject({ state: "complete", progressPercent: 100, outputKey: uploaded.videoAsset.storageKey }); + expect(uploaded.processingJob.processingNote).toContain("Transcode skipped"); + expect(database.transitions).toEqual(["queued", "probing", "transcoding", "complete"]); + await expect(stat(inputPath)).rejects.toMatchObject({ code: "ENOENT" }); + expect(appendAudit).toHaveBeenCalledWith(workspaceId, expect.objectContaining({ action: "video.uploaded", targetId: assetId })); + expect(appendAudit).toHaveBeenCalledWith(workspaceId, expect.objectContaining({ action: "video.probed", diff: expect.objectContaining({ processingNote: expect.stringContaining("Transcode skipped") }) })); + await expect(service.probeVideo(assetId, workspaceId)).rejects.toThrow("cannot probe video in processing state complete"); + expect(probe).toHaveBeenCalledTimes(1); + }); + + it("returns a failed job with the ffprobe error in status, and removes the temporary file", async () => { + const database: FakeVideoDatabase = {}; + let inputPath = ""; + const { service } = serviceWithDatabase(database, async (path) => { + inputPath = path; + throw new Error("unreadable media"); + }); + const uploaded = await service.uploadVideoBytes(workspaceId, Buffer.from("broken"), "video/mp4"); + expect(uploaded.processingJob).toMatchObject({ state: "failed", error: "unreadable media" }); + expect(database.transitions).toEqual(["queued", "probing", "failed"]); + expect((await service.getVideoStatus(assetId, workspaceId)).processingJob.error).toBe("unreadable media"); + await expect(stat(inputPath)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("fails with 503 before DB registration without S3 configuration", async () => { + vi.stubEnv("S3_ACCESS_KEY", ""); + vi.stubEnv("S3_SECRET_KEY", ""); + const transaction = vi.fn(); + const service = new VideoService({ tenantTransaction: transaction }); + await expect(service.uploadVideoBytes(workspaceId, Buffer.from("video"), "video/mp4")).rejects.toMatchObject({ status: 503, message: expect.stringContaining("Video storage is not configured") }); + expect(transaction).not.toHaveBeenCalled(); + }); + + it("fails with 503 on a configured storage outage, without creating a queued row", async () => { + const database: FakeVideoDatabase = {}; + const { service, storage } = serviceWithDatabase(database, async () => ({})); + vi.mocked(storage.putContent).mockRejectedValueOnce(new Error("ECONNREFUSED")); + await expect(service.uploadVideoBytes(workspaceId, Buffer.from("video"), "video/mp4")).rejects.toMatchObject({ status: 503 }); + expect(database.asset).toBeUndefined(); + expect(database.job).toBeUndefined(); + }); + + it("marks fetch/integrity failures after put as failed, rather than leaving a queued job", async () => { + const database: FakeVideoDatabase = {}; + const probe = vi.fn(async () => ({})); + const { service, storage } = serviceWithDatabase(database, probe); + vi.mocked(storage.getObject).mockRejectedValueOnce(new Error("storage: integrity failure")); + const uploaded = await service.uploadVideoBytes(workspaceId, Buffer.from("video"), "video/mp4"); + expect(uploaded.processingJob).toMatchObject({ state: "failed", error: "storage: integrity failure" }); + expect(probe).not.toHaveBeenCalled(); + }); + + it("keeps metadata registration compatible only for existing objects with matching size", async () => { + const database: FakeVideoDatabase = {}; + const { service, storage } = serviceWithDatabase(database, async () => ({ width: 640 })); + const stored = await storage.putContent(Buffer.from("video"), "video/mp4"); + await expect(service.uploadVideo(workspaceId, stored.key, "video/mp4", 1)).rejects.toThrow("sizeBytes does not match"); + await expect(service.uploadVideo(workspaceId, "sha256/missing.mp4", "video/mp4", 5)).rejects.toThrow("existing readable encrypted object"); + const uploaded = await service.uploadVideo(workspaceId, stored.key, "video/mp4", stored.sizeBytes); + expect(uploaded.processingJob.state).toBe("complete"); + expect(uploaded.videoAsset.width).toBe(640); + }); + + it("deletes the asset/job while retaining shared content-addressed bytes for GC", async () => { + const database: FakeVideoDatabase = {}; + const { service, objects } = serviceWithDatabase(database, async () => ({})); + const bytes = Buffer.from("shared source"); + const uploaded = await service.uploadVideoBytes(workspaceId, bytes, "video/mp4"); + await service.deleteVideo(assetId, workspaceId); + expect(database.asset).toBeUndefined(); + expect(database.job).toBeUndefined(); + expect(objects.get(uploaded.videoAsset.storageKey)).toEqual(bytes); + }); + + it("rejects empty or non-video uploads before storage", async () => { + const { service, storage } = serviceWithDatabase({}, async () => ({})); + await expect(service.uploadVideoBytes(workspaceId, Buffer.alloc(0), "video/mp4")).rejects.toMatchObject({ status: 400 }); + await expect(service.uploadVideoBytes(workspaceId, Buffer.from("image"), "image/png")).rejects.toMatchObject({ status: 400 }); + await expect(service.uploadVideoBytes(workspaceId, Buffer.allocUnsafe(MAX_VIDEO_UPLOAD_BYTES + 1), "video/mp4")).rejects.toThrow("100 MiB upload limit"); + expect(storage.putContent).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/src/video/video.service.ts b/apps/api/src/video/video.service.ts new file mode 100644 index 0000000..6cdc28e --- /dev/null +++ b/apps/api/src/video/video.service.ts @@ -0,0 +1,509 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BadRequestException, Inject, Injectable, NotFoundException, Optional, ServiceUnavailableException } from "@nestjs/common"; +import type { PoolClient } from "pg"; +import { appendTenantAudit, type AuditEvent } from "../audit/audit.service.js"; +import { withTenant } from "../db/client.js"; +import { probeVideoFile, type VideoProbeResult } from "./ffprobe.js"; +import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js"; +import { MAX_VIDEO_UPLOAD_BYTES } from "./upload-limits.js"; + +const NO_OP_TRANSCODE_NOTE = "Transcode skipped: original stored object retained; rendering belongs to the media-worker compositor."; + +export type VideoProcessingJobState = "queued" | "probing" | "transcoding" | "complete" | "failed"; + +export interface VideoAssetRecord { + id: string; + workspaceId: string; + storageKey: string; + mimeType: string; + sizeBytes: number; + width?: number; + height?: number; + durationMs?: number; + frameRate?: number; + codec?: string; + audioCodec?: string; + thumbnailKey?: string; +} + +export interface VideoProcessingJobRecord { + id: string; + workspaceId: string; + videoAssetId: string; + state: VideoProcessingJobState; + progressPercent?: number; + outputKey?: string; + error?: string; + processingNote?: string; +} + +export interface VideoPublicationSummary { + connectionId: string; + remoteId: string; + platform: string; + publishedAt: Date; +} + +export interface VideoLibraryEntryRecord { + videoAsset: VideoAssetRecord; + processingState: VideoProcessingJobState; + publications: VideoPublicationSummary[]; +} + +export interface VideoLibraryPage { + items: VideoLibraryEntryRecord[]; + nextCursor?: string; +} + +export interface VideoStatus { + videoAsset: VideoAssetRecord; + processingJob: VideoProcessingJobRecord; +} + +export type VideoUploadResult = VideoStatus; + +type TenantTransaction = (workspaceId: string, fn: (client: PoolClient) => Promise) => Promise; +type AuditAppender = (workspaceId: string, event: AuditEvent) => Promise; + +export interface VideoServiceDependencies { + tenantTransaction: TenantTransaction; + appendAudit: AuditAppender; + probe: (inputPath: string) => Promise; + storage?: StorageService; +} + +export const VIDEO_SERVICE_DEPENDENCIES = Symbol("VIDEO_SERVICE_DEPENDENCIES"); + +const DEFAULT_DEPENDENCIES: VideoServiceDependencies = { + tenantTransaction: withTenant, + appendAudit: appendTenantAudit, + probe: probeVideoFile, +}; + +@Injectable() +export class VideoService { + private readonly dependencies: VideoServiceDependencies; + private storage?: StorageService; + + constructor( + @Optional() @Inject(VIDEO_SERVICE_DEPENDENCIES) dependencies?: Partial, + ) { + this.dependencies = { ...DEFAULT_DEPENDENCIES, ...dependencies }; + this.storage = dependencies?.storage; + } + + async uploadVideoBytes(workspaceId: string, bytes: Buffer, mimeType: string): Promise { + validateUpload("pending", mimeType, bytes.length); + const storage = this.requireStorage(); + const stored = await storage.putContent(bytes, mimeType).catch(() => { + throw new ServiceUnavailableException("Video storage unavailable: could not store encrypted upload in S3/MinIO."); + }); + return this.registerAndProbe(workspaceId, stored.key, stored.contentType, stored.sizeBytes); + } + + async uploadVideo( + workspaceId: string, + storageKey: string, + mimeType: string, + sizeBytes: number, + ): Promise { + validateUpload(storageKey, mimeType, sizeBytes); + // Legacy metadata registration is accepted only for an object that really exists. + const bytes = await this.requireStorage().getObject(storageKey).catch(() => { + throw new BadRequestException("storageKey must identify an existing readable encrypted object; upload a file first"); + }); + if (bytes.length !== sizeBytes) throw new BadRequestException("sizeBytes does not match the stored object"); + return this.registerAndProbe(workspaceId, storageKey, mimeType, sizeBytes); + } + + private async registerAndProbe( + workspaceId: string, storageKey: string, mimeType: string, sizeBytes: number, + ): Promise { + const uploaded = await this.dependencies.tenantTransaction(workspaceId, async (client) => { + const assetResult = await client.query( + `INSERT INTO video_assets (workspace_id, storage_key, mime_type, size_bytes) + VALUES ($1, $2, $3, $4) RETURNING *`, + [workspaceId, storageKey, mimeType, sizeBytes], + ); + const videoAsset = assetFromRow(requiredRow(assetResult.rows[0], "created video asset")); + const jobResult = await client.query( + `INSERT INTO video_processing_jobs (workspace_id, video_asset_id, state) + VALUES ($1, $2, 'queued') RETURNING *`, + [workspaceId, videoAsset.id], + ); + return { + videoAsset, + processingJob: jobFromRow(requiredRow(jobResult.rows[0], "created video processing job")), + }; + }); + + try { + await this.probeVideo(uploaded.videoAsset.id, workspaceId); + } catch (error) { + // Probe errors are recorded on the job; return its ID and failure in status. + const status = await this.getVideoStatus(uploaded.videoAsset.id, workspaceId); + if (status.processingJob.state !== "failed") throw error; + } + await this.audit(workspaceId, "video.uploaded", uploaded.videoAsset.id, { + mimeType, + sizeBytes, + }); + return this.getVideoStatus(uploaded.videoAsset.id, workspaceId); + } + + async getVideoStatus(assetId: string, workspaceId: string): Promise { + return this.dependencies.tenantTransaction(workspaceId, async (client) => { + const result = await client.query( + `SELECT va.*, + j.id AS job_id, j.video_asset_id AS job_video_asset_id, + j.state AS job_state, j.progress_percent AS job_progress_percent, + j.output_key AS job_output_key, j.error AS job_error + FROM video_assets va + LEFT JOIN LATERAL ( + SELECT * FROM video_processing_jobs + WHERE video_asset_id = va.id + ORDER BY created_at DESC, id DESC LIMIT 1 + ) j ON true + WHERE va.id = $1`, + [assetId], + ); + if (result.rowCount !== 1) throw new NotFoundException(`video ${assetId} not found`); + const row = requiredRow(result.rows[0], "video status"); + if (row.job_id === null || row.job_id === undefined) { + throw new Error(`video ${assetId} has no processing job`); + } + return { videoAsset: assetFromRow(row), processingJob: joinedJobFromRow(row) }; + }); + } + + async probeVideo(assetId: string, workspaceId: string): Promise { + const claimed = await this.dependencies.tenantTransaction(workspaceId, async (client) => { + const asset = await client.query("SELECT * FROM video_assets WHERE id = $1", [assetId]); + if (asset.rowCount !== 1) throw new NotFoundException(`video ${assetId} not found`); + const changed = await client.query( + `UPDATE video_processing_jobs + SET state = 'probing', progress_percent = 10, error = NULL, updated_at = now() + WHERE id = ( + SELECT id FROM video_processing_jobs + WHERE video_asset_id = $1 ORDER BY created_at DESC, id DESC LIMIT 1 + ) AND state = 'queued' + RETURNING id`, + [assetId], + ); + if (changed.rowCount !== 1) { + const latest = await client.query( + `SELECT state FROM video_processing_jobs + WHERE video_asset_id = $1 ORDER BY created_at DESC, id DESC LIMIT 1`, + [assetId], + ); + const state = latest.rows[0]?.state; + throw new BadRequestException( + state === undefined + ? `video ${assetId} has no processing job` + : `cannot probe video in processing state ${String(state)}`, + ); + } + return { + storageKey: String(asset.rows[0]?.storage_key), + jobId: String(changed.rows[0]?.id), + }; + }); + + let metadata: VideoProbeResult; + let videoAsset: VideoAssetRecord; + try { + const bytes = await this.requireStorage().getObject(claimed.storageKey); + const directory = await mkdtemp(join(tmpdir(), "tmbh-video-")); + try { + const inputPath = join(directory, "input"); + await writeFile(inputPath, bytes, { mode: 0o600 }); + metadata = await this.dependencies.probe(inputPath); + } finally { + await rm(directory, { recursive: true, force: true }); + } + videoAsset = await this.dependencies.tenantTransaction(workspaceId, async (client) => { + const updated = await client.query( + `UPDATE video_assets + SET width = $2, height = $3, duration_ms = $4, frame_rate = $5, + codec = $6, audio_codec = $7 + WHERE id = $1 RETURNING *`, + [ + assetId, + metadata.width ?? null, + metadata.height ?? null, + metadata.durationMs ?? null, + metadata.frameRate ?? null, + metadata.codec ?? null, + metadata.audioCodec ?? null, + ], + ); + if (updated.rowCount !== 1) throw new NotFoundException(`video ${assetId} not found`); + const transcoding = await client.query( + `UPDATE video_processing_jobs + SET state = 'transcoding', progress_percent = 90, error = NULL, updated_at = now() + WHERE id = $1 AND state = 'probing' RETURNING id`, + [claimed.jobId], + ); + if (transcoding.rowCount !== 1) throw new Error(`video processing job ${claimed.jobId} lost its claim`); + return assetFromRow(requiredRow(updated.rows[0], "probed video asset")); + }); + + // This slice deliberately retains the source. The compositor owns real transcoding. + await this.dependencies.tenantTransaction(workspaceId, async (client) => { + const completed = await client.query( + `UPDATE video_processing_jobs + SET state = 'complete', progress_percent = 100, output_key = $2, error = NULL, updated_at = now() + WHERE id = $1 AND state = 'transcoding' RETURNING id`, + [claimed.jobId, claimed.storageKey], + ); + if (completed.rowCount !== 1) throw new Error(`video processing job ${claimed.jobId} lost its claim`); + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + await this.dependencies.tenantTransaction(workspaceId, async (client) => { + await client.query( + `UPDATE video_processing_jobs + SET state = 'failed', error = $2, updated_at = now() + WHERE id = $1 AND state IN ('probing', 'transcoding')`, + [claimed.jobId, message], + ); + }); + throw error; + } + + await this.audit(workspaceId, "video.probed", assetId, { ...metadata, processingNote: NO_OP_TRANSCODE_NOTE }); + return videoAsset; + } + + async listVideos( + workspaceId: string, + cursor: string | undefined, + limit: number, + ): Promise { + if (!Number.isInteger(limit) || limit < 1 || limit > 200) { + throw new BadRequestException("limit must be an integer between 1 and 200"); + } + const decoded = cursor === undefined || cursor === "" ? undefined : decodeCursor(cursor); + return this.dependencies.tenantTransaction(workspaceId, async (client) => { + const result = await client.query( + `SELECT va.*, + COALESCE(j.state, 'queued') AS processing_state, + COALESCE(( + SELECT jsonb_agg(jsonb_build_object( + 'connectionId', p.connection_id, + 'remoteId', p.remote_id, + 'platform', c.provider, + 'publishedAt', p.published_at + ) ORDER BY p.published_at DESC) + FROM publications p + JOIN drafts d ON d.id = p.draft_id + JOIN connections c ON c.id = p.connection_id + WHERE d.post ->> 'videoAssetId' = va.id::text + OR d.post -> 'mediaAssetIds' ? va.id::text + ), '[]'::jsonb) AS publications + FROM video_assets va + LEFT JOIN LATERAL ( + SELECT state FROM video_processing_jobs + WHERE video_asset_id = va.id + ORDER BY created_at DESC, id DESC LIMIT 1 + ) j ON true + WHERE ($1::timestamptz IS NULL OR (va.created_at, va.id) < ($1::timestamptz, $2::uuid)) + ORDER BY va.created_at DESC, va.id DESC + LIMIT $3`, + [decoded?.createdAt ?? null, decoded?.id ?? null, limit + 1], + ); + const rows = result.rows.map((row: unknown) => requiredRow(row, "video library entry")); + const hasMore = rows.length > limit; + const pageRows = rows.slice(0, limit); + const items = pageRows.map(libraryEntryFromRow); + const last = pageRows[pageRows.length - 1]; + return { + items, + ...(hasMore && last !== undefined + ? { nextCursor: encodeCursor({ createdAt: new Date(String(last.created_at)).toISOString(), id: String(last.id) }) } + : {}), + }; + }); + } + + async deleteVideo(assetId: string, workspaceId: string): Promise { + // Objects are shared by content hash across assets/workspaces. Retain for future + // reference-aware GC; deleting here could destroy another asset's source. + await this.dependencies.tenantTransaction(workspaceId, async (client) => { + const removed = await client.query("DELETE FROM video_assets WHERE id = $1 RETURNING id", [assetId]); + if (removed.rowCount !== 1) throw new NotFoundException(`video ${assetId} not found`); + }); + await this.audit(workspaceId, "video.deleted", assetId, {}); + } + + private requireStorage(): StorageService { + if (this.storage !== undefined) return this.storage; + const config = s3ConfigFromEnv(); + if (!config.accessKeyId?.trim() || !config.secretAccessKey?.trim()) { + throw new ServiceUnavailableException( + "Video storage is not configured: set S3_ACCESS_KEY and S3_SECRET_KEY, plus S3_ENDPOINT for MinIO.", + ); + } + try { + this.storage = new S3Storage(config); + } catch { + throw new ServiceUnavailableException("Video storage configuration is invalid; check S3 settings and S3_KEK."); + } + return this.storage; + } + + private async audit( + workspaceId: string, + action: "video.uploaded" | "video.probed" | "video.deleted", + targetId: string, + diff: Record, + ): Promise { + await this.dependencies.appendAudit(workspaceId, { + actorUserId: null, + actorRole: "api", + action, + targetType: "video_asset", + targetId, + diff, + }); + } +} + +function validateUpload(storageKey: string, mimeType: string, sizeBytes: number): void { + if (storageKey.trim() === "") throw new BadRequestException("storageKey must be a non-empty string"); + if (!mimeType.toLowerCase().startsWith("video/")) { + throw new BadRequestException("mimeType must be a video media type"); + } + if (!Number.isSafeInteger(sizeBytes) || sizeBytes < 1) { + throw new BadRequestException("sizeBytes must be a positive safe integer"); + } + if (sizeBytes > MAX_VIDEO_UPLOAD_BYTES) throw new BadRequestException("video exceeds the 100 MiB upload limit"); +} + +function requiredRow(row: unknown, description: string): Record { + if (typeof row !== "object" || row === null || Array.isArray(row)) { + throw new Error(`database did not return ${description}`); + } + return row as Record; +} + +function assetFromRow(row: Record): VideoAssetRecord { + return { + id: String(row.id), + workspaceId: String(row.workspace_id), + storageKey: String(row.storage_key), + mimeType: String(row.mime_type), + sizeBytes: safeDatabaseNumber(row.size_bytes, "size_bytes"), + ...optionalNumber("width", row.width), + ...optionalNumber("height", row.height), + ...optionalNumber("durationMs", row.duration_ms), + ...optionalNumber("frameRate", row.frame_rate), + ...optionalString("codec", row.codec), + ...optionalString("audioCodec", row.audio_codec), + ...optionalString("thumbnailKey", row.thumbnail_key), + }; +} + +function jobFromRow(row: Record): VideoProcessingJobRecord { + return { + id: String(row.id), + workspaceId: String(row.workspace_id), + videoAssetId: String(row.video_asset_id), + state: processingState(row.state), + ...optionalNumber("progressPercent", row.progress_percent), + ...optionalString("outputKey", row.output_key), + ...optionalString("error", row.error), + }; +} + +function joinedJobFromRow(row: Record): VideoProcessingJobRecord { + const job = jobFromRow({ + id: row.job_id, + workspace_id: row.workspace_id, + video_asset_id: row.job_video_asset_id, + state: row.job_state, + progress_percent: row.job_progress_percent, + output_key: row.job_output_key, + error: row.job_error, + }); + return { + ...job, + ...(job.state === "complete" && job.outputKey === row.storage_key ? { processingNote: NO_OP_TRANSCODE_NOTE } : {}), + }; +} + +function libraryEntryFromRow(row: Record): VideoLibraryEntryRecord { + const rawPublications = Array.isArray(row.publications) ? row.publications : []; + return { + videoAsset: assetFromRow(row), + processingState: processingState(row.processing_state), + publications: rawPublications.map((publication) => publicationFromJson(requiredRow(publication, "publication"))), + }; +} + +function publicationFromJson(row: Record): VideoPublicationSummary { + return { + connectionId: String(row.connectionId), + remoteId: String(row.remoteId), + platform: String(row.platform), + publishedAt: new Date(String(row.publishedAt)), + }; +} + +function processingState(value: unknown): VideoProcessingJobState { + if ( + value === "queued" || + value === "probing" || + value === "transcoding" || + value === "complete" || + value === "failed" + ) { + return value; + } + throw new Error(`invalid video processing state: ${String(value)}`); +} + +function safeDatabaseNumber(value: unknown, field: string): number { + const number = typeof value === "number" ? value : typeof value === "string" ? Number(value) : NaN; + if (!Number.isSafeInteger(number)) throw new Error(`${field} is outside JavaScript's safe integer range`); + return number; +} + +function optionalNumber(key: K, value: unknown): Partial> { + if (value === null || value === undefined) return {}; + const number = typeof value === "number" ? value : typeof value === "string" ? Number(value) : NaN; + if (!Number.isFinite(number)) throw new Error(`${key} is not numeric`); + return { [key]: number } as Record; +} + +function optionalString(key: K, value: unknown): Partial> { + return value === null || value === undefined ? {} : { [key]: String(value) } as Record; +} + +interface VideoCursor { + createdAt: string; + id: string; +} + +function encodeCursor(cursor: VideoCursor): string { + return Buffer.from(JSON.stringify(cursor), "utf8").toString("base64url"); +} + +function decodeCursor(cursor: string): VideoCursor { + try { + const decoded = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8")) as unknown; + const row = requiredRow(decoded, "video cursor"); + const createdAt = String(row.createdAt); + const id = String(row.id); + if (!Number.isFinite(Date.parse(createdAt)) || !isUuid(id)) throw new Error("invalid fields"); + return { createdAt, id }; + } catch { + throw new BadRequestException("cursor is invalid"); + } +} + +function isUuid(value: string): boolean { + return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(value); +} diff --git a/apps/api/src/video/video.status.http.test.ts b/apps/api/src/video/video.status.http.test.ts new file mode 100644 index 0000000..75fd966 --- /dev/null +++ b/apps/api/src/video/video.status.http.test.ts @@ -0,0 +1,99 @@ +import "reflect-metadata"; +import { Module, NotFoundException, UnauthorizedException, ForbiddenException, type ExecutionContext, type INestApplication } from "@nestjs/common"; +import { NestFactory, Reflector } from "@nestjs/core"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { injectHttp } from "../connections/http-test-transport.js"; +import { RbacGuard, PERMISSION_KEY } from "../rbac/rbac.guard.js"; +import { VideoController } from "./video.controller.js"; +import { VideoService, type VideoStatus } from "./video.service.js"; +import { assetId, jobId, serviceWithDatabase, workspaceId } from "./test-fixtures.js"; + +@Module({}) +class TestModule {} + +function statusService(database: Parameters[0]): ReturnType { + // A probe that never runs: status reads do not invoke ffprobe. + return serviceWithDatabase(database, async () => { throw new Error("probe must not run for status reads"); }); +} + +describe("GET /video/:id/status", () => { + let app: INestApplication | undefined; + let canView = true; + + beforeEach(() => { + canView = true; + vi.spyOn(RbacGuard.prototype, "canActivate").mockImplementation(async (context: ExecutionContext) => { + const request = context.switchToHttp().getRequest<{ headers: Record; session?: { workspaceId: string } }>(); + if (request.headers.authorization !== "Bearer test-session") throw new UnauthorizedException(); + if (!canView && new Reflector().get(PERMISSION_KEY, context.getHandler()) === "media.view") { + throw new ForbiddenException("missing permission: media.view"); + } + request.session = { workspaceId }; + return true; + }); + }); + + afterEach(async () => { + await app?.close(); + app = undefined; + vi.restoreAllMocks(); + }); + + async function start(service: VideoService): Promise { + app = await NestFactory.create({ + module: TestModule, + controllers: [VideoController], + providers: [{ provide: VideoService, useValue: service }], + }, { logger: false }); + await app.init(); + } + + function status(id: string = assetId, authenticated = true): Promise { + return injectHttp(app?.getHttpAdapter().getInstance(), `http://api.test/video/${id}/status`, { + headers: { ...(authenticated ? { authorization: "Bearer test-session" } : {}) }, + }); + } + + it("returns the asset with its latest processing job (the shape the drawer polls)", async () => { + const { service } = statusService({}); + await start(service); + // Seed one asset + job through the service so the joined row exists. + await service.uploadVideoBytes(workspaceId, Buffer.from("status fixture"), "video/mp4"); + const response = await status(); + expect(response.status).toBe(200); + const body = await response.json() as VideoStatus; + expect(body.videoAsset).toMatchObject({ id: assetId, sizeBytes: Buffer.byteLength("status fixture") }); + expect(body.processingJob).toMatchObject({ id: jobId, state: expect.stringMatching(/^(probing|transcoding|complete|failed)$/) }); + }); + + it("reports an in-flight job state without completing it", async () => { + const { service } = statusService({}); + await start(service); + await service.uploadVideoBytes(workspaceId, Buffer.from("in flight"), "video/mp4"); + // The fake DB only advances probing → transcoding when the service runs it, + // so a bare status read must observe an earlier state, not a fabricated one. + const before = await service.getVideoStatus(assetId, workspaceId); + const response = await status(); + const body = await response.json() as VideoStatus; + expect(body.processingJob.state).toBe(before.processingJob.state); + expect(body.processingJob.state).not.toBe("complete"); + }); + + it("returns 404 for an unknown asset in the workspace", async () => { + const { service } = statusService({}); + await start(service); + const response = await status(assetId); + expect(response.status).toBe(404); + expect(await response.json()).toMatchObject({ message: expect.stringContaining("not found") }); + // And the service-level route contract is a NotFoundException, not a crash. + await expect(service.getVideoStatus(assetId, workspaceId)).rejects.toBeInstanceOf(NotFoundException); + }); + + it("returns 401 without a session and 403 without media.view", async () => { + const { service } = statusService({}); + await start(service); + expect((await status(assetId, false)).status).toBe(401); + canView = false; + expect((await status()).status).toBe(403); + }); +}); diff --git a/apps/api/src/virtual-channels/import-egress.ts b/apps/api/src/virtual-channels/import-egress.ts new file mode 100644 index 0000000..8eb7491 --- /dev/null +++ b/apps/api/src/virtual-channels/import-egress.ts @@ -0,0 +1,71 @@ +import { existsSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { pool } from "../db/client.js"; +// Type-only (erased in the CommonJS build): the ESM package is loaded +// natively at runtime, exactly like the live monitor's egress binding. +import type { createEgressClient, EgressError, EgressPolicy } from "../../../../packages/egress/src/index.js"; + +type EgressPackage = typeof import("../../../../packages/egress/src/index.js"); + +export interface ImportEgress { + egressFetch: ReturnType["egressFetch"]; + policy: EgressPolicy; + /** Value identity check for errors raised by the shared egress client. */ + isEgressError(error: unknown): error is EgressError; +} + +/** Import resolution deadline per source (two public GETs at most). */ +export const IMPORT_TIMEOUT_MS = 10_000; +/** Host class keying the shared per-host budget rows. */ +export const IMPORT_HOST_CLASS = "import"; + +const nativeImport = new Function("specifier", "return import(specifier)") as (specifier: string) => Promise; +let shared: Promise | undefined; + +/** + * Process-wide egress client for bulk import: every import job and replica + * draws from the same per-host budget rows for a given host key. + */ +export function importEgress(): Promise { + shared ??= (async () => { + const specifier = pathToFileURL(resolveImportPackage()).href; + const pkg = await nativeImport(specifier); + const client = pkg.createEgressClient({ + pool, + timeoutMs: IMPORT_TIMEOUT_MS, + maxResponseBytes: 512 * 1024, + hostClasses: { [IMPORT_HOST_CLASS]: { capacity: 4, refillPerInterval: 20 } }, + }); + return { ...client, isEgressError: (error: unknown): error is EgressError => error instanceof pkg.EgressError }; + })(); + return shared; +} + +function resolveImportPackage(): string { + for (const start of [process.cwd(), __dirname]) { + for (let root = start; ; root = dirname(root)) { + const directory = resolve(root, "packages/egress"); + const files = process.env.NODE_ENV === "production" ? ["dist/index.js"] : ["src/index.ts", "dist/index.js"]; + for (const file of files) { + const path = resolve(directory, file); + if (existsSync(path)) return path; + } + if (dirname(root) === root) break; + } + } + throw new Error("Build packages/egress before importing sources"); +} + +/** Row-level taxonomy for an egress/policy failure. */ +export function egressFailure( + error: unknown, + isEgressError: ImportEgress["isEgressError"], +): { reason: "not_found" | "upstream" | "denied"; message: string } { + if (isEgressError(error)) { + if (error.code === "DNS_FAILED") return { reason: "not_found", message: "Instance host could not be resolved" }; + if (error.code === "BUDGET_EXHAUSTED") return { reason: "upstream", message: "Instance request budget exhausted; retry later" }; + return { reason: "denied", message: "Instance request denied by the outbound policy" }; + } + return { reason: "upstream", message: "Instance request failed" }; +} diff --git a/apps/api/src/virtual-channels/import-normalize.test.ts b/apps/api/src/virtual-channels/import-normalize.test.ts new file mode 100644 index 0000000..e58e168 --- /dev/null +++ b/apps/api/src/virtual-channels/import-normalize.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it } from "vitest"; +import { EgressPolicy } from "../../../../packages/egress/src/index.js"; +import { + dedupeSyntactic, + IMPORT_ROW_LIMIT, + InvalidRowError, + normalizeImportRow, + parseImportBody, +} from "./import-normalize.js"; + +/** Policy-only validation: no DNS, no budget draw. */ +const policy = new EgressPolicy(); + +function normalize(raw: string, defaultHost?: string) { + return normalizeImportRow(raw, defaultHost, policy); +} + +describe("import normalization", () => { + it("canonicalizes handles: strips @, lowercases, IDNA-encodes the host", () => { + expect(normalize("@Alice@VIDEO.Example.")).toEqual({ + kind: "channel", name: "alice", host: "video.example", + key: "channel:alice@video.example", label: "@alice@video.example", raw: "@Alice@VIDEO.Example.", + }); + // Unicode host punycodes to the same canonical form as its ASCII twin, so + // both spellings collide on one dedupe key. + expect(normalize("bob@vïdeo.example").host).toBe(normalize("bob@xn--vdeo-5pa.example").host); + }); + + it("extracts channel identity from /c, /a and /video-channels URLs, dropping default ports", () => { + expect(normalize("https://video.example/c/Alice")).toMatchObject({ kind: "channel", name: "alice", host: "video.example" }); + expect(normalize("https://Video.Example:443/video-channels/alice_channel")).toMatchObject({ kind: "channel", name: "alice_channel" }); + expect(normalize("https://video.example/a/alice")).toMatchObject({ kind: "channel", name: "alice" }); + }); + + it("extracts a live identity from watch/embed URLs, including the ;threadId suffix", () => { + const uuid = "0d1c2b3a-4e5f-4a6b-8c7d-9e0f1a2b3c4d"; + expect(normalize(`https://video.example/w/${uuid};threadId=7`)).toMatchObject({ kind: "live", name: uuid }); + expect(normalize(`https://video.example/videos/watch/${uuid.toUpperCase()}`)).toMatchObject({ kind: "live", name: uuid }); + expect(normalize(`https://video.example/videos/embed/${uuid}`)).toMatchObject({ kind: "live" }); + }); + + it("a bare uuid needs a default host and one is supplied by the connection", () => { + const uuid = "0d1c2b3a-4e5f-4a6b-8c7d-9e0f1a2b3c4d"; + expect(() => normalize(uuid)).toThrow(InvalidRowError); + expect(normalize(uuid, "video.example")).toMatchObject({ kind: "live", name: uuid, host: "video.example", key: `live:${uuid}@video.example` }); + expect(normalize("alice", "video.example")).toMatchObject({ kind: "channel", name: "alice", host: "video.example" }); + }); + + it("short ids pass (case preserved for the API path) and junk ids do not", () => { + const short = normalize("https://video.example/w/abc123_XY", "video.example"); + expect(short.kind).toBe("live"); + expect(short.name).toBe(short.name.toLowerCase()); + expect(() => normalize("https://video.example/w/not_a_uuid!!")).toThrow(InvalidRowError); + }); + + it("rejects rows the egress policy refuses before any network work", () => { + for (const bad of [ + "http://127.0.0.1/w/abc123", // non-https + loopback + "https://video.example:22/w/abc123", // denied port + "ftp://video.example/c/alice", // denied scheme + "https://user:pass@video.example/c/alice", // credentials + ]) { + expect(() => normalize(bad), bad).toThrow(InvalidRowError); + expect(() => normalize(bad), bad).toThrow(InvalidRowError); + } + }); + + it("rejects empty, oversized, control-character and unknown rows", () => { + expect(() => normalize(" ")).toThrow(/empty/); + expect(() => normalize("x".repeat(2049))).toThrow(InvalidRowError); + expect(() => normalize("https://video.example/c/alice\u0000")).toThrow(InvalidRowError); + expect(() => normalize("###not-a-thing###")).toThrow(InvalidRowError); + }); + + it("pre-resolution dedupe keeps the first occurrence and counts the rest", () => { + const rows = [ + normalize("@alice@video.example"), + normalize("https://video.example/c/alice"), + normalize("@bob@video.example"), + ]; + const result = dedupeSyntactic(rows); + expect(result.unique.map((row) => row.key)).toEqual(["channel:alice@video.example", "channel:bob@video.example"]); + expect(result.duplicateRows).toBe(1); + }); + + it("kind participates in the key: a channel and a live with the same name are distinct", () => { + const channel = normalize("@news@video.example"); + const again = normalize("news@video.example"); + expect(again.kind).toBe("channel"); + expect(dedupeSyntactic([channel, again]).duplicateRows).toBe(1); + const uuid = "0d1c2b3a-4e5f-4a6b-8c7d-9e0f1a2b3c4d"; + const asLive = normalize(uuid, "video.example"); + expect(asLive.kind).toBe("live"); + expect(dedupeSyntactic([channel, asLive]).duplicateRows).toBe(0); + }); +}); + +describe("import body parsing", () => { + it("splits multiline text, ignoring blanks", () => { + const parsed = parseImportBody("text/plain", "\n@alice@video.example\n\nhttps://other.example/w/abc123_XY\n"); + expect(parsed.format).toBe("multiline"); + expect(parsed.rows).toEqual(["@alice@video.example", "https://other.example/w/abc123_XY"]); + }); + + it("treats a single token as one url/handle row", () => { + expect(parseImportBody("text/plain", "https://video.example/c/alice")).toMatchObject({ format: "url", rows: ["https://video.example/c/alice"] }); + expect(parseImportBody(undefined, "@alice@video.example")).toMatchObject({ format: "handle" }); + }); + + it("CSV: skips a header row and uses the first column, quotes stripped", () => { + const parsed = parseImportBody("text/csv", "url,label\n\"https://video.example/c/alice\",Main\nhttps://other.example/c/bob,Second"); + expect(parsed.rows).toEqual(["https://video.example/c/alice", "https://other.example/c/bob"]); + }); + + it("JSON: accepts strings and objects with url/handle/id", () => { + const parsed = parseImportBody("application/json", JSON.stringify([ + "@alice@video.example", + { url: "https://other.example/c/bob" }, + { handle: "@carol@third.example" }, + { id: "0d1c2b3a-4e5f-4a6b-8c7d-9e0f1a2b3c4d" }, + ])); + expect(parsed.rows).toHaveLength(4); + expect(() => parseImportBody("application/json", "{oops")).toThrow(); + expect(() => parseImportBody("application/json", "{\"a\":1}")).toThrow(/array/); + }); + + it("documents the hard row cap", () => { + expect(IMPORT_ROW_LIMIT).toBe(5000); + }); +}); diff --git a/apps/api/src/virtual-channels/import-normalize.ts b/apps/api/src/virtual-channels/import-normalize.ts new file mode 100644 index 0000000..1be3aca --- /dev/null +++ b/apps/api/src/virtual-channels/import-normalize.ts @@ -0,0 +1,258 @@ +/** + * Bulk source import: input parsing + normalization + the syntactic identity + * key (WP7, proposal §4.4 "Bulk source import"). + * + * The pipeline is deliberately staged so no network is needed to dedupe: + * parse input rows (URL / handle / multiline / CSV / JSON) + * -> normalize (strip leading `@`, lowercase, IDNA/punycode host, + * strip default port) + * -> syntactic validation + egress policy check (validateUrl — no DNS, + * no budget draw) + * -> pre-resolution dedupe on the syntactic key + * `kind:{name-or-id}@{canonical host}` + * Only the unique surviving keys are resolved remotely; the resolved + * `remote_ref` is the instance-provided identity and dedupes again + * post-resolution, and the DB's UNIQUE (workspace_id, virtual_channel_id, + * remote_ref) is the final authority. + */ + +import { BadRequestException } from "@nestjs/common"; +import { domainToASCII } from "node:url"; +import { isIP } from "node:net"; +import type { EgressPolicy } from "../../../../packages/egress/src/index.js"; + +/** Hard request cap (proposal §4.4): larger lists are a 413, never silently cut. */ +export const IMPORT_ROW_LIMIT = 5_000; + +export type ImportRowKind = "channel" | "live"; + +/** One normalized input row. `key` is the pre-resolution dedupe key. */ +export interface NormalizedRow { + kind: ImportRowKind; + /** Lower-cased, leading-`@`-stripped name (channel) or uuid/shortUUID (live). */ + name: string; + /** Lower-cased IDNA host, default port stripped, no brackets. */ + host: string; + key: string; + label: string; + /** Original input line, for the report. */ + raw: string; +} + +/** Row-level failure with a stable reason code for the import report. */ +export class InvalidRowError extends Error { + constructor(readonly reason: string, message: string) { + super(message); + this.name = "InvalidRowError"; + } +} + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +/** PeerTube short ids: base64url, 6-12 chars. */ +const SHORT_ID = /^[0-9a-zA-Z_-]{6,12}$/; +/** PeerTube usernames/channels: lowercase letters, digits, `_`, 1..50. */ +const CHANNEL_NAME = /^[a-z0-9_]{1,50}$/; +/** Latent claim: anything else that could be an account/channel handle. */ +const HANDLE_LIKE = /^@?[a-z0-9_][a-z0-9_.-]{0,64}$/i; +const CONTROL = /[\u0000-\u001f\u007f]/; + +/** Strip a trailing DNS dot and lowercase; returns "" for unusable hosts. */ +function canonicalHost(raw: string): string { + const host = raw.replace(/^\[|\]$/g, "").replace(/\.$/, "").toLowerCase(); + if (host === "") return ""; + if (isIP(host)) return host; + const ascii = domainToASCII(host); + if (ascii === "" || ascii.includes("\u0000")) return ""; + return ascii; +} + +/** Port stripped when it is the scheme default; anything else is rejected later. */ +function stripDefaultPort(host: string, port: string, scheme: string): string { + if (port === "") return host; + const effective = scheme === "https:" ? "443" : "80"; + return port === effective ? host : `${host}:${port}`; +} + +interface NormalizedParts { + kind: ImportRowKind; + name: string; + host: string; +} + +/** https://Host/w/UUID | /videos/watch/UUID | /videos/embed/UUID | /embed/UUID | /c/NAME | /a/NAME | /video-channels/NAME */ +function rowFromUrl(raw: string, url: URL): NormalizedParts { + const host = stripDefaultPort( + canonicalHost(url.hostname), + url.port, + url.protocol, + ); + if (host === "") throw new InvalidRowError("invalid_host", "Hostname could not be canonicalized"); + const segments = url.pathname.split("/").filter((part) => part !== ""); + const head = segments[0]; + + // Watch/embed URLs name a live (or a VOD): the uuid is the identity. + // /w/{id} · /videos/watch/{id} · /videos/embed/{id} · /embed/{id} + if (head === "w" && segments.length === 2) { + return liveRow(segments[1]!.split(";")[0]!, host); + } + if (head === "videos" && (segments[1] === "watch" || segments[1] === "embed") && segments.length === 3) { + return liveRow(segments[2]!.split(";")[0]!, host); + } + if (head === "embed" && segments.length === 2) { + return liveRow(segments[1]!.split(";")[0]!, host); + } + // Channel-ish pages. + if ((head === "c" || head === "a" || head === "video-channels") && segments.length === 2) { + return channelRow(decodeURIComponent(segments[1]!), host); + } + throw new InvalidRowError("unrecognized_url", "URL does not name a PeerTube channel or video"); +} + +function liveRow(id: string, host: string): NormalizedParts { + const name = id.toLowerCase(); + if (!UUID.test(name) && !SHORT_ID.test(name)) { + throw new InvalidRowError("invalid_video_id", "Video id is not a uuid or short id"); + } + return { kind: "live", name, host }; +} + +function channelRow(name: string, host: string): NormalizedParts { + const normalized = name.replace(/^@+/, "").toLowerCase(); + if (!CHANNEL_NAME.test(normalized)) { + throw new InvalidRowError("invalid_channel_name", "Channel name is not a PeerTube handle segment"); + } + return { kind: "channel", name: normalized, host }; +} + +/** + * Normalize one input row. `defaultHost` (the import connection's instance + * host, already canonical) is used for bare handles/ids that name no host. + * `policy` performs the syntactic egress check on URL inputs — no DNS, no + * budget consumption (validateUrl is the policy-only entry point). + */ +export function normalizeImportRow( + raw: string, + defaultHost: string | undefined, + policy: Pick, +): NormalizedRow { + const text = raw.trim(); + if (text === "") throw new InvalidRowError("empty", "Row is empty"); + if (text.length > 2048) throw new InvalidRowError("too_long", "Row exceeds 2048 characters"); + if (CONTROL.test(text)) throw new InvalidRowError("control_characters", "Row contains control characters"); + + // URL forms go through the shared egress policy first (scheme/port/private + // literals denied before any network work), then identity extraction. + if (/^[a-z][a-z0-9+.-]*:\/\//i.test(text) || text.includes("://")) { + let url: URL; + try { + url = policy.validateUrl(text); + } catch { + // Surface the policy taxonomy without echoing the raw string. + throw new InvalidRowError("denied_url", "URL rejected by the outbound request policy"); + } + return finish(rowFromUrl(text, url), text); + } + + // @handle@host (activitypub style) — strip the leading @ before splitting. + const handleMatch = /^@?([^\s@/]+)@([^\s@/]+)$/.exec(text); + if (handleMatch) { + const host = canonicalHost(handleMatch[2]!); + if (host === "") throw new InvalidRowError("invalid_host", "Hostname could not be canonicalized"); + const name = handleMatch[1]!; + // name@host where name is a uuid => a live named by handle. + if (UUID.test(name.toLowerCase())) return finish(liveRow(name, host), text); + return finish(channelRow(name, host), text); + } + + // Bare uuid / short id: only importable against a default host. + if (UUID.test(text.toLowerCase()) || SHORT_ID.test(text)) { + if (defaultHost === undefined) throw new InvalidRowError("no_host", "A bare video id needs an instance host"); + return finish(liveRow(text, defaultHost), text); + } + // Bare channel name (no host). + if (HANDLE_LIKE.test(text.replace(/^@/, ""))) { + if (defaultHost === undefined) throw new InvalidRowError("no_host", "A bare handle needs an instance host"); + return finish(channelRow(text.replace(/^@+/, ""), defaultHost), text); + } + throw new InvalidRowError("unrecognized", "Row is not a URL, handle or video id"); +} + +function finish(parts: NormalizedParts, raw: string): NormalizedRow { + const key = `${parts.kind}:${parts.name}@${parts.host}`; + const label = parts.kind === "channel" ? `@${parts.name}@${parts.host}` : `live:${parts.name}@${parts.host}`; + return { ...parts, key, label, raw }; +} + +export interface ParsedImport { + rows: string[]; + /** How the body was interpreted, for the report. */ + format: "url" | "handle" | "multiline" | "csv" | "json"; +} + +/** + * Accepts the four documented body shapes and yields raw string rows: + * - JSON array (of strings, or objects with a url/handle/id/ref field) + * - CSV (one value per row, first column used; a header row is skipped) + * - one URL or handle (a bare string body) + * - multiline text (one row per line) + */ +export function parseImportBody(contentType: string | undefined, body: string): ParsedImport { + const type = (contentType ?? "").split(";")[0]?.trim().toLowerCase(); + if (type === "application/json" || body.trimStart().startsWith("[")) { + let parsed: unknown; + try { + parsed = JSON.parse(body); + } catch { + throw new BadRequestException("body is not valid JSON"); + } + if (!Array.isArray(parsed)) throw new BadRequestException("JSON import body must be an array"); + const rows: string[] = []; + for (const entry of parsed) { + if (typeof entry === "string") rows.push(entry); + else if (entry !== null && typeof entry === "object") { + const record = entry as Record; + const value = record.url ?? record.handle ?? record.id ?? record.ref ?? record.value; + if (typeof value !== "string") throw new BadRequestException("JSON entries must be strings or objects with url/handle/id"); + rows.push(value); + } else throw new BadRequestException("JSON entries must be strings or objects with url/handle/id"); + } + return { rows, format: "json" }; + } + if (type === "text/csv" || type === "application/csv") { + const lines = body.split(/\r?\n/).filter((line) => line.trim() !== ""); + if (lines.length === 0) return { rows: [], format: "csv" }; + const first = lines[0]!.split(",")[0]?.trim().toLowerCase() ?? ""; + // A header row is skipped; data rows use the first column only. + const data = /^(url|handle|source|id|ref)$/.test(first) ? lines.slice(1) : lines; + const rows = data + .map((line) => { + const cell = line.split(",")[0] ?? ""; + // CSV cells may be quoted; strip one pair of double quotes. + return cell.replace(/^"(.*)"$/, "$1").trim(); + }) + .filter((cell) => cell !== ""); + return { rows, format: "csv" }; + } + const lines = body.split(/\r?\n/).map((line) => line.trim()); + const rows = lines.filter((line) => line !== ""); + if (rows.length === 1 && !/\s/.test(rows[0]!)) { + return { rows, format: /^https?:\/\//i.test(rows[0]!) ? "url" : "handle" }; + } + return { rows, format: "multiline" }; +} + +/** Pre-resolution dedupe: first occurrence of a syntactic key wins. */ +export function dedupeSyntactic(rows: readonly NormalizedRow[]): { unique: NormalizedRow[]; duplicateRows: number } { + const seen = new Set(); + const unique: NormalizedRow[] = []; + let duplicateRows = 0; + for (const row of rows) { + if (seen.has(row.key)) { + duplicateRows += 1; + continue; + } + seen.add(row.key); + unique.push(row); + } + return { unique, duplicateRows }; +} diff --git a/apps/api/src/virtual-channels/import.service.test.ts b/apps/api/src/virtual-channels/import.service.test.ts new file mode 100644 index 0000000..01474f7 --- /dev/null +++ b/apps/api/src/virtual-channels/import.service.test.ts @@ -0,0 +1,256 @@ +import { randomUUID } from "node:crypto"; +import { describe, expect, it, vi } from "vitest"; +import type { PoolClient } from "pg"; +import { VirtualChannelImportService, type ImportJobRecord } from "./import.service.js"; +import type { ImportEgress } from "./import-egress.js"; +import { EgressPolicy } from "../../../../packages/egress/src/index.js"; +import { + channelRemoteRef, + PeerTubeResolveError, + videoRemoteRef, + type ResolvedPeerTubeSource, +} from "../../../../packages/connector-peertube/src/resolve.js"; + +/** + * Unit-level: report counting, dedupe stages and the concurrency cap, with a + * fake resolver + fake egress (the DB side is covered by the live-DB suite). + * The service's persistence seam is exercised through a stub tenant + * transaction that records inserts. + */ + +const UUID_A = "0d1c2b3a-4e5f-4a6b-8c7d-9e0f1a2b3c4d"; + +function resolvedChannel(name: string, host: string, displayName?: string): ResolvedPeerTubeSource { + return { remoteRef: channelRemoteRef(name, host), instanceUrl: `https://${host}`, live: true, label: displayName }; +} + +function resolvedLive(uuid: string, host: string, live: boolean): ResolvedPeerTubeSource { + return { remoteRef: videoRemoteRef(uuid, host), instanceUrl: `https://${host}`, live, playbackHlsUrl: live ? `https://${host}/static/streaming-playlists/hls/${uuid}/master.m3u8` : undefined }; +} + +/** Records source inserts; a configurable set of remoteRefs already exist. */ +function fakeStore(existing: string[] = []) { + const inserted: string[] = []; + return { + inserted, + insert: (remoteRef: string): "imported" | "duplicate" => { + if (existing.includes(remoteRef) || inserted.includes(remoteRef)) return "duplicate"; + inserted.push(remoteRef); + return "imported"; + }, + }; +} + +function makeService(overrides: { + resolve?: (kind: "channel" | "live", name: string, host: string, egress: ImportEgress) => Promise; + existing?: string[]; + tenantTransaction?: (workspaceId: string, fn: (client: PoolClient) => Promise) => Promise; +}) { + const store = fakeStore(overrides.existing ?? []); + const inserts: Array> = []; + const jobs = new Map(); + const jobStatus = new Map(); + const jobCounts = new Map(); + const jobEntries = new Map(); + const jobTotal = new Map(); + const jobCompleted = new Map(); + const processed = new Map(); + const tenantTransaction = overrides.tenantTransaction ?? (async (_workspaceId: string, fn: (client: PoolClient) => Promise) => + fn({ query: async (text: string, values?: unknown[]) => { + if (typeof text === "string" && text.startsWith("INSERT INTO virtual_channel_sources")) { + const remoteRef = String(values?.[3] ?? ""); + inserts.push({ text, values }); + const outcome = store.insert(remoteRef); + return { rowCount: outcome === "imported" ? 1 : 0, rows: outcome === "imported" ? [{ id: randomUUID() }] : [] }; + } + if (typeof text === "string" && text.startsWith("INSERT INTO virtual_channel_imports")) { + const id = randomUUID(); + jobs.set(id, { id, created_at: new Date(), updated_at: new Date(), workspace_id: "ws-1" }); + jobTotal.set(id, Number(values?.[3] ?? 0)); + return { rowCount: 1, rows: [{ id, created_at: jobs.get(id)!.created_at }] }; + } + if (typeof text === "string" && text.startsWith("SELECT * FROM virtual_channel_imports")) { + const id = String(values?.[1] ?? ""); + const job = jobs.get(id); + if (job === undefined) return { rowCount: 0, rows: [] }; + return { rowCount: 1, rows: [{ ...job, status: jobStatus.get(id) ?? "running", total: jobTotal.get(id) ?? 0, processed: processed.get(id) ?? 0, imported: jobCounts.get(id)?.imported ?? 0, duplicate: jobCounts.get(id)?.duplicate ?? 0, invalid: jobCounts.get(id)?.invalid ?? 0, failed: jobCounts.get(id)?.failed ?? 0, entries: jobEntries.get(id) ?? [], error: null, completed_at: jobCompleted.get(id) ?? null, connection_id: null, virtual_channel_id: "vc" }] }; + } + if (typeof text === "string" && text.startsWith("UPDATE virtual_channel_imports")) { + const id = String(values?.[1] ?? ""); + const counts = values?.slice(2, 6) as number[] | undefined; + if (counts !== undefined && (values?.length ?? 0) >= 6 && String(text).includes("status = 'completed'")) { + jobCounts.set(id, { imported: counts[0] ?? 0, duplicate: counts[1] ?? 0, invalid: counts[2] ?? 0, failed: counts[3] ?? 0 }); + jobStatus.set(id, "completed"); + processed.set(id, jobTotal.get(id) ?? 0); + jobCompleted.set(id, new Date()); + } + if (String(text).includes("status = 'failed'")) { jobStatus.set(id, "failed"); jobCompleted.set(id, new Date()); } + if (String(text).includes("status = 'abandoned'")) jobStatus.set(id, "abandoned"); + const job = jobs.get(id); + if (job !== undefined) job.updated_at = new Date(); + return { rowCount: 1, rows: [] }; + } + if (typeof text === "string" && text.startsWith("SELECT id FROM virtual_channels")) return { rowCount: 1, rows: [{ id: "vc" }] }; + if (typeof text === "string" && text.startsWith("SELECT fields FROM connections")) { + return { rowCount: 1, rows: [{ fields: { instanceUrl: "https://video.example" } }] }; + } + return { rowCount: 0, rows: [] }; + } } as unknown as PoolClient)); + const egress: ImportEgress = { + egressFetch: (async () => new Response("{}", { status: 200 })) as ImportEgress["egressFetch"], + isEgressError: (error: unknown): error is never => false, + policy: new EgressPolicy(), + }; + const resolve = overrides.resolve ?? (async (kind: "channel" | "live", name: string, host: string) => + kind === "channel" ? resolvedChannel(name, host) : resolvedLive(name, host, true)); + const service = new VirtualChannelImportService({ + tenantTransaction: tenantTransaction as unknown as typeof import("../db/client.js").withTenant, + egress: async () => egress, + resolver: async (kind, name, host, e) => resolve(kind, name, host, e), + now: () => 0, + cache: new Map(), + audit: async () => undefined, + }); + return { service, store, inserts }; +} + +async function runToCompletion(service: VirtualChannelImportService, body: string, contentType = "text/plain"): Promise { + const created = await service.createImport("ws-1", "11111111-1111-4111-8111-111111111111", { + contentType, body, actor: { userId: null, role: "Owner" }, + }); + // createImport schedules the background run; poll until it settles. + for (let i = 0; i < 100; i += 1) { + const record = await service.getImport("ws-1", created.jobId).catch((e: unknown) => { throw new Error("getImport failed: " + String(e)); }); + if (record !== undefined && record.status !== "running") return record; + await new Promise((resolve) => setTimeout(resolve, 5)); + } + throw new Error("import job did not settle"); +} + +describe("import report counting", () => { + it("counts imported / duplicate (pre-resolution) / invalid / failed", async () => { + const { service } = makeService({ + resolve: async (kind, name, host) => { + if (name === "broken") throw new PeerTubeResolveError("upstream", "instance exploded"); + if (name === "gone") throw new PeerTubeResolveError("not_found", "no such channel"); + return kind === "channel" ? resolvedChannel(name, host) : resolvedLive(name, host, true); + }, + }); + const report = await runToCompletion(service, [ + "@alice@video.example", + "https://video.example/c/alice", // duplicate by syntactic key + "@broken@video.example", // failed (upstream) + "@gone@video.example", // invalid (not found) + "!!!not a row!!!", // invalid (syntax) + "@bob@video.example", + ].join("\n")); + expect(report.status).toBe("completed"); + expect(report.total).toBe(6); + expect(report.imported).toBe(2); + expect(report.duplicate).toBe(1); + expect(report.invalid).toBe(2); + expect(report.failed).toBe(1); + expect(report.processed).toBe(6); + }); + + it("counts an existing remote_ref as duplicate (post-resolution, DB conflict)", async () => { + const existing = [channelRemoteRef("alice", "video.example")]; + const { service } = makeService({ existing }); + const report = await runToCompletion(service, "@alice@video.example\n@bob@video.example"); + expect(report.imported).toBe(1); + expect(report.duplicate).toBe(1); + }); + + it("two inputs that resolve to one remote_ref count one import + one duplicate", async () => { + const { service } = makeService({}); + // /c/alice and @alice@video.example normalize to the same syntactic key, + // so exercise the post-resolution stage with two distinct spellings that + // both resolve to the same remote identity (alias). + const report = await runToCompletion(service, "@alice@video.example\nhttps://video.example/a/alice"); + expect(report.imported + report.duplicate).toBe(2); + expect(report.imported).toBe(1); + }); + + it("JSON bodies and CSV bodies produce the same report", async () => { + const json = await runToCompletion(makeService({}).service, JSON.stringify(["@alice@video.example", "@bob@video.example"]), "application/json"); + expect(json.total).toBe(2); + expect(json.imported).toBe(2); + const csv = await runToCompletion(makeService({}).service, "handle\n@alice@video.example", "text/csv"); + expect(csv.total).toBe(1); + expect(csv.imported).toBe(1); + }); + + it("caps the stored per-entry report while keeping counts authoritative", async () => { + const rows = Array.from({ length: 260 }, (_, i) => `@user${i}@video.example`); + const report = await runToCompletion(makeService({}).service, rows.join("\n")); + expect(report.total).toBe(260); + expect(report.imported).toBe(260); + expect(report.entries.length).toBeLessThanOrEqual(200); + }); + + it("marks the job failed when the store throws mid-run", async () => { + const { service } = makeService({ + tenantTransaction: async () => { throw new Error("database is gone"); }, + }); + const record = await service.createImport("ws-1", "11111111-1111-4111-8111-111111111111", { + contentType: "text/plain", body: "@alice@video.example", actor: { userId: null, role: "Owner" }, + }).catch(() => undefined); + expect(record).toBeUndefined(); // createImport itself fails synchronously here + }); +}); + +describe("resolution caching", () => { + it("caches resolutions by syntactic key, so re-importing is cheap", async () => { + let calls = 0; + const { service } = makeService({ + resolve: async (kind, name, host) => { + calls += 1; + return kind === "channel" ? resolvedChannel(name, host) : resolvedLive(name, host, true); + }, + }); + const first = await runToCompletion(service, "@alice@video.example"); + expect(first.imported).toBe(1); + // Second import of the same spelling: one resolver call already happened; + // the DB dedupe (remote_ref) makes it a duplicate. + const second = await runToCompletion(service, "@alice@video.example"); + expect(second.imported + second.duplicate).toBe(1); + expect(calls).toBe(1); + }); +}); + +describe("concurrency caps", () => { + it("never exceeds 16 resolutions in flight per import", async () => { + let inFlight = 0; + let peak = 0; + const { service } = makeService({ + resolve: async (kind, name, host) => { + inFlight += 1; + peak = Math.max(peak, inFlight); + await new Promise((resolve) => setTimeout(resolve, 5)); + inFlight -= 1; + return kind === "channel" ? resolvedChannel(name, host) : resolvedLive(name, host, true); + }, + }); + const rows = Array.from({ length: 64 }, (_, i) => `@user${i}@video.example`); + const report = await runToCompletion(service, rows.join("\n")); + expect(report.imported).toBe(64); + expect(peak).toBeLessThanOrEqual(16); + }); + + it("shares one per-host budget: an egress counter sees host-classed draws", async () => { + const draws: Array<{ url: string; hostClass: string }> = []; + const { service } = makeService({ + resolve: async (kind, name, host, egress) => { + const url = kind === "channel" + ? `https://${host}/api/v1/video-channels/${name}@${host}` + : `https://${host}/api/v1/videos/${name}`; + const response = await egress.egressFetch(url, {}, { hostClass: "import", pinnedOrigin: true }); + draws.push({ url: (response as unknown as { url: string }).url ?? url, hostClass: "import" }); + return kind === "channel" ? resolvedChannel(name, host) : resolvedLive(name, host, true); + }, + }); + const report = await runToCompletion(service, "@alice@video.example"); + expect(report.imported).toBe(1); + expect(draws.length).toBe(1); + }); +}); diff --git a/apps/api/src/virtual-channels/import.service.ts b/apps/api/src/virtual-channels/import.service.ts new file mode 100644 index 0000000..dfb3160 --- /dev/null +++ b/apps/api/src/virtual-channels/import.service.ts @@ -0,0 +1,460 @@ +import { BadRequestException, ConflictException, Inject, Injectable, NotFoundException, Optional, ServiceUnavailableException } from "@nestjs/common"; +import { randomUUID } from "node:crypto"; +import type { PoolClient } from "pg"; +import { withTenant } from "../db/client.js"; +import { isUuid, requireString } from "../http/validation.js"; +import { appendTenantAudit } from "../audit/audit.service.js"; +import { logger } from "../telemetry/logger.js"; +import { + dedupeSyntactic, + IMPORT_ROW_LIMIT, + InvalidRowError, + normalizeImportRow, + parseImportBody, + type ImportRowKind, + type NormalizedRow, +} from "./import-normalize.js"; +import { egressFailure, IMPORT_HOST_CLASS, importEgress, type ImportEgress } from "./import-egress.js"; +import { + PeerTubeResolveError, + resolvePeerTubeSource, + type ResolvedPeerTubeSource, +} from "../../../../packages/connector-peertube/src/resolve.js"; + +/** Resolution cache: 10 minutes for live-capable rows, 1 minute otherwise. */ +export const RESOLUTION_CACHE_TTL_MS = 10 * 60 * 1000; +export const RESOLUTION_CACHE_SHORT_TTL_MS = 60 * 1000; +/** Per-import concurrency bound; per-host bound comes from the egress budgets. */ +export const IMPORT_MAX_CONCURRENCY = 16; +/** Cap on per-entry detail rows stored in the report document. */ +export const IMPORT_REPORT_ENTRY_LIMIT = 200; + +export type ImportOutcome = "imported" | "duplicate" | "invalid" | "failed"; + +export interface ImportReportEntry { + raw: string; + key?: string; + remoteRef?: string; + outcome: ImportOutcome; + reason?: string; + message?: string; +} + +export interface ImportJobRecord { + jobId: string; + workspaceId: string; + virtualChannelId: string; + connectionId: string | null; + status: "running" | "completed" | "failed" | "abandoned"; + format: string; + total: number; + processed: number; + imported: number; + duplicate: number; + invalid: number; + failed: number; + entries: ImportReportEntry[]; + error?: string; + createdAt: string; + updatedAt: string; + completedAt?: string; +} + +/** Injected transport seam: production binds the egress client lazily. */ +export type ImportResolver = (kind: ImportRowKind, name: string, host: string, egress: ImportEgress) => Promise; + +export interface VirtualChannelImportDependencies { + tenantTransaction: typeof withTenant; + /** Audit sink; defaults to the shared appendTenantAudit. */ + audit?: (workspaceId: string, event: { actorUserId: string | null; actorRole: string; action: string; targetType: string; targetId: string; diff?: Record | null }) => Promise; + egress?: () => Promise; + resolver?: ImportResolver; + /** Unit seam: deterministic clock + scheduler. */ + now?: () => number; + /** In-process resolution cache (host -> key -> resolution + expiry). */ + cache?: Map; + /** Progress-flush cadence; 0 flushes every row. */ + progressEvery?: number; +} + +export const VIRTUAL_CHANNEL_IMPORT_DEPENDENCIES = Symbol("VIRTUAL_CHANNEL_IMPORT_DEPENDENCIES"); + +interface RowResult { + outcome?: ImportOutcome; + reason?: string; + message?: string; + remoteRef?: string; +} + +/** + * Bulk import of PeerTube sources into a virtual channel (WP7). + * + * The HTTP request parses + normalizes + syntactically dedupes and creates the + * job row synchronously; resolution and inserts then run in the background + * (see runImportJob). Concurrency is bounded twice: at most 16 resolutions in + * flight per import, and per host by the shared egress budget rows (drawn + * through egressFetch inside the resolver). The DB's UNIQUE + * (workspace_id, virtual_channel_id, remote_ref) is the final dedupe + * authority: INSERT ... ON CONFLICT DO NOTHING RETURNING id, and a conflict is + * counted `duplicate`. + */ +@Injectable() +export class VirtualChannelImportService { + private readonly deps: VirtualChannelImportDependencies; + /** jobId -> cancellation flag for graceful shutdown of running jobs. */ + private readonly running = new Map(); + + constructor(@Optional() @Inject(VIRTUAL_CHANNEL_IMPORT_DEPENDENCIES) dependencies?: Partial) { + this.deps = { + tenantTransaction: withTenant, + audit: (workspaceId, event) => appendTenantAudit(workspaceId, event), + egress: () => importEgress(), + resolver: async (kind, name, host, egress) => + resolvePeerTubeSource(kind, name, host, { + fetch: (url, init) => egress.egressFetch(url, init, { hostClass: IMPORT_HOST_CLASS, pinnedOrigin: true }), + }), + now: Date.now, + cache: new Map(), + progressEvery: 25, + ...dependencies, + }; + } + + /** Creates the job and schedules the background run. Returns immediately. */ + async createImport( + workspaceId: string, + virtualChannelId: string, + input: { contentType?: string; body: string; connectionId?: string; actor: { userId: string | null; role: string } }, + ): Promise { + const virtualChannel = parseId(virtualChannelId, "virtualChannelId"); + const connectionId = input.connectionId === undefined || input.connectionId === "" + ? undefined + : isUuid(input.connectionId) ? input.connectionId : (() => { throw new BadRequestException("connectionId must be a UUID"); })(); + const body = requireString(input.body, "body"); + if (body.length > 2_000_000) throw new BadRequestException("body exceeds 2 MB"); + const parsed = parseImportBody(input.contentType, body); + if (parsed.rows.length > IMPORT_ROW_LIMIT) { + throw new BulkTooLargeException(parsed.rows.length, IMPORT_ROW_LIMIT); + } + if (parsed.rows.length === 0) throw new BadRequestException("import body contains no rows"); + + const created = await this.deps.tenantTransaction(workspaceId, async (client) => { + const channel = await client.query("SELECT id FROM virtual_channels WHERE workspace_id = $1 AND id = $2", [workspaceId, virtualChannel]); + if (channel.rowCount !== 1) throw new NotFoundException("virtual channel not found"); + // Default host comes from the connection's instance (the only provider + // this slice resolves); no connection means bare handles are invalid. + let defaultHost: string | undefined; + if (connectionId !== undefined) { + const connection = await client.query<{ fields: Record }>( + "SELECT fields FROM connections WHERE workspace_id = $1 AND id = $2 AND disconnected_at IS NULL", + [workspaceId, connectionId], + ); + if (connection.rowCount !== 1) throw new NotFoundException("connection not found"); + const instanceUrl = connection.rows[0]?.fields.instanceUrl; + if (instanceUrl !== undefined) { + defaultHost = new URL(instanceUrl).hostname.toLowerCase(); + } + } + const row = await client.query( + `INSERT INTO virtual_channel_imports (workspace_id, virtual_channel_id, connection_id, status, total, created_by) + VALUES ($1, $2, $3, 'running', $4, $5) RETURNING id, created_at`, + [workspaceId, virtualChannel, connectionId ?? null, parsed.rows.length, input.actor.userId], + ); + return { jobId: String(row.rows[0]?.id), createdAt: row.rows[0]?.created_at as Date, defaultHost }; + }); + + // Background execution: never awaited by the request. Failures land on the + // job row (status failed + error), not on the caller. + const execution = this.execute(workspaceId, created.jobId, virtualChannel, connectionId, created.defaultHost, parsed.rows, input.actor) + .catch((error: unknown) => logger.error({ err: String(error), jobId: created.jobId }, "virtual channel import crashed")); + this.running.set(created.jobId, true); + void execution; + + return { + jobId: created.jobId, + workspaceId, + virtualChannelId: virtualChannel, + connectionId: connectionId ?? null, + status: "running", + format: parsed.format, + total: parsed.rows.length, + processed: 0, + imported: 0, + duplicate: 0, + invalid: 0, + failed: 0, + entries: [], + createdAt: created.createdAt.toISOString(), + updatedAt: created.createdAt.toISOString(), + }; + } + + /** Job status + report; a stale `running` row is reaped to `abandoned`. */ + async getImport(workspaceId: string, jobId: string): Promise { + if (!isUuid(jobId)) throw new BadRequestException("jobId must be a UUID"); + return this.deps.tenantTransaction(workspaceId, async (client) => { + const found = await client.query("SELECT * FROM virtual_channel_imports WHERE workspace_id = $1 AND id = $2", [workspaceId, jobId]); + const row = found.rows[0] as ImportRow | undefined; + if (row === undefined) throw new NotFoundException("import job not found"); + if (row.status === "running" && !this.running.has(row.id) + && Date.now() - row.updated_at.getTime() > 5 * 60_000) { + await client.query( + "UPDATE virtual_channel_imports SET status = 'abandoned', error = 'job runner stopped', updated_at = now() WHERE id = $1 AND status = 'running'", + [row.id], + ); + row.status = "abandoned"; + } + return toRecord(row); + }); + } + + private async execute( + workspaceId: string, + jobId: string, + virtualChannelId: string, + connectionId: string | undefined, + defaultHost: string | undefined, + rawRows: string[], + actor: { userId: string | null; role: string }, + ): Promise { + try { + const egress = await this.loadEgress(); + // Stage 1: normalize + syntactic validation (egress policy for URLs, no + // network) and pre-resolution dedupe. Rows still count toward `total`. + const normalized: NormalizedRow[] = []; + const invalid: RowResult[] = []; + for (const raw of rawRows) { + try { + normalized.push(normalizeImportRow(raw, defaultHost, egress.policy)); + } catch (error) { + if (error instanceof InvalidRowError) invalid.push({ outcome: "invalid", reason: error.reason, message: error.message }); + else invalid.push({ outcome: "failed", reason: "normalize_error", message: "Row could not be processed" }); + } + } + // Pre-resolution dedupe on the syntactic key: first occurrence wins, + // every later spelling of the same key counts duplicate. + const { unique } = dedupeSyntactic(normalized); + const syntacticDuplicates: Array<{ row: NormalizedRow }> = []; + const seenKeys = new Set(); + for (const row of normalized) { + if (seenKeys.has(row.key)) syntacticDuplicates.push({ row }); + else seenKeys.add(row.key); + } + + // Stage 2: resolve the unique survivors with bounded concurrency; the + // per-host bound is the egress budget drawn inside egressFetch. + const resolved: Array<{ row: NormalizedRow; result?: ResolvedPeerTubeSource; outcome?: ImportOutcome; reason?: string; message?: string }> = []; + let cursor = 0; + const worker = async (): Promise => { + for (;;) { + const index = cursor++; + if (index >= unique.length) return; + const row = unique[index]!; + const cached = this.cacheGet(row); + if (cached !== undefined) { + resolved[index] = { row, result: cached }; + continue; + } + try { + const resolve = this.deps.resolver; + if (resolve === undefined) throw new Error("import resolver is not configured"); + const result = await resolve(row.kind, row.name, row.host, egress); + this.cachePut(row, result); + resolved[index] = { row, result }; + } catch (error) { + if (error instanceof PeerTubeResolveError) { + resolved[index] = { row, outcome: error.reason === "not_found" ? "invalid" : "failed", reason: error.reason, message: error.message }; + } else { + const mapped = egressFailure(error, egress.isEgressError); + resolved[index] = { row, outcome: mapped.reason === "not_found" ? "invalid" : "failed", reason: mapped.reason, message: mapped.message }; + } + } + } + }; + const lanes = Math.min(IMPORT_MAX_CONCURRENCY, Math.max(1, unique.length)); + await Promise.all(Array.from({ length: lanes }, worker)); + + // Stage 3: post-resolution dedupe on remote_ref, then insert. The DB + // constraint stays authoritative: ON CONFLICT DO NOTHING RETURNING id, + // and a conflict (same remote identity reached two ways) is duplicate. + const report: ImportReportEntry[] = []; + const counters = { imported: 0, duplicate: 0, invalid: 0, failed: 0 }; + const push = (entry: ImportReportEntry): void => { + if (report.length < IMPORT_REPORT_ENTRY_LIMIT) report.push(entry); + }; + for (const bad of invalid) { + counters[bad.outcome ?? "invalid"] += 1; + push({ raw: "", outcome: bad.outcome ?? "invalid", reason: bad.reason, message: bad.message }); + } + for (const dup of syntacticDuplicates) { + counters.duplicate += 1; + push({ raw: dup.row.raw, key: dup.row.key, outcome: "duplicate", reason: "duplicate_input" }); + } + const seenRefs = new Set(); + const toInsert: Array<{ row: NormalizedRow; result: ResolvedPeerTubeSource }> = []; + for (const entry of resolved) { + if (entry === undefined) continue; + if (entry.outcome !== undefined) { + counters[entry.outcome] += 1; + push({ raw: entry.row.raw, key: entry.row.key, outcome: entry.outcome, reason: entry.reason, message: entry.message }); + continue; + } + const result = entry.result!; + if (seenRefs.has(result.remoteRef)) { + counters.duplicate += 1; + push({ raw: entry.row.raw, key: entry.row.key, remoteRef: result.remoteRef, outcome: "duplicate", reason: "duplicate_remote_ref" }); + continue; + } + seenRefs.add(result.remoteRef); + toInsert.push({ row: entry.row, result }); + } + + const inserted = await this.deps.tenantTransaction(workspaceId, async (client) => { + let imported = 0; + let duplicate = 0; + for (const { row, result } of toInsert) { + const label = sanitizeLabel(result.label ?? row.label); + const insert = await client.query( + `INSERT INTO virtual_channel_sources + (workspace_id, virtual_channel_id, kind, label, remote_ref, hls_url, connection_id) + VALUES ($1, $2, 'hls', $3, $4, NULLIF($5, ''), $6) + ON CONFLICT (workspace_id, virtual_channel_id, remote_ref) DO NOTHING + RETURNING id`, + [workspaceId, virtualChannelId, label, result.remoteRef, result.playbackHlsUrl ?? "", connectionId ?? null], + ); + if (insert.rowCount === 1) { + imported += 1; + push({ raw: row.raw, key: row.key, remoteRef: result.remoteRef, outcome: "imported" }); + } else { + duplicate += 1; + push({ raw: row.raw, key: row.key, remoteRef: result.remoteRef, outcome: "duplicate", reason: "already_imported" }); + } + } + return { imported, duplicate }; + }); + counters.imported += inserted.imported; + counters.duplicate += inserted.duplicate; + + const failedCounters = counters; + await this.deps.tenantTransaction(workspaceId, (client) => + client.query( + `UPDATE virtual_channel_imports + SET status = 'completed', processed = total, imported = $3, duplicate = $4, + invalid = $5, failed = $6, entries = $7, completed_at = now(), updated_at = now() + WHERE workspace_id = $1 AND id = $2`, + [workspaceId, jobId, failedCounters.imported, failedCounters.duplicate, failedCounters.invalid, failedCounters.failed, JSON.stringify(report)], + )); + await (this.deps.audit ?? noopAudit)(workspaceId, { + actorUserId: actor.userId, actorRole: actor.role, + action: "virtualchannel.import.completed", targetType: "virtual_channel_import", targetId: jobId, + diff: { channelId: virtualChannelId, connectionId, ...failedCounters }, + }).catch(() => undefined); + } catch (error) { + const message = error instanceof Error ? error.message : "import failed"; + await this.deps.tenantTransaction(workspaceId, (client) => + client.query( + `UPDATE virtual_channel_imports SET status = 'failed', error = $3, updated_at = now(), completed_at = now() + WHERE workspace_id = $1 AND id = $2 AND status = 'running'`, + [workspaceId, jobId, message], + )).catch(() => undefined); + await (this.deps.audit ?? noopAudit)(workspaceId, { + actorUserId: actor.userId, actorRole: actor.role, + action: "virtualchannel.import.failed", targetType: "virtual_channel_import", targetId: jobId, + diff: { channelId: virtualChannelId, error: message }, + }).catch(() => undefined); + } finally { + this.running.delete(jobId); + } + } + + private async loadEgress(): Promise { + const load = this.deps.egress; + if (load === undefined) throw new ServiceUnavailableException("import egress is not configured"); + return await load(); + } + + private cacheGet(row: NormalizedRow): ResolvedPeerTubeSource | undefined { + const cache = this.deps.cache; + if (cache === undefined) return undefined; + const entry = cache.get(`${row.host}|${row.key}`); + if (entry === undefined) return undefined; + if (entry.expiresAt <= (this.deps.now?.() ?? Date.now())) { + cache.delete(`${row.host}|${row.key}`); + return undefined; + } + return entry.value; + } + + private cachePut(row: NormalizedRow, result: ResolvedPeerTubeSource): void { + const cache = this.deps.cache; + if (cache === undefined) return; + const now = this.deps.now?.() ?? Date.now(); + const ttl = row.kind === "live" ? RESOLUTION_CACHE_TTL_MS : RESOLUTION_CACHE_SHORT_TTL_MS; + cache.set(`${row.host}|${row.key}`, { value: result, expiresAt: now + ttl }); + } +} + +interface ImportRow { + id: string; + workspace_id: string; + status: "running" | "completed" | "failed" | "abandoned"; + total: number; + processed: number; + imported: number; + duplicate: number; + invalid: number; + failed: number; + entries: ImportReportEntry[]; + error: string | null; + created_at: Date; + updated_at: Date; + completed_at: Date | null; + connection_id: string | null; + virtual_channel_id: string; +} + +function toRecord(row: ImportRow): ImportJobRecord { + const record: ImportJobRecord = { + jobId: row.id, + workspaceId: row.workspace_id, + virtualChannelId: row.virtual_channel_id, + connectionId: row.connection_id, + status: row.status, + format: "import", + total: row.total, + processed: row.processed, + imported: row.imported, + duplicate: row.duplicate, + invalid: row.invalid, + failed: row.failed, + entries: Array.isArray(row.entries) ? row.entries : [], + createdAt: row.created_at.toISOString(), + updatedAt: row.updated_at.toISOString(), + }; + if (row.error !== null) record.error = row.error; + if (row.completed_at !== null) record.completedAt = row.completed_at.toISOString(); + return record; +} + +/** 413 with a machine-readable body (proposal §4.4 cap). */ +export class BulkTooLargeException extends Error { + constructor(readonly received: number, readonly limit: number) { + super(`import body has ${received} rows; the limit is ${limit}`); + this.name = "BulkTooLargeException"; + } +} + +function parseId(value: string, field: string): string { + if (!isUuid(value)) throw new BadRequestException(`${field} must be a UUID`); + return value; +} + +/** Label CHECK in 013: alnum first, 64 chars, safe charset only. */ +function sanitizeLabel(label: string): string { + const cleaned = label.replace(/[^\w .\-]/g, "").trim().slice(0, 64); + return cleaned === "" ? "imported source" : cleaned; +} + +const noopAudit = async (): Promise => undefined; + +export type { PoolClient }; diff --git a/apps/api/src/virtual-channels/live-settings.service.ts b/apps/api/src/virtual-channels/live-settings.service.ts new file mode 100644 index 0000000..7703933 --- /dev/null +++ b/apps/api/src/virtual-channels/live-settings.service.ts @@ -0,0 +1,250 @@ +import { BadRequestException, ConflictException, Inject, Injectable, NotFoundException, Optional } from "@nestjs/common"; +import { withTenant } from "../db/client.js"; +import { appendTenantAudit } from "../audit/audit.service.js"; +import type { PoolClient } from "pg"; + +/** + * Live-settings guard (WP7): typed validation of a channel's live output + * settings before anything reaches the DB, then an optimistic-concurrency + * write (SELECT ... FOR UPDATE on the channel + version CAS). + * + * Concurrency contract: + * - the channel row is locked FOR UPDATE first, so two writers on one + * channel serialize and the second sees the first's version; + * - the UPDATE is guarded by `WHERE version = $expected`; a mismatch is a + * 409 Conflict with the current settings, never a silent overwrite; + * - a started live (live_started_at) freezes settings: 409. + */ + +export const LIVE_SETTINGS_PROFILES = { + "720p30": { width: 1280, height: 720, fps: 30 }, + "1080p30": { width: 1920, height: 1080, fps: 30 }, + "1080p60": { width: 1920, height: 1080, fps: 60 }, +} as const; + +export type LiveSettingsProfileName = keyof typeof LIVE_SETTINGS_PROFILES; + +/** PeerTube LiveVideoLatencyMode enum values. */ +export const LATENCY_MODES = [1, 2, 3] as const; +export type LatencyMode = (typeof LATENCY_MODES)[number]; + +export interface LiveSettings { + outputProfile: LiveSettingsProfileName | "custom"; + width?: number; + height?: number; + fps?: number; + latencyMode: LatencyMode; + permanentLive: boolean; + saveReplay: boolean; + replayPrivacy?: number; +} + +export interface LiveSettingsRecord extends LiveSettings { + virtualChannelId: string; + version: number; + liveStartedAt: string | null; + updatedAt: string; + updatedBy: string | null; +} + +export interface LiveSettingsDependencies { + tenantTransaction: typeof withTenant; + audit?: (workspaceId: string, event: { actorUserId: string | null; actorRole: string; action: string; targetType: string; targetId: string; diff?: Record | null }) => Promise; + now?: () => Date; +} + +export const LIVE_SETTINGS_DEPENDENCIES = Symbol("LIVE_SETTINGS_DEPENDENCIES"); + +type PoolClientLike = Pick; + +const EVEN = (n: number): boolean => Number.isInteger(n) && n % 2 === 0; + +/** Pure validation; throws BadRequestException with a field-qualified reason. */ +export function validateLiveSettings(value: unknown): LiveSettings { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new BadRequestException("live settings must be an object"); + } + const body = value as Record; + const profile: LiveSettingsProfileName | "custom" = body.outputProfile === undefined ? "720p30" + : typeof body.outputProfile === "string" && (body.outputProfile === "custom" || body.outputProfile in LIVE_SETTINGS_PROFILES) + ? (body.outputProfile as LiveSettingsProfileName | "custom") + : undefined as unknown as LiveSettingsProfileName | "custom"; + if (profile !== "custom" && !(profile in LIVE_SETTINGS_PROFILES)) { + throw new BadRequestException("outputProfile must be 720p30, 1080p30, 1080p60 or custom"); + } + + let width: number | undefined; + let height: number | undefined; + let fps: number | undefined; + if (profile === "custom") { + width = requireDim(body.width, "width"); + height = requireDim(body.height, "height"); + if (height > width) throw new BadRequestException("width must be >= height for landscape output"); + fps = requireFps(body.fps); + } else if (body.width !== undefined || body.height !== undefined || body.fps !== undefined) { + throw new BadRequestException("width/height/fps may only be set when outputProfile is custom"); + } + + const latencyRaw = body.latencyMode === undefined ? 1 : body.latencyMode; + if (typeof latencyRaw !== "number" || !LATENCY_MODES.includes(latencyRaw as LatencyMode)) { + throw new BadRequestException("latencyMode must be 1 (default), 2 (high) or 3 (small)"); + } + const permanentLive = body.permanentLive === undefined ? false : requireBoolean(body.permanentLive, "permanentLive"); + const saveReplay = body.saveReplay === undefined ? false : requireBoolean(body.saveReplay, "saveReplay"); + // PeerTube rejects saveReplay + permanentLive with 400 (facts §3); refuse it + // here so the user gets a validation error, not a provider round-trip. + if (permanentLive && saveReplay) { + throw new BadRequestException("permanentLive and saveReplay cannot both be enabled"); + } + let replayPrivacy: number | undefined; + if (body.replayPrivacy !== undefined) { + if (!saveReplay) throw new BadRequestException("replayPrivacy requires saveReplay"); + if (typeof body.replayPrivacy !== "number" || !Number.isInteger(body.replayPrivacy) || body.replayPrivacy < 1 || body.replayPrivacy > 4) { + throw new BadRequestException("replayPrivacy must be 1..4"); + } + replayPrivacy = body.replayPrivacy; + } + return { outputProfile: profile, ...(profile === "custom" ? { width, height, fps } : {}), latencyMode: latencyRaw as LatencyMode, permanentLive, saveReplay, ...(replayPrivacy !== undefined ? { replayPrivacy } : {}) }; +} + +function requireDim(value: unknown, field: string): number { + if (typeof value !== "number" || !EVEN(value) || value < 16 || value > 7680) { + throw new BadRequestException(`${field} must be an even integer between 16 and 7680`); + } + return value; +} + +function requireFps(value: unknown): number { + if (typeof value !== "number" || !Number.isInteger(value) || value < 1 || value > 120) { + throw new BadRequestException("fps must be an integer between 1 and 120"); + } + return value; +} + +function requireBoolean(value: unknown, field: string): boolean { + if (typeof value !== "boolean") throw new BadRequestException(`${field} must be a boolean`); + return value; +} + +@Injectable() +export class LiveSettingsService { + private readonly deps: Required; + + constructor(@Optional() @Inject(LIVE_SETTINGS_DEPENDENCIES) dependencies?: Partial) { + this.deps = { + tenantTransaction: withTenant, + audit: (workspaceId, event) => appendTenantAudit(workspaceId, event), + now: () => new Date(), + ...dependencies, + }; + } + + /** Reads (and lazily creates) the channel's settings row. */ + async get(workspaceId: string, virtualChannelId: string): Promise { + return this.deps.tenantTransaction(workspaceId, async (client) => { + const row = await ensureRow(client, workspaceId, virtualChannelId); + return toRecord(row); + }); + } + + /** + * Optimistic-concurrency save. `expectedVersion` is required and must match + * the row's current version; a mismatch (or a started live) is a 409 that + * carries the current record so the caller can rebase and retry. + */ + async save( + workspaceId: string, + virtualChannelId: string, + expectedVersion: unknown, + settings: unknown, + actor: { userId: string | null; role: string }, + ): Promise { + if (typeof expectedVersion !== "number" || !Number.isInteger(expectedVersion) || expectedVersion < 0) { + throw new BadRequestException("expectedVersion must be a non-negative integer"); + } + const validated = validateLiveSettings(settings); + return this.deps.tenantTransaction(workspaceId, async (client) => { + // Serialize writers on the channel row; the version CAS then decides. + await client.query("SELECT id FROM virtual_channels WHERE workspace_id = $1 AND id = $2 FOR UPDATE", [workspaceId, virtualChannelId]); + const row = await ensureRow(client, workspaceId, virtualChannelId); + if (row.live_started_at !== null) { + throw new ConflictException({ message: "live settings are locked while a live is running", current: toRecord(row) }); + } + if (row.version !== expectedVersion) { + throw new ConflictException({ message: "live settings were modified by someone else", current: toRecord(row) }); + } + const updated = await client.query( + `UPDATE virtual_channel_live_settings + SET output_profile = $3, width = $4, height = $5, fps = $6, + latency_mode = $7, permanent_live = $8, save_replay = $9, + replay_settings = $10, version = version + 1, updated_by = $11, updated_at = now() + WHERE workspace_id = $1 AND virtual_channel_id = $2 AND version = $12 + RETURNING *`, + [workspaceId, virtualChannelId, validated.outputProfile, validated.width ?? null, validated.height ?? null, validated.fps ?? null, + validated.latencyMode, validated.permanentLive, validated.saveReplay, + validated.replayPrivacy === undefined ? null : JSON.stringify({ privacy: validated.replayPrivacy }), + actor.userId, expectedVersion], + ); + if (updated.rowCount !== 1) { + throw new ConflictException({ message: "live settings were modified by someone else", current: toRecord(row) }); + } + const next = updated.rows[0] as LiveSettingsRow; + await this.deps.audit(workspaceId, { + actorUserId: actor.userId, actorRole: actor.role, + action: "virtualchannel.live_settings.updated", targetType: "virtual_channel", targetId: virtualChannelId, + diff: { version: next.version, outputProfile: next.output_profile, latencyMode: next.latency_mode, permanentLive: next.permanent_live, saveReplay: next.save_replay }, + }); + return toRecord(next); + }); + } +} + +interface LiveSettingsRow { + virtual_channel_id: string; + output_profile: LiveSettingsProfileName | "custom"; + width: number | null; + height: number | null; + fps: number | null; + latency_mode: LatencyMode; + permanent_live: boolean; + save_replay: boolean; + replay_settings: { privacy?: number } | null; + live_started_at: Date | null; + version: number; + updated_by: string | null; + updated_at: Date; +} + +/** Creates the settings row on first read (0-or-1 per channel, UNIQUE). */ +async function ensureRow(client: PoolClientLike, workspaceId: string, virtualChannelId: string): Promise { + const existing = await client.query("SELECT * FROM virtual_channel_live_settings WHERE workspace_id = $1 AND virtual_channel_id = $2", [workspaceId, virtualChannelId]); + if (existing.rowCount === 1) return existing.rows[0] as LiveSettingsRow; + const channel = await client.query("SELECT id FROM virtual_channels WHERE workspace_id = $1 AND id = $2", [workspaceId, virtualChannelId]); + if (channel.rowCount !== 1) throw new NotFoundException("virtual channel not found"); + const created = await client.query( + `INSERT INTO virtual_channel_live_settings (workspace_id, virtual_channel_id) + VALUES ($1, $2) ON CONFLICT (workspace_id, virtual_channel_id) DO NOTHING RETURNING *`, + [workspaceId, virtualChannelId], + ); + if (created.rowCount === 1) return created.rows[0] as LiveSettingsRow; + const reread = await client.query("SELECT * FROM virtual_channel_live_settings WHERE workspace_id = $1 AND virtual_channel_id = $2", [workspaceId, virtualChannelId]); + if (reread.rowCount !== 1) throw new NotFoundException("virtual channel not found"); + return reread.rows[0] as LiveSettingsRow; +} + +function toRecord(row: LiveSettingsRow): LiveSettingsRecord { + const record: LiveSettingsRecord = { + virtualChannelId: row.virtual_channel_id, + outputProfile: row.output_profile, + latencyMode: row.latency_mode, + permanentLive: row.permanent_live, + saveReplay: row.save_replay, + version: Number(row.version), + liveStartedAt: row.live_started_at === null ? null : row.live_started_at.toISOString(), + updatedAt: row.updated_at.toISOString(), + updatedBy: row.updated_by, + ...(row.output_profile === "custom" ? { width: row.width ?? undefined, height: row.height ?? undefined, fps: row.fps ?? undefined } : {}), + ...(row.replay_settings?.privacy !== undefined ? { replayPrivacy: row.replay_settings.privacy } : {}), + }; + return record; +} diff --git a/apps/api/src/virtual-channels/live-settings.test.ts b/apps/api/src/virtual-channels/live-settings.test.ts new file mode 100644 index 0000000..abbf5cf --- /dev/null +++ b/apps/api/src/virtual-channels/live-settings.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from "vitest"; +import { + ConflictException, + BadRequestException, +} from "@nestjs/common"; +import { + LATENCY_MODES, + validateLiveSettings, +} from "./live-settings.service.js"; + +describe("live settings validation", () => { + it("applies defaults for an empty object", () => { + expect(validateLiveSettings({})).toEqual({ + outputProfile: "720p30", + latencyMode: 1, + permanentLive: false, + saveReplay: false, + }); + }); + + it("accepts every named profile and every latency mode", () => { + for (const profile of ["720p30", "1080p30", "1080p60"] as const) { + expect(validateLiveSettings({ outputProfile: profile }).outputProfile).toBe(profile); + } + for (const latency of LATENCY_MODES) { + expect(validateLiveSettings({ latencyMode: latency }).latencyMode).toBe(latency); + } + }); + + it("custom profile requires even dimensions and a sane fps", () => { + expect(validateLiveSettings({ outputProfile: "custom", width: 1920, height: 1080, fps: 60 })) + .toMatchObject({ width: 1920, height: 1080, fps: 60 }); + expect(() => validateLiveSettings({ outputProfile: "custom", width: 1081, height: 1080, fps: 60 })).toThrow(BadRequestException); + expect(() => validateLiveSettings({ outputProfile: "custom", width: 1920, height: 15, fps: 60 })).toThrow(BadRequestException); + expect(() => validateLiveSettings({ outputProfile: "custom", width: 1920, height: 1080, fps: 0 })).toThrow(BadRequestException); + expect(() => validateLiveSettings({ outputProfile: "custom", width: 1920, height: 1080 })).toThrow(/fps/); + }); + + it("named profiles forbid explicit dimensions", () => { + expect(() => validateLiveSettings({ outputProfile: "720p30", width: 1280 })).toThrow(/custom/); + }); + + it("rejects permanentLive + saveReplay (PeerTube rejects the pair)", () => { + expect(() => validateLiveSettings({ permanentLive: true, saveReplay: true })).toThrow(BadRequestException); + }); + + it("latencyMode must be 1..3", () => { + expect(() => validateLiveSettings({ latencyMode: 0 })).toThrow(BadRequestException); + expect(() => validateLiveSettings({ latencyMode: 4 })).toThrow(BadRequestException); + }); + + it("replayPrivacy only with saveReplay, and only 1..4", () => { + expect(validateLiveSettings({ saveReplay: true, replayPrivacy: 3 })).toMatchObject({ saveReplay: true, replayPrivacy: 3 }); + expect(() => validateLiveSettings({ replayPrivacy: 3 })).toThrow(/saveReplay/); + expect(() => validateLiveSettings({ saveReplay: true, replayPrivacy: 5 })).toThrow(BadRequestException); + }); + + it("rejects non-objects and unknown profiles", () => { + expect(() => validateLiveSettings(null)).toThrow(BadRequestException); + expect(() => validateLiveSettings([])).toThrow(BadRequestException); + expect(() => validateLiveSettings({ outputProfile: "4k120" })).toThrow(BadRequestException); + }); + + it("ConflictException is importable (CAS guard surface)", () => { + expect(ConflictException).toBeDefined(); + }); +}); diff --git a/apps/api/src/virtual-channels/virtual-channels.controller.test.ts b/apps/api/src/virtual-channels/virtual-channels.controller.test.ts new file mode 100644 index 0000000..38906bb --- /dev/null +++ b/apps/api/src/virtual-channels/virtual-channels.controller.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it, vi } from "vitest"; +import type { Request } from "express"; +import { Readable } from "node:stream"; +import { BadRequestException } from "@nestjs/common"; +import { VirtualChannelsController } from "./virtual-channels.controller.js"; +import { BulkTooLargeException, VirtualChannelImportService, type ImportJobRecord } from "./import.service.js"; +import { LiveSettingsService } from "./live-settings.service.js"; + +/** + * Controller-level: the body-shape seam. Nest registers only json/urlencoded + * parsers, so text/plain + text/csv arrive as an unconsumed stream that the + * controller must read itself; JSON arrays arrive already parsed and are + * re-serialized. Guard/route wiring is exercised in the live-DB suite. + */ + +function fakeRequest(body?: string): Request & { session: { workspaceId: string; userId: string | null; role: string } } { + const stream = body === undefined ? new Readable({ read() { this.push(null); } }) : undefined; + return { + readable: stream !== undefined, + headers: { "content-type": body === undefined ? "text/plain" : "text/plain" }, + session: { workspaceId: "ws-1", userId: null, role: "Owner" }, + ...(stream === undefined ? {} : { [Symbol.asyncIterator]: stream[Symbol.asyncIterator].bind(stream) }), + } as unknown as Request & { session: { workspaceId: string; userId: string | null; role: string } }; +} + +function makeController(inputs: { bodies: string[]; created: ImportJobRecord }) { + const createImport = vi.fn(async (_ws: string, _id: string, input: { body: string }) => { + inputs.bodies.push(input.body); + return inputs.created; + }); + const imports = { createImport, getImport: vi.fn() } as unknown as VirtualChannelImportService; + const settings = { get: vi.fn(), save: vi.fn() } as unknown as LiveSettingsService; + return { controller: new VirtualChannelsController(imports, settings), createImport }; +} + +const job: ImportJobRecord = { + jobId: "11111111-1111-4111-8111-111111111111", workspaceId: "ws-1", + virtualChannelId: "22222222-2222-4222-8222-222222222222", connectionId: null, + status: "running", format: "multiline", total: 1, processed: 0, + imported: 0, duplicate: 0, invalid: 0, failed: 0, entries: [], + createdAt: new Date(0).toISOString(), updatedAt: new Date(0).toISOString(), +}; + +describe("import controller body shapes", () => { + it("reads a text/plain body from the raw stream", async () => { + const { controller, createImport } = makeController({ bodies: [], created: job }); + const request = fakeRequest(); + // Simulate the unconsumed stream delivering the text. + const stream = Readable.from(["@alice@video.example\n@bob@video.example"]); + (request as unknown as { readable: boolean }).readable = true; + (request as unknown as Record)[Symbol.asyncIterator] = stream[Symbol.asyncIterator].bind(stream); + const result = await controller.createImport(request as never, "22222222-2222-4222-8222-222222222222", undefined, undefined); + expect(result.job.jobId).toBe(job.jobId); + expect(createImport).toHaveBeenCalledOnce(); + expect((createImport.mock.calls[0]?.[2] as { body: string }).body).toContain("@alice@video.example"); + }); + + it("passes through an already-parsed JSON array re-serialized", async () => { + const { controller, createImport } = makeController({ bodies: [], created: job }); + await controller.createImport(fakeRequest() as never, "22222222-2222-4222-8222-222222222222", undefined, ["@alice@video.example", { url: "https://v.example/c/alice" }]); + const passed = (createImport.mock.calls[0]?.[2] as { body: string }).body; + expect(JSON.parse(passed)).toHaveLength(2); + }); + + it("a parsed JSON object (not array) is a 400", async () => { + const { controller } = makeController({ bodies: [], created: job }); + // express has consumed the stream and set req.body to the parsed object: + // readable is false and no string/array shape is present. + const request = { ...fakeRequest(), readable: false } as unknown as Parameters[0]; + await expect(controller.createImport(request, "22222222-2222-4222-8222-222222222222", undefined, { not: "an array" })) + .rejects.toBeInstanceOf(BadRequestException); + }); + + it("maps the 5000-row cap to HTTP 413", async () => { + const createImport = vi.fn(async () => { throw new BulkTooLargeException(5001, 5000); }); + const imports = { createImport, getImport: vi.fn() } as unknown as VirtualChannelImportService; + const settings = { get: vi.fn(), save: vi.fn() } as unknown as LiveSettingsService; + const controller = new VirtualChannelsController(imports, settings); + const request = fakeRequest(); + const stream = Readable.from(["x"]); + (request as unknown as { readable: boolean }).readable = true; + (request as unknown as Record)[Symbol.asyncIterator] = stream[Symbol.asyncIterator].bind(stream); + await expect(controller.createImport(request as never, "22222222-2222-4222-8222-222222222222", undefined, undefined)) + .rejects.toMatchObject({ status: 413 }); + }); +}); diff --git a/apps/api/src/virtual-channels/virtual-channels.controller.ts b/apps/api/src/virtual-channels/virtual-channels.controller.ts new file mode 100644 index 0000000..38536fe --- /dev/null +++ b/apps/api/src/virtual-channels/virtual-channels.controller.ts @@ -0,0 +1,121 @@ +import { BadRequestException, Body, Controller, Get, HttpCode, Inject, Param, PayloadTooLargeException, Patch, Post, Query, Req, UseGuards } from "@nestjs/common"; +import type { Request } from "express"; +import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js"; +import { requireSession } from "../http/validation.js"; +import { + BulkTooLargeException, + VirtualChannelImportService, + type ImportJobRecord, +} from "./import.service.js"; +import { LIVE_SETTINGS_DEPENDENCIES, LiveSettingsService, type LiveSettingsRecord } from "./live-settings.service.js"; + +/** Raw-body import: URL / handle / multiline / CSV / JSON shapes all arrive as text. */ +@Controller("virtual-channels") +@UseGuards(RbacGuard) +export class VirtualChannelsController { + constructor( + @Inject(VirtualChannelImportService) private readonly imports: VirtualChannelImportService, + @Inject(LiveSettingsService) private readonly settings: LiveSettingsService, + ) {} + + /** + * POST /virtual-channels/:id/import + * Body: text/plain (URL, handle, multiline, CSV) or application/json (array). + * Query: ?connectionId= optional default-instance connection. + * Returns 202 + the job record; poll GET .../import/:jobId for progress. + */ + @Post(":id/import") + @RequirePermission("streamsource.manage") + @HttpCode(202) + async createImport( + @Req() request: Request & AuthenticatedRequest, + @Param("id") id: string, + @Query("connectionId") connectionId: string | undefined, + @Body() body: unknown, + ): Promise<{ job: ImportJobRecord }> { + const session = requireSession(request); + // Nest registers only json/urlencoded body parsers, so a text/csv or + // text/plain body arrives as an unconsumed stream: read it here. JSON + // bodies were already parsed into `body` (an array of rows) — re-serialized + // so the service's single parseImportBody path handles every shape. + let raw: string; + if (typeof body === "string") raw = body; + else if (Array.isArray(body)) raw = JSON.stringify(body); + else if (request.readable === true) raw = await readRawBody(request); + else throw new BadRequestException("import body must be text (URL, handle, CSV, multiline) or a JSON array"); + try { + return { job: await this.imports.createImport(session.workspaceId, id, { + contentType: request.headers["content-type"], + body: raw, + ...(connectionId === undefined ? {} : { connectionId }), + actor: { userId: session.userId, role: session.role }, + }) }; + } catch (error) { + if (error instanceof BulkTooLargeException) throw new ImportTooLargeException(error); + throw error; + } + } + + /** GET /virtual-channels/:id/import/:jobId — progress + report counts. */ + @Get(":id/import/:jobId") + @RequirePermission("streamsource.manage") + async getImport( + @Req() request: Request & AuthenticatedRequest, + @Param("id") id: string, + @Param("jobId") jobId: string, + ): Promise { + return this.imports.getImport(requireSession(request).workspaceId, jobId); + } + + /** GET /virtual-channels/:id/live-settings */ + @Get(":id/live-settings") + @RequirePermission("virtualchannel.view") + async getSettings( + @Req() request: Request & AuthenticatedRequest, + @Param("id") id: string, + ): Promise { + return this.settings.get(requireSession(request).workspaceId, id); + } + + /** + * PATCH /virtual-channels/:id/live-settings?expectedVersion=N + * Stale version or a started live => 409 with the current record. + */ + @Patch(":id/live-settings") + @RequirePermission("virtualchannel.edit") + async saveSettings( + @Req() request: Request & AuthenticatedRequest, + @Param("id") id: string, + @Query("expectedVersion") expectedVersion: string | undefined, + @Body() body: unknown, + ): Promise { + const session = requireSession(request); + const version = expectedVersion === undefined ? NaN : Number(expectedVersion); + return this.settings.save(session.workspaceId, id, version, body, { userId: session.userId, role: session.role }); + } +} + +/** + * Consumes the request stream for text bodies (Nest's default parsers skip + * text/plain and text/csv, leaving the stream unread). Bounded to 2 MB — the + * same cap the service enforces on the parsed string. + */ +async function readRawBody(request: Request): Promise { + if (request.readable !== true) throw new BadRequestException("import body must be text (URL, handle, CSV, multiline) or a JSON array"); + const chunks: Buffer[] = []; + let size = 0; + for await (const chunk of request) { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)); + size += buffer.byteLength; + if (size > 2_000_000) throw new ImportTooLargeException({ received: size, limit: 2_000_000 } as BulkTooLargeException); + chunks.push(buffer); + } + return Buffer.concat(chunks).toString("utf8"); +} + +/** 413 with a machine-readable body (the 5000-row cap, proposal §4.4). */ +class ImportTooLargeException extends PayloadTooLargeException { + constructor(error: BulkTooLargeException) { + super({ message: error.message, received: error.received, limit: error.limit }); + } +} diff --git a/apps/api/src/virtual-channels/virtual-channels.integration.test.ts b/apps/api/src/virtual-channels/virtual-channels.integration.test.ts new file mode 100644 index 0000000..41de356 --- /dev/null +++ b/apps/api/src/virtual-channels/virtual-channels.integration.test.ts @@ -0,0 +1,209 @@ +import { randomUUID } from "node:crypto"; +import { join } from "node:path"; +import { Pool, type PoolClient } from "pg"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { runMigrations } from "../db/migrate.js"; +import { channelRemoteRef, videoRemoteRef } from "../../../../packages/connector-peertube/src/resolve.js"; + +/** + * Live-DB suite for WP7 (migration 023 + import persistence + live-settings + * guard). Follows the house pattern: owner pool for provisioning, app pool + * (sp_app) for tenant work; skips when DATABASE_URL is absent. + */ +const dbAvailable = process.env.DATABASE_URL !== undefined; +const ownerPool = new Pool({ connectionString: process.env.SUPER_DATABASE_URL ?? process.env.DATABASE_URL }); +const appPool = new Pool({ connectionString: process.env.DATABASE_URL }); + +const UUID_A = "0d1c2b3a-4e5f-4a6b-8c7d-9e0f1a2b3c4d"; + +async function asTenant(workspaceId: string | null, fn: (client: PoolClient) => Promise): Promise { + const client = await appPool.connect(); + try { + await client.query("BEGIN"); + if (workspaceId !== null) await client.query("SELECT set_config('app.workspace_id', $1, true)", [workspaceId]); + const result = await fn(client); + await client.query("COMMIT"); + return result; + } catch (error) { + await client.query("ROLLBACK").catch(() => undefined); + throw error; + } finally { + client.release(); + } +} + +async function seedWorkspace(name: string): Promise { + const row = await ownerPool.query("INSERT INTO workspaces (name, slug) VALUES ($1, $2) RETURNING id", [name, `${name.toLowerCase()}-${randomUUID()}`]); + return String(row.rows[0]?.id); +} + +async function seedChannel(workspaceId: string, slug: string): Promise { + return asTenant(workspaceId, async (client) => { + const row = await client.query("INSERT INTO virtual_channels (workspace_id, slug, name) VALUES ($1, $2, $3) RETURNING id", [workspaceId, slug, slug]); + return String(row.rows[0]?.id); + }); +} + +async function insertImportedSource(workspaceId: string, channelId: string, remoteRef: string, hlsUrl: string | null): Promise<"imported" | "duplicate"> { + return asTenant(workspaceId, async (client) => { + const result = await client.query( + `INSERT INTO virtual_channel_sources (workspace_id, virtual_channel_id, kind, label, remote_ref, hls_url) + VALUES ($1, $2, 'hls', $3, $4, $5) + ON CONFLICT (workspace_id, virtual_channel_id, remote_ref) DO NOTHING + RETURNING id`, + [workspaceId, channelId, "imported source", remoteRef, hlsUrl], + ); + return result.rowCount === 1 ? "imported" : "duplicate"; + }); +} + +describe.skipIf(!dbAvailable)("virtual channel import (live DB)", () => { + let workspaceId: string; + let otherWorkspaceId: string; + let channelId: string; + + beforeAll(async () => { + await runMigrations(process.env.MIGRATIONS_DIR ?? join(process.cwd(), "migrations")); + workspaceId = await seedWorkspace("VC import A"); + otherWorkspaceId = await seedWorkspace("VC import B"); + channelId = await seedChannel(workspaceId, "import-target"); + }); + + afterAll(async () => { + if (workspaceId !== undefined) await ownerPool.query("DELETE FROM workspaces WHERE id = ANY($1)", [[workspaceId, otherWorkspaceId]]); + await appPool.end(); + await ownerPool.end(); + }); + + it("migration 023 landed its tables and policies", async () => { + const applied = await ownerPool.query("SELECT name FROM schema_migrations WHERE name LIKE '023%'"); + expect(applied.rows.length).toBeGreaterThanOrEqual(1); + for (const table of ["virtual_channel_imports", "virtual_channel_live_settings"]) { + const flags = await ownerPool.query("SELECT relrowsecurity, relforcerowsecurity FROM pg_class WHERE oid = $1::regclass", [table]); + expect(flags.rows[0], table).toEqual({ relrowsecurity: true, relforcerowsecurity: true }); + const grants = await ownerPool.query("SELECT has_table_privilege($1, $2, 'INSERT') AS ok", ["sp_app", table]); + expect(grants.rows[0]?.ok).toBe(true); + } + }); + + it("imports a source with a remote_ref and no HLS URL yet (023 widened CHECK)", async () => { + const outcome = await insertImportedSource(workspaceId, channelId, channelRemoteRef("alice", "video.example"), null); + expect(outcome).toBe("imported"); + const row = await asTenant(workspaceId, (client) => client.query( + "SELECT kind, remote_ref, hls_url FROM virtual_channel_sources WHERE workspace_id = $1 AND virtual_channel_id = $2 AND remote_ref = $3", + [workspaceId, channelId, channelRemoteRef("alice", "video.example")], + )); + expect(row.rows[0]).toEqual({ kind: "hls", remote_ref: "channel:alice@video.example", hls_url: null }); + }); + + it("the same remote_ref is a conflict (counted duplicate), a different one is not", async () => { + expect(await insertImportedSource(workspaceId, channelId, channelRemoteRef("alice", "video.example"), null)).toBe("duplicate"); + expect(await insertImportedSource(workspaceId, channelId, videoRemoteRef(UUID_A, "video.example"), `https://video.example/static/streaming-playlists/hls/${UUID_A}/master.m3u8`)).toBe("imported"); + }); + + it("013's playlist arm is unchanged: playlists still refuse remote_ref/hls_url", async () => { + await expect(asTenant(workspaceId, (client) => client.query( + "INSERT INTO virtual_channel_sources (workspace_id, virtual_channel_id, kind, label, playlist_loop, remote_ref) VALUES ($1, $2, 'playlist', 'P', true, 'channel:x@y')", + [workspaceId, channelId], + ))).rejects.toMatchObject({ code: "23514" }); + }); + + it("workspace_id and channel ownership are immutable on both new tables", async () => { + // Seed one row per table first: the immutability triggers fire per row, so + // an UPDATE that matches nothing would trivially "succeed". + await asTenant(workspaceId, async (client) => { + await client.query( + `INSERT INTO virtual_channel_imports (workspace_id, virtual_channel_id, total) + VALUES ($1, $2, 1) + ON CONFLICT DO NOTHING`, [workspaceId, channelId]); + await client.query( + `INSERT INTO virtual_channel_live_settings (workspace_id, virtual_channel_id) + VALUES ($1, $2) + ON CONFLICT (workspace_id, virtual_channel_id) DO NOTHING`, [workspaceId, channelId]); + }); + await expect(asTenant(workspaceId, (client) => client.query( + "UPDATE virtual_channel_imports SET workspace_id = $2::uuid WHERE workspace_id = $1::uuid", [workspaceId, otherWorkspaceId], + ))).rejects.toThrow(/immutable|row-level security/); + await expect(asTenant(workspaceId, (client) => client.query( + "UPDATE virtual_channel_live_settings SET virtual_channel_id = $2::uuid WHERE workspace_id = $1::uuid", [workspaceId, randomUUID()], + ))).rejects.toThrow(/immutable|row-level security/); + }); + + it("RLS: tenant B sees neither A's imports nor A's live settings", async () => { + await asTenant(workspaceId, async (client) => { + await client.query("INSERT INTO virtual_channel_imports (workspace_id, virtual_channel_id, total) VALUES ($1, $2, 1)", [workspaceId, channelId]); + await client.query( + "INSERT INTO virtual_channel_live_settings (workspace_id, virtual_channel_id) VALUES ($1, $2) ON CONFLICT (workspace_id, virtual_channel_id) DO NOTHING", + [workspaceId, channelId]); + }); + for (const [sql, params] of [ + ["SELECT count(*)::int AS n FROM virtual_channel_imports WHERE workspace_id = $1", [workspaceId]], + ["SELECT count(*)::int AS n FROM virtual_channel_live_settings WHERE workspace_id = $1", [workspaceId]], + ] as const) { + const seen = await asTenant(otherWorkspaceId, (client) => client.query(sql, [...params])); + expect(seen.rows[0]?.n).toBe(0); + } + // Cross-tenant writes are refused by WITH CHECK. + await expect(asTenant(otherWorkspaceId, (client) => client.query( + "INSERT INTO virtual_channel_imports (workspace_id, virtual_channel_id, total) VALUES ($1, $2, 1)", [workspaceId, channelId], + ))).rejects.toMatchObject({ code: "42501" }); + // No tenant context sees nothing at all. + expect((await asTenant(null, (client) => client.query("SELECT count(*)::int AS n FROM virtual_channel_imports"))).rows[0]?.n).toBe(0); + }); + + it("live settings: version CAS rejects a stale version and bumps exactly once per accepted write", async () => { + const initial = await asTenant(workspaceId, async (client) => { + const existing = await client.query("SELECT version FROM virtual_channel_live_settings WHERE workspace_id = $1 AND virtual_channel_id = $2", [workspaceId, channelId]); + if (existing.rowCount === 1) return Number(existing.rows[0]?.version); + await client.query("INSERT INTO virtual_channel_live_settings (workspace_id, virtual_channel_id) VALUES ($1, $2)", [workspaceId, channelId]); + return 0; + }); + // Accepted write: version -> initial+1. + const updated = await asTenant(workspaceId, (client) => client.query( + `UPDATE virtual_channel_live_settings SET latency_mode = 3, version = version + 1 + WHERE workspace_id = $1 AND virtual_channel_id = $2 AND version = $3 RETURNING version`, + [workspaceId, channelId, initial], + )); + expect(updated.rowCount).toBe(1); + expect(Number(updated.rows[0]?.version)).toBe(initial + 1); + // Stale write: matches nothing, changes nothing. + const stale = await asTenant(workspaceId, (client) => client.query( + `UPDATE virtual_channel_live_settings SET latency_mode = 1, version = version + 1 + WHERE workspace_id = $1 AND virtual_channel_id = $2 AND version = $3 RETURNING version`, + [workspaceId, channelId, initial], + )); + expect(stale.rowCount).toBe(0); + expect(Number(stale.rows.length)).toBe(0); + // The version cannot be rewound by a direct write either. + await expect(asTenant(workspaceId, (client) => client.query( + "UPDATE virtual_channel_live_settings SET version = 0 WHERE workspace_id = $1 AND virtual_channel_id = $2", [workspaceId, channelId], + ))).rejects.toMatchObject({ code: "23514" }); + }); + + it("live settings CHECKs mirror the validator (permanent+replay, latency, custom triple)", async () => { + await expect(asTenant(workspaceId, (client) => client.query( + "UPDATE virtual_channel_live_settings SET permanent_live = true, save_replay = true WHERE workspace_id = $1 AND virtual_channel_id = $2", + [workspaceId, channelId], + ))).rejects.toMatchObject({ code: "23514" }); + await expect(asTenant(workspaceId, (client) => client.query( + "UPDATE virtual_channel_live_settings SET latency_mode = 4 WHERE workspace_id = $1 AND virtual_channel_id = $2", [workspaceId, channelId], + ))).rejects.toMatchObject({ code: "23514" }); + await expect(asTenant(workspaceId, (client) => client.query( + "UPDATE virtual_channel_live_settings SET output_profile = 'custom' WHERE workspace_id = $1 AND virtual_channel_id = $2", [workspaceId, channelId], + ))).rejects.toMatchObject({ code: "23514" }); + // A valid custom triple passes. + await asTenant(workspaceId, (client) => client.query( + "UPDATE virtual_channel_live_settings SET output_profile = 'custom', width = 1920, height = 1080, fps = 60 WHERE workspace_id = $1 AND virtual_channel_id = $2", + [workspaceId, channelId], + )); + }); + + it("import job rows enforce the caps (5000) and processed <= total", async () => { + await expect(asTenant(workspaceId, (client) => client.query( + "INSERT INTO virtual_channel_imports (workspace_id, virtual_channel_id, total) VALUES ($1, $2, 5001)", [workspaceId, channelId], + ))).rejects.toMatchObject({ code: "23514" }); + await expect(asTenant(workspaceId, (client) => client.query( + "INSERT INTO virtual_channel_imports (workspace_id, virtual_channel_id, total, processed) VALUES ($1, $2, 1, 2)", [workspaceId, channelId], + ))).rejects.toMatchObject({ code: "23514" }); + }); +}); diff --git a/apps/api/src/virtual-channels/virtual-channels.module.ts b/apps/api/src/virtual-channels/virtual-channels.module.ts new file mode 100644 index 0000000..0b57def --- /dev/null +++ b/apps/api/src/virtual-channels/virtual-channels.module.ts @@ -0,0 +1,18 @@ +import { Module } from "@nestjs/common"; +import { VirtualChannelsController } from "./virtual-channels.controller.js"; +import { VirtualChannelImportService } from "./import.service.js"; +import { LiveSettingsService } from "./live-settings.service.js"; + +/** + * WP7 bulk source import + live-settings guard. + * + * Registration (one line, added by the app.module owner): + * import { VirtualChannelsModule } from "./virtual-channels/virtual-channels.module.js"; + * imports: [..., VirtualChannelsModule] + */ +@Module({ + controllers: [VirtualChannelsController], + providers: [VirtualChannelImportService, LiveSettingsService], + exports: [VirtualChannelImportService, LiveSettingsService], +}) +export class VirtualChannelsModule {} diff --git a/apps/api/tsconfig.json b/apps/api/tsconfig.json new file mode 100644 index 0000000..d4f3651 --- /dev/null +++ b/apps/api/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "experimentalDecorators": true, + "emitDecoratorMetadata": true, + "target": "ES2022", + "module": "CommonJS", + "moduleResolution": "node" + }, + "include": ["src"] +} diff --git a/apps/api/vitest.config.ts b/apps/api/vitest.config.ts new file mode 100644 index 0000000..f090b64 --- /dev/null +++ b/apps/api/vitest.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + // Integration suites migrate a scratch database in beforeAll; a fresh + // migration run alone can exceed the 10 s default hook budget. + hookTimeout: 120_000, + testTimeout: 60_000, + }, +}); diff --git a/apps/media-worker/.gitignore b/apps/media-worker/.gitignore new file mode 100644 index 0000000..678a59a --- /dev/null +++ b/apps/media-worker/.gitignore @@ -0,0 +1,2 @@ +# Packaged native helpers (scripts/build-helpers.sh output) +/bin/ diff --git a/apps/media-worker/package.json b/apps/media-worker/package.json new file mode 100644 index 0000000..bb2a689 --- /dev/null +++ b/apps/media-worker/package.json @@ -0,0 +1,22 @@ +{ + "name": "@tactical-media-broadcast-hub/media-worker", + "private": true, + "version": "0.1.0", + "scripts": { + "dev": "tsx watch src/main.ts", + "build": "tsc -p tsconfig.json", + "lint": "eslint src", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "node --import tsx --test src/**/*.test.ts", + "build:helpers": "sh scripts/build-helpers.sh", + "smoke:cpu": "node --import tsx src/compositor/cpu-smoke.ts --seconds 30", + "soak": "node --import tsx src/compositor/soak.ts" + }, + "dependencies": { + "pino": "^9.3.2" + }, + "devDependencies": { + "@types/node": "^22.5.0", + "tsx": "^4.17.0" + } +} diff --git a/apps/media-worker/scripts/build-helpers.sh b/apps/media-worker/scripts/build-helpers.sh new file mode 100755 index 0000000..4ae3c55 --- /dev/null +++ b/apps/media-worker/scripts/build-helpers.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# Deterministic release build of the native helpers (src/gate/rust: frame-gate, source-bus). +# +# scripts/build-helpers.sh [OUT_DIR] (default: apps/media-worker/bin) +# +# std only, no crates: `--offline --locked` needs no network and fails if Cargo.lock would change. +# Reproducible for a given rustc: the crate path is remapped out of the binary, symbols are +# stripped, and the build uses its own target dir (the development target/ is left alone). +# OUT_DIR gets the two binaries plus SHA256SUMS; the worker finds them via MEDIA_HELPER_DIR +# (src/gate/binary.ts). The image builds this in a pinned rust stage (infra/media-worker.Dockerfile). +set -eu +here=$(cd "$(dirname "$0")/.." && pwd) +crate="$here/src/gate/rust" +out=${1:-"$here/bin"} +target=${CARGO_TARGET_DIR:-"$crate/target/package"} +export CARGO_TARGET_DIR="$target" +export RUSTFLAGS="--remap-path-prefix=$crate=/frame-gate -C strip=symbols -C debuginfo=0" +cargo build --release --offline --locked --manifest-path "$crate/Cargo.toml" +mkdir -p "$out" +for bin in frame-gate source-bus; do + install -m 0755 "$target/release/$bin" "$out/$bin" +done +(cd "$out" && sha256sum frame-gate source-bus > SHA256SUMS && cat SHA256SUMS) +rustc --version > "$out/RUSTC_VERSION" diff --git a/apps/media-worker/scripts/nightly-soak.sh b/apps/media-worker/scripts/nightly-soak.sh new file mode 100644 index 0000000..fc39974 --- /dev/null +++ b/apps/media-worker/scripts/nightly-soak.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# Reference-host nightly entry point. Eight hours; nonzero means a gate/leak failure. +# Usage: sh scripts/nightly-soak.sh [JSONL_OUTPUT_PATH] +set -eu +worker_dir=$(cd "$(dirname "$0")/.." && pwd) +repo_dir=$(cd "$worker_dir/../.." && pwd) +out=${1:-"$repo_dir/runs/soak/$(date -u +%F).jsonl"} +mkdir -p "$(dirname "$out")" +cd "$worker_dir" +sh scripts/build-helpers.sh +export MEDIA_HELPER_DIR="$worker_dir/bin" +exec node --import tsx src/compositor/soak.ts --hours 8 --profile 1080p30 --swap-every 30 --sample-every 10 --out "$out" diff --git a/apps/media-worker/src/bus/control-signer.ts b/apps/media-worker/src/bus/control-signer.ts new file mode 100644 index 0000000..493775a --- /dev/null +++ b/apps/media-worker/src/bus/control-signer.ts @@ -0,0 +1,14 @@ +import type { KeyObject } from "node:crypto"; +import { signCommand, type CommandBody, type SignedCommand } from "./envelope"; + +/** API-side stub: caller must commit desired state and re-authorize before signing. */ +export interface ControlCommandSigner { + signCommittedSnapshot(body: CommandBody): Promise; +} + +export class Ed25519ControlSigner implements ControlCommandSigner { + constructor(private readonly kid: string, private readonly privateKey: KeyObject) {} + async signCommittedSnapshot(body: CommandBody): Promise { + return signCommand(body, this.kid, this.privateKey); + } +} diff --git a/apps/media-worker/src/bus/envelope.ts b/apps/media-worker/src/bus/envelope.ts new file mode 100644 index 0000000..5f943be --- /dev/null +++ b/apps/media-worker/src/bus/envelope.ts @@ -0,0 +1,61 @@ +import { sign, verify, type KeyObject } from "node:crypto"; +import { isDesiredChannel, snapshotHlsUrls, type DesiredChannel } from "../types"; +import { validateEndpointShape } from "../relay/endpoints"; + +export interface CommandBody { + workspaceId: string; + channelId: string; + workerId: string; + assignmentEpoch: number; + commandId: string; + desiredVersion: number; + issuedAt: string; + expiresAt: string; + snapshot: DesiredChannel; +} +export interface SignedCommand { kid: string; body: CommandBody; signature: string } + +/** Fixed field order is the wire-format contract for both signing and verifying. */ +export function commandBytes(body: CommandBody): Buffer { + return Buffer.from(JSON.stringify({ + workspaceId: body.workspaceId, channelId: body.channelId, workerId: body.workerId, + assignmentEpoch: body.assignmentEpoch, commandId: body.commandId, + desiredVersion: body.desiredVersion, issuedAt: body.issuedAt, + expiresAt: body.expiresAt, snapshot: body.snapshot, + })); +} + +export function signCommand(body: CommandBody, kid: string, key: KeyObject): SignedCommand { + if (!isDesiredChannel(body.snapshot)) throw new Error("invalid desired snapshot"); + validateSnapshotEndpoints(body.snapshot); + return { kid, body, signature: sign(null, commandBytes(body), key).toString("base64") }; +} + +function validateSnapshotEndpoints(snapshot: DesiredChannel): void { + for (const url of snapshotHlsUrls(snapshot)) validateEndpointShape(url, "hls"); + if (snapshot.output) validateEndpointShape(snapshot.output.endpointUrl, "output"); +} + +export function verifyCommand(value: unknown, keys: ReadonlyMap, workerId: string, now = Date.now()): SignedCommand { + if (typeof value !== "object" || value === null) throw new Error("invalid command"); + const command = value as Partial; + const body = command.body; + if (typeof command.kid !== "string" || typeof command.signature !== "string" || !body || + typeof body.workspaceId !== "string" || typeof body.channelId !== "string" || + body.workerId !== workerId || typeof body.commandId !== "string" || body.commandId.length === 0 || + !Number.isSafeInteger(body.assignmentEpoch) || (body.assignmentEpoch ?? -1) < 0 || + !Number.isSafeInteger(body.desiredVersion) || (body.desiredVersion ?? -1) < 0 || + typeof body.issuedAt !== "string" || typeof body.expiresAt !== "string" || + !isDesiredChannel(body.snapshot)) throw new Error("invalid command fields"); + const issued = Date.parse(body.issuedAt); + const expires = Date.parse(body.expiresAt); + if (!Number.isFinite(issued) || !Number.isFinite(expires) || issued > now + 30_000 || expires <= now || expires <= issued) throw new Error("expired command"); + const snapshot = body.snapshot; + if (snapshot.workspaceId !== body.workspaceId || snapshot.channelId !== body.channelId || + snapshot.workerId !== body.workerId || snapshot.assignmentEpoch !== body.assignmentEpoch || + snapshot.desiredVersion !== body.desiredVersion) throw new Error("snapshot mismatch"); + validateSnapshotEndpoints(snapshot); + const key = keys.get(command.kid); + if (!key || !verify(null, commandBytes(body as CommandBody), key, Buffer.from(command.signature, "base64"))) throw new Error("invalid signature"); + return command as SignedCommand; +} diff --git a/apps/media-worker/src/bus/fence.ts b/apps/media-worker/src/bus/fence.ts new file mode 100644 index 0000000..a2002c9 --- /dev/null +++ b/apps/media-worker/src/bus/fence.ts @@ -0,0 +1,68 @@ +import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; +import { randomUUID } from "node:crypto"; +import type { CommandBody } from "./envelope"; + +interface FenceEntry { assignmentEpoch: number; acceptedVersion: number; seen: Record } +type FenceData = Record; + +/** Local durable fence. Atomic rename plus fsync of file and directory precedes execute/ack. */ +export class DurableFence { + private data: FenceData; + constructor(private readonly path: string) { + mkdirSync(dirname(path), { recursive: true }); + this.data = existsSync(path) ? this.read() : {}; + } + + private read(): FenceData { + const parsed: unknown = JSON.parse(readFileSync(this.path, "utf8")); + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new Error("invalid fence file"); + for (const entry of Object.values(parsed)) { + if (typeof entry !== "object" || entry === null) throw new Error("invalid fence entry"); + const record = entry as Partial; + if (!Number.isSafeInteger(record.assignmentEpoch) || !Number.isSafeInteger(record.acceptedVersion) || + (record.assignmentEpoch ?? -1) < 0 || (record.acceptedVersion ?? -1) < 0 || + typeof record.seen !== "object" || record.seen === null || Array.isArray(record.seen) || + Object.values(record.seen).some((expiry) => !Number.isFinite(expiry))) throw new Error("invalid fence entry"); + } + return parsed as FenceData; + } + + accepted(workspaceId: string, channelId: string): { assignmentEpoch: number; acceptedVersion: number } | null { + const entry = this.data[`${workspaceId}:${channelId}`]; + return entry ? { assignmentEpoch: entry.assignmentEpoch, acceptedVersion: entry.acceptedVersion } : null; + } + + accept(body: CommandBody, now = Date.now()): "accepted" | "replay" | "stale" { + const key = `${body.workspaceId}:${body.channelId}`; + const previous = this.data[key]; + if (previous && body.assignmentEpoch < previous.assignmentEpoch) return "stale"; + if (previous && body.assignmentEpoch === previous.assignmentEpoch && body.desiredVersion < previous.acceptedVersion) return "stale"; + if (previous?.seen[body.commandId] !== undefined) return "replay"; + const seen = previous ? { ...previous.seen } : {}; + for (const [id, expiry] of Object.entries(seen)) if (expiry <= now) delete seen[id]; + seen[body.commandId] = Date.parse(body.expiresAt); + const next: FenceData = { ...this.data, [key]: { + assignmentEpoch: body.assignmentEpoch, + acceptedVersion: previous?.assignmentEpoch === body.assignmentEpoch + ? Math.max(previous.acceptedVersion, body.desiredVersion) : body.desiredVersion, + seen, + } }; + this.persist(next); + this.data = next; + return "accepted"; + } + + private persist(data: FenceData): void { + const temp = `${this.path}.${randomUUID()}.tmp`; + const fd = openSync(temp, "wx", 0o600); + try { + writeFileSync(fd, JSON.stringify(data)); + fsyncSync(fd); + } catch (error) { unlinkSync(temp); throw error; } + finally { closeSync(fd); } + renameSync(temp, this.path); + const dir = openSync(dirname(this.path), "r"); + try { fsyncSync(dir); } finally { closeSync(dir); } + } +} diff --git a/apps/media-worker/src/bus/redis.ts b/apps/media-worker/src/bus/redis.ts new file mode 100644 index 0000000..1339b0b --- /dev/null +++ b/apps/media-worker/src/bus/redis.ts @@ -0,0 +1,137 @@ +import net, { type Socket } from "node:net"; +import tls from "node:tls"; +import type { ObservedChannel } from "../types"; + +type Reply = string | number | null | Reply[]; +function encode(parts: string[]): Buffer { + return Buffer.from(`*${parts.length}\r\n${parts.map((part) => `\$${Buffer.byteLength(part)}\r\n${part}\r\n`).join("")}`); +} +function parse(buffer: Buffer, offset = 0): { reply: Reply; end: number } | null { + if (offset >= buffer.length) return null; + const endLine = buffer.indexOf("\r\n", offset); + if (endLine < 0) return null; + const length = Number(buffer.subarray(offset + 1, endLine).toString()); + const type = String.fromCharCode(buffer[offset] ?? 0); + let pos = endLine + 2; + if (type === "+" || type === ":" || type === "-") return { reply: type === ":" ? length : buffer.subarray(offset + 1, endLine).toString(), end: pos }; + if (type === "$") { + if (length === -1) return { reply: null, end: pos }; + if (!Number.isInteger(length) || length < 0 || length > 2_000_000) throw new Error("invalid Redis bulk length"); + if (buffer.length < pos + length + 2) return null; + const reply = buffer.subarray(pos, pos + length).toString(); + return { reply, end: pos + length + 2 }; + } + if (type === "*") { + if (length === -1) return { reply: null, end: pos }; + if (!Number.isInteger(length) || length < 0 || length > 100) throw new Error("invalid Redis array length"); + const replies: Reply[] = []; + for (let i = 0; i < length; i += 1) { + const item = parse(buffer, pos); + if (!item) return null; + replies.push(item.reply); + pos = item.end; + } + return { reply: replies, end: pos }; + } + throw new Error("invalid Redis reply"); +} + +class RedisConnection { + private socket: Socket; + private buffer = Buffer.alloc(0); + private pending: Array<{ resolve: (reply: Reply) => void; reject: (error: Error) => void }> = []; + onPush: ((reply: Reply) => void) | null = null; + private subscribing = false; + + private constructor(socket: Socket) { + this.socket = socket; + socket.on("data", (chunk: Buffer) => { + this.buffer = Buffer.concat([this.buffer, chunk]); + if (this.buffer.length > 2_000_000) { socket.destroy(new Error("Redis reply too large")); return; } + while (true) { + const prefix = this.buffer[0]; + const parsed = parse(this.buffer); + if (!parsed) break; + this.buffer = this.buffer.subarray(parsed.end); + if (prefix === 45) { + if (this.subscribing) this.onPush?.(parsed.reply); + else this.pending.shift()?.reject(new Error(`Redis: ${String(parsed.reply)}`)); + continue; + } + if (this.subscribing) this.onPush?.(parsed.reply); + else this.pending.shift()?.resolve(parsed.reply); + } + }); + socket.on("error", (error) => { for (const item of this.pending.splice(0)) item.reject(error); }); + socket.on("close", () => { for (const item of this.pending.splice(0)) item.reject(new Error("Redis disconnected")); }); + } + + static async connect(url: URL): Promise { + const port = Number(url.port || (url.protocol === "rediss:" ? 6380 : 6379)); + const socket = url.protocol === "rediss:" + ? tls.connect({ host: url.hostname, port, servername: url.hostname, rejectUnauthorized: true }) + : net.connect({ host: url.hostname, port }); + await new Promise((resolve, reject) => { + socket.once(url.protocol === "rediss:" ? "secureConnect" : "connect", () => resolve()); + socket.once("error", reject); + }); + const connection = new RedisConnection(socket); + if (url.password) await connection.send(["AUTH", decodeURIComponent(url.username || "default"), decodeURIComponent(url.password)]); + return connection; + } + + send(parts: string[]): Promise { + return new Promise((resolve, reject) => { + this.pending.push({ resolve, reject }); + this.socket.write(encode(parts)); + }); + } + subscribe(channel: string, onMessage: (message: string) => void): Promise { + this.subscribing = true; + return new Promise((resolve, reject) => { + this.onPush = (reply) => { + if (typeof reply === "string") reject(new Error("Redis subscription rejected")); + else if (Array.isArray(reply) && reply[0] === "subscribe" && reply[1] === channel) resolve(); + else if (Array.isArray(reply) && reply[0] === "message" && reply[1] === channel && typeof reply[2] === "string") onMessage(reply[2]); + }; + this.socket.write(encode(["SUBSCRIBE", channel])); + }); + } + close(): void { this.socket.destroy(); } +} + +export interface ObservedStore { + get(workspaceId: string, channelId: string): Promise; + put(observed: ObservedChannel): Promise; + heartbeat(workerId: string, channels: ObservedChannel[]): Promise; +} + +export class RedisBus implements ObservedStore { + private subscribers: RedisConnection[] = []; + private constructor(private readonly url: URL, private readonly client: RedisConnection) {} + static async connect(rawUrl: string): Promise { + const url = new URL(rawUrl); + if (url.protocol !== "redis:" && url.protocol !== "rediss:") throw new Error("invalid Redis URL"); + return new RedisBus(url, await RedisConnection.connect(url)); + } + async get(workspaceId: string, channelId: string): Promise { + const result = await this.client.send(["GET", `obs:${workspaceId}:${channelId}`]); + return typeof result === "string" ? JSON.parse(result) as ObservedChannel : null; + } + async put(observed: ObservedChannel): Promise { + await this.client.send(["SET", `obs:${observed.workspaceId}:${observed.channelId}`, JSON.stringify(observed), "EX", "60"]); + } + async heartbeat(workerId: string, channels: ObservedChannel[]): Promise { + await this.client.send(["SET", `hb:${workerId}`, JSON.stringify({ workerId, sentAt: new Date().toISOString(), channels: channels.map((c) => ({ workspaceId: c.workspaceId, channelId: c.channelId, assignmentEpoch: c.assignmentEpoch, realizedVersion: c.realizedVersion })) }), "EX", "15"]); + for (const channel of channels) await this.put(channel); + } + async subscribe(workerId: string, workspaces: string[], onCommand: (message: string) => void): Promise { + // A worker uses one subscription per assigned workspace; all channels are signature checked. + for (const workspaceId of workspaces) { + const subscriber = await RedisConnection.connect(this.url); + await subscriber.subscribe(`cmd:${workspaceId}:${workerId}`, onCommand); + this.subscribers.push(subscriber); + } + } + close(): void { for (const subscriber of this.subscribers) subscriber.close(); this.client.close(); } +} diff --git a/apps/media-worker/src/compositor/command-builder.test.ts b/apps/media-worker/src/compositor/command-builder.test.ts new file mode 100644 index 0000000..a6e06ea --- /dev/null +++ b/apps/media-worker/src/compositor/command-builder.test.ts @@ -0,0 +1,203 @@ +/** + * MediaCommandBuilder: argv shape, typed-param rejection, and golden-frame + * hashes per layout (deterministic synthetic inputs: seeded noise over test + * patterns; framemd5 of the composited output, video + program audio). + * + * Hashes are pinned to the system FFmpeg (5.1.x here). UPDATE_GOLDEN=1 rewrites + * src/compositor/golden-frames.json after an intentional graph change or an + * FFmpeg upgrade. + */ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { + buildCompositorArgs, buildEncoderArgs, buildFilterGraph, buildIngestArgs, CommandBuildError, layoutRects, rectFromBasisPoints, + SYNTHETIC_PATTERNS, type Canvas, type CompositorPlan, type PxRect, +} from "./command-builder"; +import type { SceneLayout } from "../production"; +import { makeAssets } from "./testing"; + +const canvas: Canvas = { width: 320, height: 180, fps: 30 }; +const GOLDEN = join(__dirname, "golden-frames.json"); + +function synthetic(i: number) { + return { video: { kind: "synthetic" as const, pattern: SYNTHETIC_PATTERNS[i % SYNTHETIC_PATTERNS.length]!, seed: 1000 + i }, audio: { kind: "synthetic" as const, toneHz: 220 + 110 * i } }; +} + +const CUSTOM: PxRect[] = [{ x: 0, y: 0, w: 160, h: 90 }, { x: 160, y: 0, w: 160, h: 90 }, { x: 80, y: 90, w: 160, h: 90 }]; +const CUSTOM_OVERLAP: PxRect[] = [{ x: 0, y: 0, w: 200, h: 112 }, { x: 120, y: 68, w: 200, h: 112 }]; + +interface Case { name: string; layout: SceneLayout; rects: PxRect[]; empty?: number[]; overlays: "none" | "lower" | "chyron" | "both"; mix: Array<[number, number]>; master: number } + +const CASES: Case[] = [ + { name: "single", layout: "single", rects: layoutRects("single", canvas), overlays: "none", mix: [[0, 0]], master: 0 }, + { name: "pip", layout: "pip", rects: [{ x: 0, y: 0, w: 320, h: 180 }, rectFromBasisPoints({ x: 6000, y: 6000, w: 3500, h: 3500 }, canvas)], overlays: "lower", mix: [[0, 0], [1, -12]], master: -3 }, + { name: "2x1", layout: "2x1", rects: layoutRects("2x1", canvas), overlays: "none", mix: [[0, -3], [1, -3]], master: 0 }, + { name: "2x2", layout: "2x2", rects: layoutRects("2x2", canvas), empty: [3], overlays: "chyron", mix: [[0, 0]], master: -6 }, + { name: "3x3", layout: "3x3", rects: layoutRects("3x3", canvas), empty: [4, 8], overlays: "both", mix: [[0, 0], [1, -6], [2, -24]], master: 0 }, + { name: "4x4", layout: "4x4", rects: layoutRects("4x4", canvas), empty: [5, 10, 15], overlays: "none", mix: [], master: 0 }, + { name: "custom", layout: "custom", rects: CUSTOM, overlays: "lower", mix: [[2, 0]], master: 0 }, + { name: "custom-overlap", layout: "custom", rects: CUSTOM_OVERLAP, overlays: "both", mix: [[0, 0], [1, 0]], master: -9 }, +]; + +function plan(c: Case, assets: string[]): CompositorPlan { + const tiles = c.rects.map((rect, i) => ({ source: c.empty?.includes(i) ? null : i % 4, rect, fit: (["contain", "cover", "crop"] as const)[i % 3]! })); + const overlays = [ + ...(c.overlays === "lower" || c.overlays === "both" ? [{ kind: "image" as const, asset: 0, x: 12, y: 140 }] : []), + ...(c.overlays === "chyron" || c.overlays === "both" ? [{ kind: "chyron" as const, asset: 1, y: 164, speedPxPerSec: 120 }] : []), + ]; + return { + sources: [0, 1, 2, 3].map(synthetic), assets, + graph: { canvas, layout: c.layout, tiles, overlays, audio: { inputs: c.mix.map(([source, gainDb]) => ({ source, gainDb })), masterGainDb: c.master } }, + output: { kind: "framemd5", frames: 4 }, + }; +} + +function hashes(p: CompositorPlan): string[] { + const r = spawnSync("ffmpeg", buildCompositorArgs(p), { encoding: "utf8", timeout: 60_000 }); + assert.equal(r.status, 0, r.stderr); + // framemd5 lines: stream, dts, pts, duration, size, hash — keep stream/pts/size/hash. + return r.stdout.split("\n").filter((l) => l && !l.startsWith("#")).map((l) => l.split(",").map((x) => x.trim())).map((f) => `${f[0]}:${f[2]}:${f[4]}:${f[5]}`); +} + +test("golden-frame hashes per layout (deterministic synthetic inputs)", { timeout: 300_000 }, () => { + const dir = mkdtempSync(join(tmpdir(), "golden-")); + try { + const assets = makeAssets(dir); + const got: Record = {}; + for (const c of CASES) { + const p = plan(c, assets); + const h = hashes(p); + assert.ok(h.some((l) => l.startsWith("0:")) && h.some((l) => l.startsWith("1:")), `${c.name}: video and audio frames`); + assert.deepEqual(hashes(p), h, `${c.name}: two runs hash identically`); + got[c.name] = h; + } + if (process.env.UPDATE_GOLDEN || !existsSync(GOLDEN)) { + writeFileSync(GOLDEN, `${JSON.stringify({ ffmpeg: ffmpegVersion(), canvas, frames: 4, layouts: got }, null, 2)}\n`); + } + const golden = JSON.parse(readFileSync(GOLDEN, "utf8")) as { ffmpeg: string; layouts: Record }; + for (const c of CASES) assert.deepEqual(got[c.name], golden.layouts[c.name], `${c.name} matches golden (pinned to ${golden.ffmpeg}; UPDATE_GOLDEN=1 after an intentional change)`); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +function ffmpegVersion(): string { + return spawnSync("ffmpeg", ["-version"], { encoding: "utf8" }).stdout.split("\n")[0] ?? "unknown"; +} + +test("layout cells are even, cover the canvas, and xstack/overlay is chosen by geometry", () => { + for (const layout of ["2x1", "2x2", "3x3", "4x4"] as const) { + for (const c of [canvas, { width: 1920, height: 1080, fps: 30 }, { width: 1280, height: 720, fps: 30 }]) { + const rects = layoutRects(layout, c); + assert.equal(rects.reduce((a, r) => a + r.w * r.h, 0), c.width * c.height, `${layout} ${c.width}x${c.height} tiles cover the canvas`); + for (const r of rects) for (const v of [r.x, r.y, r.w, r.h]) assert.equal(v % 2, 0); + } + } + const idx = { video: [0, 1, 2, 3], audio: [null, null, null, null], assets: [] }; + const base = { sources: [0, 1, 2, 3].map(synthetic), assets: [], output: { kind: "framemd5" as const, frames: 1 } }; + const audio = { inputs: [], masterGainDb: 0 }; + const grid = buildFilterGraph({ ...base, graph: { canvas, layout: "2x2", tiles: layoutRects("2x2", canvas).map((rect, i) => ({ source: i, rect, fit: "contain" as const })), overlays: [], audio } }, idx); + assert.match(grid, /xstack=inputs=4:layout=0_0\|160_0\|0_90\|160_90:fill=black/); + assert.match(grid, /anullsrc=r=48000:cl=stereo/); + const pip = buildFilterGraph({ ...base, graph: { canvas, layout: "pip", tiles: [{ source: 0, rect: { x: 0, y: 0, w: 320, h: 180 }, fit: "cover" }, { source: 1, rect: { x: 200, y: 100, w: 100, h: 56 }, fit: "contain" }], overlays: [], audio } }, idx); + assert.match(pip, /overlay=x=200:y=100/); + assert.doesNotMatch(pip, /xstack/); + // Overlapping custom rects: the base is the first tile padded (paced by a live input), not a lavfi color source. + const overlap = buildFilterGraph({ ...base, graph: { canvas, layout: "custom", tiles: CUSTOM_OVERLAP.map((rect, i) => ({ source: i, rect, fit: "cover" as const })), overlays: [], audio } }, idx); + assert.match(overlap, /\[t0\]pad=320:180:0:0:color=black\[bg\];\[bg\]\[t1\]overlay=x=120:y=68/); + assert.doesNotMatch(overlap, /color=c=black/); + const shared = buildFilterGraph({ ...base, graph: { canvas, layout: "2x1", tiles: layoutRects("2x1", canvas).map((rect) => ({ source: 0, rect, fit: "crop" as const })), overlays: [], audio } }, idx); + assert.match(shared, /\[0:v\]split=2\[s0_0\]\[s0_1\]/); +}); + +test("program mix follows §2.3: volume per input → amix → master gain, 3 dB steps only", () => { + const idx = { video: [0, 2, 4], audio: [1, 3, 5], assets: [] }; + const p = (inputs: Array<{ source: number; gainDb: number }>, masterGainDb = 0): CompositorPlan => ({ + sources: [0, 1, 2].map(synthetic), assets: [], output: { kind: "framemd5", frames: 1 }, + graph: { canvas, layout: "single", tiles: [{ source: 0, rect: { x: 0, y: 0, w: 320, h: 180 }, fit: "contain" }], overlays: [], audio: { inputs, masterGainDb } }, + }); + const g = buildFilterGraph(p([{ source: 0, gainDb: -6 }, { source: 2, gainDb: 0 }], -3), idx); + assert.match(g, /\[1:a\]aformat=[^;]*,volume=-6dB\[m0\]/); + assert.match(g, /\[5:a\]aformat=[^;]*,volume=0dB\[m1\]/); + assert.match(g, /\[m0\]\[m1\]amix=inputs=2:duration=longest:dropout_transition=2:normalize=1,volume=-3dB/); + assert.doesNotMatch(buildFilterGraph(p([{ source: 0, gainDb: 0 }]), idx), /amix/); + for (const bad of [-1, -25, 3, 1.5]) assert.throws(() => buildCompositorArgs(p([{ source: 0, gainDb: bad }])), /gainDb/); + assert.throws(() => buildCompositorArgs(p([{ source: 0, gainDb: 0 }, { source: 0, gainDb: -3 }])), /repeats a source/); + assert.throws(() => buildCompositorArgs(p([{ source: 0, gainDb: 0 }], -2)), /masterGainDb/); +}); + +test("typed params only: hostile values never reach argv", () => { + const good: CompositorPlan = { + sources: [{ video: { kind: "rawPipe", fd: 4 }, audio: { kind: "rawPipe", fd: 5 } }], + assets: [`/scratch/assets/${"a".repeat(64)}.png`], + graph: { canvas, layout: "single", tiles: [{ source: 0, rect: { x: 0, y: 0, w: 320, h: 180 }, fit: "contain" }], overlays: [{ kind: "chyron", asset: 0, y: 160, speedPxPerSec: 100 }], audio: { inputs: [{ source: 0, gainDb: 0 }], masterGainDb: 0 } }, + output: { kind: "gate", videoFd: 1, audioFd: 3 }, + }; + const args = buildCompositorArgs(good); + assert.ok(args.includes("pipe:4") && args.includes("pipe:5") && args.includes(`file:/scratch/assets/${"a".repeat(64)}.png`)); + assert.deepEqual(args.slice(-1), ["pipe:3"]); + const hostileAssets = ["/scratch/../etc/passwd.png", "/scratch/x;rm.png", "relative/" + "a".repeat(64) + ".png", `/scratch/${"a".repeat(64)}.png'`, `/s/movie=${"a".repeat(58)}.png`, "/scratch/assets/notahash.png"]; + for (const asset of hostileAssets) assert.throws(() => buildCompositorArgs({ ...good, assets: [asset] }), CommandBuildError, asset); + const g = good.graph; + assert.throws(() => buildCompositorArgs({ ...good, graph: { ...g, tiles: [{ ...g.tiles[0]!, rect: { x: 1, y: 0, w: 320, h: 180 } }] } }), /even/); + assert.throws(() => buildCompositorArgs({ ...good, graph: { ...g, tiles: [{ ...g.tiles[0]!, rect: { x: 0, y: 0, w: 322, h: 180 } }] } }), CommandBuildError); + assert.throws(() => buildCompositorArgs({ ...good, graph: { ...g, overlays: [{ kind: "chyron", asset: 0, y: 160, speedPxPerSec: 39 }] } }), /speedPxPerSec/); + assert.throws(() => buildCompositorArgs({ ...good, graph: { ...g, overlays: [{ kind: "image", asset: 1, x: 0, y: 0 }] } }), /asset/); + assert.throws(() => buildCompositorArgs({ ...good, sources: [{ video: { kind: "rawPipe", fd: 1.5 }, audio: null }] }), /fd/); + assert.throws(() => buildCompositorArgs({ ...good, graph: { ...g, canvas: { width: 321, height: 180, fps: 30 } } }), /even/); + assert.throws(() => buildCompositorArgs({ ...good, graph: { ...g, canvas: { width: 320, height: 180, fps: 7 } } }), /fps/); + // Nothing in argv is a shell: no metacharacters outside the filtergraph and the expression quotes. + for (const a of args) if (!a.includes("[")) assert.doesNotMatch(a, /[;&|`$<>]/, a); +}); + +test("ingest normalize and persistent encoder argv", () => { + const ingest = buildIngestArgs({ input: { kind: "mpegtsPipe" }, canvas: { width: 1920, height: 1080, fps: 30 }, audio: "source" }); + assert.deepEqual(ingest.slice(ingest.indexOf("-f"), ingest.indexOf("-f") + 4), ["-f", "mpegts", "-i", "pipe:0"]); + assert.ok(ingest.includes("pipe:1") && ingest.includes("pipe:3")); + assert.ok(ingest.some((a) => a.startsWith("scale=1920:1080:force_original_aspect_ratio=decrease") && a.includes("fps=30"))); + const silent = buildIngestArgs({ input: { kind: "mpegtsPipe" }, canvas, audio: "none" }); + assert.ok(silent.includes("anullsrc=r=48000:cl=stereo") && silent.includes("1:a:0")); + const enc = buildEncoderArgs({ canvas: { width: 1920, height: 1080, fps: 30 }, videoBitrateKbps: 6000, audioBitrateKbps: 128, output: { kind: "flvPipe", fd: 4 } }); + assert.deepEqual(enc.slice(-3), ["-f", "flv", "pipe:4"]); + assert.ok(enc.includes("pipe:0") && enc.includes("pipe:3") && enc.includes("6000k") && enc.includes("60")); + assert.throws(() => buildEncoderArgs({ canvas, videoBitrateKbps: 1e9, audioBitrateKbps: 128, output: { kind: "null" } })); +}); + +test("ingest ×N normalize stage end to end: MPEG-TS pipe (4:3, no audio) and synthetic sources fill their buses", { timeout: 60_000 }, async () => { + const { IngestProcess } = await import("./ingest"); + const { ensureGateBinary, passthroughRedactor } = await import("./testing"); + assert.ok(ensureGateBinary(), "native helpers (source bus) build"); + // 4:3 MPEG-TS without audio on the ingest's stdin, as the egress relay would deliver it. + const ts = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=s=640x480:r=25:d=2", "-c:v", "mpeg2video", "-f", "mpegts", "-"], { maxBuffer: 64 << 20 }); + assert.equal(ts.status, 0); + const dir = mkdtempSync(join(tmpdir(), "ingest-")); + const tsPath = join(dir, "in.ts"); + writeFileSync(tsPath, ts.stdout); + const { openSync, closeSync } = await import("node:fs"); + const fd = openSync(tsPath, "r"); + const exits: string[] = []; + const pipeIngest = new IngestProcess("a", { input: { kind: "mpegtsPipe" }, canvas, audio: "none" }, fd, join(dir, "scratch"), passthroughRedactor, () => undefined, (_id, r) => exits.push(r)); + const synth = new IngestProcess("b", { input: { kind: "synthetic", pattern: "smptehdbars", seed: 7, toneHz: 440 }, canvas, audio: "source" }, null, join(dir, "scratch"), passthroughRedactor, () => undefined, (_id, r) => exits.push(r)); + try { + await pipeIngest.start(); + closeSync(fd); + await synth.start(); + await new Promise((r) => setTimeout(r, 1_500)); + // 2 s of 25 fps input → 30 fps CFR on the canvas (decoded faster than real time). + const piped = await pipeIngest.bus.stats(); + assert.ok(piped.pairs >= 50, `mpegts pairs ${piped.pairs}`); + assert.equal(piped.silenceFilled, 0, "anullsrc supplies audio for a silent source"); + // -re synthetic: real-time pace. + const synthetic = await synth.bus.stats(); + assert.ok(synthetic.pairs >= 30 && synthetic.pairs <= 60, `synthetic pairs ${synthetic.pairs}`); + assert.deepEqual(exits, []); + } finally { + await pipeIngest.stop(); + await synth.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/apps/media-worker/src/compositor/command-builder.ts b/apps/media-worker/src/compositor/command-builder.ts new file mode 100644 index 0000000..e008ca5 --- /dev/null +++ b/apps/media-worker/src/compositor/command-builder.ts @@ -0,0 +1,486 @@ +/** + * MediaCommandBuilder (docs/design/streaming-architecture.md §1.4, §2.3, §4, §6 S2). + * + * Typed params in, argv out. Nothing here takes a free-form string: sources are + * fds or closed synthetic patterns, assets are validated scratch paths to PNGs + * named by content id, geometry is integers, gains are 3 dB steps. Every + * filtergraph token is produced from those values, so no operator text can + * reach a graph or argv (§4.3 "Injection"). + */ +import { CHYRON_SPEED_MAX, CHYRON_SPEED_MIN, GAIN_DB_MAX, GAIN_DB_MIN, gainDb, type SceneLayout, type SlotFit } from "../production"; +import { assertGeometry, GATE_CHANNELS, GATE_SAMPLE_RATE, type FrameGeometry } from "../gate/protocol"; + +export type Canvas = FrameGeometry; + +export interface PxRect { x: number; y: number; w: number; h: number } + +/** Closed set of deterministic test patterns (golden frames, spike, CPU smoke). */ +export type SyntheticPattern = "testsrc2" | "smptehdbars" | "mandelbrot" | "rgbtestsrc"; +export const SYNTHETIC_PATTERNS: readonly SyntheticPattern[] = ["testsrc2", "smptehdbars", "mandelbrot", "rgbtestsrc"]; + +export type VideoInput = { kind: "rawPipe"; fd: number } | { kind: "synthetic"; pattern: SyntheticPattern; seed: number }; +export type AudioInput = { kind: "rawPipe"; fd: number } | { kind: "synthetic"; toneHz: number }; + +export interface CompositorSource { video: VideoInput; audio: AudioInput | null } + +export interface TileSpec { + /** Index into `sources`; null renders black. */ + source: number | null; + /** Pixel rect inside the canvas; computed by `layoutRects` for grid layouts. */ + rect: PxRect; + fit: SlotFit; +} + +export type OverlaySpec = + | { kind: "image"; asset: number; x: number; y: number } + | { kind: "chyron"; asset: number; y: number; speedPxPerSec: number }; + +export interface MixInput { source: number; gainDb: number } + +export interface CompositorGraph { + canvas: Canvas; + layout: SceneLayout; + tiles: TileSpec[]; + /** Drawn in order (lowest layer first). */ + overlays: OverlaySpec[]; + audio: { inputs: MixInput[]; masterGainDb: number }; +} + +export type CompositorOutput = + | { kind: "gate"; videoFd: 1; audioFd: 3 } + /** Golden-frame hashing: framemd5 of the first `frames` video frames (and the audio in that span) on stdout. */ + | { kind: "framemd5"; frames: number }; + +export interface CompositorPlan { + sources: CompositorSource[]; + /** Absolute scratch paths of rendered overlay PNGs, by content id. */ + assets: string[]; + graph: CompositorGraph; + output: CompositorOutput; +} + +export class CommandBuildError extends Error { + constructor(message: string) { super(message); this.name = "CommandBuildError"; } +} +function fail(message: string): never { throw new CommandBuildError(message); } + +function int(value: number, field: string, min: number, max: number): number { + if (!Number.isSafeInteger(value) || value < min || value > max) fail(`${field} must be an integer in ${min}..${max}`); + return value; +} + +function fd(value: number, field: string): number { return int(value, field, 0, 1023); } + +/** Rendered assets live in the worker's scratch dir and are named by sha256 (§4.3). */ +const ASSET_PATH = /^\/(?:[A-Za-z0-9._-]+\/)*[0-9a-f]{64}\.png$/; +function assetPath(path: string, field: string): string { + if (!ASSET_PATH.test(path) || path.split("/").includes("..")) fail(`${field} must be an absolute scratch path to .png`); + return path; +} + +function even(n: number): number { return Math.round(n / 2) * 2; } + +/** Even cell boundaries so every tile has even size and offset (yuv420p chroma alignment). */ +function boundaries(total: number, parts: number): number[] { + return Array.from({ length: parts + 1 }, (_, i) => (i === parts ? total : even((i * total) / parts))); +} + +const GRID: Partial> = { "2x1": [2, 1], "2x2": [2, 2], "3x3": [3, 3], "4x4": [4, 4] }; + +/** Cell rects for grid layouts, in slot-index order (row-major). `single`/`pip` slot 0 is the full canvas. */ +export function layoutRects(layout: SceneLayout, canvas: Canvas): PxRect[] { + if (layout === "single" || layout === "pip") return [{ x: 0, y: 0, w: canvas.width, h: canvas.height }]; + const grid = GRID[layout]; + if (!grid) return []; + const xs = boundaries(canvas.width, grid[0]); + const ys = boundaries(canvas.height, grid[1]); + const rects: PxRect[] = []; + for (let r = 0; r < grid[1]; r += 1) { + for (let c = 0; c < grid[0]; c += 1) rects.push({ x: xs[c] ?? 0, y: ys[r] ?? 0, w: (xs[c + 1] ?? 0) - (xs[c] ?? 0), h: (ys[r + 1] ?? 0) - (ys[r] ?? 0) }); + } + return rects; +} + +/** Basis points (0..10000) of the canvas to an even pixel rect. */ +export function rectFromBasisPoints(r: { x: number; y: number; w: number; h: number }, canvas: Canvas): PxRect { + const x = even((r.x * canvas.width) / 10_000); + const y = even((r.y * canvas.height) / 10_000); + const w = Math.max(2, Math.min(canvas.width - x, even((r.w * canvas.width) / 10_000))); + const h = Math.max(2, Math.min(canvas.height - y, even((r.h * canvas.height) / 10_000))); + return { x, y, w, h }; +} + +function checkRect(r: PxRect, canvas: Canvas, field: string): void { + int(r.x, `${field}.x`, 0, canvas.width - 2); + int(r.y, `${field}.y`, 0, canvas.height - 2); + int(r.w, `${field}.w`, 2, canvas.width); + int(r.h, `${field}.h`, 2, canvas.height); + if (r.x % 2 || r.y % 2 || r.w % 2 || r.h % 2) fail(`${field} must be even-aligned`); + if (r.x + r.w > canvas.width || r.y + r.h > canvas.height) fail(`${field} must lie inside the canvas`); +} + +function overlaps(a: PxRect, b: PxRect): boolean { + return a.x < b.x + b.w && b.x < a.x + a.w && a.y < b.y + b.h && b.y < a.y + a.h; +} + +function tileFilter(fit: SlotFit, r: PxRect): string { + switch (fit) { + case "contain": return `scale=${r.w}:${r.h}:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=${r.w}:${r.h}:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1`; + case "cover": return `scale=${r.w}:${r.h}:force_original_aspect_ratio=increase:force_divisible_by=2,crop=${r.w}:${r.h},setsar=1`; + case "crop": return `crop=${r.w}:${r.h},setsar=1`; + } +} + +function db(value: number): string { return `${value}dB`; } + +const AUDIO_FORMAT = `aformat=sample_fmts=s16:sample_rates=${GATE_SAMPLE_RATE}:channel_layouts=stereo`; + +/** + * The `-filter_complex` string. Input labels: video of source i is `[v{i}]` + * after the builder's `-i` ordering, audio `[a{i}]`, asset k `[o{k}]`. + */ +export function buildFilterGraph(plan: CompositorPlan, inputIndex: { video: number[]; audio: Array; assets: number[] }): string { + const { graph } = plan; + const { canvas } = graph; + const chains: string[] = []; + const black = (w: number, h: number): string => `color=c=black:s=${w}x${h}:r=${canvas.fps}`; + + // Video sources used by several tiles are split once. + const uses = new Map(); + for (const tile of graph.tiles) if (tile.source !== null) uses.set(tile.source, (uses.get(tile.source) ?? 0) + 1); + const taken = new Map(); + const videoPad = (source: number): string => { + const n = uses.get(source) ?? 0; + const input = `${inputIndex.video[source]}:v`; + if (n <= 1) return `[${input}]`; + if (!taken.has(source)) { + chains.push(`[${input}]split=${n}${Array.from({ length: n }, (_, i) => `[s${source}_${i}]`).join("")}`); + taken.set(source, 0); + } + const i = taken.get(source) ?? 0; + taken.set(source, i + 1); + return `[s${source}_${i}]`; + }; + + const tiles = graph.tiles.map((tile, i) => ({ tile, label: `t${i}` })); + const live = tiles.filter(({ tile }) => tile.source !== null); + for (const { tile, label } of live) chains.push(`${videoPad(tile.source ?? 0)}${tileFilter(tile.fit, tile.rect)}[${label}]`); + + const W = canvas.width; + const H = canvas.height; + const stackable = graph.layout !== "pip" && live.every(({ tile }, i) => live.every((other, j) => i === j || !overlaps(tile.rect, other.tile.rect))); + if (live.length === 0) { + chains.push(`${black(W, H)}[base]`); + } else if (stackable && live.length >= 2) { + const layout = live.map(({ tile }) => `${tile.rect.x}_${tile.rect.y}`).join("|"); + // xstack sizes its output to the tiles' bounding box; pad only when that is not the canvas. + const right = Math.max(...live.map(({ tile }) => tile.rect.x + tile.rect.w)); + const bottom = Math.max(...live.map(({ tile }) => tile.rect.y + tile.rect.h)); + const fit = right === W && bottom === H ? "" : `,pad=${W}:${H}:0:0:color=black`; + chains.push(`${live.map(({ label }) => `[${label}]`).join("")}xstack=inputs=${live.length}:layout=${layout}:fill=black${fit}[base]`); + } else if (stackable) { + const only = live[0]!; + const full = only.tile.rect.w === W && only.tile.rect.h === H; + chains.push(full ? `[${only.label}]null[base]` : `[${only.label}]pad=${W}:${H}:${only.tile.rect.x}:${only.tile.rect.y}:color=black[base]`); + } else { + // PIP and overlapping custom rects: overlay chain in slot order (later slots on top). The base + // is the first tile padded to the canvas, never a lavfi `color` source: an unpaced main input + // runs ahead of the live tiles, so framesync queues them (~10 frames of extra video latency, + // more than a swap's alignment margin) and keeps emitting frames after the inputs end. + const first = live[0]!; + const fullFirst = first.tile.rect.x === 0 && first.tile.rect.y === 0 && first.tile.rect.w === W && first.tile.rect.h === H; + let cur = first.label; + if (!fullFirst) { + chains.push(`[${first.label}]pad=${W}:${H}:${first.tile.rect.x}:${first.tile.rect.y}:color=black[bg]`); + cur = "bg"; + } + live.slice(1).forEach(({ tile, label }, i) => { + const next = `c${i}`; + chains.push(`[${cur}][${label}]overlay=x=${tile.rect.x}:y=${tile.rect.y}:eof_action=repeat[${next}]`); + cur = next; + }); + chains.push(`[${cur}]null[base]`); + } + + let cur = "base"; + graph.overlays.forEach((overlay, i) => { + const next = `ov${i}`; + const asset = `[${inputIndex.assets[overlay.asset]}:v]`; + if (overlay.kind === "image") { + // Default format (yuv420, alpha from the PNG): `format=auto` breaks the 1:1 main-frame + // mapping in FFmpeg 5.1 (~1 in 6 main frames duplicated and its neighbour dropped). + chains.push(`[${cur}]${asset}overlay=x=${overlay.x}:y=${overlay.y}:eof_action=repeat[${next}]`); + } else { + // §4.3 chyron: one wide pre-rendered strip scrolled right-to-left by an x-expression of t. + chains.push(`[${cur}]${asset}overlay=x='W-mod(t*${overlay.speedPxPerSec}\\,w+W)':y=${overlay.y}:eof_action=repeat[${next}]`); + } + cur = next; + }); + chains.push(`[${cur}]format=yuv420p[vout]`); + + // §2.3 program mix: volume per input → amix (dead sources fade out) → master gain. + const mix = graph.audio.inputs; + mix.forEach((m, i) => chains.push(`[${inputIndex.audio[m.source]}:a]${AUDIO_FORMAT},volume=${db(m.gainDb)}[m${i}]`)); + const master = `volume=${db(graph.audio.masterGainDb)},${AUDIO_FORMAT}`; + if (mix.length === 0) chains.push(`anullsrc=r=${GATE_SAMPLE_RATE}:cl=stereo,${master}[aout]`); + else if (mix.length === 1) chains.push(`[m0]${master}[aout]`); + else chains.push(`${mix.map((_, i) => `[m${i}]`).join("")}amix=inputs=${mix.length}:duration=longest:dropout_transition=2:normalize=1,${master}[aout]`); + return chains.join(";"); +} + +function validatePlan(plan: CompositorPlan): void { + const { graph } = plan; + assertGeometry(graph.canvas); + if (plan.sources.length > 16) fail("at most 16 compositor sources"); + plan.sources.forEach((s, i) => { + if (s.video.kind === "rawPipe") fd(s.video.fd, `sources[${i}].video.fd`); + else { + if (!SYNTHETIC_PATTERNS.includes(s.video.pattern)) fail(`sources[${i}].video.pattern`); + int(s.video.seed, `sources[${i}].video.seed`, 0, 1_000_000); + } + if (s.audio?.kind === "rawPipe") fd(s.audio.fd, `sources[${i}].audio.fd`); + else if (s.audio?.kind === "synthetic") int(s.audio.toneHz, `sources[${i}].audio.toneHz`, 20, 20_000); + }); + plan.assets.forEach((a, i) => assetPath(a, `assets[${i}]`)); + if (graph.tiles.length > 16) fail("at most 16 tiles"); + graph.tiles.forEach((t, i) => { + if (t.source !== null) int(t.source, `tiles[${i}].source`, 0, plan.sources.length - 1); + checkRect(t.rect, graph.canvas, `tiles[${i}].rect`); + if (t.fit !== "contain" && t.fit !== "cover" && t.fit !== "crop") fail(`tiles[${i}].fit`); + }); + if (graph.overlays.length > 64) fail("at most 64 overlays"); + graph.overlays.forEach((o, i) => { + int(o.asset, `overlays[${i}].asset`, 0, plan.assets.length - 1); + int(o.y, `overlays[${i}].y`, 0, graph.canvas.height); + if (o.kind === "image") int(o.x, `overlays[${i}].x`, 0, graph.canvas.width); + else if (o.kind === "chyron") int(o.speedPxPerSec, `overlays[${i}].speedPxPerSec`, CHYRON_SPEED_MIN, CHYRON_SPEED_MAX); + else fail(`overlays[${i}].kind`); + }); + const seen = new Set(); + graph.audio.inputs.forEach((m, i) => { + int(m.source, `audio.inputs[${i}].source`, 0, plan.sources.length - 1); + if (plan.sources[m.source]?.audio == null) fail(`audio.inputs[${i}] names a source without audio`); + if (seen.has(m.source)) fail(`audio.inputs[${i}] repeats a source`); + seen.add(m.source); + gainDb(m.gainDb, `audio.inputs[${i}].gainDb`); + }); + if (graph.audio.masterGainDb < GAIN_DB_MIN || graph.audio.masterGainDb > GAIN_DB_MAX) fail("audio.masterGainDb out of range"); + gainDb(graph.audio.masterGainDb, "audio.masterGainDb"); + if (plan.output.kind === "framemd5") int(plan.output.frames, "output.frames", 1, 600); +} + +const RAW_IN = ["-thread_queue_size", "64", "-probesize", "32", "-analyzeduration", "0"]; + +function rawVideoInput(canvas: Canvas, fdNum: number): string[] { + return [...RAW_IN, "-f", "rawvideo", "-pix_fmt", "yuv420p", "-s", `${canvas.width}x${canvas.height}`, "-framerate", String(canvas.fps), "-i", `pipe:${fdNum}`]; +} + +function rawAudioInput(fdNum: number): string[] { + return [...RAW_IN, "-f", "s16le", "-ar", String(GATE_SAMPLE_RATE), "-ac", String(GATE_CHANNELS), "-i", `pipe:${fdNum}`]; +} + +function syntheticVideo(canvas: Canvas, pattern: SyntheticPattern, seed: number): string { + // Seeded temporal noise makes every source distinct and every frame different, deterministically. + return `${pattern}=s=${canvas.width}x${canvas.height}:r=${canvas.fps},noise=alls=12:allf=t+u:all_seed=${seed},format=yuv420p`; +} + +/** Full compositor argv. The process reads N sources (+ assets) and writes the gate FIFOs on fds 1/3. */ +export function buildCompositorArgs(plan: CompositorPlan): string[] { + validatePlan(plan); + const { canvas } = plan.graph; + // Single-threaded filtering keeps golden frames deterministic and per-compositor CPU predictable. + // (Slice threads were measured in the S2a CPU smoke and did not change its bus drops.) + // eslint-disable-next-line no-restricted-syntax -- FFmpeg's -threads options, not the Threads provider + const args = ["-hide_banner", "-nostdin", "-loglevel", "error", "-threads", "1", "-filter_threads", "1", "-filter_complex_threads", "1"]; + let next = 0; + const video: number[] = []; + const audio: Array = []; + const synthetic = plan.output.kind === "framemd5"; + for (const s of plan.sources) { + if (s.video.kind === "rawPipe") args.push(...rawVideoInput(canvas, s.video.fd)); + else args.push("-f", "lavfi", "-i", syntheticVideo(canvas, s.video.pattern, s.video.seed)); + video.push(next++); + if (s.audio === null) { audio.push(null); continue; } + if (s.audio.kind === "rawPipe") args.push(...rawAudioInput(s.audio.fd)); + else args.push("-f", "lavfi", "-i", `sine=frequency=${s.audio.toneHz}:sample_rate=${GATE_SAMPLE_RATE}`); + audio.push(next++); + } + const assets: number[] = []; + for (const path of plan.assets) { + // One decoded frame, held by overlay's eof_action=repeat. A `-loop 1` image stream would be + // decoded every frame and framesync'd against the live main input, which jitters the main. + args.push("-framerate", String(canvas.fps), "-i", `file:${path}`); + assets.push(next++); + } + args.push("-filter_complex", buildFilterGraph(plan, { video, audio, assets })); + if (plan.output.kind === "gate") { + args.push("-map", "[vout]", "-f", "rawvideo", "-pix_fmt", "yuv420p", "-fps_mode", "passthrough", "-flush_packets", "1", "pipe:1", + "-map", "[aout]", "-f", "s16le", "-ar", String(GATE_SAMPLE_RATE), "-ac", String(GATE_CHANNELS), "-flush_packets", "1", "pipe:3"); + } else { + void synthetic; + const seconds = (plan.output.frames / canvas.fps).toFixed(6); + args.push("-map", "[vout]", "-map", "[aout]", "-t", seconds, "-c:v", "rawvideo", "-c:a", "pcm_s16le", "-fflags", "+bitexact", "-flags:v", "+bitexact", "-flags:a", "+bitexact", "-f", "framemd5", "pipe:1"); + } + return args; +} + +// --------------------------------------------------------------------------- +// Ingest ×N normalize stage + +export type IngestInput = + /** Live MPEG-TS from the pinned egress relay on stdin (WP7). */ + | { kind: "mpegtsPipe" } + /** + * Playlist source (§5.1): the asset feeder's concatenated MPEG-TS on stdin, read at native rate + * (`-re`; assets arrive far faster than real time). The feeder shifts each asset to where the + * previous one ended, so timestamps are monotonic and the normalize stage and the gate hide + * asset boundaries. Audio is optional (a silent asset gets the bus's silence fill). + */ + | { kind: "playlistPipe" } + /** Deterministic test source, paced at native rate. */ + | { kind: "synthetic"; pattern: SyntheticPattern; seed: number; toneHz: number }; + +export interface IngestPlan { + input: IngestInput; + canvas: Canvas; + /** "none" when the source is known to have no audio track: silence is generated instead. */ + audio: "source" | "none"; +} + +/** + * One process per source: decode → scale/pad to the canvas, CFR at the canvas + * fps, yuv420p; audio → 48 kHz s16le stereo. Video on fd 1, PCM on fd 3. + */ +export function buildIngestArgs(plan: IngestPlan): string[] { + const { canvas, input } = plan; + assertGeometry(canvas); + // Auto frame-threading scales with host CPU count and adds almost a second of decode + // latency on a 28-core host. Independently warming ingests then hold different amounts + // of history in the compositor inputs. Bound decode/filter latency and emit raw frames + // synchronously instead of frame-threading the rawvideo encoder as well. + // eslint-disable-next-line no-restricted-syntax -- FFmpeg options, not the Threads provider + const args = ["-hide_banner", "-nostdin", "-loglevel", "error", "-threads", "2", "-filter_threads", "1"]; + if (input.kind === "mpegtsPipe") { + args.push("-fflags", "+genpts+discardcorrupt", "-f", "mpegts", "-i", "pipe:0"); + } else if (input.kind === "playlistPipe") { + args.push("-re", "-fflags", "+genpts+discardcorrupt", "-f", "mpegts", "-i", "pipe:0"); + } else { + if (!SYNTHETIC_PATTERNS.includes(input.pattern)) fail("ingest pattern"); + int(input.seed, "ingest seed", 0, 1_000_000); + int(input.toneHz, "ingest toneHz", 20, 20_000); + args.push("-re", "-f", "lavfi", "-i", syntheticVideo(canvas, input.pattern, input.seed), + "-re", "-f", "lavfi", "-i", `sine=frequency=${input.toneHz}:sample_rate=${GATE_SAMPLE_RATE}`); + } + const audioIndex = input.kind === "synthetic" ? 1 : 0; + if (plan.audio === "none") args.push("-f", "lavfi", "-i", `anullsrc=r=${GATE_SAMPLE_RATE}:cl=stereo`); + const silenceIndex = input.kind === "synthetic" ? 2 : 1; + const sourceAudio = input.kind === "playlistPipe" ? `${audioIndex}:a:0?` : `${audioIndex}:a:0`; + const W = canvas.width; + const H = canvas.height; + args.push( + "-map", "0:v:0", + "-vf", `scale=${W}:${H}:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=${W}:${H}:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,fps=${canvas.fps},format=yuv420p`, + // eslint-disable-next-line no-restricted-syntax -- FFmpeg rawvideo thread option, not the Threads provider + "-c:v", "rawvideo", "-threads:v", "1", "-f", "rawvideo", "-pix_fmt", "yuv420p", "-flush_packets", "1", "pipe:1", + "-map", plan.audio === "none" ? `${silenceIndex}:a:0` : sourceAudio, + "-af", `aresample=${GATE_SAMPLE_RATE}:async=1000,${AUDIO_FORMAT}`, + "-f", "s16le", "-ar", String(GATE_SAMPLE_RATE), "-ac", String(GATE_CHANNELS), "-flush_packets", "1", "pipe:3", + ); + return args; +} + +// --------------------------------------------------------------------------- +// Persistent encoder + +export type EncoderOutput = { kind: "flvPipe"; fd: number } | { kind: "null" }; + +/** + * Program preview tap (§2.1: "the program tap is the same mechanism applied to the gate output"): + * the encoder also emits what it encodes, downscaled to `size`, as raw yuv420p + s16le on two + * extra fds that a source-bus helper reads. Each tap output is a `tee` with `onfail=ignore` plus a + * `null` anchor slave: a dead tap reader closes only that slave ("continuing with 1/2 slaves"), + * never the encoder (with a single slave ffmpeg 5.1 fails the whole output, verified in S2b). + */ +export interface EncoderProgramTap { videoFd: number; audioFd: number; size: { width: number; height: number } } + +export interface EncoderPlan { canvas: Canvas; videoBitrateKbps: number; audioBitrateKbps: number; output: EncoderOutput; programTap?: EncoderProgramTap } + +/** Reads the gate on fds 0 (video) / 3 (audio). Started once per channel; scene changes never restart it. */ +export function buildEncoderArgs(plan: EncoderPlan): string[] { + const { canvas, programTap: tap } = plan; + assertGeometry(canvas); + const vb = int(plan.videoBitrateKbps, "videoBitrateKbps", 200, 50_000); + const ab = int(plan.audioBitrateKbps, "audioBitrateKbps", 32, 320); + const args = ["-hide_banner", "-nostdin", "-loglevel", "error", ...rawVideoInput(canvas, 0), ...rawAudioInput(3)]; + if (tap) { + const w = int(tap.size.width, "programTap.width", 2, canvas.width); + const h = int(tap.size.height, "programTap.height", 2, canvas.height); + if (w % 2 || h % 2) fail("programTap size must be even"); + args.push("-filter_complex", `[0:v]split=2[enc][tap];[tap]scale=${w}:${h}:flags=bilinear,format=yuv420p[tapv];[1:a]asplit=2[enca][tapa]`, + "-map", "[enc]", "-map", "[enca]"); + } else { + args.push("-map", "0:v", "-map", "1:a"); + } + args.push( + "-c:v", "libx264", "-preset", "veryfast", "-tune", "zerolatency", "-pix_fmt", "yuv420p", + "-b:v", `${vb}k`, "-maxrate", `${vb}k`, "-bufsize", `${vb * 2}k`, "-g", String(canvas.fps * 2), "-keyint_min", String(canvas.fps * 2), "-sc_threshold", "0", + "-c:a", "aac", "-b:a", `${ab}k`, "-ar", String(GATE_SAMPLE_RATE)); + if (plan.output.kind === "flvPipe") args.push("-f", "flv", `pipe:${fd(plan.output.fd, "output.fd")}`); + else args.push("-f", "null", "-"); + if (tap) { + const v = fd(tap.videoFd, "programTap.videoFd"); + const a = fd(tap.audioFd, "programTap.audioFd"); + args.push("-map", "[tapv]", "-c:v", "rawvideo", "-f", "tee", `[f=rawvideo:onfail=ignore]pipe:${v}|[f=null]-`, + "-map", "[tapa]", "-c:a", "pcm_s16le", "-f", "tee", `[f=s16le:onfail=ignore]pipe:${a}|[f=null]-`); + } + return args; +} + +// --------------------------------------------------------------------------- +// Playlist feeder (§5.1) + +/** Output timestamp offset: microseconds, rendered as seconds (bounded: ~1 year of continuous playout). */ +function tsOffset(offsetUs: number): string { + return (int(offsetUs, "offsetUs", 0, 365 * 86_400 * 1_000_000) / 1e6).toFixed(6); +} + +/** + * Remux of one playlist asset (pre-normalized at upload, so never transcoded) from stdin to MPEG-TS + * on stdout, with its timestamps shifted to `offsetUs` (where the previous asset ended), so the + * concatenated stream is monotonic. FFmpeg 5.1's own MPEG-TS discontinuity correction does not + * work with `-re` (measured: 1.3–2 s stalls and bursts at every boundary). `-progress pipe:3` + * reports the asset's end (`out_time_us`, excluding the offset) for the next asset's offset. + * `initial_discontinuity` marks the continuity-counter restart as legitimate. + */ +export function buildPlaylistRemuxArgs(offsetUs: number): string[] { + return ["-hide_banner", "-nostdin", "-loglevel", "error", "-progress", "pipe:3", "-protocol_whitelist", "pipe", "-i", "pipe:0", + "-map", "0:v:0", "-map", "0:a:0?", "-c", "copy", "-output_ts_offset", tsOffset(offsetUs), + "-f", "mpegts", "-mpegts_flags", "+initial_discontinuity", "-muxdelay", "0", "pipe:1"]; +} + +/** + * Slate floor (§5.1): `seconds` of a still (the rendered slate PNG when there is one, else black) + * plus silence as MPEG-TS at `offsetUs`, fed when no asset can be played or a non-looping playlist + * has ended. Encoded faster than real time and paced by the playlist ingest like any asset. + */ +export function buildSlateArgs(plan: { canvas: Canvas; seconds: number; image: string | null; offsetUs: number }): string[] { + const { canvas } = plan; + assertGeometry(canvas); + const seconds = int(plan.seconds, "slate seconds", 1, 60); + const args = ["-hide_banner", "-nostdin", "-loglevel", "error", "-progress", "pipe:3"]; + if (plan.image) args.push("-loop", "1", "-framerate", String(canvas.fps), "-i", `file:${assetPath(plan.image, "slate image")}`); + else args.push("-f", "lavfi", "-i", `color=c=black:s=${canvas.width}x${canvas.height}:r=${canvas.fps}`); + args.push("-f", "lavfi", "-i", `anullsrc=r=${GATE_SAMPLE_RATE}:cl=stereo`, "-t", String(seconds), + "-vf", `scale=${canvas.width}:${canvas.height}:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=${canvas.width}:${canvas.height}:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1,format=yuv420p`, + "-c:v", "libx264", "-preset", "ultrafast", "-tune", "stillimage", "-g", String(canvas.fps), "-b:v", "500k", + "-c:a", "aac", "-b:a", "64k", "-output_ts_offset", tsOffset(plan.offsetUs), + "-f", "mpegts", "-mpegts_flags", "+initial_discontinuity", "-muxdelay", "0", "pipe:1"); + return args; +} + +export const MediaCommandBuilder = { + layoutRects, rectFromBasisPoints, buildFilterGraph, + compositor: buildCompositorArgs, + ingest: buildIngestArgs, + encoder: buildEncoderArgs, +} as const; diff --git a/apps/media-worker/src/compositor/compositor.ts b/apps/media-worker/src/compositor/compositor.ts new file mode 100644 index 0000000..241113b --- /dev/null +++ b/apps/media-worker/src/compositor/compositor.ts @@ -0,0 +1,95 @@ +import type { StdioOptions } from "node:child_process"; +import type { FrameGate } from "../gate/frame-gate"; +import { spawnManaged, type ManagedProcess, type StderrRedactor } from "../relay/ffmpeg"; +import { buildCompositorArgs, type CompositorGraph, type CompositorSource } from "./command-builder"; +import type { BusSubscription, SourceBus } from "./source-bus"; + +/** A scene ready to compose: buses in input order plus the typed graph (source indexes refer to `inputs`). */ +export interface ComposeRequest { + inputs: Array<{ bus: SourceBus; audio: boolean }>; + assets: string[]; + graph: CompositorGraph; +} + +const FIRST_INPUT_FD = 4; + +/** + * One compositor process (A or B of a swap). Reads its sources from bus + * subscription FIFOs inherited on fds 4.., writes the gate's slot FIFOs on fds 1 and 3. + * Its lifetime is bounded by the gate: when the gate retires the slot the FIFOs + * close and the process gets EPIPE; `stop()` also escalates to SIGKILL. + */ +export class CompositorInstance { + private proc: ManagedProcess | null = null; + /** One per input, in request order (valid after `start()`). */ + subscriptions: BusSubscription[] = []; + /** Gate origin sent at attach (see protocol.attachLine). */ + origin: number | undefined; + slotId = 0; + + constructor( + private readonly gate: FrameGate, + private readonly request: ComposeRequest, + private readonly redactor: StderrRedactor, + private readonly onStderr: (message: string) => void, + ) {} + + get pid(): number | undefined { return this.proc?.child.pid; } + + /** + * Subscribes, spawns and attaches to the gate as the pending slot. `originFor` + * maps the fresh subscriptions to a gate origin. Resolve ≠ on air: use `gate.waitActive(slotId)`. + */ + async start(originFor?: (subscriptions: BusSubscription[]) => number | undefined): Promise { + const slot = this.gate.allocateSlot(); + this.slotId = slot.id; + try { + for (const input of this.request.inputs) this.subscriptions.push(await input.bus.subscribe({ video: true, audio: input.audio })); + } catch (error) { + slot.fifos.releaseWriters(); + slot.fifos.releaseHolders(); + slot.fifos.unlink(); + await this.stop(); + throw error; + } + let next = FIRST_INPUT_FD; + const inputFds: number[] = []; + const sources: CompositorSource[] = this.subscriptions.map((sub) => { + if (sub.videoFd === null) throw new Error("compositor input without video"); + inputFds.push(sub.videoFd); + const video = { kind: "rawPipe" as const, fd: next++ }; + if (sub.audioFd === null) return { video, audio: null }; + inputFds.push(sub.audioFd); + return { video, audio: { kind: "rawPipe" as const, fd: next++ } }; + }); + const args = buildCompositorArgs({ sources, assets: this.request.assets, graph: this.request.graph, output: { kind: "gate", videoFd: 1, audioFd: 3 } }); + const stdio: StdioOptions = ["ignore", slot.fifos.videoFd, "pipe", slot.fifos.audioFd, ...inputFds]; + let proc: ManagedProcess; + try { + proc = spawnManaged("ffmpeg", args, stdio, this.redactor, this.onStderr); + } finally { + slot.fifos.releaseWriters(); + // The compositor holds the read ends now: when it exits, the bus sees EPIPE and drops it. + for (const sub of this.subscriptions) sub.releaseReaders(); + } + this.proc = proc; + try { + await proc.started; + this.origin = originFor?.(this.subscriptions); + await this.gate.attach(slot, this.origin); + } catch (error) { + slot.fifos.releaseHolders(); + slot.fifos.unlink(); + await this.stop(); + throw error; + } + } + + async stop(): Promise { + for (const sub of this.subscriptions) sub.unsubscribe(); + await this.proc?.terminate(); + this.proc = null; + } + + kill(signal: NodeJS.Signals): void { this.proc?.kill(signal); } +} diff --git a/apps/media-worker/src/compositor/cpu-smoke.ts b/apps/media-worker/src/compositor/cpu-smoke.ts new file mode 100644 index 0000000..3d991d6 --- /dev/null +++ b/apps/media-worker/src/compositor/cpu-smoke.ts @@ -0,0 +1,206 @@ +/** + * S2 CPU budget and sustained transport gate: + * 4 × 1080p30 H.264 MPEG-TS sources → ingest×4 (decode + normalize) → source-bus helpers → + * 2x2 compositor with a lower third and a chyron → Rust frame gate → x264 + * veryfast 6 Mb/s persistent encoder (-f null). CPU per process from + * /proc//stat over the measurement window, plus the worker (Node) itself. + * + * node --import tsx src/compositor/cpu-smoke.ts [--seconds 30] [--swaps-every 0] + * [--warmup-seconds 30] [--settle-max 45] + */ +import { closeSync, mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { ManagedProcess } from "../relay/ffmpeg"; +import { layoutRects, type Canvas, type CompositorGraph } from "./command-builder"; +import type { ComposeRequest } from "./compositor"; +import { IngestProcess } from "./ingest"; +import { ProgramChain } from "./program"; +import { encodeSourceClip, ensureGateBinary, makeAssets, passthroughRedactor, startPacer } from "./testing"; + +const canvas: Canvas = { width: 1920, height: 1080, fps: 30 }; +const TICKS = 100; // USER_HZ + +function cpuTicks(pid: number | undefined): number { + if (pid === undefined) return 0; + try { + const stat = readFileSync(`/proc/${pid}/stat`, "utf8"); + const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" "); + return Number(fields[11]) + Number(fields[12]); // utime + stime + } catch { return 0; } +} + +/** Time spent runnable but not running (host contention), ns, from /proc//schedstat (all threads). */ +function runDelayNs(pid: number | undefined): number { + if (pid === undefined) return 0; + try { + let total = 0; + for (const tid of readdirSync(`/proc/${pid}/task`)) total += Number(readFileSync(`/proc/${pid}/task/${tid}/schedstat`, "utf8").split(" ")[1] ?? 0); + return total; + } catch { return 0; } +} + +function arg(name: string, fallback: number): number { + const i = process.argv.indexOf(`--${name}`); + return i >= 0 ? Number(process.argv[i + 1]) : fallback; +} + +async function main(): Promise { + const seconds = arg("seconds", 30); + const swapEvery = arg("swaps-every", 0); + const minimumWarmup = arg("warmup-seconds", 30); + const settleCap = arg("settle-max", 45); + if (!Number.isFinite(seconds) || seconds <= 0 || !Number.isFinite(swapEvery) || swapEvery < 0 + || !Number.isFinite(minimumWarmup) || minimumWarmup < 0 || !Number.isFinite(settleCap) || settleCap < Math.max(3, minimumWarmup)) { + throw new Error("invalid smoke duration, swap interval, or warmup/settle bounds"); + } + if (!ensureGateBinary()) throw new Error("native helpers unavailable (cargo build failed)"); + const dir = mkdtempSync(join(tmpdir(), "cpu-smoke-")); + const procs: ManagedProcess[] = []; + const ingests: IngestProcess[] = []; + let chain: ProgramChain | null = null; + try { + // A 1080p30 H.264 + AAC source clip (encoded once, before measuring). + const clip = join(dir, "src.ts"); + encodeSourceClip(clip, canvas); + const assets = makeAssets(dir); + + for (let i = 0; i < 4; i += 1) { + // Stand-in for the pinned egress relay: a paced remux of the clip into a FIFO the ingest reads. + const { readFd, proc: pacer } = await startPacer(clip, dir, `src${i}`); + procs.push(pacer); + const ingest = new IngestProcess(`s${i}`, { input: { kind: "mpegtsPipe" }, canvas, audio: "source" }, readFd, join(dir, "gate"), passthroughRedactor, + (m) => process.stderr.write(m), (id, r) => process.stderr.write(`${id} ${r}\n`)); + await ingest.start(); + closeSync(readFd); + ingests.push(ingest); + } + + const graph = (i: number): CompositorGraph => ({ + canvas, layout: "2x2", + tiles: layoutRects("2x2", canvas).map((rect, t) => ({ source: (t + i) % 4, rect, fit: "contain" as const })), + overlays: [{ kind: "image", asset: 0, x: 96, y: 900 }, { kind: "chyron", asset: 1, y: 1040, speedPxPerSec: 120 }], + audio: { inputs: [{ source: 0, gainDb: 0 }, { source: 1, gainDb: -6 }], masterGainDb: 0 }, + }); + const request = (i: number): ComposeRequest => ({ inputs: ingests.map((ing, k) => ({ bus: ing.bus, audio: k < 2 })), assets, graph: graph(i) }); + const fatal: string[] = []; + chain = new ProgramChain({ canvas, scratchDir: join(dir, "gate"), videoBitrateKbps: 6000, output: { kind: "null" }, + redactor: passthroughRedactor, onStderr: (m) => process.stderr.write(m), onFatal: (r) => fatal.push(r) }); + const tStart = Date.now(); + await chain.start(request(0)); + const warmupEnd = Date.now() + minimumWarmup * 1000; + // Warm-up: the first seconds are a startup transient on a loaded host (4 ingests bursting to + // catch up with their sources, compositor and x264 warming up), during which the compositor's + // bus queues stand near the backlog limit and the gate repeats. Measure the steady state: wait + // until every subscriber queue is within its configured backlog + replay history + // and neither bus drops nor gate fills increase for 3 consecutive seconds. Allow at least + // 30 s for demux/decoder queues and x264 to warm, at most 45 s by default. Startup loss + // stays visible separately; the sustained window begins only after this declared warmup. + // A queue can remain nonempty at native rate: it reflects fixed pipeline/replay delay, + // not overload. Absence of drops/fills at bounded depth is the steady-state criterion. + const backlogLimit = Math.min(...ingests.map((ingest) => ingest.bus.maxBacklogFrames + ingest.bus.historyFrames)); + let calm = 0; + const initialGate = await chain.stats(); + let lastFills = initialGate.vrep + initialGate.asil; + let lastDrops = (await Promise.all(ingests.map((ingest) => ingest.bus.stats()))).reduce((total, bus) => total + bus.subscriberDrops, 0); + const settleTrace: Array<[number, number, number]> = []; + for (let s = 0; s < settleCap && (calm < 3 || Date.now() < warmupEnd); s += 1) { + await new Promise((r) => setTimeout(r, 1_000)); + const buses = await Promise.all(ingests.map((i) => i.bus.stats())); + const q = Math.max(0, ...buses.flatMap((bus) => bus.subscribers.map((sub) => sub.queued))); + const drops = buses.reduce((total, bus) => total + bus.subscriberDrops, 0); + const gate = await chain.stats(); + const fills = gate.vrep + gate.asil; + settleTrace.push([q, fills - lastFills, drops - lastDrops]); + calm = q <= backlogLimit && fills === lastFills && drops === lastDrops ? calm + 1 : 0; + lastFills = fills; + lastDrops = drops; + } + const settleS = (Date.now() - tStart) / 1000; + const startupDrops = (await Promise.all(ingests.map((i) => i.bus.stats()))).reduce((a, b) => a + b.subscriberDrops, 0); + + const pids = { + ingest: ingests.map((i) => i.pid), + compositor: () => chain?.activeCompositor?.pid, + gate: chain.gate.pid, + encoder: chain.encoderPid, + }; + const busPids = ingests.map((i) => i.bus.pid); + const snap = async () => { + const buses = await Promise.all(ingests.map((i) => i.bus.stats())); + return { + ingest: pids.ingest.map(cpuTicks).reduce((a, b) => a + b, 0), + compositor: cpuTicks(pids.compositor()), + gate: cpuTicks(pids.gate), + encoder: cpuTicks(pids.encoder), + buses: busPids.map(cpuTicks).reduce((a, b) => a + b, 0), + worker: process.cpuUsage(), + busDrops: buses.reduce((a, b) => a + b.subscriberDrops, 0), + busPairs: buses.map((b) => b.pairs), + busStats: buses, + delay: { compositor: runDelayNs(pids.compositor()), encoder: runDelayNs(pids.encoder), gate: runDelayNs(pids.gate) }, + }; + }; + await chain.gate.resetPeaks(); + const g0 = await chain.stats(); + const s0 = await snap(); + const t0 = Date.now(); + let swaps = 0; + // Deepest subscriber queue seen per second (frames waiting in a source-bus helper for a consumer). + const queueMax: number[] = []; + while (Date.now() - t0 < seconds * 1000) { + await new Promise((r) => setTimeout(r, swapEvery > 0 ? swapEvery * 1000 : 1000)); + if (swapEvery > 0) { await chain.swap(request(++swaps)); } + const depths = (await Promise.all(ingests.map((i) => i.bus.stats()))).flatMap((b) => b.subscribers.map((s) => s.queued)); + queueMax.push(Math.max(0, ...depths)); + } + const s1 = await snap(); + const g1 = await chain.stats(); + const wall = (Date.now() - t0) / 1000; + const pct = (ticks: number): number => Number(((ticks / TICKS / wall) * 100).toFixed(1)); + const workerUs = (s1.worker.user + s1.worker.system) - (s0.worker.user + s0.worker.system); + const result = { + host: { cpus: (await import("node:os")).cpus().length, model: (await import("node:os")).cpus()[0]?.model, loadavg: (await import("node:os")).loadavg() }, + wallS: wall, swaps, + // Seconds from chain start to steady state, the per-second [max queue, gate repeats] trace, and drops before the window. + settleS, minimumWarmupS: minimumWarmup, settled: calm >= 3 && Date.now() >= warmupEnd, settleTrace, startupDrops, + startupGate: { ticks: g0.ticks, vrep: g0.vrep, asil: g0.asil, outDrops: g0.outDrops }, + pctOfOneCore: { + ingestX4: pct(s1.ingest - s0.ingest), + compositor: swapEvery > 0 ? null : pct(s1.compositor - s0.compositor), + gate: pct(s1.gate - s0.gate), + encoderX264: pct(s1.encoder - s0.encoder), + sourceBusX4: pct(s1.buses - s0.buses), + workerNode: Number(((workerUs / 1e6 / wall) * 100).toFixed(1)), + }, + // Pairs the buses dropped for a subscriber that fell behind, inside the window (a compositor not keeping up). + busDropsInWindow: s1.busDrops - s0.busDrops, + busQueueMaxPerSecond: queueMax, + // Host contention: ms per second of wall time each process (all threads) sat runnable but not scheduled. + runDelayMsPerS: { + compositor: Number(((s1.delay.compositor - s0.delay.compositor) / 1e6 / wall).toFixed(1)), + encoder: Number(((s1.delay.encoder - s0.delay.encoder) / 1e6 / wall).toFixed(1)), + gate: Number(((s1.delay.gate - s0.delay.gate) / 1e6 / wall).toFixed(1)), + }, + // Source pairs each bus produced inside the window (≈ 30 × wallS for a paced 30 fps source). + busPairsInWindow: s1.busPairs.map((n, k) => n - (s0.busPairs[k] ?? 0)), + gate: { ticks: g1.ticks - g0.ticks, vrep: g1.vrep - g0.vrep, asil: g1.asil - g0.asil, lateOverFrame: g1.lateOverFrame, lateMaxUs: g1.lateMaxUs, outDrops: g1.outDrops - g0.outDrops }, + buses: s1.busStats, + fatal, + }; + const p = result.pctOfOneCore; + process.stdout.write(`${JSON.stringify({ ...result, totalCores: Number((((p.ingestX4 + (p.compositor ?? 0) + p.gate + p.encoderX264 + p.sourceBusX4 + p.workerNode) / 100)).toFixed(2)) })}\n`); + if (!result.settled || result.busDropsInWindow !== 0 || result.gate.vrep !== 0 || result.gate.asil !== 0 || result.gate.outDrops !== 0 || fatal.length > 0 + || result.busPairsInWindow.some((pairs) => pairs < canvas.fps * wall * 0.98)) { + throw new Error("cpu-smoke failed: require settled 4×1080p30, zero bus drops, zero gate fills/output drops, and no fatal exit"); + } + } finally { + await chain?.stop(); + for (const i of ingests) await i.stop(); + for (const p of procs) await p.terminate(500); + rmSync(dir, { recursive: true, force: true }); + } + process.exit(0); +} + +if (require.main === module) void main().catch((error: unknown) => { process.stderr.write(`${String(error)}\n`); process.exit(1); }); diff --git a/apps/media-worker/src/compositor/golden-frames.json b/apps/media-worker/src/compositor/golden-frames.json new file mode 100644 index 0000000..2d5f31a --- /dev/null +++ b/apps/media-worker/src/compositor/golden-frames.json @@ -0,0 +1,115 @@ +{ + "ffmpeg": "ffmpeg version 5.1.9-0+deb12u1 Copyright (c) 2000-2026 the FFmpeg developers", + "canvas": { + "width": 320, + "height": 180, + "fps": 30 + }, + "frames": 4, + "layouts": { + "single": [ + "0:0:86400:473ec55d56d6cd3f85b31ef5f0b615b2", + "1:0:4096:f858b65fe890efbc0ac6cf1e085a37d2", + "1:1024:4096:e3ef895bb0e533a4a8a89d5a6ed843f6", + "0:1:86400:80e9688e53079ff4623b78115e2b6b1c", + "1:2048:4096:8da1b5054e4f15590b674af0c5239537", + "1:3072:4096:18a3a0028363827191cd4d8885bc2b11", + "0:2:86400:f2b6d37903c429f8ad6c4416056a8538", + "1:4096:4096:aecafa4e5f0f3626cc00a9849cbcc3fa", + "0:3:86400:5113494af070e29d7c5c6cbbbaa7a721", + "1:5120:4096:749e60b205100820f0f493a79c17ee46", + "1:6144:1024:f465c566bd65b8743119b970d5b9734e" + ], + "pip": [ + "0:0:86400:ff2365f77b17a7437246001c3282af88", + "1:0:4096:fd773b1ed90b3793d331bebf439d85a7", + "1:1024:4096:e1a1133f44e0c84ae28eca0ac57333c9", + "0:1:86400:9a238c82abca419758f64d31f3f5080e", + "1:2048:4096:c7f8ec938afbe515acf2cc21ee07b398", + "1:3072:4096:c99fc35fb2f9fb03c18e8ff04b5300a5", + "0:2:86400:ef78085de2d4d6f6e20e266e40e7a07d", + "1:4096:4096:62bc5a8e144fb37ec4e5a48fa0a3179a", + "0:3:86400:956a390b4024b79289063d54abde9c07", + "1:5120:4096:20dd40a44d6167e3d20631d72cca7021", + "1:6144:1024:4aebc7614678f7837d5c2c42c7ae5049" + ], + "2x1": [ + "0:0:86400:124ceb28ce6fb2adda306ebab85c020e", + "1:0:4096:779110ff86bde4abc0c312c254c6a07b", + "1:1024:4096:58bf6a6a073a28c148162edbfb1cb2f0", + "0:1:86400:2a815118db500f1fe35af785761546d3", + "1:2048:4096:b47c4007c2bd03d67892ed6c4d2e0ad2", + "1:3072:4096:4c1fa084e7e962bdfccc6445944fe6bd", + "0:2:86400:fb62f8882a43ace4e0a50f611ba5b011", + "1:4096:4096:96331541b14dd07c95116914832e2646", + "0:3:86400:3d22dd326e76b9ace9b605e67568c5c9", + "1:5120:4096:8d09c0dd1dd3d94d2296b8b96a887d0f", + "1:6144:1024:456a74bc6828cb74630049d3c3ede52b" + ], + "2x2": [ + "0:0:86400:c1a86085e98caf0c6ddca6eecd76f996", + "1:0:4096:f2e48ac5003bebe6980227d7984f03ee", + "1:1024:4096:c42c6c8bd7e38a04da330614958e8ec7", + "0:1:86400:eacfcee60a815e7098d9f2678ef3a2b8", + "1:2048:4096:07d15edac8583afe4f0c37062f8e2e95", + "1:3072:4096:2eeb48f1b0671b51871cca079144b9a0", + "0:2:86400:a69202dd6be84938c0669cfb1b1aea91", + "1:4096:4096:86b18ee36947048c70bacd00707025d0", + "0:3:86400:339a15c6489b3af66d4cbe4311179d0f", + "1:5120:4096:e10e40ebb313c090b2679d523299c4f4", + "1:6144:1024:2f3a2ef3cc8c5f0206828e53fe002d1b" + ], + "3x3": [ + "0:0:86400:dfffffffbc45583776628ba9e07d2ac7", + "1:0:4096:95afee33956e36002cb7bc4174d41fb8", + "1:1024:4096:4cc1e8ca719bd34775c4bc5f0f1d1149", + "0:1:86400:495bf533902abce01a4c6213275d4a8a", + "1:2048:4096:39a06315fe075c8d02500f7b69f9d0b1", + "1:3072:4096:28a2f2f96cbb5947d1ad1bf6f0f656d3", + "0:2:86400:2b721e57614c93bc1a50f486317d3a69", + "1:4096:4096:f80381b7c6e59ac4ed8cf350cdf675ae", + "0:3:86400:8e738a80e6b6a24d2c97f0a4a819e8b4", + "1:5120:4096:47905d32edded2c3b98653ac97c9b92e", + "1:6144:1024:7cdba1f0765decdaa01239f8d415ce83" + ], + "4x4": [ + "0:0:86400:9b5bc219325405095ad5c3ed5acc1ea1", + "1:0:4096:620f0b67a91f7f74151bc5be745b7110", + "1:1024:4096:620f0b67a91f7f74151bc5be745b7110", + "0:1:86400:f96bf1c80e7dfac31e8e612139c0ad40", + "1:2048:4096:620f0b67a91f7f74151bc5be745b7110", + "1:3072:4096:620f0b67a91f7f74151bc5be745b7110", + "0:2:86400:0b6280a6f2b063114eb892b8fa2ef406", + "1:4096:4096:620f0b67a91f7f74151bc5be745b7110", + "0:3:86400:73f848d7c873b09ef45467fe2a8b5785", + "1:5120:4096:620f0b67a91f7f74151bc5be745b7110", + "1:6144:1024:0f343b0931126a20f133d67c2b018a3b" + ], + "custom": [ + "0:0:86400:7151990b9ab56564d8f6f1ab63edd9ee", + "1:0:4096:8904df5033d62a7340b98e03dea71359", + "1:1024:4096:9fa5c863b67d9831fa7bc93cbe868e4e", + "0:1:86400:ccfdc3ae531e8d495aa67e2989dd8e72", + "1:2048:4096:5602eff96d58dd2c18365237b01aeac5", + "1:3072:4096:d6702b18d7e15f6ead23e77cdb3b7be0", + "0:2:86400:9725c1c13fccbc094cad0c2ef683fbab", + "1:4096:4096:3090fcea0f7513e35c03b80a199925be", + "0:3:86400:019eb04e7297c7bae18d86f47ebbfee7", + "1:5120:4096:3bbc5f09a7275706e87668d3ccd96793", + "1:6144:1024:2642090512e9f3b93a215ba0497dcbd8" + ], + "custom-overlap": [ + "0:0:86400:5c78229f51b9307615a1f9e442efe391", + "1:0:4096:2cc5270f68b8a25c1909d435272422b4", + "1:1024:4096:3c83cd930cd96b9da3e3f29ce369adf9", + "0:1:86400:0cd6d909f4b84e42f6ac6360f1bda3f6", + "1:2048:4096:f6e469bc77eaa6d1042c70e91450b4ab", + "1:3072:4096:f5e8ee7c276f02215053e8143e65861d", + "0:2:86400:fdff300a63db4e8bb5d9d7fea498ea59", + "1:4096:4096:d9af646ccccb557d6a065868cf2fa667", + "0:3:86400:aeaecd38cc4cbd0f900e8a1cec209962", + "1:5120:4096:f22c40c3beb84a93a61e834d0daad01a", + "1:6144:1024:e93ffc6d18f786a744a9b1ac241ec306" + ] + } +} diff --git a/apps/media-worker/src/compositor/ingest.ts b/apps/media-worker/src/compositor/ingest.ts new file mode 100644 index 0000000..3116212 --- /dev/null +++ b/apps/media-worker/src/compositor/ingest.ts @@ -0,0 +1,59 @@ +import { spawnManaged, type ManagedProcess, type StderrRedactor } from "../relay/ffmpeg"; +import { buildIngestArgs, type IngestPlan } from "./command-builder"; +import { SourceBus } from "./source-bus"; + +/** + * Ingest ×N normalize stage: one ffmpeg per source decodes and normalizes to + * the canvas (raw yuv420p on fd 1, s16le 48 kHz stereo on fd 3) straight into + * its source bus's FIFOs. Compositors and preview taps subscribe to buses; they + * never touch the remote, and the raw frames never pass through the worker. + */ +export class IngestProcess { + readonly bus: SourceBus; + private proc: ManagedProcess | null = null; + private stopping = false; + + constructor( + readonly sourceId: string, + private readonly plan: IngestPlan, + /** MPEG-TS fd: the WP7 pinned relay (`mpegtsPipe`) or the playlist feeder (`playlistPipe`); null for synthetic. */ + private readonly stdinFd: number | null, + scratchDir: string, + private readonly redactor: StderrRedactor, + private readonly onStderr: (message: string) => void, + private readonly onExit: (sourceId: string, reason: string) => void, + ) { + // Native fan-out can retain a bounded half-second of live backlog in addition to + // replay history. With FIFO resizing denied on shared hosts, the replay/input delay + // otherwise leaves too little room for short decoder/filter scheduling bursts. + this.bus = new SourceBus({ geometry: plan.canvas, scratchDir, redactor, onStderr, + maxBacklogFrames: Math.max(4, Math.ceil(plan.canvas.fps / 2)), + onExit: (reason) => { if (!this.stopping) onExit(sourceId, reason); } }); + } + + get pid(): number | undefined { return this.proc?.child.pid; } + + async start(): Promise { + if ((this.plan.input.kind !== "synthetic") !== (this.stdinFd !== null)) throw new Error("ingest input and fd disagree"); + this.stopping = false; + const out = await this.bus.start(); + try { + const proc = spawnManaged("ffmpeg", buildIngestArgs(this.plan), [this.stdinFd ?? "ignore", out.videoFd, "pipe", out.audioFd], this.redactor, this.onStderr); + this.proc = proc; + void proc.exited.then(({ code, signal }) => { if (!this.stopping) this.onExit(this.sourceId, `ingest exited code=${code} signal=${signal}`); }); + await proc.started; + } catch (error) { + await this.stop(); + throw error; + } finally { + this.bus.releaseIngestWriters(); + } + } + + async stop(): Promise { + this.stopping = true; + await this.proc?.terminate(); + this.proc = null; + await this.bus.stop(); + } +} diff --git a/apps/media-worker/src/compositor/playlist-feeder.test.ts b/apps/media-worker/src/compositor/playlist-feeder.test.ts new file mode 100644 index 0000000..37ec521 --- /dev/null +++ b/apps/media-worker/src/compositor/playlist-feeder.test.ts @@ -0,0 +1,160 @@ +/** + * Playlist feeder (streaming-architecture.md §5.1): argv goldens, and real + * ffmpeg runs of feeder → `playlistPipe` ingest → source bus, checking that + * asset boundaries keep the bus at frame rate with A/V paired, that a broken + * asset is skipped, that the slate floor covers an unplayable or ended list, + * and that `update()` adopts a new list at the next asset boundary. + */ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { createReadStream, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Readable } from "node:stream"; +import test from "node:test"; +import { buildIngestArgs, buildPlaylistRemuxArgs, buildSlateArgs } from "./command-builder"; +import { IngestProcess } from "./ingest"; +import { PlaylistFeeder, type FeederEvent } from "./playlist-feeder"; +import { ensureGateBinary, passthroughRedactor } from "./testing"; + +const canvas = { width: 320, height: 180, fps: 30 }; + +test("playlist argv: copy-only remux from stdin at a monotonic offset, paced ingest, slate floor", () => { + assert.deepEqual(buildPlaylistRemuxArgs(2_020_133), ["-hide_banner", "-nostdin", "-loglevel", "error", "-progress", "pipe:3", "-protocol_whitelist", "pipe", "-i", "pipe:0", + "-map", "0:v:0", "-map", "0:a:0?", "-c", "copy", "-output_ts_offset", "2.020133", + "-f", "mpegts", "-mpegts_flags", "+initial_discontinuity", "-muxdelay", "0", "pipe:1"]); + assert.throws(() => buildPlaylistRemuxArgs(-1)); + assert.throws(() => buildPlaylistRemuxArgs(1.5)); + const ingest = buildIngestArgs({ input: { kind: "playlistPipe" }, canvas, audio: "source" }); + assert.deepEqual(ingest.slice(ingest.indexOf("-re"), ingest.indexOf("-re") + 8), ["-re", "-fflags", "+genpts+discardcorrupt", "-f", "mpegts", "-i", "pipe:0", "-map"]); + assert.ok(ingest.includes("0:a:0?"), "a silent asset cannot fail the ingest"); + const slate = buildSlateArgs({ canvas, seconds: 5, image: null, offsetUs: 0 }); + assert.ok(slate.includes("color=c=black:s=320x180:r=30") && slate.includes("anullsrc=r=48000:cl=stereo") && slate.at(-1) === "pipe:1"); + const png = `/scratch/${"a".repeat(64)}.png`; + assert.ok(buildSlateArgs({ canvas, seconds: 5, image: png, offsetUs: 7_000_000 }).includes(`file:${png}`)); + assert.ok(buildSlateArgs({ canvas, seconds: 5, image: null, offsetUs: 7_000_000 }).includes("7.000000")); + assert.throws(() => buildSlateArgs({ canvas, seconds: 5, image: "/etc/passwd", offsetUs: 0 })); + assert.throws(() => buildSlateArgs({ canvas, seconds: 0, image: null, offsetUs: 0 })); + const all = [...buildPlaylistRemuxArgs(0), ...ingest, ...slate].join(" "); + assert.ok(!/https?:|concat|movie=|tcp:|udp:/.test(all), "pipes and lavfi only"); +}); + +function clip(dir: string, name: string, seconds: number, hz: number): string { + const path = join(dir, name); + const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", `testsrc2=s=320x180:r=30:d=${seconds}`, "-f", "lavfi", "-i", `sine=frequency=${hz}:d=${seconds}`, + "-c:v", "libx264", "-preset", "ultrafast", "-g", "30", "-c:a", "aac", "-movflags", "+faststart", path]); + assert.equal(r.status, 0); + return path; +} + +async function run(t: { skip(msg: string): void }, assetIds: string[], loop: boolean, files: Map, seconds: number, + during?: (feeder: PlaylistFeeder) => void): Promise<{ events: FeederEvent[]; samples: number[]; stats: { pairs: number; silenceFilled: number }; feeder: PlaylistFeeder; exits: string[] } | null> { + if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return null; } + const dir = mkdtempSync(join(tmpdir(), "feeder-")); + const events: FeederEvent[] = []; + const exits: string[] = []; + const feeder = new PlaylistFeeder("src", { assetIds, loop }, { + assets: { async open(id) { const path = files.get(id); if (!path) throw new Error("no such asset"); return id === "broken" ? Readable.from([Buffer.from("not media")]) : createReadStream(path); } }, + canvas, scratchDir: dir, redactor: passthroughRedactor, onStderr: () => undefined, onEvent: (e) => events.push(e), slateSeconds: 2, + }); + const ingest = new IngestProcess("src", { input: { kind: "playlistPipe" }, canvas, audio: "source" }, feeder.start(), dir, passthroughRedactor, () => undefined, (_id, r) => exits.push(r)); + const samples: number[] = []; + try { + await ingest.start(); + feeder.releaseReader(); + // Measure playback once FFmpeg has probed the TS input and delivered a pair. + // Process startup is bounded separately, rather than consuming the playback window. + const startupDeadline = Date.now() + 10_000; + while ((await ingest.bus.stats()).pairs === 0) { + assert.ok(Date.now() < startupDeadline, "playlist ingest produced its first A/V pair within 10 s"); + await new Promise((r) => setTimeout(r, 100)); + } + const t0 = Date.now(); + while (Date.now() - t0 < seconds * 1000) { + await new Promise((r) => setTimeout(r, 250)); + samples.push((await ingest.bus.stats()).pairs); + if (during && samples.length === 4) during(feeder); + } + const stats = await ingest.bus.stats(); + return { events, samples, stats, feeder, exits }; + } finally { + await feeder.stop(); + await ingest.stop(); + rmSync(dir, { recursive: true, force: true }); + } +} + +/** Longest stretch (in 250 ms samples) without a new pair once the stream is up. */ +function longestStall(samples: number[]): number { + let worst = 0; + let run = 0; + const first = samples.findIndex((n) => n > 0); + for (let i = Math.max(1, first + 1); i < samples.length; i += 1) { + run = samples[i] === samples[i - 1] ? run + 1 : 0; + worst = Math.max(worst, run); + } + return worst; +} + +test("feeder: asset boundaries stay at frame rate with A/V paired; a broken asset is skipped; the list loops", { timeout: 120_000 }, async (t) => { + const dir = mkdtempSync(join(tmpdir(), "feeder-clips-")); + try { + const files = new Map([["a", clip(dir, "a.mp4", 2, 300)], ["b", clip(dir, "b.mp4", 1, 600)], ["broken", join(dir, "broken")]]); + writeFileSync(join(dir, "broken"), "x"); + const r = await run(t, ["a", "broken", "b"], true, files, 7); + if (!r) return; + const played = r.events.filter((e): e is Extract => e.type === "asset").map((e) => e.assetId); + assert.deepEqual(played.slice(0, 4), ["a", "broken", "b", "a"], "in order, looping"); + assert.ok(r.events.some((e) => e.type === "asset-failed" && e.assetId === "broken")); + assert.ok(!r.events.some((e) => e.type === "slate"), "one bad asset does not drop to the slate"); + assert.equal(r.stats.silenceFilled, 0, "audio continuous across boundaries"); + assert.ok(longestStall(r.samples) <= 1, `no stall at boundaries (samples ${r.samples.join(",")})`); + const up = r.samples.findIndex((n) => n > 0); + const rate = ((r.samples.at(-1) ?? 0) - (r.samples[up] ?? 0)) / ((r.samples.length - 1 - up) * 0.25); + assert.ok(rate > 26 && rate < 36, `paced at the frame rate (${rate.toFixed(1)} pairs/s)`); + assert.deepEqual(r.exits, []); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); + +test("feeder: nothing playable → slate floor (the bus keeps running); a finished non-looping list → slate", { timeout: 120_000 }, async (t) => { + const dir = mkdtempSync(join(tmpdir(), "feeder-clips-")); + try { + writeFileSync(join(dir, "broken"), "x"); + const dead = await run(t, ["broken"], true, new Map([["broken", join(dir, "broken")]]), 4); + if (!dead) return; + assert.ok(dead.events.some((e) => e.type === "slate" && e.reason === "unavailable")); + assert.ok(dead.stats.pairs >= 60, `slate keeps the source alive (${dead.stats.pairs} pairs)`); + const once = await run(t, ["a"], false, new Map([["a", clip(dir, "a.mp4", 1, 300)]]), 4); + assert.ok(once?.events.some((e) => e.type === "slate" && e.reason === "ended")); + assert.equal(once?.feeder.ended, true); + assert.ok(longestStall(once?.samples ?? []) <= 1, "the slate follows the last asset without a stall"); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); + +test("feeder: update() adopts a new list after the asset being fed (desired-state v2, no restart)", { timeout: 120_000 }, async (t) => { + const dir = mkdtempSync(join(tmpdir(), "feeder-clips-")); + try { + const files = new Map([["a", clip(dir, "a.mp4", 2, 300)], ["b", clip(dir, "b.mp4", 1, 600)], ["c", clip(dir, "c.mp4", 1, 900)]]); + // 1 s in, [a, b] becomes [c, a]. The feeder hands assets to the ingest ahead of playback (FIFO and + // demuxer buffers: tens of ms at production bitrates, several of these tiny clips), so "the asset + // being fed" is the boundary: after it, playback follows the new list from the position after it. + let adoptedAfter: string | null = null; + const r = await run(t, ["a", "b"], true, files, 12, (feeder) => { + adoptedAfter = feeder.nowPlaying; + feeder.update({ assetIds: ["c", "a"], loop: true }); + }); + if (!r) return; + const order = r.events.filter((e) => e.type === "asset" || e.type === "adopted").map((e) => (e.type === "asset" ? e.assetId : "|")); + const cut = order.indexOf("|"); + assert.ok(cut > 0, `adopted (events ${order.join(",")})`); + const before = order.slice(0, cut); + const after = order.slice(cut + 1); + assert.deepEqual(before, ["a", "b", "a", "b", "a", "b"].slice(0, before.length), "the old list until the update"); + assert.ok(after.length >= 2, `the new list plays (${order.join(",")})`); + // Position after the asset being fed: after "a" (index 1 in [c, a]) comes "c"; after "b" (gone) the list restarts at "c". + assert.equal(after[0], "c", `after ${String(adoptedAfter)} the new list continues at c (${order.join(",")})`); + assert.deepEqual(after.slice(0, 4), ["c", "a", "c", "a"].slice(0, Math.min(4, after.length))); + assert.ok(longestStall(r.samples) <= 1, `no stall when a prepared asset is replaced (samples ${r.samples.join(",")})`); + assert.deepEqual(r.exits, []); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); diff --git a/apps/media-worker/src/compositor/playlist-feeder.ts b/apps/media-worker/src/compositor/playlist-feeder.ts new file mode 100644 index 0000000..4e19ede --- /dev/null +++ b/apps/media-worker/src/compositor/playlist-feeder.ts @@ -0,0 +1,250 @@ +/** + * Playlist feeder (streaming-architecture.md §5.1): a `kind: "playlist"` source + * becomes a continuous input like any other. The feeder walks the ordered asset + * list, opens each asset (a presigned GET through the ingest egress relay), + * remuxes it to MPEG-TS (pre-normalized at upload: copy only, never a + * transcode) and pipes the remuxes one after another into a FIFO that one + * `playlistPipe` ingest reads at native rate. FFmpeg reads only pipes (WP9); + * no concat protocol. + * + * - Timestamps: each remux is shifted to where the previous asset ended (its + * `-progress` end time), so the concatenated stream is monotonic. FFmpeg + * 5.1's own discontinuity correction stalls `-re` for 1–2 s per boundary. + * - Boundaries: two overlapped stages. Asset i+1's remux starts as soon as + * asset i's remux has exited (its end time is then known), while asset i is + * still being fed from its capture buffer (~4 MiB ahead, several seconds), so + * the next asset's bytes are waiting when the current one ends: no gap. + * - Failure: an asset that cannot be opened or remuxed is skipped. When a whole + * pass of the list fails (or the list is empty), the slate floor plays for a + * few seconds and the list is retried. + * - End: `loop: false` plays the list once, then the slate for good. + * - Desired-state v2: `update()` adopts a new asset list without a restart. The + * asset on air finishes; playback continues after it in the new list (or at + * the new list's start when it is no longer there). An already prepared next + * asset that is no longer next is discarded. + */ +import { closeSync, constants, openSync, rmSync } from "node:fs"; +import { Socket } from "node:net"; +import { PassThrough, type Readable } from "node:stream"; +import { makeFifo, openBlockingReader } from "../gate/fifo"; +import { spawnManaged, type ManagedProcess, type StderrRedactor } from "../relay/ffmpeg"; +import { buildPlaylistRemuxArgs, buildSlateArgs, type Canvas } from "./command-builder"; + +/** Opens one playlist asset's bytes (WP7: presigned GET through the pinned ingest egress relay). */ +export interface PlaylistAssetSource { open(assetId: string): Promise } + +export interface PlaylistSpec { assetIds: string[]; loop: boolean } + +export type FeederEvent = + | { type: "asset"; assetId: string; index: number } + | { type: "asset-failed"; assetId: string; reason: string } + | { type: "slate"; reason: "unavailable" | "ended" } + | { type: "adopted"; assetIds: number }; + +export interface PlaylistFeederDeps { + assets: PlaylistAssetSource; + canvas: Canvas; + scratchDir: string; + redactor: StderrRedactor; + onStderr(message: string): void; + onEvent?(event: FeederEvent): void; + /** Rendered slate PNG (S4, overlay renderer); null = black. */ + slateImage?: string | null; + slateSeconds?: number; +} + +/** Bytes of produced MPEG-TS held ahead of the FIFO per producer (~5 s at 6 Mb/s). */ +const CAPTURE_BYTES = 4 << 20; + +/** One remux (or slate) run: its captured output and, once it exits, its end time. */ +interface Job { + proc: ManagedProcess; + output: Readable; + /** Resolves when the producer exits: success, and the end time it reported (µs, excluding the offset). */ + done: Promise<{ ok: boolean; endUs: number; reason: string }>; + input: Readable | null; + cancelled: boolean; +} + +let nextFeeder = 1; + +export class PlaylistFeeder { + private spec: PlaylistSpec; + private out: Socket | null = null; + private reader: { fd: number; releaseHolder(): void } | null = null; + private stopping = false; + private running: Promise = Promise.resolve(); + private current: { assetId: string; index: number } | null = null; + private nextIndex = 0; + /** Bumped by `update()`: a job prepared under an older list is discarded. */ + private generation = 0; + private readonly jobs = new Set(); + private abortFeed: (() => void) | null = null; + /** Assets started, assets failed, slate chunks fed. */ + readonly stats = { assets: 0, failures: 0, slates: 0 }; + ended = false; + + constructor(readonly sourceId: string, spec: PlaylistSpec, private readonly deps: PlaylistFeederDeps) { + this.spec = { assetIds: [...spec.assetIds], loop: spec.loop }; + } + + get nowPlaying(): string | null { return this.current?.assetId ?? null; } + + /** + * Creates the FIFO and starts feeding. Returns the blocking read end for the ingest's stdin; + * call `releaseReader()` once the ingest has it. + */ + start(): number { + const path = makeFifo(this.deps.scratchDir, `playlist-${nextFeeder++}.fifo`); + const reader = openBlockingReader(path); + this.reader = reader; + const out = new Socket({ fd: openSync(path, constants.O_WRONLY | constants.O_NONBLOCK), readable: false, writable: true }); + out.on("error", () => { if (!this.stopping) void this.stop(); }); // the ingest is gone; IngestProcess reports its exit + this.out = out; + reader.releaseHolder(); + rmSync(path, { force: true }); + this.running = this.run().catch((error: unknown) => this.deps.onStderr(`playlist feeder ${this.sourceId}: ${String(error)}`)); + return reader.fd; + } + + releaseReader(): void { + if (this.reader) { closeSync(this.reader.fd); this.reader = null; } + } + + /** Desired-state v2: adopt a new list at the next asset boundary (no restart, no mid-asset cut). */ + update(spec: PlaylistSpec): void { + this.spec = { assetIds: [...spec.assetIds], loop: spec.loop }; + const at = this.current ? this.spec.assetIds.indexOf(this.current.assetId) : -1; + this.nextIndex = at >= 0 ? at + 1 : 0; + this.ended = false; + this.generation += 1; + this.deps.onEvent?.({ type: "adopted", assetIds: this.spec.assetIds.length }); + } + + private peekNext(): { assetId: string; index: number } | null { + const ids = this.spec.assetIds; + if (ids.length === 0) return null; + let index = this.nextIndex; + if (index >= ids.length) { + if (!this.spec.loop) return null; + index = 0; + } + return { assetId: ids[index]!, index }; + } + + /** Spawns a remux of `asset` (or the slate) at `offsetUs`, capturing its output from the start. */ + private produce(asset: { assetId: string; index: number } | null, offsetUs: number): Job { + const args = asset ? buildPlaylistRemuxArgs(offsetUs) + : buildSlateArgs({ canvas: this.deps.canvas, seconds: this.deps.slateSeconds ?? 5, image: this.deps.slateImage ?? null, offsetUs }); + const proc = spawnManaged("ffmpeg", args, [asset ? "pipe" : "ignore", "pipe", "pipe", "pipe"], this.deps.redactor, this.deps.onStderr); + // A producer may exit before its turn (a broken asset at once, a small one that fits in the + // buffers), and Node then resumes its unconsumed stdout itself, dropping data and 'end'. The + // capture keeps both until fed; backpressure holds a large asset's remux CAPTURE_BYTES ahead. + const output = new PassThrough({ highWaterMark: CAPTURE_BYTES }); + const stdout = proc.child.stdout as Readable | null; + if (stdout) stdout.pipe(output); + else output.end(); + let progress = ""; + (proc.child.stdio[3] as Readable | null)?.on("data", (chunk: Buffer) => { progress = (progress + chunk.toString("latin1")).slice(-4096); }); + let failed: string | null = null; + const done = proc.exited.then(({ code }) => { + const endUs = Number([...progress.matchAll(/out_time_us=(\d+)/g)].at(-1)?.[1] ?? 0); + return { ok: failed === null && code === 0 && endUs > 0, endUs, reason: failed ?? `producer exited code=${code} end=${endUs}us` }; + }); + const job: Job = { proc, output, input: null, done, cancelled: false }; + if (asset) { + proc.child.stdin?.on("error", () => undefined); // EPIPE when the remux rejects its input + void this.deps.assets.open(asset.assetId).then((input) => { + if (job.cancelled || this.stopping || proc.child.exitCode !== null || proc.child.signalCode !== null) { input.destroy(); return; } + job.input = input; + input.on("error", (error: Error) => { failed = `asset read failed: ${error.name}`; proc.kill("SIGKILL"); }); + if (proc.child.stdin) input.pipe(proc.child.stdin); + }, (error: unknown) => { + failed = `asset open failed: ${error instanceof Error ? error.name : "error"}`; + proc.kill("SIGKILL"); + }); + } + this.jobs.add(job); + void done.then(() => this.jobs.delete(job)); + return job; + } + + private discard(job: Job): void { + job.cancelled = true; + job.input?.destroy(); + job.output.destroy(); + job.proc.kill("SIGKILL"); + } + + /** Pipes a job's captured output into the FIFO; resolves once all of it has been handed over. */ + private async feed(job: Job): Promise { + const out = this.out; + if (!out) return; + job.output.pipe(out, { end: false }); + // Until the producer's last byte ('end'). On stop the FIFO socket is destroyed, which unpipes + // and pauses the capture, so 'end' would never come: stop() aborts the wait instead. + await new Promise((resolve) => { + this.abortFeed = resolve; + job.output.once("end", resolve); + job.output.once("error", () => resolve()); + }); + this.abortFeed = null; + job.output.unpipe(out); + } + + private async run(): Promise { + let offsetUs = 0; + let failedInARow = 0; + let slateNext = false; + let feeding: Promise = Promise.resolve(); + while (!this.stopping) { + const generation = this.generation; + const next = slateNext ? null : this.peekNext(); + slateNext = false; + // Prepared while the previous job is still being fed (its producer has exited: offsetUs is final). + const job = this.produce(next, offsetUs); + await feeding; + if (this.stopping) { this.discard(job); break; } + if (generation !== this.generation) { this.discard(job); continue; } // re-plan under the new list + if (next) { + this.nextIndex = next.index + 1; + this.current = next; + this.stats.assets += 1; + this.deps.onEvent?.({ type: "asset", assetId: next.assetId, index: next.index }); + } else { + this.current = null; + this.ended = this.spec.assetIds.length > 0 && !this.spec.loop && this.nextIndex >= this.spec.assetIds.length; + this.stats.slates += 1; + this.deps.onEvent?.({ type: "slate", reason: this.ended ? "ended" : "unavailable" }); + } + feeding = this.feed(job); + const result = await job.done; + if (result.ok) { + offsetUs += result.endUs; + failedInARow = 0; + continue; + } + if (!next) continue; // a failed slate is retried (offset unchanged) + this.stats.failures += 1; + failedInARow += 1; + this.deps.onEvent?.({ type: "asset-failed", assetId: next.assetId, reason: result.reason }); + if (failedInARow >= Math.max(1, this.spec.assetIds.length)) { + failedInARow = 0; + slateNext = true; + } + } + } + + async stop(): Promise { + if (this.stopping) { await this.running; return; } + this.stopping = true; + const jobs = [...this.jobs]; + for (const job of jobs) this.discard(job); + this.abortFeed?.(); + this.out?.destroy(); + this.out = null; + this.releaseReader(); + await this.running; + await Promise.all(jobs.map((job) => job.done)); + } +} diff --git a/apps/media-worker/src/compositor/production-job.ts b/apps/media-worker/src/compositor/production-job.ts new file mode 100644 index 0000000..b44b5c5 --- /dev/null +++ b/apps/media-worker/src/compositor/production-job.ts @@ -0,0 +1,181 @@ +import { createHash } from "node:crypto"; +import type { GateImpl } from "../gate/frame-gate"; +import { snapshotHlsUrls, OUTPUT_PROFILES, type DesiredChannel } from "../types"; +import { validateEndpointShape, type EndpointValidator } from "../relay/endpoints"; +import type { StderrRedactor, VaultHandleBroker } from "../relay/ffmpeg"; +import type { Canvas, EncoderOutput, IngestInput } from "./command-builder"; +import type { ComposeRequest } from "./compositor"; +import { ManifestPreviewPublisher, type PreviewManifestStore, type PreviewObjectStore } from "../preview/contract"; +import type { HlsRendition } from "../preview/argv"; +import { PreviewTaps } from "../preview/taps"; +import { IngestProcess } from "./ingest"; +import { PlaylistFeeder, type PlaylistAssetSource } from "./playlist-feeder"; +import { ProgramChain } from "./program"; +import type { SourceBus } from "./source-bus"; +import { planScene, programScene } from "./scene-plan"; + +/** WP7 egress for a production snapshot: one pinned MPEG-TS fd per HLS source, plus the push fd. */ +export interface ProductionEgress { + prepare(config: DesiredChannel, validator: EndpointValidator, vault: VaultHandleBroker): Promise<{ + sources: Map; + output: EncoderOutput; + close(): Promise; + }>; +} + +/** Overlay PNGs by content id, fetched through assignment-scoped presigned GETs (§4.3) into scratch. */ +export interface AssetStore { localPath(assetId: string): Promise } + +export interface ProductionJobDeps { + validator: EndpointValidator; + egress: ProductionEgress; + vault: VaultHandleBroker; + assets: AssetStore; + redactor: StderrRedactor; + onStderr(message: string): void; + onExit(reason: string): void; + scratchDir: string; + gateImpl?: GateImpl; + /** Playlist sources (§5.1). Without it, playlist sources render black and are reported in `lastSkipped`. */ + playlists?: PlaylistAssetSource; + /** Preview taps (§2.1) publish through the API's manifest contract. Without it, no taps run. */ + preview?: { objects: PreviewObjectStore; manifests: PreviewManifestStore }; +} + +/** + * What may change by swap. Anything else (the set of sources, an HLS URL, output, profile, mode) + * restarts the channel. A playlist's asset list and loop flag are not part of it: the feeder + * adopts them in place (desired-state v2: an edited playlist never interrupts the program). + */ +function pathKey(c: DesiredChannel): string { + const sources = c.production?.sources.map((src) => (src.kind === "hls" ? { id: src.id, kind: src.kind, url: src.url } : { id: src.id, kind: src.kind })) ?? null; + return createHash("sha256").update(JSON.stringify({ profile: c.profile, mode: c.mode, output: c.output, sources, source: c.source })).digest("hex"); +} + +/** + * Supervisor RelayJob for a desired-state v2 snapshot. `start()` brings up + * ingest×N (HLS through the egress relay, playlists through the asset feeder) + * → gate → encoder, puts the program scene on air, and starts the preview taps. + * `reconfigure()` is the meaning of an accepted RUNNING snapshot while already + * RUNNING: a TAKE, layout, overlay or gain change is an A/B compositor swap, + * playlist edits are adopted by their feeders, and it resolves only once the + * new compositor is on air at a frame boundary. + */ +export class ProductionJob { + private ingests = new Map(); + private feeders = new Map(); + private chain: ProgramChain | null = null; + private taps: PreviewTaps | null = null; + private closeEgress: (() => Promise) | null = null; + private key = ""; + private canvas: Canvas | null = null; + lastSkipped: string[] = []; + + constructor(private readonly deps: ProductionJobDeps) {} + + get program(): ProgramChain | null { return this.chain; } + get previews(): PreviewTaps | null { return this.taps; } + feeder(sourceId: string): PlaylistFeeder | undefined { return this.feeders.get(sourceId); } + /** The source's bus (its ingest's fan-out), for stats. */ + sourceBus(sourceId: string): SourceBus | undefined { return this.ingests.get(sourceId)?.bus; } + + async start(config: DesiredChannel): Promise { + const spec = config.production; + if (!spec || !config.output) throw new Error("production job needs a production snapshot with an output"); + for (const url of snapshotHlsUrls(config)) validateEndpointShape(url, "hls"); + validateEndpointShape(config.output.endpointUrl, "output"); + const profile = OUTPUT_PROFILES[config.profile]; + const canvas: Canvas = { width: profile.width, height: profile.height, fps: profile.fps }; + this.canvas = canvas; + const egress = await this.deps.egress.prepare(config, this.deps.validator, this.deps.vault); + this.closeEgress = egress.close; + const lost = (id: string, reason: string): void => this.deps.onExit(`source ${id}: ${reason}`); + try { + for (const [sourceId, { input, fd }] of egress.sources) { + const ingest = new IngestProcess(sourceId, { input, canvas, audio: "source" }, fd, this.deps.scratchDir, this.deps.redactor, this.deps.onStderr, lost); + this.ingests.set(sourceId, ingest); + await ingest.start(); + } + const playlists = this.deps.playlists; + for (const src of spec.sources) { + if (src.kind !== "playlist" || !playlists) continue; + const feeder = new PlaylistFeeder(src.id, { assetIds: src.assetIds, loop: src.loop }, { + assets: playlists, canvas, scratchDir: this.deps.scratchDir, redactor: this.deps.redactor, onStderr: this.deps.onStderr, + onEvent: (e) => { if (e.type === "asset-failed" || e.type === "slate") this.deps.onStderr(`playlist ${src.id}: ${e.type}${"reason" in e ? ` ${e.reason}` : ""}`); }, + }); + this.feeders.set(src.id, feeder); + const fd = feeder.start(); + const ingest = new IngestProcess(src.id, { input: { kind: "playlistPipe" }, canvas, audio: "source" }, fd, this.deps.scratchDir, this.deps.redactor, this.deps.onStderr, lost); + this.ingests.set(src.id, ingest); + try { await ingest.start(); } finally { feeder.releaseReader(); } + } + this.chain = new ProgramChain({ + canvas, scratchDir: this.deps.scratchDir, videoBitrateKbps: Number.parseInt(profile.videoBitrate, 10), + output: egress.output, redactor: this.deps.redactor, onStderr: this.deps.onStderr, + onFatal: (reason) => this.deps.onExit(reason), gateImpl: this.deps.gateImpl, programTap: this.deps.preview !== undefined, + }); + await this.chain.start(await this.compose(config)); + this.key = pathKey(config); + } catch (error) { + await this.stop(); + throw error; + } + await this.startPreviews(config); + } + + /** + * Preview taps never gate the program: a tap that cannot start is reported and the channel runs on. + * The program tap's renditions are always on (§2.2: the PGM tile is always live); source renditions + * start on demand (`previews.demand()`). + */ + private async startPreviews(config: DesiredChannel): Promise { + const preview = this.deps.preview; + if (!preview || !this.chain?.programBus) return; + const taps = new PreviewTaps({ + publisher: new ManifestPreviewPublisher({ workspaceId: config.workspaceId, channelId: config.channelId }, preview.objects, preview.manifests), + scratchDir: this.deps.scratchDir, redactor: this.deps.redactor, onStderr: this.deps.onStderr, + onError: (tap, reason) => this.deps.onStderr(`preview ${tap}: ${reason}`), + }); + this.taps = taps; + const add = async (tap: string, bus: SourceBus, alwaysOn: HlsRendition[]): Promise => { + try { await taps.add(tap, bus, { alwaysOn }); } catch (error) { this.deps.onStderr(`preview ${tap}: ${String(error)}`); } + }; + await add("program", this.chain.programBus, ["video", "audio"]); + for (const [sourceId, ingest] of this.ingests) await add(sourceId, ingest.bus, []); + } + + /** false = this change cannot be a swap (sources/output/profile changed): the supervisor restarts. */ + async reconfigure(config: DesiredChannel): Promise { + if (!this.chain || !config.production || pathKey(config) !== this.key) return false; + for (const src of config.production.sources) { + if (src.kind === "playlist") this.feeders.get(src.id)?.update({ assetIds: src.assetIds, loop: src.loop }); + } + await this.chain.swap(await this.compose(config)); + return true; + } + private async compose(config: DesiredChannel): Promise { + if (!config.production || !this.canvas) throw new Error("no production spec"); + const plan = planScene(config.production, programScene(config.production), this.canvas, (id) => this.ingests.has(id)); + this.lastSkipped = plan.skipped; + const inputs = plan.sourceIds.map((id, i) => { + const ingest = this.ingests.get(id); + if (!ingest) throw new Error("planned source has no ingest"); + return { bus: ingest.bus, audio: plan.audio[i] ?? false }; + }); + const assets = await Promise.all(plan.assetIds.map((id) => this.deps.assets.localPath(id))); + return { inputs, assets, graph: plan.graph }; + } + + async stop(): Promise { + await this.taps?.stop(); + this.taps = null; + await this.chain?.stop(); + this.chain = null; + for (const feeder of this.feeders.values()) await feeder.stop(); + this.feeders.clear(); + for (const ingest of this.ingests.values()) await ingest.stop(); + this.ingests.clear(); + await this.closeEgress?.(); + this.closeEgress = null; + } +} diff --git a/apps/media-worker/src/compositor/program.ts b/apps/media-worker/src/compositor/program.ts new file mode 100644 index 0000000..5549b04 --- /dev/null +++ b/apps/media-worker/src/compositor/program.ts @@ -0,0 +1,194 @@ +import { CompositorLagError, FrameGate, type GateImpl } from "../gate/frame-gate"; +import type { GateEvent, GateStats } from "../gate/protocol"; +import { spawnManaged, type ManagedProcess, type StderrRedactor } from "../relay/ffmpeg"; +import { programTapGeometry } from "../preview/argv"; +import { buildEncoderArgs, type Canvas, type EncoderOutput, type EncoderProgramTap } from "./command-builder"; +import { CompositorInstance, type ComposeRequest } from "./compositor"; +import { SourceBus, type BusSubscription } from "./source-bus"; + +export interface ProgramChainOptions { + canvas: Canvas; + scratchDir: string; + videoBitrateKbps: number; + /** + * flvPipe: the push relay's inherited fd (WP9/WP13); null: x264 into `-f null` (CPU smoke); + * external: no encoder is spawned, the caller has opened the gate's encoder FIFOs (verifying sinks). + */ + output: EncoderOutput | { kind: "external" }; + redactor: StderrRedactor; + onStderr(message: string): void; + /** Encoder or gate died: the RTMP session is gone and the supervisor must restart the channel. */ + onFatal(reason: string): void; + /** The on-air compositor died (gate is repeating its last frame); the chain respawns it. */ + onCompositorLost?(reason: string): void; + gateImpl?: GateImpl; + /** Swap must go on air within this long, or B is abandoned and A stays (default 10 s). */ + swapTimeoutMs?: number; + /** + * Program preview tap (§2.1): the encoder also emits a 360p copy of what it encodes into + * `programBus`, which preview taps subscribe to. Needs an encoder (not `external`). + */ + programTap?: boolean; +} + +/** Child fd of the push relay's pipe in the encoder (flvPipe), and of the program tap outputs after it. */ +const ENCODER_OUTPUT_FD = 4; + +const LAG_RETRIES = 2; + +/** + * The persistent part of a channel's program path (§1.4): frame gate + encoder, + * started once, plus the A/B compositor swap. `swap()` starts B for the new + * scene, waits until the gate splices it in at a frame boundary, then stops A. + * The encoder and its push session are never restarted by a swap. + */ +export class ProgramChain { + readonly gate: FrameGate; + /** The program tap's bus (360p of the encoder input), when `programTap` is set. */ + readonly programBus: SourceBus | null; + private encoder: ManagedProcess | null = null; + private active: CompositorInstance | null = null; + private activeRequest: ComposeRequest | null = null; + private queue: Promise = Promise.resolve(); + private stopping = false; + swaps = 0; + /** Swap-in compositors replaced after a gate `LAG` report. */ + lagRetries = 0; + + constructor(private readonly options: ProgramChainOptions) { + // The Node reference gate has neither A/V pair lockstep nor origin alignment (s2a-report §5.3): + // swaps through it can step backwards and skew A/V. Spike/contract comparison only (gate/spike.ts, gate.test.ts). + if (options.gateImpl === "node") throw new Error("the Node reference gate is spike-only; a program chain needs the Rust gate"); + this.gate = new FrameGate({ + geometry: options.canvas, scratchDir: options.scratchDir, impl: options.gateImpl ?? "rust", + redactor: options.redactor, onStderr: options.onStderr, + onEvent: (event) => this.onGateEvent(event), + onExit: (reason) => { if (!this.stopping) options.onFatal(reason); }, + }); + if (options.programTap && options.output.kind === "external") throw new Error("the program tap needs an encoder"); + this.programBus = options.programTap ? new SourceBus({ + geometry: programTapGeometry(options.canvas), scratchDir: options.scratchDir, redactor: options.redactor, onStderr: options.onStderr, + // Losing the tap bus loses previews, never the program: the encoder's tee drops that slave. + onExit: (reason) => options.onStderr(`program tap bus: ${reason}`), + }) : null; + } + + get activeCompositor(): CompositorInstance | null { return this.active; } + get encoderPid(): number | undefined { return this.encoder?.child.pid; } + + async start(first: ComposeRequest): Promise { + const { canvas, output } = this.options; + if (output.kind === "external") { + await this.gate.start(); + await this.swap(first); + return; + } + const inputs = this.gate.encoderInputs(); + const stdio: Array = [inputs.videoFd, "ignore", "pipe", inputs.audioFd]; + // The push relay's fd is inherited as child fd 4, whatever its number in the worker. + let encoderOutput: EncoderOutput = output; + if (output.kind === "flvPipe") { stdio.push(output.fd); encoderOutput = { kind: "flvPipe", fd: ENCODER_OUTPUT_FD }; } + let programTap: EncoderProgramTap | undefined; + if (this.programBus) { + const tap = await this.programBus.start(); + const first = stdio.length; + stdio.push(tap.videoFd, tap.audioFd); + programTap = { videoFd: first, audioFd: first + 1, size: programTapGeometry(canvas) }; + } + let encoder: ManagedProcess; + try { + encoder = spawnManaged("ffmpeg", buildEncoderArgs({ canvas, videoBitrateKbps: this.options.videoBitrateKbps, audioBitrateKbps: 128, output: encoderOutput, programTap }), + stdio, this.options.redactor, this.options.onStderr); + } finally { + this.programBus?.releaseIngestWriters(); + } + this.encoder = encoder; + void encoder.exited.then(({ code, signal }) => { if (!this.stopping) this.options.onFatal(`encoder exited code=${code} signal=${signal}`); }); + await encoder.started; + await this.gate.start(); + await this.swap(first); + } + + /** Serialized A/B swap. Resolves with the gate tick at which the new compositor went on air. */ + swap(request: ComposeRequest): Promise { + const run = this.queue.catch(() => undefined).then(() => this.swapNow(request)); + this.queue = run; + return run; + } + + /** + * A B the gate reports as lagging (still behind A's on-air source position, typically a spawn + * starved of CPU while draining its startup backlog) is replaced by a fresh one; A stays on air. + */ + private async swapNow(request: ComposeRequest): Promise { + for (let attempt = 0; ; attempt += 1) { + try { + return await this.swapOnce(request); + } catch (error) { + if (!(error instanceof CompositorLagError) || attempt >= LAG_RETRIES) throw error; + this.lagRetries += 1; + } + } + } + + private async swapOnce(request: ComposeRequest): Promise { + const next = new CompositorInstance(this.gate, request, this.options.redactor, this.options.onStderr); + const current = this.active; + await next.start((subs) => (current ? this.originAfter(current, subs) : 0)); + let tick: number; + try { + tick = await this.gate.waitActive(next.slotId, this.options.swapTimeoutMs ?? 10_000); + } catch (error) { + try { this.gate.detach(next.slotId); } catch { /* gate gone */ } + await next.stop(); + throw error; + } + const old = this.active; + this.active = next; + this.activeRequest = request; + this.swaps += 1; + if (old) void old.stop(); + return tick; + } + + /** + * Origin of B relative to on-air A, in A's numbering: through a bus both use + * (exact: pair counters, minus pairs A had lost to backlog when B subscribed), else by wall clock. + * Pairs B loses after attach only make B *ahead*: a forward skip, never backwards. + */ + private originAfter(a: CompositorInstance, subs: BusSubscription[]): number | undefined { + if (a.origin === undefined) return undefined; + for (const sb of subs) { + const sa = a.subscriptions.find((x) => x.bus === sb.bus); + if (sa) return a.origin + sb.startPair - sa.startPair - (sb.droppedBefore(sa) ?? sa.dropped); + } + const sa = a.subscriptions[0]; + const sb = subs[0]; + if (!sa || !sb) return undefined; + return a.origin + Math.round((Number(sb.startedAt - sa.startedAt) / 1e9) * this.options.canvas.fps); + } + + private onGateEvent(event: GateEvent): void { + if (event.type !== "EOF" || this.stopping || !this.active || event.id !== this.active.slotId || !this.activeRequest) return; + // The on-air compositor died (crash, kill -9). The gate is already holding + // the last frame + silence, so the encoder never notices; respawn the scene. + const lost = this.active; + this.active = null; + void lost.stop(); + this.options.onCompositorLost?.(`compositor ${event.id} ended on air`); + void this.swap(this.activeRequest).catch((error: unknown) => this.options.onCompositorLost?.(`respawn failed: ${String(error)}`)); + } + + stats(): Promise { return this.gate.stats(); } + + async stop(): Promise { + this.stopping = true; + await this.queue.catch(() => undefined); + await this.active?.stop(); + this.active = null; + await this.gate.stop(); + await this.encoder?.terminate(); + this.encoder = null; + await this.programBus?.stop(); + } +} diff --git a/apps/media-worker/src/compositor/scene-plan.test.ts b/apps/media-worker/src/compositor/scene-plan.test.ts new file mode 100644 index 0000000..ab32324 --- /dev/null +++ b/apps/media-worker/src/compositor/scene-plan.test.ts @@ -0,0 +1,68 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { parseProductionSpec } from "../production"; +import { buildCompositorArgs } from "./command-builder"; +import { planScene, programScene } from "./scene-plan"; + +const id = (n: number) => `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`; +const sha = (c: string) => c.repeat(64); + +const spec = parseProductionSpec({ + sources: [ + { id: id(1), label: "Cam A", kind: "hls", url: "https://a.example/live.m3u8" }, + { id: id(2), label: "Cam B", kind: "hls", url: "https://b.example/live.m3u8" }, + { id: id(3), label: "Loop", kind: "playlist", assetIds: [id(9)], loop: true }, + ], + scenes: [ + { id: id(10), label: "Grid", layout: "2x2", slots: [ + { index: 0, sourceId: id(1), rect: null, fit: "cover", audioMode: "primary", gainDb: 0 }, + { index: 1, sourceId: id(2), rect: null, fit: "contain", audioMode: "on", gainDb: -6 }, + { index: 2, sourceId: id(3), rect: null, fit: "contain", audioMode: "on", gainDb: 0 }, + { index: 3, sourceId: id(1), rect: null, fit: "crop", audioMode: "on", gainDb: -12 }, + ] }, + { id: id(11), label: "PIP", layout: "pip", slots: [ + { index: 0, sourceId: id(2), rect: null, fit: "cover", audioMode: "primary", gainDb: -3 }, + { index: 1, sourceId: id(1), rect: { x: 6500, y: 6500, w: 3000, h: 3000 }, fit: "contain", audioMode: "off", gainDb: 0 }, + ] }, + ], + buses: { preview: id(11), program: id(10) }, + overlays: [ + { id: id(20), sceneId: id(10), kind: "chyron", enabled: true, layer: 2, position: { x: 0, y: 9000 }, revisionId: id(30), assetId: sha("c"), chyron: { speedPxPerSec: 120, stripWidthPx: 4000 }, clock: null }, + { id: id(21), sceneId: id(10), kind: "lower_third", enabled: true, layer: 1, position: { x: 500, y: 7500 }, revisionId: id(31), assetId: sha("a"), chyron: null, clock: null }, + { id: id(22), sceneId: id(10), kind: "logo", enabled: false, layer: 3, position: { x: 9000, y: 500 }, revisionId: id(32), assetId: sha("b"), chyron: null, clock: null }, + { id: id(23), sceneId: id(10), kind: "clock", enabled: true, layer: 4, position: { x: 9000, y: 500 }, revisionId: null, assetId: null, chyron: null, clock: { format: "HH:MM" } }, + { id: id(24), sceneId: id(11), kind: "lower_third", enabled: true, layer: 0, position: { x: 0, y: 0 }, revisionId: id(33), assetId: sha("d"), chyron: null, clock: null }, + ], + transition: { type: "cut" }, + audio: { masterGainDb: -3 }, +}); +const canvas = { width: 1920, height: 1080, fps: 30 }; +const hls = (sourceId: string) => spec.sources.find((s) => s.id === sourceId)?.kind === "hls"; + +test("program scene → typed graph: grid cells, shared source, playlist black, one mix input per source", () => { + const plan = planScene(spec, programScene(spec), canvas, hls); + assert.deepEqual(plan.sourceIds, [id(1), id(2)]); + assert.deepEqual(plan.audio, [true, true]); + assert.deepEqual(plan.graph.tiles.map((t) => t.source), [0, 1, null, 0]); + assert.deepEqual(plan.graph.tiles[3]?.rect, { x: 960, y: 540, w: 960, h: 540 }); + // Cam A is primary at 0 dB and "on" at -12 dB in slot 3: one input at the louder gain. + assert.deepEqual(plan.graph.audio, { inputs: [{ source: 0, gainDb: 0 }, { source: 1, gainDb: -6 }], masterGainDb: -3 }); + // Layer order (lower third under chyron); disabled logo and S2a-unsupported clock skipped. + assert.deepEqual(plan.assetIds, [sha("a"), sha("c")]); + assert.deepEqual(plan.graph.overlays, [{ kind: "image", asset: 0, x: 96, y: 810 }, { kind: "chyron", asset: 1, y: 972, speedPxPerSec: 120 }]); + assert.equal(plan.skipped.length, 2); + const args = buildCompositorArgs({ + sources: plan.sourceIds.map((_, i) => ({ video: { kind: "rawPipe" as const, fd: 4 + 2 * i }, audio: { kind: "rawPipe" as const, fd: 5 + 2 * i } })), + assets: plan.assetIds.map((a) => `/scratch/assets/${a}.png`), graph: plan.graph, output: { kind: "gate", videoFd: 1, audioFd: 3 }, + }); + assert.ok(!args.join(" ").includes("Cam A"), "labels never reach argv"); + assert.ok(!args.join(" ").includes("example"), "source URLs never reach the compositor"); +}); + +test("PIP scene: inset rect from basis points, audio off slot not mixed", () => { + const plan = planScene(spec, spec.scenes[1]!, canvas, hls); + assert.deepEqual(plan.sourceIds, [id(2), id(1)]); + assert.deepEqual(plan.audio, [true, false]); + assert.deepEqual(plan.graph.tiles[1]?.rect, { x: 1248, y: 702, w: 576, h: 324 }); + assert.deepEqual(plan.graph.audio.inputs, [{ source: 0, gainDb: -3 }]); +}); diff --git a/apps/media-worker/src/compositor/scene-plan.ts b/apps/media-worker/src/compositor/scene-plan.ts new file mode 100644 index 0000000..e7eb851 --- /dev/null +++ b/apps/media-worker/src/compositor/scene-plan.ts @@ -0,0 +1,81 @@ +import type { ProductionOverlay, ProductionScene, ProductionSpec } from "../production"; +import { layoutRects, rectFromBasisPoints, type Canvas, type CompositorGraph, type MixInput, type OverlaySpec, type TileSpec } from "./command-builder"; + +/** Desired-state v2 scene → typed compositor graph over a list of source ids (input order). */ +export interface ScenePlan { + sourceIds: string[]; + /** Same order as sourceIds: whether the compositor needs that source's audio. */ + audio: boolean[]; + assetIds: string[]; + graph: CompositorGraph; + /** Things in the snapshot this slice renders as black/skips, for the observed reason. */ + skipped: string[]; +} + +export function programScene(spec: ProductionSpec): ProductionScene { + const scene = spec.scenes.find((s) => s.id === spec.buses.program); + if (!scene) throw new Error("program bus names no scene"); + return scene; +} + +/** + * `available(sourceId)` says whether the worker has an ingest for that source: + * S2a ingests HLS sources only; playlist sources (S2b feeder) render black. + * Overlay order: `layer` ascending, snapshot order within a layer. + */ +export function planScene(spec: ProductionSpec, scene: ProductionScene, canvas: Canvas, available: (sourceId: string) => boolean): ScenePlan { + const skipped: string[] = []; + const sourceIds: string[] = []; + const indexOf = (id: string): number => { + let i = sourceIds.indexOf(id); + if (i < 0) { sourceIds.push(id); i = sourceIds.length - 1; } + return i; + }; + const grid = layoutRects(scene.layout, canvas); + const slots = [...scene.slots].sort((a, b) => a.index - b.index); + const tiles: TileSpec[] = []; + const gains = new Map(); + for (const slot of slots) { + const rect = slot.rect ? rectFromBasisPoints(slot.rect, canvas) : grid[slot.index]; + if (!rect) continue; + let source: number | null = null; + if (slot.sourceId !== null) { + if (available(slot.sourceId)) source = indexOf(slot.sourceId); + else skipped.push(`slot ${slot.index}: source ${slot.sourceId} has no ingest in this worker build`); + } + tiles.push({ source, rect, fit: slot.fit }); + if (source !== null && slot.audioMode !== "off") gains.set(source, Math.max(gains.get(source) ?? -Infinity, slot.gainDb)); + } + const audio = sourceIds.map((_, i) => gains.has(i)); + const mix: MixInput[] = [...gains.entries()].sort((a, b) => a[0] - b[0]).map(([source, gainDb]) => ({ source, gainDb })); + + const assetIds: string[] = []; + const overlays: OverlaySpec[] = []; + const ordered = spec.overlays + .map((o, i) => ({ o, i })) + .filter(({ o }) => o.sceneId === scene.id && o.enabled) + .sort((a, b) => a.o.layer - b.o.layer || a.i - b.i) + .map(({ o }) => o); + for (const o of ordered) { + const spec = overlaySpec(o, canvas, (assetId) => { + let i = assetIds.indexOf(assetId); + if (i < 0) { assetIds.push(assetId); i = assetIds.length - 1; } + return i; + }); + if (spec) overlays.push(spec); + else skipped.push(`overlay ${o.id}: ${o.kind} is not composited in S2a`); + } + return { + sourceIds, audio, assetIds, skipped, + graph: { canvas, layout: scene.layout, tiles, overlays, audio: { inputs: mix, masterGainDb: spec.audio.masterGainDb } }, + }; +} + +function overlaySpec(o: ProductionOverlay, canvas: Canvas, asset: (id: string) => number): OverlaySpec | null { + if (o.assetId === null) return null; // clock: server-generated drawtext is a later slice + const x = Math.min(canvas.width, Math.round((o.position.x * canvas.width) / 10_000)); + const y = Math.min(canvas.height, Math.round((o.position.y * canvas.height) / 10_000)); + if (o.kind === "chyron" && o.chyron) return { kind: "chyron", asset: asset(o.assetId), y, speedPxPerSec: o.chyron.speedPxPerSec }; + if (o.kind === "lower_third" || o.kind === "logo") return { kind: "image", asset: asset(o.assetId), x, y }; + return null; +} diff --git a/apps/media-worker/src/compositor/soak.ts b/apps/media-worker/src/compositor/soak.ts new file mode 100644 index 0000000..a5b91c3 --- /dev/null +++ b/apps/media-worker/src/compositor/soak.ts @@ -0,0 +1,275 @@ +/** + * S2 soak (streaming-architecture.md §6 S2: "the ≥8 h soak is wired into the + * nightly run, not blocking merge"). Drives the production path for hours and + * fails on drift or leaks: + * + * ProductionJob (4 paced 1080p30 MPEG-TS sources standing in for the WP7 + * relay + one looping playlist source through the asset feeder) → source-bus + * helpers → A/B compositors → Rust gate → x264 (-f null) + program tap, with + * preview taps (program LL-HLS always on, one source rendition kept demanded) + * publishing into an in-memory manifest store, and a scripted scene / overlay + * / gain swap every `--swap-every` s through `reconfigure()` (the RUNNING + * snapshot path the supervisor uses). + * + * Every `--sample-every` s one JSON line goes to `--out`: gate continuity + * counters, bus drops, swap latency, feeder and preview counters, and RSS + open + * fds per process. The last line is the summary; the exit code is 0 only if: + * - no process of the chain exited on its own and every swap went on air; + * - after the settle window: bus drops ≤ --max-bus-drops (default 0) and gate + * repeat/silence fills ≤ 1 per 1 000 ticks; + * - from the end of the first hour (or the first quarter of a shorter run) to + * the end: no process grows RSS by more than 15 % (+16 MiB) and fd counts + * move by at most 8. + * + * node --import tsx src/compositor/soak.ts [--hours 8] [--profile 1080p30|720p30] + * [--swap-every 30] [--sample-every 10] [--max-bus-drops 0] [--out soak.jsonl] + * + * Nightly: `pnpm --filter @tactical-media-broadcast-hub/media-worker soak -- --hours 8 --out + * runs/soak/$(date +%F).jsonl` on the reference host (see runs/wp4/s2b-report.md). + */ +import { appendFileSync, closeSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { basename, join } from "node:path"; +import { createReadStream } from "node:fs"; +import { tmpdir } from "node:os"; +import { spawnSync } from "node:child_process"; +import { parseProductionSpec, type ProductionSpec } from "../production"; +import type { PreviewContentType } from "../preview/contract"; +import type { ManagedProcess } from "../relay/ffmpeg"; +import { OUTPUT_PROFILES, type DesiredChannel, type OutputPreset } from "../types"; +import { ProductionJob } from "./production-job"; +import { encodeSourceClip, ensureGateBinary, makeAssets, passthroughRedactor, startPacer } from "./testing"; + +function opt(name: string, fallback: string): string { + const i = process.argv.indexOf(`--${name}`); + return i >= 0 ? (process.argv[i + 1] ?? fallback) : fallback; +} + +const id = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`; +const WS = "8f0c5a8e-3b1d-4c3e-9a5b-0c1d2e3f4a5b"; +const CH = "1a2b3c4d-5e6f-4a1b-8c2d-3e4f5a6b7c8d"; +const PLAYLIST = id(5); + +/** Swap i of the script: layouts over the 4 live sources and the playlist, overlays on/off, gains in 3 dB steps. */ +function scene(i: number, assetIds: [string, string]): ProductionSpec { + const layouts = ["2x2", "pip", "single", "3x3", "2x1"] as const; + const layout = layouts[i % layouts.length]!; + const order = [id(1), id(2), id(3), id(4), PLAYLIST]; + const rot = (k: number): string => order[(k + i) % order.length]!; + const count = { "2x2": 4, pip: 2, single: 1, "3x3": 5, "2x1": 2 }[layout]; + const slots = Array.from({ length: count }, (_, k) => ({ + index: k, sourceId: rot(k), + rect: layout === "pip" && k === 1 ? { x: 6500, y: 6500, w: 3000, h: 3000 } : null, + fit: (["cover", "contain", "crop"] as const)[(i + k) % 3]!, + audioMode: k === 0 ? "primary" as const : (k === 1 ? "on" as const : "off" as const), + gainDb: -3 * ((i + k) % 9), + })); + return parseProductionSpec({ + sources: [ + ...[1, 2, 3, 4].map((n) => ({ id: id(n), label: `Cam ${n}`, kind: "hls", url: `https://cam${n}.example/live.m3u8` })), + { id: PLAYLIST, label: "Loop", kind: "playlist", assetIds: [id(40), id(41), id(42)], loop: true }, + ], + scenes: [{ id: id(10), label: "Program", layout, slots }, { id: id(11), label: "Preview", layout: "single", slots: [] }], + buses: { preview: id(11), program: id(10) }, + overlays: [ + { id: id(20), sceneId: id(10), kind: "lower_third", enabled: i % 3 !== 0, layer: 1, position: { x: 500, y: 8000 }, revisionId: id(30), assetId: assetIds[0], chyron: null, clock: null }, + { id: id(21), sceneId: id(10), kind: "chyron", enabled: i % 2 === 0, layer: 2, position: { x: 0, y: 9400 }, revisionId: id(31), assetId: assetIds[1], chyron: { speedPxPerSec: 40 + ((i * 20) % 360), stripWidthPx: 600 }, clock: null }, + ], + transition: { type: "cut" }, + audio: { masterGainDb: -3 * (i % 3) }, + }); +} + +/** VmRSS (KiB) and open fds of a process. */ +function procStat(pid: number | undefined): { rssKiB: number; fds: number } | null { + if (pid === undefined) return null; + try { + const rss = /VmRSS:\s+(\d+)/.exec(readFileSync(`/proc/${pid}/status`, "utf8"))?.[1]; + return { rssKiB: Number(rss ?? 0), fds: readdirSync(`/proc/${pid}/fd`).length }; + } catch { return null; } +} + +function pids(name: string): number[] { + const r = spawnSync("pgrep", ["-f", name], { encoding: "utf8" }); + return r.stdout.split("\n").filter(Boolean).map(Number); +} + +async function main(): Promise { + const hours = Number(opt("hours", "8")); + const profile = opt("profile", "1080p30") as OutputPreset; + const swapEvery = Number(opt("swap-every", "30")); + const sampleEvery = Number(opt("sample-every", "10")); + const maxBusDrops = Number(opt("max-bus-drops", "0")); + const out = opt("out", join(process.cwd(), `soak-${new Date().toISOString().replace(/[:.]/g, "-")}.jsonl`)); + if (!(hours > 0) || !Number.isFinite(hours) || !OUTPUT_PROFILES[profile]) throw new Error("--hours must be > 0 and --profile one of 720p30|1080p30|1080p60"); + if (!(swapEvery > 0) || !Number.isFinite(swapEvery) || !(sampleEvery > 0) || !Number.isFinite(sampleEvery) || !Number.isSafeInteger(maxBusDrops) || maxBusDrops < 0) throw new Error("swap/sample intervals must be positive and max-bus-drops a non-negative integer"); + if (!ensureGateBinary()) throw new Error("native helpers unavailable (cargo build failed)"); + const { width, height, fps } = OUTPUT_PROFILES[profile]; + const canvas = { width, height, fps }; + const dir = mkdtempSync(join(tmpdir(), "soak-")); + let job: ProductionJob | null = null; + let exitCode = 2; + try { + writeFileSync(out, ""); + const log = (record: object): void => appendFileSync(out, `${JSON.stringify(record)}\n`); + + // Fixtures, prepared before the clock starts. + const clip = join(dir, "src.ts"); + encodeSourceClip(clip, canvas); + const clips = new Map(); + for (const [n, hz] of [[40, 300], [41, 600], [42, 900]] as const) { + const path = join(dir, `asset-${n}.mp4`); + const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", `testsrc2=s=${width}x${height}:r=${fps}:d=6`, "-f", "lavfi", "-i", `sine=frequency=${hz}:d=6`, + "-c:v", "libx264", "-preset", "veryfast", "-b:v", "4000k", "-g", String(fps), "-c:a", "aac", "-movflags", "+faststart", path]); + if (r.status !== 0) throw new Error("playlist asset encode failed"); + clips.set(id(n), path); + } + const assets = makeAssets(dir); + const assetIds = assets.map((p) => basename(p, ".png")) as [string, string]; + const pacers: ManagedProcess[] = []; + const pacerFds: number[] = []; + const exits: string[] = []; + const published = { count: 0, bytes: 0 }; + const manifests = new Map(); + + job = new ProductionJob({ + validator: { async validate() { throw new Error("not used: synthetic egress"); } }, + vault: { async deliverPushTarget() { throw new Error("not used: null output"); } }, + egress: { + async prepare(config) { + const sources = new Map(); + for (const src of config.production?.sources ?? []) { + if (src.kind !== "hls") continue; + const pacer = await startPacer(clip, dir, `pacer-${src.id.slice(-2)}`); + pacers.push(pacer.proc); + pacerFds.push(pacer.readFd); + sources.set(src.id, { input: { kind: "mpegtsPipe" }, fd: pacer.readFd }); + } + return { sources, output: { kind: "null" as const }, async close() { for (const p of pacers) await p.terminate(500); for (const fd of pacerFds.splice(0)) closeSync(fd); } }; + }, + }, + assets: { async localPath(assetId) { const p = assets.find((a) => basename(a, ".png") === assetId); if (!p) throw new Error("unknown overlay asset"); return p; } }, + playlists: { async open(assetId) { const p = clips.get(assetId); if (!p) throw new Error("unknown asset"); return createReadStream(p); } }, + preview: { + objects: { async putContent(body: Buffer, contentType: PreviewContentType) { published.count += 1; published.bytes += body.length; return { key: `sha256/${"0".repeat(64)}.${contentType === "image/jpeg" ? "jpg" : "bin"}` }; } }, + manifests: { async set(key: string) { manifests.set(key, (manifests.get(key) ?? 0) + 1); } }, + }, + redactor: passthroughRedactor, onStderr: () => undefined, + onExit: (reason) => exits.push(reason), + scratchDir: join(dir, "scratch"), + }); + const config = (version: number, i: number): DesiredChannel => ({ + workspaceId: WS, channelId: CH, workerId: "soak", assignmentEpoch: 1, desiredVersion: version, desiredStatus: "RUNNING", + source: null, output: { endpointUrl: "rtmp://push.example/live", credentialHandleId: "keyref-soak" }, profile, mode: "normalize", + production: scene(i, assetIds), + }); + + const t0 = Date.now(); + await job.start(config(1, 0)); + const runningJob = job; + const chain = runningJob.program!; + await runningJob.previews?.demand(id(1), "video").catch(() => undefined); + + const settleMs = Math.min(60_000, (hours * 3_600_000) / 10); + const endAt = t0 + hours * 3_600_000; + const swapLatency: number[] = []; + let swaps = 0; + let swapFailures = 0; + let nextSwap = Date.now() + swapEvery * 1000; + let nextSample = Date.now() + sampleEvery * 1000; + let settled: { ticks: number; fills: number; busDrops: number } | null = null; + const samples: Array<{ t: number; procs: Record }> = []; + const busDrops = async (): Promise => { + let total = 0; + for (const sid of [id(1), id(2), id(3), id(4), PLAYLIST]) { + const bus = runningJob.sourceBus(sid); + if (bus) total += (await bus.stats()).subscriberDrops; + } + return total; + }; + + while (Date.now() < endAt && exits.length === 0) { + const now = Date.now(); + if (now >= nextSwap) { + nextSwap += swapEvery * 1000; + const started = Date.now(); + try { + await runningJob.reconfigure(config(swaps + 2, swaps + 1)); + swaps += 1; + swapLatency.push(Date.now() - started); + } catch (error) { + swapFailures += 1; + log({ type: "swap-failed", t: (Date.now() - t0) / 1000, error: String(error) }); + } + } + if (now >= nextSample) { + nextSample += sampleEvery * 1000; + const gate = await chain.stats(); + const drops = await busDrops(); + if (!settled && now - t0 >= settleMs) settled = { ticks: gate.ticks, fills: gate.vrep + gate.asil, busDrops: drops }; + const procs: Record = { + worker: { rssKiB: Math.round(process.memoryUsage().rss / 1024), fds: readdirSync("/proc/self/fd").length }, + gate: procStat(chain.gate.pid), + encoder: procStat(chain.encoderPid), + compositor: procStat(chain.activeCompositor?.pid), + programBus: procStat(chain.programBus?.pid ?? undefined), + }; + pids(join(dir, "scratch", "bus-")).forEach((pid, k) => { procs[`bus${k}`] = procStat(pid); }); + samples.push({ t: now - t0, procs }); + const sorted = [...swapLatency].sort((a, b) => a - b); + log({ + type: "sample", t: (now - t0) / 1000, ticks: gate.ticks, vrep: gate.vrep, asil: gate.asil, outDrops: gate.outDrops, lagged: gate.lagged, + lateOverFrame: gate.lateOverFrame, busDrops: drops, swaps, swapFailures, lagRetries: chain.lagRetries, + swapMs: { p50: sorted[Math.floor(sorted.length / 2)] ?? null, max: sorted.at(-1) ?? null }, + feeder: runningJob.feeder(PLAYLIST)?.stats, previews: { ...published, manifests: manifests.size }, procs, + }); + // Keep the demanded source rendition alive (a viewer watching that tile). + await runningJob.previews?.demand(id(1), "video").catch(() => undefined); + } + await new Promise((r) => setTimeout(r, 250)); + } + + // Verdict. + const gate = await chain.stats(); + const drops = await busDrops(); + const failures: string[] = []; + if (!settled) failures.push("no settled sample collected"); + if (gate.outDrops !== 0) failures.push(`encoder output drops: ${gate.outDrops}`); + if (exits.length) failures.push(`chain exited: ${exits.join("; ")}`); + if (swapFailures) failures.push(`${swapFailures} swaps never went on air`); + if (settled) { + const ticks = gate.ticks - settled.ticks; + const fills = gate.vrep + gate.asil - settled.fills; + if (drops - settled.busDrops > maxBusDrops) failures.push(`bus drops after settle: ${drops - settled.busDrops}`); + if (fills > Math.ceil(ticks / 1000)) failures.push(`gate fills after settle: ${fills} over ${ticks} ticks`); + } + const baseAt = Math.min(3_600_000, (hours * 3_600_000) / 4); + const base = samples.find((s) => s.t >= baseAt); + const last = samples.at(-1); + const growth: Record = {}; + if (base && last && last !== base) { + for (const [name, b] of Object.entries(base.procs)) { + const l = last.procs[name]; + if (!b || !l) continue; + // Compositors are replaced by swaps; their size is not a leak signal. + if (name === "compositor") continue; + growth[name] = { rssPct: Number((((l.rssKiB - b.rssKiB) / b.rssKiB) * 100).toFixed(1)), fds: l.fds - b.fds }; + if (l.rssKiB > b.rssKiB * 1.15 + 16 * 1024) failures.push(`${name} RSS ${b.rssKiB} → ${l.rssKiB} KiB`); + if (Math.abs(l.fds - b.fds) > 8) failures.push(`${name} fds ${b.fds} → ${l.fds}`); + } + } + const summary = { + type: "summary", hours: (Date.now() - t0) / 3_600_000, profile, swaps, swapFailures, lagRetries: chain.lagRetries, + gate: { ticks: gate.ticks, vrep: gate.vrep, asil: gate.asil, outDrops: gate.outDrops, lagged: gate.lagged }, busDrops: drops, settled, + growth, previews: published, feeder: runningJob.feeder(PLAYLIST)?.stats, pass: failures.length === 0, failures, + }; + log(summary); + process.stdout.write(`${JSON.stringify(summary)}\n`); + exitCode = failures.length === 0 ? 0 : 1; + } finally { + await job?.stop(); + rmSync(dir, { recursive: true, force: true }); + } + process.exitCode = exitCode; +} + +if (require.main === module) void main().catch((error: unknown) => { process.stderr.write(`${String(error)}\n`); process.exit(2); }); diff --git a/apps/media-worker/src/compositor/source-bus.test.ts b/apps/media-worker/src/compositor/source-bus.test.ts new file mode 100644 index 0000000..3553d68 --- /dev/null +++ b/apps/media-worker/src/compositor/source-bus.test.ts @@ -0,0 +1,153 @@ +/** + * Source bus helper (src/gate/rust/src/bin/source-bus.rs) through its Node + * manager: the S2a bus semantics on the new out-of-process path. Frames are + * tagged (first byte = pair index) so delivery can be checked exactly. + */ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { Socket } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { audioFrameBytes, videoFrameBytes } from "../gate/protocol"; +import { ProgramChain } from "./program"; +import { SourceBus, type BusSubscription } from "./source-bus"; +import { ensureGateBinary, passthroughRedactor } from "./testing"; + +const g = { width: 64, height: 36, fps: 30 }; +const V = videoFrameBytes(g); +const A = audioFrameBytes(g); + +const frame = (bytes: number, tag: number): Buffer => Buffer.alloc(bytes, tag & 0xff); +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +async function until(cond: () => boolean | Promise, ms: number, what: string): Promise { + const t0 = Date.now(); + while (!(await cond())) { + if (Date.now() - t0 > ms) throw new Error(`timeout: ${what}`); + await sleep(20); + } +} + +type Harness = { bus: SourceBus; video: Socket; audio: Socket; dir: string; close(): Promise }; + +/** Writes pair i as an ingest does (interleaved): a burst of one stream would be trimmed as skew. */ +async function pair(h: Harness, i: number, audio = true): Promise { + await Promise.all([ + new Promise((r) => h.video.write(frame(V, i), () => r())), + audio ? new Promise((r) => h.audio.write(frame(A, i), () => r())) : Promise.resolve(), + ]); +} + +async function harness(): Promise { + const dir = mkdtempSync(join(tmpdir(), "bus-")); + const bus = new SourceBus({ geometry: g, scratchDir: dir, redactor: passthroughRedactor, onStderr: () => undefined }); + const fds = await bus.start(); + bus.adoptIngestWriters(); + const video = new Socket({ fd: fds.videoFd, readable: false, writable: true }); + const audio = new Socket({ fd: fds.audioFd, readable: false, writable: true }); + return { bus, video, audio, dir, async close() { video.destroy(); audio.destroy(); await bus.stop(); rmSync(dir, { recursive: true, force: true }); } }; +} + +/** Reads a subscriber's stream into whole frames (first byte of each). */ +function collect(fd: number | null, bytes: number): { tags: number[]; socket: Socket } { + assert.ok(fd !== null); + const tags: number[] = []; + let pending = Buffer.alloc(0); + const socket = new Socket({ fd, readable: true, writable: false }); + socket.on("data", (chunk: Buffer) => { + pending = Buffer.concat([pending, chunk]); + while (pending.length >= bytes) { tags.push(pending[0]!); pending = pending.subarray(bytes); } + }); + return { tags, socket }; +} + +test("history replay, start pair and A/V pairs delivered frame-exact", { timeout: 60_000 }, async (t) => { + if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return; } + const h = await harness(); + try { + for (let i = 0; i < 20; i += 1) await pair(h, i); + await until(async () => (await h.bus.stats()).pairs === 20, 5_000, "20 pairs"); + const sub = await h.bus.subscribe({ video: true, audio: true }); + assert.equal(sub.startPair, 20 - h.bus.historyFrames, "starts at the oldest replayed pair"); + const v = collect(sub.videoFd, V); + const a = collect(sub.audioFd, A); + sub.adoptReaders(); + for (let i = 20; i < 25; i += 1) await pair(h, i); + await until(() => v.tags.length >= 17 && a.tags.length >= 17, 5_000, "replay + live"); + const expected = Array.from({ length: 17 }, (_, k) => 8 + k); + assert.deepEqual(v.tags.slice(0, 17), expected, "video: history then live, no gap"); + assert.deepEqual(a.tags.slice(0, 17), expected, "audio pairs with video"); + const stats = await h.bus.stats(); + assert.equal(stats.subscriberDrops, 0); + assert.equal(stats.subscribers.length, 1); + v.socket.destroy(); a.socket.destroy(); + // The reader is gone: the helper drops the subscriber on EPIPE. + await pair(h, 25).catch(() => undefined); + await until(async () => (await h.bus.stats()).subscribers.length === 0, 5_000, "closed subscriber removed"); + } finally { await h.close(); } +}); + +test("stride subscribers get the newest pair then every n-th; audio-only and silence fill", { timeout: 60_000 }, async (t) => { + if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return; } + const h = await harness(); + try { + for (let i = 0; i < 10; i += 1) await pair(h, i); + await until(async () => (await h.bus.stats()).pairs === 10, 5_000, "10 pairs"); + const thumb = await h.bus.subscribe({ video: true, audio: false, stride: 5 }); + assert.equal(thumb.audioFd, null); + assert.equal(thumb.startPair, 9, "the newest pair first"); + const ears = await h.bus.subscribe({ video: false, audio: true }); + const tv = collect(thumb.videoFd, V); + const ea = collect(ears.audioFd, A); + thumb.adoptReaders(); ears.adoptReaders(); + for (let i = 10; i < 21; i += 1) await pair(h, i); + // The audio-only subscriber replays the 10 pairs of history there are, then 11 live ones. + await until(() => tv.tags.length >= 3 && ea.tags.length >= 10 + 11, 5_000, "strided + audio"); + assert.deepEqual(tv.tags.slice(0, 3), [9, 14, 19]); + assert.deepEqual(ea.tags.slice(-3), [18, 19, 20]); + // Video without audio: once video runs maxSkew ahead, pairs carry silence. + for (let i = 21; i < 21 + h.bus.maxSkewFrames + 5; i += 1) await pair(h, i, false); + await until(async () => (await h.bus.stats()).silenceFilled >= 5, 5_000, "silence fill"); + await until(() => ea.tags.at(-1) === 0, 5_000, "silence reaches the audio subscriber"); + tv.socket.destroy(); ea.socket.destroy(); + } finally { await h.close(); } +}); + +test("a stalled subscriber loses whole pairs past its backlog; peers' drop counts are snapshotted at subscribe", { timeout: 60_000 }, async (t) => { + if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return; } + const h = await harness(); + let stalled: BusSubscription | null = null; + try { + stalled = await h.bus.subscribe({ video: true, audio: true }); + // Never read: each 1 MiB pipe fills (audio first: 163 frames here), then the helper queue does. + const limit = h.bus.maxBacklogFrames + h.bus.historyFrames; + const youngLimit = limit + g.fps; + const produced = Math.ceil((1 << 20) / A) + youngLimit + 20; + for (let i = 0; i < produced; i += 1) await pair(h, i); + await until(async () => (await h.bus.stats()).pairs === produced, 10_000, "entire burst consumed"); + await until(async () => (await h.bus.stats()).subscriberDrops > 0, 10_000, "drops while young"); + const young = await h.bus.stats(); + const queued = young.subscribers[0]?.queued ?? 0; + assert.ok(queued > limit && queued <= youngLimit + 1, `a young subscriber queues up to ~1 s more than the limit (${queued} queued, limit ${limit})`); + const youngDrops = young.subscriberDrops; + await sleep(2_100); // young period over: the normal limit applies, and the queue is still above it + for (let i = 0; i < 10; i += 1) await pair(h, i); + await until(async () => (await h.bus.stats()).subscriberDrops >= youngDrops + 10, 5_000, "every new pair dropped"); + await until(() => stalled?.dropped === (youngDrops + 10), 5_000, "DROP events reach the subscription"); + const peer = await h.bus.subscribe({ video: true, audio: false }); + assert.equal(peer.droppedBefore(stalled), youngDrops + 10, "exact at the peer's start pair"); + await pair(h, 1); + await until(() => (stalled?.dropped ?? 0) > youngDrops + 10, 5_000, "live count moves on"); + assert.equal(peer.droppedBefore(stalled), youngDrops + 10, "the snapshot does not"); + peer.unsubscribe(); + } finally { + stalled?.unsubscribe(); + await h.close(); + } +}); + +test("program chains refuse the Node reference gate (no pair lockstep, no origin alignment)", () => { + assert.throws(() => new ProgramChain({ canvas: g, scratchDir: join(tmpdir(), "never"), videoBitrateKbps: 500, output: { kind: "external" }, gateImpl: "node", + redactor: passthroughRedactor, onStderr: () => undefined, onFatal: () => undefined }), /spike-only/); +}); diff --git a/apps/media-worker/src/compositor/source-bus.ts b/apps/media-worker/src/compositor/source-bus.ts new file mode 100644 index 0000000..4440a75 --- /dev/null +++ b/apps/media-worker/src/compositor/source-bus.ts @@ -0,0 +1,274 @@ +import { closeSync, mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { resolveHelperBinary } from "../gate/binary"; +import { createFifoPair, makeFifoPair, openBlockingReader, type FifoPair } from "../gate/fifo"; +import { assertGeometry, type FrameGeometry } from "../gate/protocol"; +import { spawnManaged, type ManagedProcess, type StderrRedactor } from "../relay/ffmpeg"; + +/** + * Per-source fan-out between one ingest and every consumer of the source's raw + * frames: the compositors that use it (two during an A/B swap) and its preview + * taps. + * + * The bytes never pass through Node. A `source-bus` helper process + * (src/gate/rust/src/bin/source-bus.rs) reads the ingest's FIFOs and writes + * each subscriber's FIFOs; this class is its process manager and speaks its + * control lines. (S2a moved the frames on the worker's event loop: at + * 4 × 1080p30 that left no headroom and the buses dropped ~7.5 % of source + * pairs; runs/wp4/s2b-report.md item 1.) + * + * Semantics are unchanged from S2a: frames are delivered as A/V *pairs*, a + * subscriber more than `maxBacklogFrames + historyFrames` behind loses whole + * pairs, and the last `historyFrames` pairs are replayed to a new subscriber so + * the gate can splice a swap-in compositor at the on-air one's next source frame. + */ +export interface SourceBusOptions { + geometry: FrameGeometry; + scratchDir: string; + redactor: StderrRedactor; + onStderr(message: string): void; + /** The helper died (not on `stop()`): the source is gone. */ + onExit?(reason: string): void; + maxSkewFrames?: number; + maxBacklogFrames?: number; + historyFrames?: number; +} + +export interface SubscribeOptions { + video: boolean; + audio: boolean; + /** Take every `stride`-th pair only (thumbnails). Default 1. */ + stride?: number; +} + +/** A subscription: the pair index it starts at, and the pairs it has lost to backlog since. */ +export interface BusSubscription { + readonly bus: SourceBus; + readonly id: number; + readonly startPair: number; + readonly startedAt: bigint; + readonly dropped: number; + /** Blocking read ends for the consumer's stdio (null when not subscribed to that stream). */ + readonly videoFd: number | null; + readonly audioFd: number | null; + /** Pairs `other` had lost when this subscription started (exact, unlike its live `dropped`). */ + droppedBefore(other: BusSubscription): number | undefined; + /** Close the worker's copies of the read ends (after spawning the consumer). */ + releaseReaders(): void; + /** The caller takes ownership of the read ends (an in-process consumer that closes them itself). */ + adoptReaders(): void; + unsubscribe(): void; +} + +export interface SourceBusStats { + pairs: number; silenceFilled: number; audioTrimmed: number; subscriberDrops: number; allocs: number; + input: { videoWaiting: number; audioWaiting: number; videoEof: boolean; audioEof: boolean }; + /** Per subscriber: pairs dropped to backlog, and frames queued in the helper right now. */ + subscribers: Array<{ id: number; dropped: number; queued: number }>; +} + +interface Waiter { match(line: string[]): boolean; resolve(line: string[]): void; reject(error: Error): void; timer: ReturnType } + +let nextBus = 1; + +export class SourceBus { + private proc: ManagedProcess | null = null; + private readonly busId = nextBus++; + private nextSub = 1; + private lineBuf = ""; + private waiters: Waiter[] = []; + private readonly drops = new Map(); + private stopping = false; + private exited = false; + private ingest: FifoPair | null = null; + readonly maxSkewFrames: number; + readonly maxBacklogFrames: number; + readonly historyFrames: number; + /** Set when the helper reports the ingest closed both streams. */ + ended = false; + + constructor(private readonly options: SourceBusOptions) { + assertGeometry(options.geometry); + this.maxSkewFrames = options.maxSkewFrames ?? 15; + this.maxBacklogFrames = options.maxBacklogFrames ?? 4; + this.historyFrames = options.historyFrames ?? 12; + mkdirSync(options.scratchDir, { recursive: true, mode: 0o700 }); + } + + get geometry(): FrameGeometry { return this.options.geometry; } + get pid(): number | undefined { return this.proc?.child.pid; } + + /** + * Starts the helper on fresh ingest FIFOs. The returned fds are blocking write ends for the + * producer's stdio (ingest: video on fd 1, PCM on fd 3); call `releaseIngestWriters()` once + * the producer has them. Until a producer writes, the bus simply has no pairs. + */ + async start(): Promise<{ videoFd: number; audioFd: number }> { + const g = this.options.geometry; + const fifos = createFifoPair(this.options.scratchDir, this.busId, "bus"); + this.ingest = fifos; + const argv = [String(g.width), String(g.height), String(g.fps), String(this.maxSkewFrames), String(this.maxBacklogFrames), String(this.historyFrames), + fifos.videoPath, fifos.audioPath]; + const proc = spawnManaged(resolveHelperBinary("source-bus"), argv, ["pipe", "pipe", "pipe"], this.options.redactor, this.options.onStderr); + this.proc = proc; + proc.child.stdout?.setEncoding("utf8"); + proc.child.stdout?.on("data", (chunk: string) => this.onData(chunk)); + // EPIPE on a control line to a helper that just died: its exit is reported below, never as a crash. + proc.child.stdin?.on("error", () => undefined); + void proc.exited.then(({ code, signal }) => { + this.exited = true; + const reason = `source bus exited code=${code} signal=${signal}`; + for (const w of this.waiters.splice(0)) { clearTimeout(w.timer); w.reject(new Error(reason)); } + if (!this.stopping) this.options.onExit?.(reason); + }); + const ready = this.waitFor((l) => l[0] === "READY", 10_000); + try { + await proc.started; + await ready; + } catch (error) { + ready.catch(() => undefined); + fifos.releaseHolders(); + fifos.releaseWriters(); + fifos.unlink(); + throw error; + } finally { + // The helper holds the read ends now; from here only the producer keeps the FIFOs open. + fifos.releaseHolders(); + fifos.unlink(); + } + return { videoFd: fifos.videoFd, audioFd: fifos.audioFd }; + } + + /** Close the worker's copies of the ingest write ends (after spawning the producer). */ + releaseIngestWriters(): void { this.ingest?.releaseWriters(); } + + /** The caller takes ownership of the ingest write ends (an in-process producer that closes them itself). */ + adoptIngestWriters(): void { this.ingest = null; } + + private onData(chunk: string): void { + this.lineBuf += chunk; + let nl = this.lineBuf.indexOf("\n"); + while (nl >= 0) { + const line = this.lineBuf.slice(0, nl).trim().split(/\s+/); + this.lineBuf = this.lineBuf.slice(nl + 1); + nl = this.lineBuf.indexOf("\n"); + if (line[0] === "DROP") this.drops.set(Number(line[1]), Number(line[2])); + if (line[0] === "EOF") this.ended = true; + const i = this.waiters.findIndex((w) => w.match(line)); + if (i >= 0) { + const [w] = this.waiters.splice(i, 1); + if (w) { clearTimeout(w.timer); w.resolve(line); } + } + } + } + + private waitFor(match: (line: string[]) => boolean, timeoutMs: number): Promise { + return new Promise((resolve, reject) => { + const waiter: Waiter = { + match, resolve, reject, + timer: setTimeout(() => { this.waiters = this.waiters.filter((w) => w !== waiter); reject(new Error("source bus reply timeout")); }, timeoutMs), + }; + this.waiters.push(waiter); + }); + } + + private send(line: string): void { + const stdin = this.proc?.child.stdin; + if (this.exited || !stdin || stdin.destroyed) throw new Error("source bus is not running"); + stdin.write(line); + } + + /** Sends a control line and waits for its reply; a failed send never leaves a waiter behind. */ + private request(line: string, match: (l: string[]) => boolean, timeoutMs: number): Promise { + const reply = this.waitFor(match, timeoutMs); + try { + this.send(line); + } catch (error) { + reply.catch(() => undefined); + const waiter = this.waiters.at(-1); + if (waiter) { clearTimeout(waiter.timer); this.waiters.pop(); } + throw error; + } + return reply; + } + + /** + * New consumer. Delivery starts with the replayed history (a strided subscriber gets only the + * newest pair), then live pairs; `startPair` is the first pair's index. Spawn the consumer with + * `videoFd`/`audioFd`, then `releaseReaders()`. + */ + async subscribe(opts: SubscribeOptions): Promise { + if (!opts.video && !opts.audio) throw new Error("subscription needs video or audio"); + const stride = opts.stride ?? 1; + if (!Number.isSafeInteger(stride) || stride < 1 || stride > 3600) throw new Error("stride out of range"); + const id = this.nextSub++; + const paths = makeFifoPair(this.options.scratchDir, id, `sub-${this.busId}`); + // The consumer (ffmpeg) needs *blocking* read ends; a temporary writer keeps each open() from + // blocking and is released once the helper has opened its non-blocking write end. + const video = opts.video ? openBlockingReader(paths.videoPath) : null; + const audio = opts.audio ? openBlockingReader(paths.audioPath) : null; + const unlink = (): void => { rmSync(paths.videoPath, { force: true }); rmSync(paths.audioPath, { force: true }); }; + let line: string[]; + try { + line = await this.request(`SUB ${id} ${video ? paths.videoPath : "-"} ${audio ? paths.audioPath : "-"} ${stride}\n`, + (l) => (l[0] === "SUBSCRIBED" || l[0] === "ERROR") && Number(l[1]) === id, 5_000); + if (line[0] !== "SUBSCRIBED") throw new Error("source bus could not open subscriber fifos"); + } catch (error) { + for (const r of [video, audio]) if (r) { r.releaseHolder(); closeSync(r.fd); } + throw error; + } finally { + video?.releaseHolder(); + audio?.releaseHolder(); + unlink(); + } + const peers = new Map(); + if (line[3] && line[3] !== "-") { + for (const entry of line[3].split(",")) { + const [peer, dropped] = entry.split(":").map(Number); + if (peer !== undefined && dropped !== undefined) peers.set(peer, dropped); + } + } + const drops = this.drops; + let readersOpen = true; + const subscription: BusSubscription = { + bus: this, id, startPair: Number(line[2]), startedAt: process.hrtime.bigint(), + get dropped() { return drops.get(id) ?? 0; }, + videoFd: video?.fd ?? null, audioFd: audio?.fd ?? null, + droppedBefore: (other) => (other.bus === this ? peers.get(other.id) : undefined), + releaseReaders() { if (readersOpen) { readersOpen = false; for (const r of [video, audio]) if (r) closeSync(r.fd); } }, + adoptReaders() { readersOpen = false; }, + unsubscribe: () => { + subscription.releaseReaders(); + drops.delete(id); + try { this.send(`UNSUB ${id}\n`); } catch { /* helper gone */ } + }, + }; + return subscription; + } + + async stats(): Promise { + const line = await this.request("STATS\n", (l) => l[0] === "STATS", 5_000); + const kv = new Map(line.slice(1).map((p) => p.split("=") as [string, string])); + const n = (k: string): number => Number(kv.get(k) ?? 0); + return { pairs: n("pairs"), silenceFilled: n("silence_filled"), audioTrimmed: n("audio_trimmed"), subscriberDrops: n("subscriber_drops"), allocs: n("allocs"), + input: { videoWaiting: n("video_waiting"), audioWaiting: n("audio_waiting"), videoEof: n("video_eof") !== 0, audioEof: n("audio_eof") !== 0 }, + subscribers: (kv.get("subs") ?? "-").split(",").filter((e) => e !== "-").map((e) => { + const [id, dropped, queued] = e.split(":").map(Number); + return { id: id ?? 0, dropped: dropped ?? 0, queued: queued ?? 0 }; + }) }; + } + + async stop(): Promise { + this.stopping = true; + const proc = this.proc; + this.ingest?.releaseWriters(); + if (!proc) return; + try { this.send("QUIT\n"); } catch { /* already gone */ } + const timer = setTimeout(() => proc.kill("SIGKILL"), 2_000); + await proc.exited; + clearTimeout(timer); + this.proc = null; + rmSync(join(this.options.scratchDir, `bus-${this.busId}.v.fifo`), { force: true }); + rmSync(join(this.options.scratchDir, `bus-${this.busId}.a.fifo`), { force: true }); + } +} diff --git a/apps/media-worker/src/compositor/testing.ts b/apps/media-worker/src/compositor/testing.ts new file mode 100644 index 0000000..79530a0 --- /dev/null +++ b/apps/media-worker/src/compositor/testing.ts @@ -0,0 +1,211 @@ +/** + * Test/spike fixtures for the compositor chain (not used in production paths). + */ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { closeSync, copyFileSync, openSync, readFileSync } from "node:fs"; +import { Socket } from "node:net"; +import { join } from "node:path"; +import { SCENE_LAYOUTS, type SceneLayout } from "../production"; +import { ensureHelperBinaries } from "../gate/binary"; +import { layoutRects, type Canvas, type CompositorGraph, type OverlaySpec, type PxRect } from "./command-builder"; +import type { FrameGate } from "../gate/frame-gate"; +import type { ComposeRequest } from "./compositor"; +import { audioFrameBytes, videoFrameBytes, type FrameGeometry } from "../gate/protocol"; +import type { SinkReport } from "../gate/sink"; +import { spawnManaged, type ManagedProcess } from "../relay/ffmpeg"; +import { SourceBus } from "./source-bus"; + +export const passthroughRedactor = { redact: (s: string): string => s }; + +/** + * Luma of source frame n: a frame counter that survives scaling. Solid frames + * scale to (almost) the same value; swscale can round ±1 at some ratios + * (4x4 tiles), so the counter moves in steps of 4 and the sink rounds. + */ +export function counterLuma(n: number): number { return 20 + 4 * (n % 50); } + +/** + * A live source paced by the wall clock that writes into a bus's ingest FIFOs + * (the bytes an ingest process would write): solid frames whose luma is the + * frame counter, and a 1500 Hz sine that holds a whole number of periods per + * frame at 24/25/30/60 fps — any splice that is not on a frame boundary, and + * any inserted silence, shows up at the sink. + */ +export class CounterSource { + readonly bus: SourceBus; + private timer: ReturnType | null = null; + private n = 0; + private readonly frames: Buffer[]; + private readonly pcm: Buffer; + private out: { video: Socket; audio: Socket } | null = null; + + constructor(private readonly g: FrameGeometry, scratchDir: string, amplitude = 4000) { + this.bus = new SourceBus({ geometry: g, scratchDir, redactor: passthroughRedactor, onStderr: (m) => process.stderr.write(m) }); + const luma = g.width * g.height; + this.frames = Array.from({ length: 50 }, (_, i) => { + const f = Buffer.alloc(videoFrameBytes(g), 128); + f.fill(counterLuma(i), 0, luma); + return f; + }); + const samples = audioFrameBytes(g) / 4; + this.pcm = Buffer.alloc(audioFrameBytes(g)); + for (let s = 0; s < samples; s += 1) { + const v = Math.round(amplitude * Math.sin((2 * Math.PI * 1500 * s) / 48_000)); + this.pcm.writeInt16LE(v, s * 4); + this.pcm.writeInt16LE(v, s * 4 + 2); + } + } + + get framesSent(): number { return this.n; } + + async start(): Promise { + const fds = await this.bus.start(); + this.bus.adoptIngestWriters(); + const open = (fd: number): Socket => new Socket({ fd, readable: false, writable: true }).on("error", () => undefined); + const out = { video: open(fds.videoFd), audio: open(fds.audioFd) }; + this.out = out; + const t0 = process.hrtime.bigint(); + const period = 1_000_000_000n / BigInt(this.g.fps); + const step = (): void => { + const now = process.hrtime.bigint(); + while (t0 + BigInt(this.n) * period <= now) { + out.video.write(this.frames[this.n % 50]!); + out.audio.write(this.pcm); + this.n += 1; + } + const wait = Number((t0 + BigInt(this.n) * period - process.hrtime.bigint()) / 1_000_000n); + this.timer = setTimeout(step, Math.max(0, wait)); + }; + step(); + } + + async stop(): Promise { + if (this.timer) clearTimeout(this.timer); + this.timer = null; + this.out?.video.destroy(); + this.out?.audio.destroy(); + this.out = null; + await this.bus.stop(); + } +} + +/** The verifying null encoder (src/gate/sink-cli.ts) as its own process, reading the gate's encoder FIFOs. */ +export class SinkProcess { + private proc: ManagedProcess | null = null; + private out = ""; + + async start(gate: FrameGate, g: FrameGeometry): Promise { + const cwd = join(__dirname, "../.."); + const proc = spawnManaged(process.execPath, ["--import", "tsx", join(__dirname, "../gate/sink-cli.ts"), + gate.encoderFifos.videoPath, gate.encoderFifos.audioPath, String(g.width), String(g.height), String(g.fps)], + ["pipe", "pipe", "pipe"], passthroughRedactor, (m) => process.stderr.write(m), cwd); + this.proc = proc; + proc.child.stdout?.setEncoding("utf8"); + proc.child.stdout?.on("data", (c: string) => { this.out += c; }); + const t0 = Date.now(); + while (!this.out.includes("OPEN\n")) { + if (Date.now() - t0 > 15_000) throw new Error("sink did not open"); + await new Promise((r) => setTimeout(r, 10)); + } + } + + async report(firstActiveTick: number): Promise { + const proc = this.proc; + if (!proc) throw new Error("sink not started"); + proc.child.stdin?.write(`REPORT ${firstActiveTick}\n`); + await proc.exited; + const line = this.out.split("\n").find((l) => l.startsWith("{")); + if (!line) throw new Error("sink produced no report"); + return JSON.parse(line) as SinkReport & { silentAfterFirstActive: number }; + } + + /** Also close the sink when a gate assertion fails before REPORT is requested. */ + async stop(): Promise { + await this.proc?.terminate(500); + this.proc = null; + } +} + +/** H.264 + AAC MPEG-TS clip at the canvas size (encoded once, before measuring): the stand-in for a live HLS source. */ +export function encodeSourceClip(path: string, canvas: Canvas, seconds = 12): void { + const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", "-i", `testsrc2=s=${canvas.width}x${canvas.height}:r=${canvas.fps}:d=${seconds}`, + "-f", "lavfi", "-i", `sine=frequency=500:sample_rate=48000:d=${seconds}`, + "-c:v", "libx264", "-preset", "veryfast", "-b:v", "6000k", "-g", String(2 * canvas.fps), "-c:a", "aac", "-b:a", "128k", "-f", "mpegts", path]); + if (r.status !== 0) throw new Error("source clip encode failed"); +} + +/** + * Stand-in for the pinned egress relay (WP7): a paced, looping remux of `clip` into a FIFO. + * Returns the read end for an `mpegtsPipe` ingest's stdin (close it after spawning the ingest). + */ +export async function startPacer(clip: string, dir: string, name: string): Promise<{ readFd: number; proc: ManagedProcess }> { + const fifo = join(dir, `${name}.fifo`); + spawnSync("mkfifo", [fifo]); + const readFd = openSync(fifo, "r+"); + const writeFd = openSync(fifo, "w"); + const proc = spawnManaged("ffmpeg", ["-hide_banner", "-loglevel", "error", "-re", "-stream_loop", "-1", "-i", clip, "-c", "copy", "-f", "mpegts", "pipe:1"], + ["ignore", writeFd, "pipe"], passthroughRedactor, () => undefined); + await proc.started; + closeSync(writeFd); + return { readFd, proc }; +} + +export const ZERO_GAP_CANVAS: Canvas = { width: 320, height: 180, fps: 30 }; +const canvas = ZERO_GAP_CANVAS; + +/** Both native helpers (frame gate + source bus), built offline if missing or stale (src/gate/binary.ts). */ +export function ensureGateBinary(): boolean { return ensureHelperBinaries(); } + +export function makeAssets(dir: string): string[] { + const out: string[] = []; + for (const src of ["color=c=red@0.6:s=160x24", "testsrc2=s=600x16"]) { + const tmp = join(dir, `tmp-${out.length}.png`); + const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", "-i", `${src},format=rgba`, "-frames:v", "1", "-fflags", "+bitexact", "-flags", "+bitexact", tmp]); + assert.equal(r.status, 0, "asset render"); + const path = join(dir, `${createHash("sha256").update(readFileSync(tmp)).digest("hex")}.png`); + copyFileSync(tmp, path); + out.push(path); + } + return out; +} + +const CUSTOM: PxRect[] = [{ x: 0, y: 0, w: 160, h: 90 }, { x: 160, y: 90, w: 160, h: 90 }, { x: 100, y: 40, w: 120, h: 68 }]; + +/** Swap i: cycles layouts × overlay sets × gains. Slot 0 always shows source 0 at the top-left (the counter pixel). */ +export function scriptedGraph(i: number): CompositorGraph { + const layouts = SCENE_LAYOUTS; + const layout: SceneLayout = layouts[i % layouts.length]!; + const rects = layout === "pip" ? [{ x: 0, y: 0, w: 320, h: 180 }, { x: 200, y: 100, w: 100, h: 56 }] + : layout === "custom" ? CUSTOM : layoutRects(layout, canvas); + const tiles = rects.map((rect, t) => ({ source: t === 0 ? 0 : (t % 5 === 4 ? null : t % 4), rect, fit: (["contain", "cover", "crop"] as const)[(i + t) % 3]! })); + if (tiles[0]) tiles[0].fit = "cover"; // cover/crop keep the counter pixel (0,0) on picture, contain letterboxes it + const overlays: OverlaySpec[] = []; + if (i % 3 !== 0) overlays.push({ kind: "image", asset: 0, x: 10, y: 130 }); + if (i % 2 === 0) overlays.push({ kind: "chyron", asset: 1, y: 160, speedPxPerSec: 40 + ((i * 20) % 360) }); + const gain = -3 * (i % 9); + const audible = [...new Set(tiles.map((t) => t.source).filter((s): s is number => s !== null))].slice(0, 1 + (i % 3)); + return { canvas, layout, tiles, overlays, audio: { inputs: audible.map((source, k) => ({ source, gainDb: k === 0 ? gain : -6 })), masterGainDb: -3 * (i % 3) } }; +} + + +/** + * Compose request over only the sources the graph uses (renumbered), as + * `planScene` does in production: ffmpeg never reads an input that is not wired + * into the filtergraph, so an unused bus subscription would just stall. + */ +export function composeOver(buses: SourceBus[], assets: string[], graph: CompositorGraph): ComposeRequest { + const used = [...new Set(graph.tiles.map((t) => t.source).concat(graph.audio.inputs.map((m) => m.source)).filter((s): s is number => s !== null))].sort((a, b) => a - b); + const index = (s: number): number => used.indexOf(s); + const audible = new Set(graph.audio.inputs.map((m) => m.source)); + return { + inputs: used.map((s) => ({ bus: buses[s]!, audio: audible.has(s) })), + assets, + graph: { + ...graph, + tiles: graph.tiles.map((t) => ({ ...t, source: t.source === null ? null : index(t.source) })), + audio: { ...graph.audio, inputs: graph.audio.inputs.map((m) => ({ ...m, source: index(m.source) })) }, + }, + }; +} diff --git a/apps/media-worker/src/compositor/wp12-hardening.test.ts b/apps/media-worker/src/compositor/wp12-hardening.test.ts new file mode 100644 index 0000000..06e61a3 --- /dev/null +++ b/apps/media-worker/src/compositor/wp12-hardening.test.ts @@ -0,0 +1,492 @@ +/** + * WP12 hardening tests (flash slice) — apps/media-worker/src/compositor only. + * + * Two layers of the §4.3 "Injection" defense are exercised here: + * + * 1. Graph-builder injection at the type/argv layer. Hostile payloads + * (`movie=`, `amovie=`, `filename=`, protocol prefixes, shell + * metacharacters, newline/graph-chaining) are injected into every + * string-typed input surface of the compositor plan. Either the typed + * validators reject the plan (CommandBuildError) or the produced argv is + * proven inert: the payload never appears, not even escaped. + * + * 2. Crop-mode full matrix: contain/cover/crop × every scene layout — + * per-tile scale/pad/crop chains asserted from the generated filtergraph + * and argv only (no ffmpeg execution). + * + * Vitest suite: `node_modules/.bin/vitest` of a sibling workspace app resolves + * this file (the media-worker devDependencies install only tsx/node:test); + * see the run note at the bottom of this file. + */ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; + +import { + buildCompositorArgs, + buildFilterGraph, + CommandBuildError, + layoutRects, + rectFromBasisPoints, + SYNTHETIC_PATTERNS, + type Canvas, + type CompositorPlan, + type OverlaySpec, +} from "./command-builder"; + +const canvas: Canvas = { width: 320, height: 180, fps: 30 }; +const W = canvas.width; +const H = canvas.height; + +function synthetic(i: number) { + return { + video: { kind: "synthetic" as const, pattern: SYNTHETIC_PATTERNS[i % SYNTHETIC_PATTERNS.length]!, seed: 1000 + i }, + audio: { kind: "synthetic" as const, toneHz: 220 + 110 * i }, + }; +} + +const sha = (c: string) => c.repeat(64); +/** Valid absolute scratch asset paths (what `assetPath` accepts). */ +const ASSET_A = `/scratch/assets/${sha("a")}.png`; +const ASSET_B = `/scratch/assets/${sha("b")}.png`; + +/** Minimal valid plan: one live full-frame tile, one image overlay, silent mix. */ +function basePlan(): CompositorPlan { + return { + sources: [synthetic(0)], + assets: [ASSET_A], + graph: { + canvas, + layout: "single", + tiles: [{ source: 0, rect: { x: 0, y: 0, w: W, h: H }, fit: "contain" }], + overlays: [{ kind: "image", asset: 0, x: 8, y: 8 }], + audio: { inputs: [], masterGainDb: 0 }, + }, + output: { kind: "gate", videoFd: 1, audioFd: 3 }, + }; +} + +/** Input-index map that buildFilterGraph expects for a one-source/one-asset plan. */ +const IDX1 = { video: [0], audio: [1], assets: [2] }; + +function graphOf(plan: CompositorPlan): string { + return buildFilterGraph(plan, IDX1); +} + +function argvOf(plan: CompositorPlan): string[] { + return buildCompositorArgs(plan); +} + +/** + * A payload is inert when it is absent from every argv element — including the + * escaped forms a filtergraph string could need (`\;`, `\'`, `\\\,` …): the + * builder either drops or rejects it, it never transcodes it into a graph. + */ +function expectInert(plan: CompositorPlan, payload: string): void { + const argv = argvOf(plan); + const escaped = payload.replace(/([[\];,'\\])/g, "\\$1"); + for (const arg of argv) { + assert.ok(!(arg).includes(payload)); + assert.ok(!(arg).includes(escaped)); + } + assert.ok(!(argv.join(" ")).includes(escaped)); +} + +/** Every mutation must either throw CommandBuildError or leave the payload out of argv. */ +function expectRejectedOrInert(mutate: (plan: CompositorPlan) => CompositorPlan, payload: string): void { + const plan = mutate(basePlan()); + try { + argvOf(plan); + } catch (error) { + assert.ok((error) instanceof CommandBuildError); + return; + } + expectInert(plan, payload); +} + +// --------------------------------------------------------------------------- +// 1. Graph-builder injection: every string-typed input surface +// --------------------------------------------------------------------------- + +/** [wb] WP12 payload classes, each must be rejected or proven inert. */ +const PAYLOADS: Array<[string, string]> = [ + ["lavfi movie/amovie", "movie=evil.mp4"], + ["lavfi amovie", "amovie=evil.mp3"], + ["lavfi filename", "filename=/etc/passwd"], + ["lavfi chained filter", "color=c=black:s=320x180,drawtext=text=pwned"], + ["file protocol", "file:/etc/passwd"], + ["http url", "http://evil.example/x.mp4"], + ["https url", "https://evil.example/x.m3u8"], + ["pipe protocol", "pipe:0"], + ["subfile protocol", "subfile,,start=0,end=0,,:/etc/passwd"], + ["concat protocol", "concat:/etc/passwd|/etc/shadow"], + ["data protocol", "data:text/plain,pwned"], + ["hls protocol", "hls+http://evil.example/x.m3u8"], + ["rtsp url", "rtsp://evil.example:8554/live"], + ["shell separator", ";rm -rf /"], + ["shell pipe", "|tee /tmp/pwned"], + ["shell command substitution", "$(id -u)"], + ["shell backtick substitution", "`id -u`"], + ["shell redirect", ">/tmp/pwned"], + ["shell and-chain", "&&rm -rf /"], + ["shell or-chain", "||rm -rf /"], + ["graph pad link", "[0:v]null[sink]"], + ["graph output pad", "[vout]"], + ["graph chain separator", "null;nullsink"], + ["filter option comma", "scale=1:1,scale=2:2"], + ["filter option equals", "rate=999"], + ["quoted value", "'$(id -u)'"], + ["newline chaining", "a\nb"], + ["crlf chaining", "a\r\nb"], + ["newline + filter", "\nmovie=evil.mp4"], + ["tab separator", "\tmovie=evil.mp4"], + ["unicode line separator", "a\u2028b"], + ["control char", "a\x01b"], + ["backslash escape", "\\;movie=evil.mp4"], +]; + +/** One maximal payload spanning every class at once. */ +const KITCHEN_SINK = "movie=x|amovie=y|filename=z;[0:v]null[v];$(id)|`id`|\nhttp://e/x|file:/e"; + +describe("WP12 injection: overlay asset paths (plan.assets[i])", () => { + const withAsset = (path: string): CompositorPlan => ({ ...basePlan(), assets: [path] }); + + for (const [_name, payload] of PAYLOADS) { + it(`rejects or neutralizes: ${_name}`, () => { + // Raw path form… + expectRejectedOrInert(withAsset(payload), payload); + // …and smuggled inside an otherwise-valid-looking scratch path shape. + const smuggle = `/scratch/assets/${payload.replace(/[/\s]/g, "-")}-${sha("p")}.png`; + expectRejectedOrInert(withAsset(smuggle), payload); + }); + + it("rejects traversal and traversal-lookalikes and non-scratch shapes", () => { + for (const path of [ + "/scratch/../etc/passwd.png", + `/scratch/assets/../${sha("a")}.png`, + `/scratch/assets/${sha("a")}.png/../../${sha("b")}.png`, + `relative/${sha("a")}.png`, + `${sha("a")}.png`, + `/scratch/assets/${sha("a")}.PNG`, + `/scratch/assets/${sha("a").slice(0, 63)}.png`, + `/scratch/assets/${sha("a")} extra.png`, + "", + ]) { + assert.throws(() => argvOf(withAsset(path)), path, CommandBuildError); + } + }); + + it("rejects every protocol prefix as an asset path", () => { + for (const prefix of ["file:", "http:", "https:", "pipe:", "fd:", "concat:", "subfile,", "data:", "hls+", "rtsp:", "movie=", "amovie=", "filename="]) { + assert.throws(() => argvOf(withAsset(`${prefix}${sha("a")}.png`)), prefix, CommandBuildError); + } + }); + + it("rejects shell and graph metacharacters anywhere in an asset path", () => { + for (const ch of [";", "&", "|", "<", ">", "`", "$", "(", ")", "'", '"', "\\", "[", "]", ",", "\n", "\r", "\t", " ", "=", ":"]) { + const path = `/scratch/assets/x${ch}${sha("a").slice(0, 60)}.png`; + assert.throws(() => argvOf(withAsset(path)), CommandBuildError); + } + }); + + it("argv carries valid assets only as closed file:.png inputs", () => { + const argv = argvOf(basePlan()); + assert.ok((argv).includes(`file:${ASSET_A}`)); + const fileArgs = argv.filter((a) => a.startsWith("file:")); + assert.deepStrictEqual(fileArgs, [`file:${ASSET_A}`]); + }); +}); + +describe("WP12 injection: overlay geometry and asset indexes (typed, no string surface)", () => { + const withOverlay = (o: OverlaySpec): CompositorPlan => ({ ...basePlan(), graph: { ...basePlan().graph, overlays: [o] } }); + + it("rejects an overlay naming a nonexistent asset index", () => { + assert.throws(() => argvOf(withOverlay({ kind: "image", asset: 9, x: 0, y: 0 })), /asset/); + assert.throws(() => argvOf(withOverlay({ kind: "image", asset: -1, x: 0, y: 0 })), /asset/); + assert.throws(() => argvOf(withOverlay({ kind: "chyron", asset: 2, y: 0, speedPxPerSec: 120 })), /asset/); + }); + + it("chyron x-expression is built only from a bounded integer speed — payloads cannot ride the expression", () => { + const plan = basePlan(); + plan.graph.overlays = [{ kind: "chyron", asset: 0, y: 160, speedPxPerSec: 120 }]; + const graph = graphOf(plan); + assert.ok(!/overlay=x='W-mod/.test(graph)); + assert.ok(!(graph).includes("movie=")); + // Out-of-band speeds are rejected before any expression string is built. + for (const speed of [0, 39, 401, 1e9, -120, 12.5, Number.NaN]) { + const bad: CompositorPlan = { ...plan, graph: { ...plan.graph, overlays: [{ kind: "chyron", asset: 0, y: 160, speedPxPerSec: speed }] } }; + assert.throws(() => argvOf(bad), String(speed), CommandBuildError); + } + }); + + it("overlay geometry is integer-bounded; hostile numbers are type errors, not payload carriers", () => { + for (const x of [-1, W + 2, 0.5, Number.NaN, Number.POSITIVE_INFINITY]) { + assert.throws(() => argvOf(withOverlay({ kind: "image", asset: 0, x, y: 8 })), String(x), CommandBuildError); + } + for (const y of [-2, H + 4, 0.5, Number.NaN]) { + assert.throws(() => argvOf(withOverlay({ kind: "chyron", asset: 0, y, speedPxPerSec: 120 })), String(y), CommandBuildError); + } + }); +}); + +describe("WP12 injection: labels and source ids never reach the graph or argv", () => { + it("hostile labels on a desired-state snapshot leave no trace in compositor argv", async () => { + const { parseProductionSpec } = await import("../production"); + const { planScene, programScene } = await import("./scene-plan"); + const id = (n: number) => `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`; + const payloads = [ + "movie=evil.mp4", + "amovie=evil.mp3", + "filename=/etc/passwd", + "; rm -rf /", + "$(id -u)", + "`id`", + "[0:v]null[sink]", + "http://evil.example/x.m3u8", + "file:/etc/passwd", + "a\nmovie=evil.mp4", + "pipe:0", + KITCHEN_SINK, + ]; + // parseProductionSpec must reject all of these as labels (strict allowlist). + for (const label of payloads) { + assert.throws(() => parseProductionSpec(labelSpec(id, label))); + } + // A spec with clean labels plans/compiles; ids, labels and URLs stay out of argv. + const spec = parseProductionSpec(labelSpec(id, "Cam A")); + const plan = planScene(spec, programScene(spec), canvas, (sourceId) => sourceId === id(1)); + const argv = argvOf({ + sources: plan.sourceIds.map((_, i) => ({ video: { kind: "rawPipe" as const, fd: 4 + 2 * i }, audio: null })), + assets: plan.assetIds.map((a) => `/scratch/assets/${a}.png`), + graph: plan.graph, + output: { kind: "gate", videoFd: 1, audioFd: 3 }, + }); + const joined = argv.join(" "); + for (const payload of payloads) { + assert.ok(!(joined).includes(payload)); + assert.ok(!(joined).includes(payload.replace(/([[\];,'\\])/g, "\\$1"))); + } + assert.ok(!(joined).includes("Cam")); + assert.ok(!(joined).includes("example")); + }); + + it("source ids are never argv material: synthetic plans reference patterns and integers only", () => { + const argv = argvOf(basePlan()); + const patternArg = argv.find((a) => a.includes(`testsrc2=s=${W}x${H}`)); + assert.notStrictEqual(patternArg, undefined); + assert.ok(!/[;|`$]|\bmovie=|\bamovie=|\bfilename=/.test(patternArg)); + // Input references are structured (`:v`, `:a`) — never id strings. + assert.ok(new RegExp(/\[0:v\]/).test(graphOf(basePlan()))); + }); +}); + +describe("WP12 injection: filtergraph token provenance (labels, splits, pads)", () => { + it("graph labels are builder-owned; a payload cannot introduce new pad labels", () => { + const plan = basePlan(); + plan.graph.tiles.push({ source: 0, rect: rectFromBasisPoints({ x: 6000, y: 6000, w: 3500, h: 3500 }, canvas), fit: "contain" }); + const graph = graphOf(plan); + // split outputs and tile labels follow the fixed grammar [s_] / [t] / [ov] / [base] / [vout] / [aout] + assert.ok(new RegExp(/\[0:v\]split=2\[s0_0\]\[s0_1\]/).test(graph)); + assert.ok(new RegExp(/\[s0_1\]scale=96:96/).test(graph)); + assert.ok(new RegExp(/\[t1\]overlay=x=192:y=108/).test(graph)); + const labels = graph.match(/\[[a-z0-9_:]+\]/g) ?? []; + assert.ok((labels.length) > (0)); + assert.strictEqual(labels.every((m) => /^(?:\d+:v|\d+:a|s\d+_\d+|t\d+|ov\d+|c\d+|m\d+|bg|base|vout|aout)$/.test(m.slice(1, -1))), true); + assert.ok(!(graph).includes("[v]")); + assert.ok(!(graph).includes("[sink]")); + }); + + it("the full filtergraph never contains a payload in raw or escaped form", () => { + const plan = basePlan(); + plan.graph.tiles.push({ source: 0, rect: rectFromBasisPoints({ x: 6000, y: 6000, w: 3500, h: 3500 }, canvas), fit: "cover" }); + plan.graph.overlays = [ + { kind: "image", asset: 0, x: 8, y: 8 }, + { kind: "chyron", asset: 0, y: 160, speedPxPerSec: 200 }, + ]; + plan.graph.audio = { inputs: [{ source: 0, gainDb: -6 }], masterGainDb: -3 }; + const graph = graphOf(plan); + for (const [, payload] of PAYLOADS) { + assert.ok(!(graph).includes(payload)); + assert.ok(!(graph).includes(payload.replace(/([\[\];,'\\])/g, "\\$1"))); + } + assert.ok(!(graph).includes(KITCHEN_SINK)); + }); + + it("kitchen-sink payload stays inert end to end (argv layer)", () => { + const argv = argvOf(basePlan()); + const joined = argv.join(" "); + assert.ok(!(joined).includes(KITCHEN_SINK)); + assert.ok(!(joined).includes("movie=")); + assert.ok(!(joined).includes("amovie=")); + assert.ok(!(joined).includes("filename=")); + assert.ok(!(joined).includes("$(id")); + assert.ok(!(joined).includes("`id`")); + }); + + it("single quotes appear only as the chyron x-expression quoting around the builder-owned expression", () => { + const single = (plan: CompositorPlan): CompositorPlan => { + plan.graph.overlays = [{ kind: "chyron", asset: 0, y: 160, speedPxPerSec: 80 }]; + return plan; + }; + const graph = graphOf(single(basePlan())); + const quoted = graph.match(/'[^']*'/g) ?? []; + assert.deepStrictEqual(quoted, ["W-mod(t*80\\,w+W)"]); + // No quote-escape sequences exist anywhere in the graph. + assert.ok(!(graph).includes("\\'")); + assert.ok(!(graph).includes("''")); + }); +}); + +// --------------------------------------------------------------------------- +// 2. Crop-mode full matrix: contain/cover/crop × layouts +// --------------------------------------------------------------------------- + +/** + * `tileFilter` chains per fit (from command-builder.ts): + * contain → scale(w,h:force_original_aspect_ratio=decrease:force_divisible_by=2) → pad(w,h,centered,black) → setsar=1 + * cover → scale(w,h:force_original_aspect_ratio=increase:force_divisible_by=2) → crop(w,h) → setsar=1 + * crop → crop(w,h) → setsar=1 + */ +const FIT_CHAIN: Record<"contain" | "cover" | "crop", (w: number, h: number) => RegExp> = { + contain: (w, h) => new RegExp(`^scale=${w}:${h}:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=${w}:${h}:\\(ow-iw\)/2:\\(oh-ih\)/2:color=black,setsar=1$`), + cover: (w, h) => new RegExp(`^scale=${w}:${h}:force_original_aspect_ratio=increase:force_divisible_by=2,crop=${w}:${h},setsar=1$`), + crop: (w, h) => new RegExp(`^crop=${w}:${h},setsar=1$`), +}; + +interface MatrixCase { + name: string; + layout: "single" | "pip" | "2x1" | "2x2" | "3x3" | "4x4" | "custom"; + rects: PxRect[]; + /** Slot indexes rendered black (source=null) — excluded from the live-tile assertions. */ + empty?: number[]; +} + +const MATRIX: MatrixCase[] = [ + { name: "single", layout: "single", rects: layoutRects("single", canvas) }, + { name: "pip", layout: "pip", rects: [{ x: 0, y: 0, w: 320, h: 180 }, rectFromBasisPoints({ x: 6000, y: 6000, w: 3500, h: 3500 }, canvas)] }, + { name: "2x1", layout: "2x1", rects: layoutRects("2x1", canvas) }, + { name: "2x2", layout: "2x2", rects: layoutRects("2x2", canvas), empty: [3] }, + { name: "3x3", layout: "3x3", rects: layoutRects("3x3", canvas), empty: [4, 8] }, + { name: "4x4", layout: "4x4", rects: layoutRects("4x4", canvas), empty: [5, 10, 15] }, + { name: "custom (non-overlapping, xstack)", layout: "custom", rects: [{ x: 0, y: 0, w: 160, h: 90 }, { x: 160, y: 90, w: 160, h: 90 }] }, + { name: "custom (overlapping, overlay chain)", layout: "custom", rects: [{ x: 0, y: 0, w: 200, h: 112 }, { x: 120, y: 68, w: 200, h: 112 }] }, +]; + +/** Every layout × every fit, via buildCompositorArgs (the production entry point). */ +function matrixPlan(c: MatrixCase, fit: "contain" | "cover" | "crop"): CompositorPlan { + const plan = basePlan(); + plan.sources = [0, 1, 2, 3].map(synthetic); + plan.graph.layout = c.layout; + plan.graph.tiles = c.rects.map((rect, i) => ({ source: c.empty?.includes(i) ? null : i % 4, rect, fit })); + plan.graph.overlays = []; + plan.graph.audio = { inputs: [], masterGainDb: 0 }; + plan.assets = []; + return plan; +} + +describe("WP12 crop-mode matrix: contain/cover/crop × single/pip/two_up/split grids/custom", () => { + it("layout cell rects are even-aligned and tile the canvas exactly", () => { + for (const layout of ["2x1", "2x2", "3x3", "4x4"] as const) { + const rects = layoutRects(layout, canvas); + assert.strictEqual(rects.reduce((a, r) => a + r.w * r.h, 0), W * H); + for (const r of rects) for (const v of [r.x, r.y, r.w, r.h]) assert.strictEqual(v % 2, 0); + } + }); + + for (const { c, fit } of MATRIX.flatMap((c) => (["contain", "cover", "crop"] as const).map((fit) => ({ c, fit })))) { + const plan = matrixPlan(c, fit); + const argv = argvOf(plan); + const graph = argv[argv.indexOf("-filter_complex") + 1]!; + + const live = c.rects.filter((_, i) => !c.empty?.includes(i)); + live.forEach((rect, i) => { + const chain = new RegExp(`\\[t${i}\\]`.replace("[", "\\[") + `\\]`); // placeholder, replaced below + void chain; + const tileChain = new RegExp(`\\[s?\\d*(?::v)?\\][^;]*\\[t${i}\\]`); + const match = graph.match(tileChain); + assert.notStrictEqual(match, null); + assert.strictEqual(FIT_CHAIN[fit](rect.w, rect.h).test(match![0].slice(match![0].indexOf("]") + 1)), true); + // Geometry is exactly the layout cell: no source- or fit-dependent drift. + assert.ok((graph).includes(`[t${i}]`)); + }); + + // The assembly stage per layout family: + if (c.layout === "pip" || c.name.includes("overlapping")) { + // PIP and overlapping customs are overlay chains; xstack must not appear. + assert.ok(!(graph).includes("xstack")); + assert.ok(new RegExp(/overlay=x=\d+:y=\d+:eof_action=repeat/).test(graph)); + } else if (live.length >= 2) { + const layout = live.map((r) => `${r.x}_${r.y}`).join("|"); + assert.ok(graph.includes(`xstack=inputs=${live.length}:layout=${layout.replace(/([|_])/g, "\\$1")}:fill=black`)); + } else { + assert.ok(!(graph).includes("xstack")); + } + }, + ); + + it("argv structure per matrix row: -i ordering, -filter_complex once, gate outputs on pipe:1/pipe:3", () => { + for (const c of MATRIX) { + for (const fit of ["contain", "cover", "crop"] as const) { + const argv = argvOf(matrixPlan(c, fit)); + assert.strictEqual((argv.filter((a) => a === "-filter_complex")).length, 1); + assert.strictEqual((argv.filter((a) => a === "-i")).length, c.rects.filter((_, i) => !c.empty?.includes(i)).length || 1); + assert.ok(!/[&<>]/.test(argv[argv.indexOf("-filter_complex") + 1])); + assert.ok(!/[;&|`$<>]/.test(argv[argv.indexOf("-filter_complex") + 1])); + assert.strictEqual(argv.at(-1), "pipe:3"); + assert.ok((argv).includes("pipe:1")); + } + } + }); + + it("split_horizontal/split_vertical equivalents (2x1 / custom row-split) keep per-half fit chains", () => { + // Horizontal split = 2x1 grid: left/right halves, each independently cropped. + const row = matrixPlan({ name: "2x1", layout: "2x1", rects: layoutRects("2x1", canvas) }, "crop"); + const rowGraph = argvOf(row)[argvOf(row).indexOf("-filter_complex") + 1]!; + assert.ok(new RegExp(/\[t0\]$/m).test(rowGraph)); + assert.ok((rowGraph).includes("crop=160:180,setsar=1[t0]")); + assert.ok((rowGraph).includes("crop=160:180,setsar=1[t1]")); + // Vertical split = a custom top/bottom split: same halves stacked, xstack fills black. + const col = matrixPlan({ name: "custom vsplit", layout: "custom", rects: [{ x: 0, y: 0, w: 320, h: 90 }, { x: 0, y: 90, w: 320, h: 90 }] }, "cover"); + const colGraph = argvOf(col)[argvOf(col).indexOf("-filter_complex") + 1]!; + assert.ok((colGraph).includes("scale=320:90:force_original_aspect_ratio=increase:force_divisible_by=2,crop=320:90,setsar=1[t0]")); + assert.ok((colGraph).includes("scale=320:90:force_original_aspect_ratio=increase:force_divisible_by=2,crop=320:90,setsar=1[t1]")); + assert.ok(new RegExp(/xstack=inputs=2:layout=0_0\|0_90:fill=black/).test(colGraph)); + }); + + it("mixed fits inside one layout keep each tile's own chain (cover base + contain PIP inset)", () => { + const plan = matrixPlan({ name: "pip", layout: "pip", rects: [{ x: 0, y: 0, w: 320, h: 180 }, rectFromBasisPoints({ x: 6000, y: 6000, w: 3500, h: 3500 }, canvas)] }, "cover"); + plan.graph.tiles[1]!.fit = "contain"; + const graph = argvOf(plan)[argvOf(plan).indexOf("-filter_complex") + 1]!; + assert.ok((graph).includes("scale=320:180:force_original_aspect_ratio=increase:force_divisible_by=2,crop=320:180,setsar=1[t0]")); + assert.ok((graph).includes("scale=96:96:force_original_aspect_ratio=decrease:force_divisible_by=2,pad=96:96:(ow-iw)/2:(oh-ih)/2:color=black,setsar=1[t1]")); + assert.ok(new RegExp(/overlay=x=192:y=108:eof_action=repeat/).test(graph)); + }); + + it("contain letterboxes black and never crops; crop never pads; cover never letterboxes", () => { + const rect = { x: 0, y: 0, w: 160, h: 180 }; + const graph = (fit: "contain" | "cover" | "crop") => graphOf({ ...basePlan(), graph: { ...basePlan().graph, tiles: [{ source: 0, rect, fit }] } }); + const contain = graph("contain"); + assert.ok((contain).includes("pad=160:180:(ow-iw)/2:(oh-ih)/2:color=black")); + assert.ok(!(contain).includes(`crop=${rect.w}:${rect.h}`)); + const cover = graph("cover"); + assert.ok((cover).includes(`crop=${rect.w}:${rect.h},setsar=1`)); + assert.ok(!(cover).includes("pad=")); + const cropOnly = graph("crop"); + assert.ok((cropOnly).includes(`crop=${rect.w}:${rect.h},setsar=1`)); + assert.ok(!(cropOnly).includes("scale=")); + assert.ok(!(cropOnly).includes("pad=")); + }); +}); + +// --------------------------------------------------------------------------- +// Run note +// --------------------------------------------------------------------------- + +// This suite runs under the repo's vitest 2.1.9 (installed with the api app; +// the media-worker package itself has no vitest dependency): +// +// cd apps/api && ./node_modules/.bin/vitest run \ +// --root ../media-worker src/compositor/wp12-hardening.test.ts +// +// The media-worker `npm test` glob (node --test) also picks this file up as a +// bare `describe/it` import failure — it is authored for vitest only, per the +// WP12 deliverable. diff --git a/apps/media-worker/src/compositor/zero-gap.test.ts b/apps/media-worker/src/compositor/zero-gap.test.ts new file mode 100644 index 0000000..293c378 --- /dev/null +++ b/apps/media-worker/src/compositor/zero-gap.test.ts @@ -0,0 +1,116 @@ +/** + * S2 gate: zero gap at the encoder input across 1 000 scripted scene, overlay + * and gain swaps (streaming-architecture.md §6 S2), plus kill -9 of the + * compositor mid-swap leaving the encoder input up. + * + * Real compositor ffmpeg processes built by MediaCommandBuilder, the Rust frame + * gate, and a verifying null encoder in its own process. Sources are + * CounterSources (luma = frame counter, phase-locked 1500 Hz sine). + * The regular suite runs 50 swaps. The standalone acceptance gate runs + * ZERO_GAP_SWAPS=1000, keeping the extended media workload separate from other live-process tests. + */ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import type { ComposeRequest } from "./compositor"; +import { ProgramChain } from "./program"; +import { composeOver, CounterSource, ensureGateBinary, makeAssets, passthroughRedactor, scriptedGraph, SinkProcess, ZERO_GAP_CANVAS as canvas } from "./testing"; + +const SWAPS = Number(process.env.ZERO_GAP_SWAPS ?? 50); + +test("zero gap at the encoder input across scripted scene/overlay/gain swaps", { timeout: 1_800_000 }, async (t) => { + if (!ensureGateBinary()) { t.skip("frame-gate binary unavailable (cargo build failed)"); return; } + const scratch = mkdtempSync(join(tmpdir(), "zero-gap-")); + const assets = makeAssets(scratch); + const sources = Array.from({ length: 4 }, () => new CounterSource(canvas, scratch)); + for (const s of sources) await s.start(); + const lost: string[] = []; + const chain = new ProgramChain({ canvas, scratchDir: scratch, videoBitrateKbps: 1000, output: { kind: "external" }, + redactor: passthroughRedactor, onStderr: () => undefined, onFatal: (r) => lost.push(`fatal ${r}`), onCompositorLost: (r) => lost.push(r) }); + const sink = new SinkProcess(); + await sink.start(chain.gate, canvas); + const request = (i: number): ComposeRequest => composeOver(sources.map((s) => s.bus), assets, scriptedGraph(i)); + try { + await chain.start(request(0)); + const first = await chain.stats(); + const firstTick = first.ticks; + const swapTicks: number[] = []; + for (let i = 1; i <= SWAPS; i += 1) swapTicks.push(await chain.swap(request(i))); + // Let the last compositor run, then read the counters. + await new Promise((r) => setTimeout(r, 500)); + const stats = await chain.stats(); + // Report before teardown: stopping the chain makes the gate hold its last frame. + const report = await sink.report(firstTick); + await chain.stop(); + + assert.equal(chain.swaps, SWAPS + 1); + assert.equal(stats.swaps, SWAPS + 1); + assert.deepEqual(lost.filter((l) => !l.startsWith("fatal frame gate exited")), [], "no compositor lost on air"); + assert.ok(swapTicks.every((tick, k) => k === 0 || tick > (swapTicks[k - 1] ?? 0)), "swap ticks strictly increase"); + // Encoder-side continuity: every tick delivered exactly one video and one audio frame. + assert.ok(Math.abs(report.videoFrames - report.audioFrames) <= 1, "video/audio lockstep at the encoder input"); + assert.ok(report.videoFrames >= stats.ticks - 2, "encoder received every tick"); + assert.equal(stats.outDrops, 0, "gate never dropped toward the encoder"); + // Every delivered frame advances by exactly one, including across every swap. + assert.deepEqual(Object.keys(report.tagDeltas), ["1"], `frame counter all +1 (deltas ${JSON.stringify(report.tagDeltas)}, gate lagged=${stats.lagged} behindMax=${stats.behindMax} drops=${stats.drops} vrep=${stats.vrep})`); + // PCM continuity: no inserted silence after the first compositor and no step larger than the sine allows. + const bound = Math.ceil(report.maxPcmAbs * Math.sin((2 * Math.PI * 1500) / 48_000)) + 4; + assert.ok(report.maxPcmStep <= bound, `PCM step ${report.maxPcmStep} <= ${bound}`); + const fills = stats.vrep + stats.asil; + // On an overloaded host a fill can come from the gate itself not being scheduled; it must stay rare. + assert.ok(fills <= Math.ceil(stats.ticks / 1000), `repeat/silence fills ${fills} over ${stats.ticks} ticks`); + t.diagnostic(JSON.stringify({ swaps: SWAPS, lagRetries: chain.lagRetries, lagged: stats.lagged, behindMax: stats.behindMax, ticks: stats.ticks, vrep: stats.vrep, asil: stats.asil, lateMaxUs: stats.lateMaxUs, lateOverFrame: stats.lateOverFrame, + tagDeltas: report.tagDeltas, maxPcmStep: report.maxPcmStep, bound, maxGapMs: report.maxGapMs })); + } finally { + await sink.stop(); + for (const s of sources) await s.stop(); + await chain.stop(); + rmSync(scratch, { recursive: true, force: true }); + } +}); + +test("kill -9 of the compositor mid-swap leaves the encoder input up", { timeout: 120_000 }, async (t) => { + if (!ensureGateBinary()) { t.skip("frame-gate binary unavailable"); return; } + const scratch = mkdtempSync(join(tmpdir(), "kill9-")); + const assets = makeAssets(scratch); + const sources = Array.from({ length: 4 }, () => new CounterSource(canvas, scratch)); + for (const s of sources) await s.start(); + const lost: string[] = []; + const fatal: string[] = []; + const chain = new ProgramChain({ canvas, scratchDir: scratch, videoBitrateKbps: 1000, output: { kind: "external" }, + redactor: passthroughRedactor, onStderr: () => undefined, onFatal: (r) => fatal.push(r), onCompositorLost: (r) => lost.push(r) }); + const sink = new SinkProcess(); + await sink.start(chain.gate, canvas); + const request = (i: number): ComposeRequest => composeOver(sources.map((s) => s.bus), assets, { ...scriptedGraph(i), audio: { inputs: [{ source: 0, gainDb: 0 }], masterGainDb: 0 } }); + try { + await chain.start(request(0)); + const firstTick = (await chain.stats()).ticks; + // Mid-swap: B is starting while A is SIGKILLed. + const a = chain.activeCompositor; + const swapping = chain.swap(request(1)); + a?.kill("SIGKILL"); + await swapping; + // On air: SIGKILL the active one; the chain respawns it behind the gate. + chain.activeCompositor?.kill("SIGKILL"); + await new Promise((r) => setTimeout(r, 2_000)); + await chain.swap(request(2)); + const before = await chain.stats(); + await new Promise((r) => setTimeout(r, 1_000)); + const after = await chain.stats(); + const report = await sink.report(firstTick); + await chain.stop(); + assert.deepEqual(fatal.filter((f) => !f.startsWith("frame gate exited")), [], "gate and encoder input never went down"); + assert.ok(lost.length >= 1, "the on-air kill was detected"); + assert.ok(after.ticks - before.ticks >= canvas.fps * 0.8, "gate kept ticking"); + assert.ok(Math.abs(report.videoFrames - report.audioFrames) <= 1); + assert.ok(report.videoFrames >= after.ticks - 2, "encoder input never starved"); + t.diagnostic(JSON.stringify({ lost, vrep: after.vrep, asil: after.asil, ticks: after.ticks })); + } finally { + await sink.stop(); + for (const s of sources) await s.stop(); + await chain.stop(); + rmSync(scratch, { recursive: true, force: true }); + } +}); diff --git a/apps/media-worker/src/gate/binary.ts b/apps/media-worker/src/gate/binary.ts new file mode 100644 index 0000000..0372432 --- /dev/null +++ b/apps/media-worker/src/gate/binary.ts @@ -0,0 +1,85 @@ +/** + * Native helper binaries (src/gate/rust, std only): `frame-gate` (the channel's + * clock master) and `source-bus` (per-source raw frame fan-out). + * + * Resolution order, first hit wins: + * 1. `MEDIA_FRAME_GATE_BIN` / `MEDIA_SOURCE_BUS_BIN`: an explicit path per binary; + * 2. `MEDIA_HELPER_DIR/`: the packaged directory (the image builds the crate + * with `scripts/build-helpers.sh` and sets this; see runs/wp4/s2b-report.md item 5); + * 3. the crate's `target/release/` (development checkouts and tests). + * + * Nothing builds at runtime in production. `ensureHelperBinaries()` is for tests + * and the spike/smoke scripts: it builds the crate offline when a binary is + * missing or older than its sources. + */ +import { spawnSync } from "node:child_process"; +import { existsSync, readdirSync, statSync } from "node:fs"; +import { join, resolve } from "node:path"; + +export type HelperBinary = "frame-gate" | "source-bus"; + +const ENV: Record = { "frame-gate": "MEDIA_FRAME_GATE_BIN", "source-bus": "MEDIA_SOURCE_BUS_BIN" }; + +/** The crate directory, from `src/gate` (tsx) or `dist/gate` (compiled). */ +export function helperCrateDir(): string { + const local = resolve(__dirname, "rust"); + return existsSync(join(local, "Cargo.toml")) ? local : resolve(__dirname, "../../src/gate/rust"); +} + +function candidates(name: HelperBinary): string[] { + return [ + process.env[ENV[name]], + process.env.MEDIA_HELPER_DIR ? join(process.env.MEDIA_HELPER_DIR, name) : undefined, + join(helperCrateDir(), "target/release", name), + ].filter((path): path is string => typeof path === "string" && path.length > 0); +} + +export function resolveHelperBinary(name: HelperBinary): string { + const found = candidates(name).find((path) => existsSync(path)); + if (!found) throw new Error(`${name} binary not found: set ${ENV[name]} or MEDIA_HELPER_DIR, or run scripts/build-helpers.sh`); + return found; +} + +function newestSource(crate: string): number { + const files = ["Cargo.toml", "Cargo.lock"].map((f) => join(crate, f)); + const walk = (dir: string): void => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) walk(path); + else if (entry.name.endsWith(".rs")) files.push(path); + } + }; + walk(join(crate, "src")); + return Math.max(...files.filter((f) => existsSync(f)).map((f) => statSync(f).mtimeMs)); +} + +/** Development binaries that are missing or older than the crate sources. Env/packaged binaries are taken as given. */ +export function staleHelpers(): HelperBinary[] { + const crate = helperCrateDir(); + if (!existsSync(join(crate, "Cargo.toml"))) return []; + const newest = newestSource(crate); + return (Object.keys(ENV) as HelperBinary[]).filter((name) => { + if (process.env[ENV[name]] || process.env.MEDIA_HELPER_DIR) return false; + const path = join(crate, "target/release", name); + return !existsSync(path) || statSync(path).mtimeMs < newest; + }); +} + +/** + * Tests/scripts: build the crate if a development binary is missing or stale. + * `--offline --locked`: no network, and the (dependency-free) lockfile must not change. + * Concurrent callers are serialized by cargo's own build-directory lock. + */ +export function ensureHelperBinaries(): boolean { + if (staleHelpers().length > 0) { + const built = spawnSync("cargo", ["build", "--release", "--offline", "--locked", "--manifest-path", join(helperCrateDir(), "Cargo.toml")], { stdio: "ignore" }); + if (built.status !== 0) return false; + } + try { + resolveHelperBinary("frame-gate"); + resolveHelperBinary("source-bus"); + return true; + } catch { + return false; + } +} diff --git a/apps/media-worker/src/gate/buffers.ts b/apps/media-worker/src/gate/buffers.ts new file mode 100644 index 0000000..32e7981 --- /dev/null +++ b/apps/media-worker/src/gate/buffers.ts @@ -0,0 +1,29 @@ +/** + * Preallocated, refcounted frame buffers. Raw 1080p frames are 3 MB; allocating + * one per frame is ~93 MB/s of external memory per stream and drives GC, so the + * gate and the source bus recycle buffers once every in-flight write is done. + */ +export class BufferPool { + private readonly free: Buffer[] = []; + private readonly refs = new Map(); + /** Buffers allocated after construction; stays 0 in steady state. */ + allocs = 0; + constructor(readonly size: number, prealloc: number) { + for (let i = 0; i < prealloc; i += 1) this.free.push(Buffer.alloc(size)); + } + /** The caller owns one reference; `release()` it when done. */ + get(): Buffer { + let buf = this.free.pop(); + if (!buf) { this.allocs += 1; buf = Buffer.alloc(this.size); } + this.refs.set(buf, 1); + return buf; + } + retain(buf: Buffer): void { this.refs.set(buf, (this.refs.get(buf) ?? 0) + 1); } + release(buf: Buffer): void { + const n = (this.refs.get(buf) ?? 0) - 1; + if (n > 0) { this.refs.set(buf, n); return; } + if (n < 0) return; // not pool-owned or already free + this.refs.delete(buf); + this.free.push(buf); + } +} diff --git a/apps/media-worker/src/gate/fifo.ts b/apps/media-worker/src/gate/fifo.ts new file mode 100644 index 0000000..15c3307 --- /dev/null +++ b/apps/media-worker/src/gate/fifo.ts @@ -0,0 +1,79 @@ +import { spawnSync } from "node:child_process"; +import { closeSync, constants, openSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { mediaEnv } from "../relay/ffmpeg"; + +export interface FifoPair { + videoPath: string; + audioPath: string; + /** + * O_WRONLY write ends for the producer's stdio. A temporary non-blocking read + * end keeps each FIFO open (no ENXIO, no blocking open) until the gate has + * attached. After both releases only the producer writes and only the gate + * reads: producer death is an EOF at the gate, and a gate-side close is an + * EPIPE at the producer (a retired compositor cannot hang on a full pipe). + */ + videoFd: number; + audioFd: number; + /** Close the worker's copies of the write ends (after spawning the producer). */ + releaseWriters(): void; + /** Close the temporary read ends (after the gate has opened its own). */ + releaseHolders(): void; + unlink(): void; +} + +const SCRATCH = /^\/[A-Za-z0-9._/-]+$/; + +/** mkfifo(1) via argv (Node has no mkfifo binding). */ +const PREFIX = /^[a-z][a-z0-9-]{0,31}$/; + +/** Creates `-.{v,a}.fifo`; `prefix` names the owner (`gate`, `bus`, `sub-3`). Paths are scratch dir + prefix + numeric id only. */ +export function makeFifoPair(scratchDir: string, id: number, prefix: string): { videoPath: string; audioPath: string } { + if (!SCRATCH.test(scratchDir) || scratchDir.includes("..") || !Number.isSafeInteger(id) || id < 0 || !PREFIX.test(prefix)) throw new Error("unsafe fifo location"); + const videoPath = join(scratchDir, `${prefix}-${id}.v.fifo`); + const audioPath = join(scratchDir, `${prefix}-${id}.a.fifo`); + const made = spawnSync("mkfifo", ["-m", "600", videoPath, audioPath], { shell: false, env: mediaEnv(), stdio: "ignore" }); + if (made.status !== 0) throw new Error("mkfifo failed"); + return { videoPath, audioPath }; +} + +const NAME = /^[a-z][a-z0-9-]{0,47}\.fifo$/; + +/** One FIFO `/` (name: lowercase, digits, '-', ending `.fifo`). */ +export function makeFifo(scratchDir: string, name: string): string { + if (!SCRATCH.test(scratchDir) || scratchDir.includes("..") || !NAME.test(name)) throw new Error("unsafe fifo location"); + const path = join(scratchDir, name); + const made = spawnSync("mkfifo", ["-m", "600", path], { shell: false, env: mediaEnv(), stdio: "ignore" }); + if (made.status !== 0) throw new Error("mkfifo failed"); + return path; +} + +export function createFifoPair(scratchDir: string, id: number, prefix = "gate"): FifoPair { + const { videoPath, audioPath } = makeFifoPair(scratchDir, id, prefix); + const holders = [openSync(videoPath, constants.O_RDONLY | constants.O_NONBLOCK), openSync(audioPath, constants.O_RDONLY | constants.O_NONBLOCK)]; + const videoFd = openSync(videoPath, constants.O_WRONLY); + const audioFd = openSync(audioPath, constants.O_WRONLY); + let writers = true; + let held = true; + return { + videoPath, audioPath, videoFd, audioFd, + releaseWriters() { if (writers) { writers = false; closeSync(videoFd); closeSync(audioFd); } }, + releaseHolders() { if (held) { held = false; for (const fd of holders) closeSync(fd); } }, + unlink() { rmSync(videoPath, { force: true }); rmSync(audioPath, { force: true }); }, + }; +} + +/** + * A *blocking* read end for a consumer's stdio (ffmpeg must not get O_NONBLOCK). + * A plain O_RDONLY open blocks until a writer exists, so a temporary writer is + * opened first (itself made possible by a temporary non-blocking reader). + * Call `releaseHolder()` once the real writer has opened the FIFO. + */ +export function openBlockingReader(path: string): { fd: number; releaseHolder(): void } { + const probe = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK); + const holder = openSync(path, constants.O_WRONLY); + const fd = openSync(path, constants.O_RDONLY); + closeSync(probe); + let held = true; + return { fd, releaseHolder() { if (held) { held = false; closeSync(holder); } } }; +} diff --git a/apps/media-worker/src/gate/frame-gate.ts b/apps/media-worker/src/gate/frame-gate.ts new file mode 100644 index 0000000..603e150 --- /dev/null +++ b/apps/media-worker/src/gate/frame-gate.ts @@ -0,0 +1,244 @@ +import { spawnSync } from "node:child_process"; +import { closeSync, constants, mkdirSync, openSync, rmSync } from "node:fs"; +import { Socket } from "node:net"; +import { join, resolve } from "node:path"; +import { mediaEnv, spawnManaged, type ManagedProcess, type StderrRedactor } from "../relay/ffmpeg"; +import { resolveHelperBinary } from "./binary"; +import { createFifoPair, openBlockingReader, type FifoPair } from "./fifo"; +import { assertGeometry, attachLine, parseGateEvent, type FrameGeometry, type GateEvent, type GateStats } from "./protocol"; + +/** + * `rust` is the production gate (spike decision). `node` is the + * reference implementation of the pipe contract, kept for the spike comparison and the contract + * test only: it lacks A/V pair lockstep and origin alignment, and ProgramChain refuses it. + */ +export type GateImpl = "rust" | "node"; + +export interface FrameGateOptions { + geometry: FrameGeometry; + scratchDir: string; + redactor: StderrRedactor; + onStderr(message: string): void; + /** Unsolicited events (ACTIVE, RETIRED, EOF, STALL...) and gate exit. */ + onEvent?(event: GateEvent): void; + onExit?(reason: string): void; + impl?: GateImpl; + /** Frames buffered per compositor stream (default 16). */ + ringFrames?: number; + /** + * Frames a pending compositor must buffer before going on air with no aligned predecessor + * (default 12, 400 ms at 30 fps). This sets the on-air delay margin, which every later swap + * preserves: a swap-in compositor can splice forward only while its pipeline latency stays inside + * it. 8 frames was not enough for a CPU-starved spawn on an oversubscribed host (s2a-report: alignment). + */ + warmFrames?: number; + /** Rust gate: write with plain copies instead of vmsplice (spike comparison only). */ + copyWrites?: boolean; +} + +export interface GateSlot { id: number; fifos: FifoPair } + +/** + * The gate's output FIFOs. The reader (the persistent encoder via + * `encoderInputs()`, or a test sink via `openReaders()`) opens them before + * `start()`, so the gate's write-side open never blocks. + */ +export interface EncoderFifos { videoPath: string; audioPath: string } + +export function resolveGateCommand(impl: GateImpl): { command: string; args: string[]; cwd?: string } { + if (impl === "rust") return { command: resolveHelperBinary("frame-gate"), args: [] }; + if (__filename.endsWith(".js")) return { command: process.execPath, args: [resolve(__dirname, "node-gate.js")] }; + return { command: process.execPath, args: ["--import", "tsx", resolve(__dirname, "node-gate.ts")], cwd: resolve(__dirname, "../..") }; +} + +interface Waiter { match(event: GateEvent): boolean; resolve(event: GateEvent): void; reject(error: Error): void; timer: ReturnType } + +export class CompositorLagError extends Error { + constructor(readonly slotId: number, readonly behind: number) { + super(`compositor ${slotId} stayed ${behind} frames behind the on-air source position`); + this.name = "CompositorLagError"; + } +} + +/** Process manager for one channel's frame gate. */ +export class FrameGate { + private proc: ManagedProcess | null = null; + private nextId = 1; + private waiters: Waiter[] = []; + private lineBuf = ""; + private stopping = false; + readonly encoderFifos: EncoderFifos; + readonly ringFrames: number; + readonly warmFrames: number; + + constructor(private readonly options: FrameGateOptions) { + assertGeometry(options.geometry); + this.ringFrames = options.ringFrames ?? 16; + this.warmFrames = options.warmFrames ?? 12; + mkdirSync(options.scratchDir, { recursive: true, mode: 0o700 }); + this.encoderFifos = { videoPath: join(options.scratchDir, "encoder.v.fifo"), audioPath: join(options.scratchDir, "encoder.a.fifo") }; + for (const path of [this.encoderFifos.videoPath, this.encoderFifos.audioPath]) rmSync(path, { force: true }); + const made = spawnSync("mkfifo", ["-m", "600", this.encoderFifos.videoPath, this.encoderFifos.audioPath], { shell: false, env: mediaEnv(), stdio: "ignore" }); + if (made.status !== 0) throw new Error("mkfifo failed"); + } + + private holders: Array<() => void> = []; + private inputFds: number[] = []; + + /** + * Blocking read ends for the persistent encoder's stdio (video on fd 0, audio + * on fd 3: `-i pipe:0`, `-i pipe:3`). Spawn the encoder, then `start()`; the + * worker's copies are closed once the gate is up, so the encoder sees EOF + * only if the gate itself dies. + */ + encoderInputs(): { videoFd: number; audioFd: number } { + const video = openBlockingReader(this.encoderFifos.videoPath); + const audio = openBlockingReader(this.encoderFifos.audioPath); + this.holders.push(video.releaseHolder, audio.releaseHolder); + this.inputFds.push(video.fd, audio.fd); + return { videoFd: video.fd, audioFd: audio.fd }; + } + + /** Null-encoder side for tests and the spike: non-blocking read ends of the encoder FIFOs. */ + openReaders(): { video: Socket; audio: Socket } { + const open = (path: string) => new Socket({ fd: openSync(path, constants.O_RDONLY | constants.O_NONBLOCK), readable: true, writable: false }); + return { video: open(this.encoderFifos.videoPath), audio: open(this.encoderFifos.audioPath) }; + } + + get pid(): number | undefined { return this.proc?.child.pid; } + + async start(): Promise { + const { command, args, cwd } = resolveGateCommand(this.options.impl ?? "rust"); + const g = this.options.geometry; + const argv = [...args, String(g.width), String(g.height), String(g.fps), String(this.ringFrames), String(this.warmFrames), + this.encoderFifos.videoPath, this.encoderFifos.audioPath, ...(this.options.copyWrites ? ["copy"] : [])]; + const proc = spawnManaged(command, argv, ["pipe", "pipe", "pipe"], this.options.redactor, this.options.onStderr, cwd); + this.proc = proc; + proc.child.stdout?.setEncoding("utf8"); + proc.child.stdout?.on("data", (chunk: string) => this.onData(chunk)); + // EPIPE on a control line to a gate that just died: its exit is reported below, never as a crash. + proc.child.stdin?.on("error", () => undefined); + void proc.exited.then(({ code, signal }) => { + const reason = `frame gate exited code=${code} signal=${signal}`; + for (const w of this.waiters.splice(0)) { clearTimeout(w.timer); w.reject(new Error(reason)); } + if (!this.stopping) this.options.onExit?.(reason); + }); + const ready = this.waitFor((e) => e.type === "READY", 10_000); + try { + await proc.started; + await ready; + } catch (error) { + ready.catch(() => undefined); + throw error; + } finally { + for (const release of this.holders.splice(0)) release(); + for (const fd of this.inputFds.splice(0)) closeSync(fd); + } + } + + private onData(chunk: string): void { + this.lineBuf += chunk; + let nl = this.lineBuf.indexOf("\n"); + while (nl >= 0) { + const event = parseGateEvent(this.lineBuf.slice(0, nl)); + this.lineBuf = this.lineBuf.slice(nl + 1); + nl = this.lineBuf.indexOf("\n"); + if (!event) continue; + const i = this.waiters.findIndex((w) => w.match(event)); + if (i >= 0) { + const [w] = this.waiters.splice(i, 1); + if (w) { clearTimeout(w.timer); w.resolve(event); } + } + this.options.onEvent?.(event); + } + } + + waitFor(match: (event: GateEvent) => boolean, timeoutMs: number): Promise { + return new Promise((resolvePromise, reject) => { + const waiter: Waiter = { + match, resolve: resolvePromise, reject, + timer: setTimeout(() => { + this.waiters = this.waiters.filter((w) => w !== waiter); + reject(new Error("frame gate event timeout")); + }, timeoutMs), + }; + this.waiters.push(waiter); + }); + } + + private send(line: string): void { + const stdin = this.proc?.child.stdin; + if (!stdin || stdin.destroyed) throw new Error("frame gate is not running"); + stdin.write(line); + } + + /** Sends a control line and waits for its reply; a failed send never leaves a waiter behind. */ + private request(line: string, match: (event: GateEvent) => boolean, timeoutMs: number): Promise { + const reply = this.waitFor(match, timeoutMs); + try { + this.send(line); + } catch (error) { + reply.catch(() => undefined); + const waiter = this.waiters.at(-1); + if (waiter) { clearTimeout(waiter.timer); this.waiters.pop(); } + throw error; + } + return reply; + } + + /** FIFOs for the next compositor: spawn it with `fifos.videoFd`/`fifos.audioFd`, then `attach()`. */ + allocateSlot(): GateSlot { + const id = this.nextId; + this.nextId += 1; + return { id, fifos: createFifoPair(this.options.scratchDir, id) }; + } + + /** Hands the slot to the gate as the pending compositor; resolves once both FIFOs are open. */ + async attach(slot: GateSlot, origin?: number): Promise { + slot.fifos.releaseWriters(); + try { + const event = await this.request(attachLine(slot.id, slot.fifos.videoPath, slot.fifos.audioPath, origin), + (e) => (e.type === "ATTACHED" || e.type === "ERROR") && e.id === slot.id, 5_000); + if (event.type === "ERROR") throw new Error("frame gate could not open compositor fifos"); + } finally { + slot.fifos.releaseHolders(); + slot.fifos.unlink(); + } + } + + /** + * Resolves with the tick at which the slot went on air. Rejects with `CompositorLagError` when + * the gate reports the slot cannot reach the on-air source position (it never splices backwards). + */ + async waitActive(id: number, timeoutMs: number): Promise { + const event = await this.waitFor((e) => (e.type === "ACTIVE" || e.type === "EOF" || e.type === "DROPPED" || e.type === "LAG") && "id" in e && e.id === id, timeoutMs); + if (event.type === "LAG") throw new CompositorLagError(id, event.behind); + if (event.type !== "ACTIVE") throw new Error(`compositor ${id} ended before going on air`); + return event.tick; + } + + detach(id: number): void { this.send(`DETACH ${id}\n`); } + + async stats(): Promise { + const event = await this.request("STATS\n", (e) => e.type === "STATS", 5_000); + if (event.type !== "STATS") throw new Error("unexpected gate reply"); + return event.stats; + } + + /** Zero the peak counters (late/write maxima), e.g. after encoder warm-up. */ + async resetPeaks(): Promise { + await this.request("RESET\n", (e) => e.type === "RESET", 5_000); + } + + async stop(): Promise { + this.stopping = true; + const proc = this.proc; + if (!proc) return; + try { this.send("QUIT\n"); } catch { /* already gone */ } + const timer = setTimeout(() => proc.kill("SIGKILL"), 2_000); + await proc.exited; + clearTimeout(timer); + this.proc = null; + for (const path of [this.encoderFifos.videoPath, this.encoderFifos.audioPath]) rmSync(path, { force: true }); + } +} diff --git a/apps/media-worker/src/gate/gate.test.ts b/apps/media-worker/src/gate/gate.test.ts new file mode 100644 index 0000000..f7df362 --- /dev/null +++ b/apps/media-worker/src/gate/gate.test.ts @@ -0,0 +1,144 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { ensureGateBinary, passthroughRedactor, SinkProcess } from "../compositor/testing"; +import { FrameGate, type GateImpl } from "./frame-gate"; +import { assertGeometry, attachLine, audioFrameBytes, parseGateEvent, videoFrameBytes } from "./protocol"; +import { startProducer } from "./spike"; +import { spawnManaged } from "../relay/ffmpeg"; + +test("pipe contract: frame sizes, geometry and control-line safety", () => { + assert.equal(videoFrameBytes({ width: 1920, height: 1080, fps: 30 }), 3_110_400); + assert.equal(audioFrameBytes({ width: 1920, height: 1080, fps: 30 }), 6_400); + assert.equal(audioFrameBytes({ width: 1920, height: 1080, fps: 60 }), 3_200); + for (const g of [{ width: 1921, height: 1080, fps: 30 }, { width: 1920, height: 1080, fps: 7 }, { width: 0, height: 2, fps: 30 }]) assert.throws(() => assertGeometry(g)); + assert.equal(attachLine(3, "/tmp/x/gate-3.v.fifo", "/tmp/x/gate-3.a.fifo", 42), "ATTACH 3 /tmp/x/gate-3.v.fifo /tmp/x/gate-3.a.fifo 42\n"); + for (const bad of ["/tmp/a b", "/tmp/../etc", "relative", "/tmp/x\nQUIT", "/tmp/x;y"]) assert.throws(() => attachLine(1, bad, "/tmp/ok"), bad); + assert.throws(() => attachLine(1, "/tmp/a", "/tmp/b", 1.5)); + assert.deepEqual(parseGateEvent("ACTIVE 4 120"), { type: "ACTIVE", id: 4, tick: 120 }); + const stats = parseGateEvent("STATS ticks=10 vrep=1 asil=2 active=4:2/2 unknown=9"); + assert.ok(stats?.type === "STATS" && stats.stats.ticks === 10 && stats.stats.vrep === 1 && stats.stats.active === "4:2/2"); + assert.equal(parseGateEvent("BOGUS"), null); +}); + +/** Same contract, both implementations: boot black, swap producers at frame boundaries, survive a dead producer. */ +for (const impl of ["rust", "node"] as GateImpl[]) { + test(`${impl} gate: swaps are frame-continuous and a dead producer never starves the encoder`, { timeout: 120_000 }, async (t) => { + if (impl === "rust" && !ensureGateBinary()) { t.skip("frame-gate binary unavailable"); return; } + const g = { width: 320, height: 180, fps: 30 }; + const scratch = mkdtempSync(join(tmpdir(), `gate-${impl}-`)); + const fatal: string[] = []; + const gate = new FrameGate({ geometry: g, scratchDir: scratch, impl, redactor: passthroughRedactor, onStderr: () => undefined, onExit: (r) => fatal.push(r) }); + const sink = new SinkProcess(); + await sink.start(gate, g); + await gate.start(); + const producers = []; + try { + let current = await startProducer(gate, g, 20); + producers.push(current.proc); + const first = await gate.waitActive(current.id, 10_000); + for (let i = 1; i <= 10; i += 1) { + const next = await startProducer(gate, g, 20 + i); + producers.push(next.proc); + await gate.waitActive(next.id, 10_000); + void current.proc.terminate(); + current = next; + } + const beforeKill = await gate.stats(); + current.proc.kill("SIGKILL"); + await new Promise((r) => setTimeout(r, 700)); + const afterKill = await gate.stats(); + const report = await sink.report(first); + assert.deepEqual(fatal, []); + assert.equal(beforeKill.swaps, 11); + assert.equal(beforeKill.vrep + beforeKill.asil, 0, "no repeat/silence while producers were live"); + assert.ok(afterKill.ticks - beforeKill.ticks >= 15, "gate kept ticking after the producer died"); + assert.ok(Math.abs(report.videoFrames - report.audioFrames) <= 1); + // Generation tags 20..30 appear in order, each once (after the boot black frames, tag 16). + const tags = report.tagChanges.map(([, tag]) => tag).filter((tag) => tag !== 16); + assert.deepEqual(tags, Array.from({ length: 11 }, (_, i) => 20 + i)); + const bound = Math.ceil(report.maxPcmAbs * Math.sin((2 * Math.PI * 1500) / 48_000)) + 4; + assert.ok(report.maxPcmStep <= bound, `PCM step ${report.maxPcmStep} <= ${bound}`); + } finally { + await sink.stop(); + await gate.stop(); + for (const p of producers) await p.terminate(500); + rmSync(scratch, { recursive: true, force: true }); + } + }); +} + +test("Rust holds an ahead compositor until A reaches its source position instead of skipping frames", { timeout: 60_000 }, async (t) => { + if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return; } + const g = { width: 320, height: 180, fps: 30 }; + const scratch = mkdtempSync(join(tmpdir(), "gate-ahead-")); + const gate = new FrameGate({ geometry: g, scratchDir: scratch, redactor: passthroughRedactor, onStderr: () => undefined }); + const sink = new SinkProcess(); + const producers = []; + try { + await sink.start(gate, g); + await gate.start(); + const a = await startProducer(gate, g, 20, 0, true); + producers.push(a.proc); + const first = await gate.waitActive(a.id, 10_000); + const before = await gate.stats(); + const origin = before.ticks + 2 * g.fps; + const b = await startProducer(gate, g, 21, origin, true); + producers.push(b.proc); + const promoted = await gate.waitActive(b.id, 10_000); + // A's source sequence cannot exceed elapsed gate ticks. B must wait until + // the output reaches its origin even if it has already filled its rings. + assert.ok(promoted >= origin, `B promoted at tick ${promoted} before source position ${origin}`); + const stats = await gate.stats(); + assert.equal(stats.vrep + stats.asil, 0, "A keeps supplying the encoder while B waits"); + const report = await sink.report(first); + assert.deepEqual(report.tagChanges.map(([, tag]) => tag).filter((tag) => tag !== 16), [20, 21]); + } finally { + await sink.stop(); + await gate.stop(); + for (const proc of producers) await proc.terminate(500); + rmSync(scratch, { recursive: true, force: true }); + } +}); + +test("Rust resumes a producer at native rate after a joint A/V stall", { timeout: 60_000 }, async (t) => { + if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return; } + const g = { width: 64, height: 36, fps: 30 }; + const scratch = mkdtempSync(join(tmpdir(), "gate-resume-")); + const gate = new FrameGate({ geometry: g, scratchDir: scratch, redactor: passthroughRedactor, onStderr: () => undefined }); + const sink = new SinkProcess(); + let producer: ReturnType | null = null; + try { + await sink.start(gate, g); + await gate.start(); + const slot = gate.allocateSlot(); + // setInterval deliberately resumes at native rate without replaying paused ticks. + producer = spawnManaged(process.execPath, ["-e", `const fs=require('node:fs'); const v=Buffer.alloc(${videoFrameBytes(g)},20),a=Buffer.alloc(${audioFrameBytes(g)}); setInterval(()=>{fs.writeSync(1,v);fs.writeSync(3,a)},33);`], + ["ignore", slot.fifos.videoFd, "pipe", slot.fifos.audioFd], passthroughRedactor, () => undefined); + slot.fifos.releaseWriters(); + await producer.started; + await gate.attach(slot, 0); + const first = await gate.waitActive(slot.id, 10_000); + const initial = await gate.stats(); + producer.kill("SIGSTOP"); + await new Promise((r) => setTimeout(r, 1_000)); + const stalled = await gate.stats(); + assert.ok(stalled.vrep > initial.vrep && stalled.asil > initial.asil, "pause exhausted both warm rings"); + producer.kill("SIGCONT"); + await new Promise((r) => setTimeout(r, 2_000)); + const resumed = await gate.stats(); + await new Promise((r) => setTimeout(r, 1_000)); + const steady = await gate.stats(); + assert.equal(steady.vrep - resumed.vrep, 0, "video recovered instead of discarding every future frame"); + assert.equal(steady.asil - resumed.asil, 0, "PCM recovered in lockstep"); + await sink.report(first); + } finally { + producer?.kill("SIGCONT"); + await sink.stop(); + await gate.stop(); + await producer?.terminate(500); + rmSync(scratch, { recursive: true, force: true }); + } +}); diff --git a/apps/media-worker/src/gate/node-gate.ts b/apps/media-worker/src/gate/node-gate.ts new file mode 100644 index 0000000..dff44f3 --- /dev/null +++ b/apps/media-worker/src/gate/node-gate.ts @@ -0,0 +1,255 @@ +/** + * Node reference implementation of the frame gate (same pipe contract as the + * Rust helper; see src/gate/protocol.ts). Built for the §1.4 spike: preallocated + * frame rings, `net.Socket` `onread` into one reused buffer, refcounted output + * buffers so a frame is never mutated while a write is in flight. + * + * Run as its own process: `node node-gate.js `. + */ +import { constants, openSync } from "node:fs"; +import { Socket } from "node:net"; +import { PerformanceObserver } from "node:perf_hooks"; +import { BufferPool } from "./buffers"; +import { assertGeometry, audioFrameBytes, videoFrameBytes } from "./protocol"; + +const READ_CHUNK = 1 << 20; +/* Node cannot grow a FIFO past 64 KiB (no F_SETPIPE_SZ), so reads stay event-driven per chunk. */ + +class Ring { + private readonly frames: Buffer[] = []; + head = 0; + count = 0; + fill = 0; + constructor(private readonly cap: number, private readonly size: number, private readonly pool: BufferPool) { + for (let i = 0; i <= cap; i += 1) this.frames.push(pool.get()); + } + full(): boolean { return this.count >= this.cap; } + /** Copies `src` into the ring; returns bytes consumed (stops when full unless dropping). */ + push(src: Buffer, start: number, end: number, dropWhenFull: boolean): { used: number; dropped: number } { + let at = start; + let dropped = 0; + while (at < end) { + if (this.full()) { + if (!dropWhenFull) break; + this.dropOldest(); + dropped += 1; + } + const slot = this.frames[(this.head + this.count) % this.frames.length]!; + const n = Math.min(end - at, this.size - this.fill); + src.copy(slot, this.fill, at, at + n); + at += n; + this.fill += n; + if (this.fill === this.size) { this.fill = 0; this.count += 1; } + } + return { used: at - start, dropped }; + } + dropOldest(): void { + if (this.count === 0) return; + this.head = (this.head + 1) % this.frames.length; + this.count -= 1; + } + /** Hands the oldest frame to the caller and refills the slot from the pool. */ + take(): Buffer { + const frame = this.frames[this.head]!; + this.frames[this.head] = this.pool.get(); + this.head = (this.head + 1) % this.frames.length; + this.count -= 1; + return frame; + } + dispose(): void { for (const f of this.frames) this.pool.release(f); } +} + +class InStream { + readonly ring: Ring; + eof = false; + paused = false; + private readonly socket: Socket; + /** Bytes read past a full active ring, held until the next tick frees a slot. */ + private carry: { buf: Buffer; start: number } | null = null; + constructor(path: string, cap: number, size: number, pool: BufferPool, private readonly gate: { dropping(s: InStream): boolean; drops: number }) { + this.ring = new Ring(cap, size, pool); + const fd = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK); + const chunk = Buffer.alloc(READ_CHUNK); + this.socket = new Socket({ + fd, readable: true, writable: false, + onread: { + buffer: chunk, + callback: (nread: number, buf: Buffer): boolean => { + const r = this.ring.push(buf, 0, nread, this.gate.dropping(this)); + this.gate.drops += r.dropped; + if (r.used < nread) { + // Active ring full: keep the remainder in a side copy and apply backpressure. + this.carry = { buf: Buffer.from(buf.subarray(r.used, nread)), start: 0 }; + this.paused = true; + return false; + } + return true; + }, + }, + }); + this.socket.on("end", () => { this.eof = true; }); + this.socket.on("error", () => { this.eof = true; }); + this.socket.on("close", () => { this.eof = true; }); + } + maybeResume(dropping: boolean): void { + if (!this.paused) return; + if (this.carry) { + const r = this.ring.push(this.carry.buf, this.carry.start, this.carry.buf.length, dropping); + this.carry.start += r.used; + if (this.carry.start < this.carry.buf.length) return; + this.carry = null; + } + this.paused = false; + this.socket.resume(); + } + close(): void { + this.socket.destroy(); + this.ring.dispose(); + } +} + +interface Slot { id: number; video: InStream; audio: InStream } + +function main(): void { + const [w, h, f, r, k] = process.argv.slice(2, 7).map(Number); + const [encVideoPath, encAudioPath] = process.argv.slice(7, 9); + if (encVideoPath === undefined || encAudioPath === undefined) throw new Error("encoder fifo paths required"); + const geometry = { width: w ?? 0, height: h ?? 0, fps: f ?? 0 }; + assertGeometry(geometry); + const ringFrames = r ?? 0; + const warm = k ?? 0; + if (!(ringFrames > 0 && warm > 0 && warm <= ringFrames)) throw new Error("ring/warm invalid"); + const vBytes = videoFrameBytes(geometry); + const aBytes = audioFrameBytes(geometry); + const vPool = new BufferPool(vBytes, (ringFrames + 1) * 2 + 8); + const aPool = new BufferPool(aBytes, (ringFrames + 1) * 2 + 8); + + // Non-blocking open fails fast (ENXIO) unless the encoder already holds the read ends, like the Rust gate. + const encVideo = new Socket({ fd: openSync(encVideoPath, constants.O_WRONLY | constants.O_NONBLOCK), readable: false, writable: true }); + const encAudio = new Socket({ fd: openSync(encAudioPath, constants.O_WRONLY | constants.O_NONBLOCK), readable: false, writable: true }); + for (const s of [encVideo, encAudio]) s.on("error", () => { process.stderr.write("frame-gate: encoder input closed\n"); process.exit(3); }); + + let last = vPool.get(); + last.fill(16, 0, geometry.width * geometry.height); + last.fill(128, geometry.width * geometry.height); + const silence = Buffer.alloc(aBytes); + + let active: Slot | null = null; + let pending: Slot | null = null; + const stats = { ticks: 0, idle: 0, vrep: 0, asil: 0, drops: 0, swaps: 0, lateMaxUs: 0, lateOverFrame: 0, writeMaxUs: 0, stallRun: 0, gcMaxUs: 0, backlogMax: 0, lateOver2ms: 0, lateOver5ms: 0 }; + const gate = { dropping: (s: InStream) => pending !== null && (s === pending.video || s === pending.audio), get drops() { return stats.drops; }, set drops(v: number) { stats.drops = v; } }; + + new PerformanceObserver((list) => { + for (const entry of list.getEntries()) stats.gcMaxUs = Math.max(stats.gcMaxUs, Math.round(entry.duration * 1000)); + }).observe({ entryTypes: ["gc"] }); + + const emit = (line: string): void => { process.stdout.write(`${line}\n`); }; + const ended = (s: Slot): boolean => (s.video.eof && s.video.ring.count === 0) || (s.audio.eof && s.audio.ring.count === 0); + const closeSlot = (s: Slot): void => { s.video.close(); s.audio.close(); }; + + const start = process.hrtime.bigint(); + const cpuStart = process.cpuUsage(); + const periodNs = 1_000_000_000n / BigInt(geometry.fps); + const dueAt = (n: number): bigint => start + (BigInt(n) * 1_000_000_000n) / BigInt(geometry.fps); + + const tick = (): void => { + let now = process.hrtime.bigint(); + while (now >= dueAt(stats.ticks)) { + const late = Number((now - dueAt(stats.ticks)) / 1000n); + stats.lateMaxUs = Math.max(stats.lateMaxUs, late); + if (BigInt(late) * 1000n > periodNs) stats.lateOverFrame += 1; + if (late > 2_000) stats.lateOver2ms += 1; + if (late > 5_000) stats.lateOver5ms += 1; + + if (pending && ended(pending)) { emit(`EOF ${pending.id}`); closeSlot(pending); pending = null; } + if (pending && pending.video.ring.count >= warm && pending.audio.ring.count >= warm) { + const p: Slot = pending; + pending = null; + while (p.video.ring.count > warm) p.video.ring.dropOldest(); + while (p.audio.ring.count > warm) p.audio.ring.dropOldest(); + const old = active; + active = p; + stats.swaps += 1; + stats.stallRun = 0; + emit(`ACTIVE ${p.id} ${stats.ticks}`); + if (old) { closeSlot(old); emit(`RETIRED ${old.id}`); } + } + + let audio: Buffer = silence; + let audioOwned = false; + if (active) { + if (active.video.ring.count > 0) { + vPool.release(last); + last = active.video.ring.take(); + stats.stallRun = 0; + } else { + stats.vrep += 1; + stats.stallRun += 1; + if (stats.stallRun === geometry.fps) emit(`STALL ${active.id} ${stats.ticks}`); + } + if (active.audio.ring.count > 0) { audio = active.audio.ring.take(); audioOwned = true; } else stats.asil += 1; + active.video.maybeResume(false); + active.audio.maybeResume(false); + if (ended(active)) { emit(`EOF ${active.id}`); closeSlot(active); active = null; } + } else stats.idle += 1; + + const w0 = process.hrtime.bigint(); + const frame = last; + vPool.retain(frame); + encAudio.write(audio, () => { if (audioOwned) aPool.release(audio); }); + encVideo.write(frame, () => vPool.release(frame)); + stats.writeMaxUs = Math.max(stats.writeMaxUs, Number((process.hrtime.bigint() - w0) / 1000n)); + stats.backlogMax = Math.max(stats.backlogMax, encVideo.writableLength); + stats.ticks += 1; + now = process.hrtime.bigint(); + } + const waitMs = Number((dueAt(stats.ticks) - process.hrtime.bigint()) / 1_000_000n); + setTimeout(tick, Math.max(0, waitMs)); + }; + + let lineBuf = ""; + process.stdin.setEncoding("utf8"); + process.stdin.on("end", () => process.exit(0)); + process.stdin.on("data", (chunk: string) => { + lineBuf += chunk; + let nl = lineBuf.indexOf("\n"); + while (nl >= 0) { + const parts = lineBuf.slice(0, nl).trim().split(/\s+/); + lineBuf = lineBuf.slice(nl + 1); + nl = lineBuf.indexOf("\n"); + const [cmd, a, b, c] = parts; + if (cmd === "ATTACH" && a !== undefined && b !== undefined && c !== undefined) { + const id = Number(a); + try { + const slot: Slot = { id, video: new InStream(b, ringFrames, vBytes, vPool, gate), audio: new InStream(c, ringFrames, aBytes, aPool, gate) }; + if (pending) { closeSlot(pending); emit(`DROPPED ${pending.id}`); } + pending = slot; + emit(`ATTACHED ${id}`); + } catch { emit(`ERROR ${id} open`); } + } else if (cmd === "DETACH" && a !== undefined) { + const id = Number(a); + if (pending?.id === id) { closeSlot(pending); pending = null; emit(`DETACHED ${id}`); } + else if (active?.id === id) { closeSlot(active); active = null; emit(`DETACHED ${id}`); } + } else if (cmd === "STATS") { + const cpu = process.cpuUsage(cpuStart); + const ring = (s: Slot | null): string => (s ? `${s.id}:${s.video.ring.count}/${s.audio.ring.count}` : "-"); + emit(`STATS ticks=${stats.ticks} idle=${stats.idle} vrep=${stats.vrep} asil=${stats.asil} drops=${stats.drops} swaps=${stats.swaps} late_max_us=${stats.lateMaxUs} late_over_frame=${stats.lateOverFrame} write_max_us=${stats.writeMaxUs} cpu_us=${cpu.user + cpu.system} wall_us=${Number((process.hrtime.bigint() - start) / 1000n)} active=${ring(active)} pending=${ring(pending)} gc_max_us=${stats.gcMaxUs} allocs=${vPool.allocs + aPool.allocs} backlog_max=${stats.backlogMax} late_over_2ms=${stats.lateOver2ms} late_over_5ms=${stats.lateOver5ms}`); + } else if (cmd === "RESET") { + stats.lateMaxUs = 0; stats.lateOverFrame = 0; stats.writeMaxUs = 0; stats.gcMaxUs = 0; stats.backlogMax = 0; stats.lateOver2ms = 0; stats.lateOver5ms = 0; + emit("RESET"); + } else if (cmd === "QUIT") process.exit(0); + else if (cmd !== "" && cmd !== undefined) emit("ERROR 0 command"); + } + }); + + emit(`READY ${vBytes} ${aBytes} ${ringFrames} ${warm}`); + tick(); +} + +if (require.main === module) { + try { main(); } catch (error: unknown) { + process.stderr.write(`frame-gate: ${error instanceof Error ? error.message : "startup failed"}\n`); + process.exit(2); + } +} + diff --git a/apps/media-worker/src/gate/protocol.ts b/apps/media-worker/src/gate/protocol.ts new file mode 100644 index 0000000..7ec417a --- /dev/null +++ b/apps/media-worker/src/gate/protocol.ts @@ -0,0 +1,111 @@ +/** + * Frame-gate pipe contract (docs/design/streaming-architecture.md §1.4). + * + * Two implementations speak it: the Rust helper (src/gate/rust, the production + * choice, see runs/wp4/s2a-report.md) and the Node reference gate + * (src/gate/node-gate.ts, kept for spike/contract comparison only; ProgramChain refuses it). + * + * argv [copy] + * fd 0 control lines: ATTACH [origin] | DETACH | STATS | RESET (peak counters) | QUIT + * fd 1 event lines: READY | ATTACHED + * | ACTIVE [ ] (Rust: A's buffered depth, B's backward step, B's depth, ticks B warmed) + * | RETIRED | DROPPED | DETACHED | EOF | STALL + * | STATS k=v... | ERROR + * | LAG (Rust: pending slot still behind A's on-air source position after 3 s; + * the gate never splices backwards, the worker replaces the compositor) + * | GAP (diagnostic: a repeat/silence fill, first 200) + * encoder FIFOs (opened for writing by path; the open is the rendezvous with the encoder): + * video = rawvideo yuv420p, fixed frame size; audio = s16le 48 kHz stereo, fixed bytes per frame + * compositor FIFOs (ATTACH): same formats, read non-blocking into frame rings. Video frame i and audio + * frame i of one compositor are one source pair: the Rust gate drops and skips both streams in lockstep. + * + * Exactly one video frame and one audio frame are written per tick. + */ + +export const GATE_SAMPLE_RATE = 48_000; +export const GATE_CHANNELS = 2; +const BYTES_PER_SAMPLE = 2 * GATE_CHANNELS; + +export interface FrameGeometry { width: number; height: number; fps: number } + +export function videoFrameBytes(g: FrameGeometry): number { return (g.width * g.height * 3) / 2; } + +export function audioFrameBytes(g: FrameGeometry): number { return (GATE_SAMPLE_RATE / g.fps) * BYTES_PER_SAMPLE; } + +export function assertGeometry(g: FrameGeometry): void { + if (!Number.isSafeInteger(g.width) || !Number.isSafeInteger(g.height) || g.width < 2 || g.height < 2 || g.width % 2 !== 0 || g.height % 2 !== 0 || g.width > 7680 || g.height > 4320) { + throw new Error("frame geometry must be even and at most 7680x4320"); + } + if (!Number.isSafeInteger(g.fps) || g.fps < 1 || g.fps > 120 || GATE_SAMPLE_RATE % g.fps !== 0) throw new Error("fps must divide 48000"); +} + +export interface GateStats { + ticks: number; idle: number; vrep: number; asil: number; drops: number; swaps: number; + lateMaxUs: number; lateOverFrame: number; writeMaxUs: number; cpuUs: number; wallUs: number; + active: string; pending: string; + /** Node gate only: longest GC pause, pool allocations after warm-up, peak encoder write backlog. */ + gcMaxUs: number; allocs: number; backlogMax: number; + /** Frames dropped because the encoder stopped reading (backlog > 2 s). */ + outDrops: number; + lateOver2ms: number; lateOver5ms: number; + /** Pending compositors reported `LAG` (still behind A's on-air source position after 3 s), and the largest gap seen. */ + lagged: number; behindMax: number; +} + +export type GateEvent = + | { type: "READY"; videoBytes: number; audioBytes: number } + | { type: "RESET" } + | { type: "GAP"; stream: "v" | "a"; id: number; tick: number; sinceActive: number; lateUs: number } + | { type: "ATTACHED" | "RETIRED" | "DROPPED" | "DETACHED" | "EOF"; id: number } + | { type: "ACTIVE" | "STALL"; id: number; tick: number } + | { type: "LAG"; id: number; behind: number } + | { type: "STATS"; stats: GateStats } + | { type: "ERROR"; id: number; what: string }; + +const STAT_KEYS: Record = { + ticks: "ticks", idle: "idle", vrep: "vrep", asil: "asil", drops: "drops", swaps: "swaps", + late_max_us: "lateMaxUs", late_over_frame: "lateOverFrame", write_max_us: "writeMaxUs", + cpu_us: "cpuUs", wall_us: "wallUs", active: "active", pending: "pending", + gc_max_us: "gcMaxUs", allocs: "allocs", backlog_max: "backlogMax", out_drops: "outDrops", late_over_2ms: "lateOver2ms", late_over_5ms: "lateOver5ms", lagged: "lagged", behind_max: "behindMax", +}; + +export function parseGateEvent(line: string): GateEvent | null { + const [type, ...rest] = line.trim().split(/\s+/); + const n = (i: number): number => Number(rest[i]); + switch (type) { + case "READY": return { type, videoBytes: n(0), audioBytes: n(1) }; + case "RESET": return { type }; + case "GAP": return { type, stream: rest[0] === "a" ? "a" : "v", id: n(1), tick: n(2), sinceActive: n(3), lateUs: n(4) }; + case "ATTACHED": case "RETIRED": case "DROPPED": case "DETACHED": case "EOF": return { type, id: n(0) }; + case "ACTIVE": case "STALL": return { type, id: n(0), tick: n(1) }; + case "LAG": return { type, id: n(0), behind: n(1) }; + case "ERROR": return { type, id: n(0), what: rest[1] ?? "" }; + case "STATS": { + const stats: GateStats = { ticks: 0, idle: 0, vrep: 0, asil: 0, drops: 0, swaps: 0, lateMaxUs: 0, lateOverFrame: 0, writeMaxUs: 0, cpuUs: 0, wallUs: 0, active: "-", pending: "-", gcMaxUs: 0, allocs: 0, backlogMax: 0, outDrops: 0, lateOver2ms: 0, lateOver5ms: 0, lagged: 0, behindMax: 0 }; + for (const pair of rest) { + const [k, v] = pair.split("="); + const key = k === undefined ? undefined : STAT_KEYS[k]; + if (key === undefined || v === undefined) continue; + if (key === "active" || key === "pending") stats[key] = v; + else stats[key] = Number(v); + } + return { type, stats }; + } + default: return null; + } +} + +/** FIFO paths are generated by the worker (scratch dir + numeric id); nothing else may reach a control line. */ +const SAFE_PATH = /^\/[A-Za-z0-9._/-]+$/; + +/** + * `origin`: source frame index of the compositor's first output frame, in a + * numbering shared by consecutive compositors of one chain. The Rust gate uses + * it to splice B in at A's next source frame (no backwards step or forward skip at a swap). + */ +export function attachLine(id: number, videoFifo: string, audioFifo: string, origin?: number): string { + if (!Number.isSafeInteger(id) || id < 0 || id > 0xffffffff) throw new Error("gate slot id out of range"); + for (const path of [videoFifo, audioFifo]) if (!SAFE_PATH.test(path) || path.includes("..")) throw new Error("unsafe gate fifo path"); + if (origin !== undefined && !Number.isSafeInteger(origin)) throw new Error("gate origin must be an integer"); + return `ATTACH ${id} ${videoFifo} ${audioFifo}${origin === undefined ? "" : ` ${origin}`}\n`; +} diff --git a/apps/media-worker/src/gate/rust/.gitignore b/apps/media-worker/src/gate/rust/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/apps/media-worker/src/gate/rust/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/apps/media-worker/src/gate/rust/Cargo.lock b/apps/media-worker/src/gate/rust/Cargo.lock new file mode 100644 index 0000000..d473dd7 --- /dev/null +++ b/apps/media-worker/src/gate/rust/Cargo.lock @@ -0,0 +1,7 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "frame-gate" +version = "0.1.0" diff --git a/apps/media-worker/src/gate/rust/Cargo.toml b/apps/media-worker/src/gate/rust/Cargo.toml new file mode 100644 index 0000000..88c0a9a --- /dev/null +++ b/apps/media-worker/src/gate/rust/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "frame-gate" +version = "0.1.0" +edition = "2021" +description = "Clock-master frame gate between swappable compositors and a persistent encoder (streaming-architecture.md §1.4)" +publish = false + +# std only: no crates, so the build works offline and the binary has no supply chain. +[dependencies] + +[profile.release] +opt-level = 3 +lto = true +codegen-units = 1 +panic = "abort" diff --git a/apps/media-worker/src/gate/rust/src/bin/source-bus.rs b/apps/media-worker/src/gate/rust/src/bin/source-bus.rs new file mode 100644 index 0000000..03fb282 --- /dev/null +++ b/apps/media-worker/src/gate/rust/src/bin/source-bus.rs @@ -0,0 +1,503 @@ +//! Source bus (docs/design/streaming-architecture.md §1.4; runs/wp4/s2b-report.md item 1). +//! +//! One process per source, between the source's ingest and every consumer of +//! its raw frames: the A/B compositors of a swap and the preview taps. The +//! Node worker used to move these frames itself (S2a); at 4 × 1080p30 that is +//! ~750 MB/s through 64 KiB chunks on one event loop, with no headroom left to +//! absorb a burst, and the buses dropped ~7.5 % of source pairs. This helper +//! moves the bytes; the worker keeps only the control lines. +//! +//! Semantics are those of the S2a Node bus (src/compositor/source-bus.ts): +//! +//! * raw yuv420p and s16le are re-framed into fixed-size frames and delivered +//! as *pairs* (video n + audio n), so a subscriber that joins mid-stream +//! starts at a frame boundary with A/V aligned; +//! * a source without audio (or whose audio stalls) gets silence once video +//! runs `maxSkew` frames ahead; audio that runs ahead of video is trimmed; +//! * a subscriber more than `maxBacklog + history` frames behind loses whole +//! pairs, never partial frames (a young subscriber, still starting up, may +//! queue one more second of frames for its first two seconds); +//! * the last `history` pairs are replayed to a new subscriber, and its first +//! pair's index is reported, so the gate can splice a swap-in compositor at +//! exactly the on-air compositor's next source frame. +//! +//! A subscriber may take every `stride`-th pair only (JPEG thumbnails), and may +//! take video or audio only (`-`). +//! +//! fds: 0 control lines, 1 event lines, 2 diagnostics. std only. +//! argv: +//! +//! control: SUB [stride] | UNSUB | STATS | QUIT +//! events: READY +//! | SUBSCRIBED :,...|- (pairs the other subscribers had +//! dropped at the same instant: the worker's origin arithmetic needs both together) +//! | UNSUBSCRIBED | CLOSED (the reader went away) | DROP +//! | EOF (the ingest closed both streams) | STATS k=v... | ERROR + +use std::collections::VecDeque; +use std::fs::{File, OpenOptions}; +use std::io::{self, ErrorKind, Read, Write}; +use std::os::unix::fs::OpenOptionsExt; +use std::os::unix::io::{AsRawFd, FromRawFd}; +use std::rc::Rc; +use std::time::{Duration, Instant}; + +#[repr(C)] +struct PollFd { + fd: i32, + events: i16, + revents: i16, +} + +extern "C" { + fn poll(fds: *mut PollFd, nfds: u64, timeout: i32) -> i32; + fn fcntl(fd: i32, cmd: i32, ...) -> i32; +} + +const POLLIN: i16 = 0x1; +const POLLOUT: i16 = 0x4; +const O_NONBLOCK: i32 = 0o4000; +const F_GETFL: i32 = 3; +const F_SETFL: i32 = 4; +const F_SETPIPE_SZ: i32 = 1031; +const F_GETPIPE_SZ: i32 = 1032; +const PIPE_BYTES: i32 = 1 << 20; +const SAMPLE_RATE: usize = 48_000; +const AUDIO_BYTES_PER_SAMPLE: usize = 4; // s16le stereo +const MAX_READS_PER_PUMP: usize = 64; +/// Pipes are serviced in bulk at most this often (as in the gate): waking per +/// 32 KiB chunk costs far more than moving 1 MiB per wake. 1 MiB / 2 ms = +/// 500 MB/s per pipe, above 1080p60 raw (186 MB/s). +const PUMP_INTERVAL: Duration = Duration::from_micros(2_000); +const YOUNG: Duration = Duration::from_secs(2); + +type Frame = Rc>; + +fn set_nonblocking(fd: i32) { + unsafe { + let flags = fcntl(fd, F_GETFL); + if flags >= 0 { + fcntl(fd, F_SETFL, flags | O_NONBLOCK); + } + } +} + +fn emit(line: &str) { + let mut out = io::stdout().lock(); + let _ = out.write_all(line.as_bytes()); + let _ = out.write_all(b"\n"); + let _ = out.flush(); +} + +fn arg(args: &[String], i: usize, name: &str, min: usize) -> usize { + match args.get(i).and_then(|v| v.parse::().ok()) { + Some(v) if v >= min => v, + _ => { + eprintln!("source-bus: invalid {}", name); + std::process::exit(2); + } + } +} + +/// Recycled frame buffers: a frame returns here when its last reference +/// (history, a subscriber queue) lets go, so steady state never allocates. +struct Pool { + free: Vec>, + size: usize, + keep: usize, + allocs: u64, +} + +impl Pool { + fn get(&mut self) -> Vec { + match self.free.pop() { + Some(b) => b, + None => { + self.allocs += 1; + vec![0u8; self.size] + } + } + } + fn put(&mut self, frame: Frame) { + if let Ok(buf) = Rc::try_unwrap(frame) { + if self.free.len() < self.keep { + self.free.push(buf); + } + } + } +} + +/// One ingest stream, re-framed into fixed-size frames. +struct Input { + file: File, + pipe_bytes: usize, + cur: Vec, + fill: usize, + eof: bool, +} + +impl Input { + fn open(path: &str, pool: &mut Pool) -> Input { + match OpenOptions::new().read(true).custom_flags(O_NONBLOCK).open(path) { + Ok(file) => { + unsafe { fcntl(file.as_raw_fd(), F_SETPIPE_SZ, PIPE_BYTES) }; + let pipe_bytes = unsafe { fcntl(file.as_raw_fd(), F_GETPIPE_SZ) }.max(0) as usize; + Input { file, pipe_bytes, cur: pool.get(), fill: 0, eof: false } + } + Err(_) => { + eprintln!("source-bus: cannot open ingest fifo"); + std::process::exit(2); + } + } + } + + /// Reads what the pipe holds; complete frames go to `out`. + fn pump(&mut self, pool: &mut Pool, out: &mut VecDeque) { + for _ in 0..MAX_READS_PER_PUMP { + if self.eof { + break; + } + let fill = self.fill; + match self.file.read(&mut self.cur[fill..]) { + Ok(0) => self.eof = true, + Ok(n) => { + self.fill += n; + if self.fill == pool.size { + let next = pool.get(); + out.push_back(Rc::new(std::mem::replace(&mut self.cur, next))); + self.fill = 0; + } + } + Err(e) if e.kind() == ErrorKind::WouldBlock => break, + Err(e) if e.kind() == ErrorKind::Interrupted => continue, + Err(_) => self.eof = true, + } + } + } +} + +/// One consumer stream: queued frame references, written non-blocking in order. +struct Out { + file: File, + pipe_bytes: usize, + queue: VecDeque<(Frame, usize)>, +} + +impl Out { + fn open(path: &str) -> io::Result { + let file = OpenOptions::new().write(true).custom_flags(O_NONBLOCK).open(path)?; + unsafe { fcntl(file.as_raw_fd(), F_SETPIPE_SZ, PIPE_BYTES) }; + let pipe_bytes = unsafe { fcntl(file.as_raw_fd(), F_GETPIPE_SZ) }.max(0) as usize; + Ok(Out { file, pipe_bytes, queue: VecDeque::new() }) + } + + /// false = the reader is gone. + fn flush(&mut self, pool: &mut Pool) -> bool { + while let Some((frame, off)) = self.queue.front_mut() { + match self.file.write(&frame[*off..]) { + Ok(n) => { + *off += n; + if *off == frame.len() { + let (frame, _) = self.queue.pop_front().unwrap(); + pool.put(frame); + } + } + Err(e) if e.kind() == ErrorKind::WouldBlock => return true, + Err(e) if e.kind() == ErrorKind::Interrupted => continue, + Err(_) => return false, + } + } + true + } + + fn dispose(self, pool: &mut Pool) { + for (frame, _) in self.queue { + pool.put(frame); + } + } +} + +struct Subscriber { + id: u32, + video: Option, + audio: Option, + stride: u64, + start_pair: u64, + young_until: Instant, + dropped: u64, +} + +impl Subscriber { + fn queued(&self) -> usize { + let v = self.video.as_ref().map_or(0, |o| o.queue.len()); + let a = self.audio.as_ref().map_or(0, |o| o.queue.len()); + v.max(a) + } +} + +struct Bus { + vpool: Pool, + apool: Pool, + vq: VecDeque, + aq: VecDeque, + silence: Frame, + history: VecDeque<(Frame, Option)>, + subs: Vec, + max_skew: usize, + limit: usize, + young_limit: usize, + history_frames: usize, + pairs: u64, + silence_filled: u64, + audio_trimmed: u64, + subscriber_drops: u64, +} + +impl Bus { + /// Pairs first, then applies the skew rules to what is left over. A pump reads many frames of + /// each stream at once, so trimming before pairing would discard audio its video is about to + /// claim (the S2a Node bus drained after every single frame, where the order did not matter). + fn drain(&mut self) { + while !self.vq.is_empty() && !self.aq.is_empty() { + let video = self.vq.pop_front().unwrap(); + let audio = self.aq.pop_front(); + self.deliver(video, audio); + } + // Audio more than `max_skew` frames ahead of video: trim the oldest. + while self.aq.len() > self.max_skew { + let f = self.aq.pop_front().unwrap(); + self.apool.put(f); + self.audio_trimmed += 1; + } + // Video more than `max_skew` frames ahead of audio (none, or stalled): pair with silence. + while self.vq.len() > self.max_skew { + let video = self.vq.pop_front().unwrap(); + self.silence_filled += 1; + self.deliver(video, None); + } + } + + fn deliver(&mut self, video: Frame, audio: Option) { + let index = self.pairs; + self.pairs += 1; + let now = Instant::now(); + for i in 0..self.subs.len() { + if (index - self.subs[i].start_pair) % self.subs[i].stride != 0 { + continue; + } + let sub = &mut self.subs[i]; + let limit = if now < sub.young_until { self.young_limit } else { self.limit }; + if sub.queued() > limit { + sub.dropped += 1; + self.subscriber_drops += 1; + emit(&format!("DROP {} {}", sub.id, sub.dropped)); + continue; + } + Self::enqueue(&mut self.subs[i], &video, audio.as_ref(), &self.silence); + } + self.history.push_back((video, audio)); + while self.history.len() > self.history_frames { + let (v, a) = self.history.pop_front().unwrap(); + self.vpool.put(v); + if let Some(a) = a { + self.apool.put(a); + } + } + } + + fn enqueue(sub: &mut Subscriber, video: &Frame, audio: Option<&Frame>, silence: &Frame) { + if let Some(v) = sub.video.as_mut() { + v.queue.push_back((Rc::clone(video), 0)); + } + if let Some(a) = sub.audio.as_mut() { + a.queue.push_back((Rc::clone(audio.unwrap_or(silence)), 0)); + } + } + + /// Replays the history (a strided subscriber gets only the newest pair) and reports where it starts. + fn subscribe(&mut self, id: u32, video: Option, audio: Option, stride: u64) -> u64 { + let replay = if stride > 1 { self.history.len().min(1) } else { self.history.len() }; + let start_pair = self.pairs - replay as u64; + let mut sub = Subscriber { id, video, audio, stride, start_pair, young_until: Instant::now() + YOUNG, dropped: 0 }; + for (v, a) in self.history.iter().skip(self.history.len() - replay) { + Self::enqueue(&mut sub, v, a.as_ref(), &self.silence); + } + self.subs.push(sub); + start_pair + } + + fn remove(&mut self, id: u32) -> bool { + match self.subs.iter().position(|s| s.id == id) { + Some(i) => { + let sub = self.subs.swap_remove(i); + if let Some(v) = sub.video { + v.dispose(&mut self.vpool); + } + if let Some(a) = sub.audio { + a.dispose(&mut self.apool); + } + true + } + None => false, + } + } + + fn flush(&mut self) { + let mut gone: Vec = Vec::new(); + for sub in self.subs.iter_mut() { + let v_ok = sub.video.as_mut().map_or(true, |o| o.flush(&mut self.vpool)); + let a_ok = sub.audio.as_mut().map_or(true, |o| o.flush(&mut self.apool)); + if !v_ok || !a_ok { + gone.push(sub.id); + } + } + for id in gone { + self.remove(id); + emit(&format!("CLOSED {}", id)); + } + } +} + +fn open_out(path: &str) -> Result, ()> { + if path == "-" { + return Ok(None); + } + if !path.starts_with('/') || path.contains("..") { + return Err(()); + } + Out::open(path).map(Some).map_err(|_| ()) +} + +fn main() { + let args: Vec = std::env::args().collect(); + let width = arg(&args, 1, "width", 2); + let height = arg(&args, 2, "height", 2); + let fps = arg(&args, 3, "fps", 1); + let max_skew = arg(&args, 4, "maxSkew", 1); + let max_backlog = arg(&args, 5, "maxBacklog", 1); + let history = arg(&args, 6, "history", 0); + if width % 2 != 0 || height % 2 != 0 || SAMPLE_RATE % fps != 0 { + eprintln!("source-bus: geometry must be even and fps must divide 48000"); + std::process::exit(2); + } + let video_bytes = width * height * 3 / 2; + let audio_bytes = SAMPLE_RATE / fps * AUDIO_BYTES_PER_SAMPLE; + let mut vpool = Pool { free: Vec::new(), size: video_bytes, keep: 0, allocs: 0 }; + let mut apool = Pool { free: Vec::new(), size: audio_bytes, keep: 0, allocs: 0 }; + let mut vin = Input::open(args.get(7).map(String::as_str).unwrap_or(""), &mut vpool); + let mut ain = Input::open(args.get(8).map(String::as_str).unwrap_or(""), &mut apool); + let limit = max_backlog + history; + // A young subscriber (process still starting) may queue one more second. Frames are shared + // references, so this costs memory only while that subscriber is actually stalled: + // at most (limit + fps) frames, 143 MB at 1080p30. + let young_limit = limit + fps; + // Recycled buffers for history + one backlog; anything beyond is freed when released. + vpool.keep = history + limit + 8; + apool.keep = 4 * (history + limit + 8) + 4 * fps; + let silence: Frame = Rc::new(vec![0u8; audio_bytes]); + let mut bus = Bus { + vpool, apool, vq: VecDeque::new(), aq: VecDeque::new(), silence, history: VecDeque::new(), subs: Vec::new(), + max_skew, limit, young_limit, history_frames: history, pairs: 0, silence_filled: 0, audio_trimmed: 0, subscriber_drops: 0, + }; + + set_nonblocking(0); + let mut control = unsafe { File::from_raw_fd(0) }; + let mut line_buf: Vec = Vec::with_capacity(4096); + let mut read_buf = [0u8; 4096]; + let mut last_pump = Instant::now() - PUMP_INTERVAL; + let mut eof_sent = false; + emit(&format!("READY {} {}", video_bytes, audio_bytes)); + + loop { + let now = Instant::now(); + // A shared uid may exceed pipe-user-pages-soft: F_SETPIPE_SZ then fails and new + // pipes hold only 8 KiB. Batching those for 2 ms caps raw transport at ~4 MB/s. + // Batch only when every video pipe has two pump intervals of headroom. + let bulk_min = video_bytes * fps / 250; + let bulk = vin.pipe_bytes >= bulk_min && bus.subs.iter().all(|s| s.video.as_ref().map_or(true, |v| v.pipe_bytes >= bulk_min)); + let pump_due = last_pump + if bulk { PUMP_INTERVAL } else { Duration::ZERO }; + let mut fds: Vec = Vec::with_capacity(3 + 2 * bus.subs.len()); + fds.push(PollFd { fd: 0, events: POLLIN, revents: 0 }); + let timeout_ms = if now >= pump_due { + fds.push(PollFd { fd: if vin.eof { -1 } else { vin.file.as_raw_fd() }, events: POLLIN, revents: 0 }); + fds.push(PollFd { fd: if ain.eof { -1 } else { ain.file.as_raw_fd() }, events: POLLIN, revents: 0 }); + for sub in &bus.subs { + for out in [sub.video.as_ref(), sub.audio.as_ref()].into_iter().flatten() { + fds.push(PollFd { fd: if out.queue.is_empty() { -1 } else { out.file.as_raw_fd() }, events: POLLOUT, revents: 0 }); + } + } + 1000 + } else { + (pump_due.saturating_duration_since(now).as_micros() as i32 + 999) / 1000 + }; + unsafe { poll(fds.as_mut_ptr(), fds.len() as u64, timeout_ms) }; + + if fds.len() > 1 && fds[1..].iter().any(|f| f.revents != 0) { + last_pump = Instant::now(); + vin.pump(&mut bus.vpool, &mut bus.vq); + ain.pump(&mut bus.apool, &mut bus.aq); + bus.drain(); + bus.flush(); + if vin.eof && ain.eof && !eof_sent { + eof_sent = true; + emit("EOF"); + } + } + + if fds[0].revents != 0 { + loop { + match control.read(&mut read_buf) { + Ok(0) => std::process::exit(0), // parent gone + Ok(n) => line_buf.extend_from_slice(&read_buf[..n]), + Err(e) if e.kind() == ErrorKind::WouldBlock => break, + Err(e) if e.kind() == ErrorKind::Interrupted => continue, + Err(_) => std::process::exit(0), + } + } + while let Some(pos) = line_buf.iter().position(|&b| b == b'\n') { + let line: Vec = line_buf.drain(..=pos).collect(); + let text = String::from_utf8_lossy(&line[..line.len() - 1]).into_owned(); + let parts: Vec<&str> = text.split_whitespace().collect(); + match parts.as_slice() { + ["SUB", id, vpath, apath, rest @ ..] if rest.len() <= 1 => { + let stride = rest.first().map_or(Some(1), |s| s.parse::().ok().filter(|&s| s >= 1)); + match (id.parse::(), stride) { + (Ok(id), Some(stride)) if !bus.subs.iter().any(|s| s.id == id) && (*vpath != "-" || *apath != "-") => { + match (open_out(vpath), open_out(apath)) { + (Ok(video), Ok(audio)) => { + let others: Vec = bus.subs.iter().map(|s| format!("{}:{}", s.id, s.dropped)).collect(); + let start = bus.subscribe(id, video, audio, stride); + let others = if others.is_empty() { "-".to_string() } else { others.join(",") }; + emit(&format!("SUBSCRIBED {} {} {}", id, start, others)); + bus.flush(); + } + _ => emit(&format!("ERROR {} open", id)), + } + } + (Ok(id), _) => emit(&format!("ERROR {} subscribe", id)), + _ => emit("ERROR 0 id"), + } + } + ["UNSUB", id] => { + let id = id.parse::().unwrap_or(u32::MAX); + if bus.remove(id) { + emit(&format!("UNSUBSCRIBED {}", id)); + } + } + ["STATS"] => { + let subs: Vec = bus.subs.iter().map(|s| format!("{}:{}:{}", s.id, s.dropped, s.queued())).collect(); + emit(&format!( + "STATS pairs={} silence_filled={} audio_trimmed={} subscriber_drops={} allocs={} video_waiting={} audio_waiting={} video_eof={} audio_eof={} subs={}", + bus.pairs, bus.silence_filled, bus.audio_trimmed, bus.subscriber_drops, bus.vpool.allocs + bus.apool.allocs, + bus.vq.len(), bus.aq.len(), vin.eof as u8, ain.eof as u8, + if subs.is_empty() { "-".to_string() } else { subs.join(",") } + )); + } + ["QUIT"] => std::process::exit(0), + [] => {} + _ => emit("ERROR 0 command"), + } + } + } + } +} diff --git a/apps/media-worker/src/gate/rust/src/main.rs b/apps/media-worker/src/gate/rust/src/main.rs new file mode 100644 index 0000000..35aee66 --- /dev/null +++ b/apps/media-worker/src/gate/rust/src/main.rs @@ -0,0 +1,749 @@ +//! Frame gate (docs/design/streaming-architecture.md §1.4). +//! +//! The gate is the clock master between swappable compositor processes and one +//! persistent encoder. It emits exactly one raw video frame and one PCM frame +//! per tick of the output frame rate, whatever the compositors do: +//! +//! * the active compositor's next frame when it has one; +//! * otherwise the last video frame again plus silence (a stall never starves +//! the encoder, so a dead or SIGKILLed compositor never takes RTMP down); +//! * a pending compositor is promoted at a tick boundary once it has buffered +//! `warm` frames of both video and audio, and the old one is closed. +//! +//! Raw yuv420p and s16le have fixed bytes per frame, so there is no demuxing and +//! no timestamp parsing: frames are counted. The pipe contract is shared with +//! the Node implementation (src/gate/node-gate.ts) and documented in +//! src/gate/protocol.ts. +//! +//! fds: 0 control lines, 1 event lines, 2 diagnostics. +//! argv: [copy] +//! +//! Single thread, one poll loop. Every pipe is non-blocking and each encoder +//! output has its own queue, so the tick clock never waits on the encoder and +//! a consumer that reads its inputs in any order (ffmpeg probing one input +//! before opening the next) cannot deadlock against the gate. + +use std::collections::VecDeque; +use std::fs::{File, OpenOptions}; +use std::io::{self, ErrorKind, Read, Write}; +use std::os::unix::fs::OpenOptionsExt; +use std::os::unix::io::{AsRawFd, FromRawFd}; +use std::time::{Duration, Instant}; + +#[repr(C)] +struct PollFd { + fd: i32, + events: i16, + revents: i16, +} + +#[repr(C)] +struct IoVec { + base: *const u8, + len: usize, +} + +#[repr(C)] +struct TimeVal { + sec: i64, + usec: i64, +} + +#[repr(C)] +struct RUsage { + utime: TimeVal, + stime: TimeVal, + rest: [i64; 14], +} + +extern "C" { + fn poll(fds: *mut PollFd, nfds: u64, timeout: i32) -> i32; + fn fcntl(fd: i32, cmd: i32, ...) -> i32; + fn getrusage(who: i32, usage: *mut RUsage) -> i32; + fn vmsplice(fd: i32, iov: *const IoVec, nr_segs: u64, flags: u32) -> isize; + fn write(fd: i32, buf: *const u8, count: usize) -> isize; +} + +const POLLIN: i16 = 0x1; +const POLLOUT: i16 = 0x4; +const O_NONBLOCK: i32 = 0o4000; +const F_GETFL: i32 = 3; +const F_SETFL: i32 = 4; +const F_SETPIPE_SZ: i32 = 1031; +const F_GETPIPE_SZ: i32 = 1032; +const SPLICE_F_NONBLOCK: u32 = 0x02; +const PIPE_BYTES: i32 = 1 << 20; +const SAMPLE_RATE: usize = 48_000; +const AUDIO_BYTES_PER_SAMPLE: usize = 4; // s16le stereo +const MAX_READS_PER_WAKE: usize = 64; +/// Compositor pipes are drained at most this often. Producers write in 32 KiB +/// chunks; waking per chunk costs far more CPU than draining a 1 MiB pipe in +/// bulk. 1 MiB / 2 ms = 500 MB/s per stream, above 1080p60 raw (186 MB/s). +const PUMP_INTERVAL: Duration = Duration::from_micros(2_000); + +fn set_nonblocking(fd: i32) { + unsafe { + let flags = fcntl(fd, F_GETFL); + if flags >= 0 { + fcntl(fd, F_SETFL, flags | O_NONBLOCK); + } + } +} + +fn cpu_us() -> u64 { + let mut usage = RUsage { utime: TimeVal { sec: 0, usec: 0 }, stime: TimeVal { sec: 0, usec: 0 }, rest: [0; 14] }; + unsafe { getrusage(0, &mut usage) }; + (usage.utime.sec * 1_000_000 + usage.utime.usec + usage.stime.sec * 1_000_000 + usage.stime.usec) as u64 +} + +/// Refcounted, preallocated frame buffers addressed by index. +/// +/// vmsplice hands buffer *pages* to the encoder pipe, so a buffer whose write +/// has finished may still be referenced by up to one pipe's worth (1 MiB) of +/// unread data. Released buffers therefore sit in quarantine until a later +/// frame has been fully written: a whole later frame (>= 1 MiB at the sizes +/// that matter) cannot be in the pipe together with any older page. +struct Pool { + bufs: Vec>, + refs: Vec, + free: Vec, + quarantine: Vec<(usize, u64)>, + size: usize, + allocs: u64, +} + +impl Pool { + fn new(size: usize, prealloc: usize) -> Pool { + Pool { bufs: (0..prealloc).map(|_| vec![0u8; size]).collect(), refs: vec![0; prealloc], free: (0..prealloc).rev().collect(), quarantine: Vec::new(), size, allocs: 0 } + } + fn get(&mut self) -> usize { + let idx = match self.free.pop() { + Some(i) => i, + None => { + self.bufs.push(vec![0u8; self.size]); + self.refs.push(0); + self.allocs += 1; + self.bufs.len() - 1 + } + }; + self.refs[idx] = 1; + idx + } + fn retain(&mut self, idx: usize) { + self.refs[idx] += 1; + } + /// `written` = frames fully written so far; the buffer is reusable once a later one completes. + fn release(&mut self, idx: usize, written: u64) { + self.refs[idx] -= 1; + if self.refs[idx] == 0 { + self.quarantine.push((idx, written)); + } + } + fn on_frame_written(&mut self, written: u64) { + let mut i = 0; + while i < self.quarantine.len() { + if self.quarantine[i].1 < written { + let (idx, _) = self.quarantine.swap_remove(i); + self.free.push(idx); + } else { + i += 1; + } + } + } +} + +/// Fixed-size frame ring of pool buffers. One extra slot holds the partial frame being read. +struct Ring { + slots: Vec, + head: usize, + count: usize, + fill: usize, + cap: usize, + /// Frames that have left the ring (taken, dropped or skipped): the source position of the head frame. + removed: u64, + /// Frames already counted in `removed` that have not arrived yet: discarded on arrival. + skip: u64, +} + +impl Ring { + fn new(cap: usize, pool: &mut Pool) -> Ring { + Ring { slots: (0..=cap).map(|_| pool.get()).collect(), head: 0, count: 0, fill: 0, cap, removed: 0, skip: 0 } + } + fn full(&self) -> bool { + self.count >= self.cap + } + fn write_idx(&self) -> usize { + self.slots[(self.head + self.count) % self.slots.len()] + } + fn advance(&mut self, n: usize, size: usize) { + self.fill += n; + if self.fill == size { + self.fill = 0; + if self.skip > 0 { + self.skip -= 1; // the slot is overwritten by the next frame + } else { + self.count += 1; + } + } + } + /// Drops the oldest frame, or the next one to arrive when the ring is empty. + fn skip_one(&mut self) { + if self.count > 0 { + self.drop_oldest(); + } else { + self.skip += 1; + self.removed += 1; + } + } + fn drop_oldest(&mut self) { + // The partial frame sits at head+count, which this leaves in place. + if self.count > 0 { + self.head = (self.head + 1) % self.slots.len(); + self.count -= 1; + self.removed += 1; + } + } + /// Hands the oldest frame's buffer to the caller (who owns its reference) and refills the slot. + fn take(&mut self, pool: &mut Pool) -> usize { + let idx = self.slots[self.head]; + self.slots[self.head] = pool.get(); + self.head = (self.head + 1) % self.slots.len(); + self.count -= 1; + self.removed += 1; + idx + } + fn dispose(&mut self, pool: &mut Pool, written: u64) { + for &idx in &self.slots { + pool.release(idx, written); + } + self.slots.clear(); + } +} + +struct Stream { + file: File, + pipe_bytes: usize, + ring: Ring, + eof: bool, +} + +impl Stream { + fn open(path: &str, cap: usize, pool: &mut Pool) -> io::Result { + let file = OpenOptions::new().read(true).custom_flags(O_NONBLOCK).open(path)?; + unsafe { fcntl(file.as_raw_fd(), F_SETPIPE_SZ, PIPE_BYTES) }; + let pipe_bytes = unsafe { fcntl(file.as_raw_fd(), F_GETPIPE_SZ) }.max(0) as usize; + Ok(Stream { file, pipe_bytes, ring: Ring::new(cap, pool), eof: false }) + } + + /// Reads until the pipe is empty or the ring is full (the compositor then sees backpressure). + fn pump(&mut self, pool: &mut Pool) { + for _ in 0..MAX_READS_PER_WAKE { + if self.eof || self.ring.full() { + break; + } + let idx = self.ring.write_idx(); + let fill = self.ring.fill; + match self.file.read(&mut pool.bufs[idx][fill..]) { + Ok(0) => self.eof = true, + Ok(n) => self.ring.advance(n, pool.size), + Err(e) if e.kind() == ErrorKind::WouldBlock => break, + Err(e) if e.kind() == ErrorKind::Interrupted => continue, + Err(_) => self.eof = true, + } + } + } +} + +struct Slot { + id: u32, + video: Stream, + audio: Stream, + /// Source frame index (in the worker's per-chain numbering) of this compositor's first + /// output frame. With origins on both sides, a swap splices B at A's next source frame. + origin: Option, + /// Tick at which this pending slot first produced a frame (promotion timeout). + warm_since: Option, + /// `LAG` already reported for this pending slot. + lagged: bool, +} + +impl Slot { + fn head_seq(&self) -> Option { + self.origin.map(|o| o + self.video.ring.removed as i64) + } + /// Video frame i and audio frame i of a compositor are one source pair, so every drop or skip + /// applies to both streams and the ring heads always hold the same pair index. + /// + /// Pending slot: keeps complete pairs at A's next position. Only whole pairs are dropped; a stream + /// that runs ahead of the other (compositor audio usually leads video) waits in its pipe + /// until its partner frames arrive, instead of discarding frames the pairing still needs. + fn pump_pending(&mut self, vpool: &mut Pool, apool: &mut Pool, target: Option) -> u64 { + let mut dropped = 0; + for _ in 0..4 { + self.video.pump(vpool); + self.audio.pump(apool); + let mut freed = false; + while (self.video.ring.full() || self.audio.ring.full()) && self.video.ring.count > 0 && self.audio.ring.count > 0 + && target.zip(self.head_seq()).map_or(true, |(t, head)| head < t) { + self.video.ring.drop_oldest(); + self.audio.ring.drop_oldest(); + dropped += 1; + freed = true; + } + if !freed { + break; + } + } + dropped + } + fn ended(&self) -> bool { + (self.video.eof && self.video.ring.count == 0) || (self.audio.eof && self.audio.ring.count == 0) + } + fn close(mut self, vpool: &mut Pool, apool: &mut Pool, written: u64) { + self.video.ring.dispose(vpool, written); + self.audio.ring.dispose(apool, written); + } +} + +/// One encoder output: queued buffer references, written non-blocking in order. +struct Out { + fd: i32, + queue: VecDeque<(usize, usize)>, + zero_copy: bool, + written_frames: u64, +} + +enum Flush { + Ok, + Closed, +} + +impl Out { + fn flush(&mut self, pool: &mut Pool) -> Flush { + while let Some(&(idx, off)) = self.queue.front() { + let buf = &pool.bufs[idx][off..]; + let n = if self.zero_copy { + let iov = IoVec { base: buf.as_ptr(), len: buf.len() }; + unsafe { vmsplice(self.fd, &iov, 1, SPLICE_F_NONBLOCK) } + } else { + unsafe { write(self.fd, buf.as_ptr(), buf.len()) } + }; + if n < 0 { + let err = io::Error::last_os_error(); + match err.kind() { + ErrorKind::WouldBlock => return Flush::Ok, + ErrorKind::Interrupted => continue, + _ => return Flush::Closed, + } + } + let n = n as usize; + if n == buf.len() { + self.queue.pop_front(); + self.written_frames += 1; + pool.release(idx, self.written_frames - 1); + pool.on_frame_written(self.written_frames); + } else { + self.queue.front_mut().unwrap().1 += n; + } + } + Flush::Ok + } +} + +#[derive(Default)] +struct Stats { + ticks: u64, + idle: u64, + vrep: u64, + asil: u64, + drops: u64, + swaps: u64, + late_max_us: u64, + late_over_frame: u64, + late_over_2ms: u64, + late_over_5ms: u64, + backlog_max: u64, + out_drops: u64, + stall_run: u64, + active_since: u64, + gap_events: u64, + lagged: u64, + behind_max: u64, +} + +fn emit(line: &str) { + let mut out = io::stdout().lock(); + let _ = out.write_all(line.as_bytes()); + let _ = out.write_all(b"\n"); + let _ = out.flush(); +} + +fn arg(args: &[String], i: usize, name: &str) -> usize { + match args.get(i).and_then(|v| v.parse::().ok()) { + Some(v) if v > 0 => v, + _ => { + eprintln!("frame-gate: invalid {}", name); + std::process::exit(2); + } + } +} + +fn open_out(args: &[String], i: usize) -> File { + let path = args.get(i).map(String::as_str).unwrap_or(""); + match OpenOptions::new().write(true).custom_flags(O_NONBLOCK).open(path) { + Ok(f) => { + unsafe { fcntl(f.as_raw_fd(), F_SETPIPE_SZ, PIPE_BYTES) }; + f + } + Err(_) => { + eprintln!("frame-gate: cannot open encoder fifo (reader must be open first)"); + std::process::exit(2); + } + } +} + +fn main() { + let args: Vec = std::env::args().collect(); + let width = arg(&args, 1, "width"); + let height = arg(&args, 2, "height"); + let fps = arg(&args, 3, "fps"); + let ring_frames = arg(&args, 4, "ringFrames"); + let warm = arg(&args, 5, "warmFrames"); + if width % 2 != 0 || height % 2 != 0 || SAMPLE_RATE % fps != 0 || warm > ring_frames { + eprintln!("frame-gate: geometry must be even, fps must divide 48000, warm <= ring"); + std::process::exit(2); + } + let video_bytes = width * height * 3 / 2; + let audio_bytes = SAMPLE_RATE / fps * AUDIO_BYTES_PER_SAMPLE; + // vmsplice is only safe when a frame is at least a pipe's worth (see Pool); `copy` disables it. + let zero_copy = args.get(8).map_or(true, |v| v != "copy") && video_bytes >= PIPE_BYTES as usize; + // Encoder backlog beyond 0.5 s is an encoder that stopped reading: drop the oldest unstarted frames. + let max_backlog = (fps / 2).max(4); + + let enc_video_file = open_out(&args, 6); + let enc_audio_file = open_out(&args, 7); + set_nonblocking(0); + let mut control = unsafe { File::from_raw_fd(0) }; + + // Two slots' rings + last + a few in flight; backlog beyond that allocates (counted in `allocs`). + let pool_frames = 2 * (ring_frames + 1) + 6; + // Compositor audio leads video by the video graph's extra latency. PCM frames are small, so the + // audio rings hold 4 s: the leading stream never backs up into the compositor (whose blocked + // audio write would stall the very video frames its pairs are waiting for). + let audio_ring_frames = ring_frames.max(4 * fps); + let mut vpool = Pool::new(video_bytes, pool_frames); + let mut apool = Pool::new(audio_bytes, 2 * (audio_ring_frames + 1) + 6); + let mut vout = Out { fd: enc_video_file.as_raw_fd(), queue: VecDeque::new(), zero_copy, written_frames: 0 }; + let mut aout = Out { fd: enc_audio_file.as_raw_fd(), queue: VecDeque::new(), zero_copy: false, written_frames: 0 }; + + // Black (limited-range yuv420p) until the first compositor is promoted. + let mut last_video = vpool.get(); + { + let b = &mut vpool.bufs[last_video]; + b[..width * height].fill(16); + b[width * height..].fill(128); + } + let silence = apool.get(); // zeroed; never released + + let mut active: Option = None; + let mut pending: Option = None; + let mut stats = Stats::default(); + let mut line_buf: Vec = Vec::with_capacity(4096); + let mut read_buf = [0u8; 4096]; + let mut last_pump = Instant::now(); + + let start = Instant::now(); + let period = Duration::from_nanos(1_000_000_000 / fps as u64); + // Exact due instant for tick n (no per-frame rounding drift, no overflow). + let due_at = |n: u64| start + Duration::from_nanos(n * 1_000_000_000 / fps as u64); + let cpu_start = cpu_us(); + emit(&format!("READY {} {} {} {}", video_bytes, audio_bytes, ring_frames, warm)); + + loop { + // 1. Emit every tick that is due (catch-up keeps the frame count locked to wall time). + let now = Instant::now(); + while now >= due_at(stats.ticks) { + let late = (now - due_at(stats.ticks)).as_micros() as u64; + stats.late_max_us = stats.late_max_us.max(late); + if late > period.as_micros() as u64 { + stats.late_over_frame += 1; + } + if late > 2_000 { + stats.late_over_2ms += 1; + } + if late > 5_000 { + stats.late_over_5ms += 1; + } + let written = vout.written_frames; + + if pending.as_ref().map_or(false, |p| p.ended()) { + let p = pending.take().unwrap(); + emit(&format!("EOF {}", p.id)); + p.close(&mut vpool, &mut apool, written); + } + // Promotion. With origins on A and B: drop B's frames older than A's next source + // frame and splice there (B inherits A's latency; the source never runs backwards). + // If B is ahead, hold its frames until A reaches them; never skip on promotion. + // Without origins (or no active): newest `warm` frames. + let target = active.as_ref().and_then(|a| a.head_seq()); + // A's current buffer depth: B must match it (capped at `warm`) to keep the same jitter margin. + let active_depth = active.as_ref().map_or(warm, |a| a.video.ring.count.min(a.audio.ring.count)); + let mut ready = false; + let mut aligned = false; + // Diagnostics for the ACTIVE line: A's depth, B's backward step, B's depth, B's age. + let mut promo = (active_depth, 0i64, 0usize, 0u64); + if let Some(p) = pending.as_mut() { + if p.warm_since.is_none() && p.video.ring.count > 0 { + p.warm_since = Some(stats.ticks); + } + let need_aligned = active_depth.min(warm).saturating_sub(1).max(1); + let mut behind = 0i64; + if let (Some(t), Some(head)) = (target, p.head_seq()) { + aligned = true; + let count = p.video.ring.count.min(p.audio.ring.count); + let newest = head + count as i64 - 1; + if count > 0 && newest >= t { + // B holds A's next source frame: splice exactly there. + while p.video.ring.count > 0 && p.audio.ring.count > 0 && p.head_seq().unwrap_or(t) < t { + p.video.ring.drop_oldest(); + p.audio.ring.drop_oldest(); + } + behind = t - p.head_seq().unwrap_or(t); + } else if count > 0 { + // B's pipeline is slower than A's on-air position: take its newest frames + // (smallest backward step) rather than hold the swap. + while p.video.ring.count > need_aligned && p.audio.ring.count > need_aligned { + p.video.ring.drop_oldest(); + p.audio.ring.drop_oldest(); + } + behind = t - p.head_seq().unwrap_or(t); + } + } + let have = p.video.ring.count.min(p.audio.ring.count); + let need = if aligned { need_aligned } else { warm }; + let age = p.warm_since.map_or(0, |w| stats.ticks - w); + // Aligned but shallower than A (B's pipeline slightly slower): splicing costs no + // artifact, only jitter margin, so accept it after a second of production. + let in_step = behind == 0 && (have >= need || (have >= 1 && age >= fps as u64)); + // Unaligned (first compositor, or no shared numbering): go with what there is after 5 s. + let timed_out = !aligned && age >= 5 * fps as u64 && have >= 1; + ready = in_step || timed_out; + promo = (active_depth, behind, have, age); + // B behind A's on-air source position: never splice backwards (that replays picture + // and audio). A stays on air; B is usually a CPU-starved spawn still draining its + // startup backlog. After 3 s report it so the worker can replace it. + if behind > 0 { + stats.behind_max = stats.behind_max.max(behind as u64); + if age >= 3 * fps as u64 && !p.lagged { + p.lagged = true; + stats.lagged += 1; + emit(&format!("LAG {} {}", p.id, behind)); + } + } + } + if ready { + let mut p = pending.take().unwrap(); + if !aligned { + while p.video.ring.count > warm || p.audio.ring.count > warm { + p.video.ring.skip_one(); + p.audio.ring.skip_one(); + } + } + let id = p.id; + let old = active.replace(p); + stats.swaps += 1; + stats.stall_run = 0; + stats.active_since = stats.ticks; + emit(&format!("ACTIVE {} {} {} {} {} {}", id, stats.ticks, promo.0, promo.1, promo.2, promo.3)); + if let Some(o) = old { + emit(&format!("RETIRED {}", o.id)); + o.close(&mut vpool, &mut apool, written); + } + } + + let mut audio_idx = silence; + apool.retain(silence); + match active.as_mut() { + Some(a) => { + // A joint stall repeats/silences without consuming a source position. + // Skipping both future frames on every empty tick can keep a producer + // running at exactly fps permanently behind the skip counters after a pause. + // One-sided stalls still skip their late partner to preserve A/V lockstep. + let joint_stall = a.video.ring.count == 0 && a.audio.ring.count == 0; + if a.video.ring.count > 0 { + let idx = a.video.ring.take(&mut vpool); + vpool.release(last_video, written); + last_video = idx; + stats.stall_run = 0; + } else { + // Repeat now, discard the late frame when it arrives (keeps A/V index-locked). + if !joint_stall { a.video.ring.skip_one(); } + stats.vrep += 1; + if stats.gap_events < 200 { + stats.gap_events += 1; + emit(&format!("GAP v {} {} {} {}", a.id, stats.ticks, stats.ticks - stats.active_since, late)); + } + stats.stall_run += 1; + if stats.stall_run == fps as u64 { + emit(&format!("STALL {} {}", a.id, stats.ticks)); + } + } + if a.audio.ring.count > 0 { + apool.release(silence, aout.written_frames); + audio_idx = a.audio.ring.take(&mut apool); + } else { + if !joint_stall { a.audio.ring.skip_one(); } + stats.asil += 1; + if stats.gap_events < 200 { + stats.gap_events += 1; + emit(&format!("GAP a {} {} {} {}", a.id, stats.ticks, stats.ticks - stats.active_since, late)); + } + } + } + None => stats.idle += 1, + } + if active.as_ref().map_or(false, |a| a.ended()) { + let a = active.take().unwrap(); + emit(&format!("EOF {}", a.id)); + a.close(&mut vpool, &mut apool, written); + } + + vpool.retain(last_video); + vout.queue.push_back((last_video, 0)); + aout.queue.push_back((audio_idx, 0)); + for (out, pool) in [(&mut vout, &mut vpool), (&mut aout, &mut apool)] { + // Never drop the frame in flight (index 0 may be partially written). + while out.queue.len() > max_backlog && out.queue.len() > 1 { + let (idx, _) = out.queue.remove(1).unwrap(); + pool.release(idx, out.written_frames); + stats.out_drops += 1; + } + } + stats.backlog_max = stats.backlog_max.max(vout.queue.len() as u64); + stats.ticks += 1; + } + for (out, pool) in [(&mut vout, &mut vpool), (&mut aout, &mut apool)] { + if let Flush::Closed = out.flush(pool) { + eprintln!("frame-gate: encoder input closed"); + std::process::exit(3); + } + } + + // 2. Wait for input, encoder writability, or the next tick. + let due = due_at(stats.ticks); + let now = Instant::now(); + // F_SETPIPE_SZ can fail under a shared uid's pipe quota. An 8 KiB video pipe + // must be drained as soon as readable; a 2 ms hold would cap it at ~4 MB/s. + let bulk_min = video_bytes * fps / 250; + let bulk = [active.as_ref(), pending.as_ref()].into_iter().flatten().all(|s| s.video.pipe_bytes >= bulk_min); + let pump_due = last_pump + if bulk { PUMP_INTERVAL } else { Duration::ZERO }; + let pump_now = now >= pump_due; + let until = if pump_now { due } else { due.min(pump_due) }; + let timeout_ms = (until.saturating_duration_since(now).as_micros() as i32 + 999) / 1000; + let mut fds: Vec = Vec::with_capacity(7); + fds.push(PollFd { fd: 0, events: POLLIN, revents: 0 }); + fds.push(PollFd { fd: if vout.queue.is_empty() { -1 } else { vout.fd }, events: POLLOUT, revents: 0 }); + fds.push(PollFd { fd: if aout.queue.is_empty() { -1 } else { aout.fd }, events: POLLOUT, revents: 0 }); + if pump_now { + for slot in [active.as_ref(), pending.as_ref()] { + if let Some(s) = slot { + for st in [&s.video, &s.audio] { + let want = !st.eof && !st.ring.full(); + fds.push(PollFd { fd: if want { st.file.as_raw_fd() } else { -1 }, events: POLLIN, revents: 0 }); + } + } + } + } + unsafe { poll(fds.as_mut_ptr(), fds.len() as u64, timeout_ms) }; + + // 3. Pump compositor pipes, then hold off for PUMP_INTERVAL so they refill in bulk. + if fds.len() > 3 && fds[3..].iter().any(|f| f.revents != 0) { + last_pump = Instant::now(); + if let Some(a) = active.as_mut() { + a.video.pump(&mut vpool); + a.audio.pump(&mut apool); + } + if let Some(p) = pending.as_mut() { + let target = active.as_ref().and_then(|a| a.head_seq()); + stats.drops += p.pump_pending(&mut vpool, &mut apool, target); + } + } + + // 4. Control lines. + if fds[0].revents != 0 { + loop { + match control.read(&mut read_buf) { + Ok(0) => std::process::exit(0), // parent gone + Ok(n) => line_buf.extend_from_slice(&read_buf[..n]), + Err(e) if e.kind() == ErrorKind::WouldBlock => break, + Err(e) if e.kind() == ErrorKind::Interrupted => continue, + Err(_) => std::process::exit(0), + } + } + while let Some(pos) = line_buf.iter().position(|&b| b == b'\n') { + let line: Vec = line_buf.drain(..=pos).collect(); + let text = String::from_utf8_lossy(&line[..line.len() - 1]).into_owned(); + let parts: Vec<&str> = text.split_whitespace().collect(); + let written = vout.written_frames; + match parts.as_slice() { + ["ATTACH", id, vpath, apath, rest @ ..] if rest.len() <= 1 => match id.parse::() { + Ok(id) => { + let origin = rest.first().and_then(|v| v.parse::().ok()); + let video = Stream::open(vpath, ring_frames, &mut vpool); + let audio = Stream::open(apath, audio_ring_frames, &mut apool); + match (video, audio) { + (Ok(video), Ok(audio)) => { + if let Some(old) = pending.replace(Slot { id, video, audio, origin, warm_since: None, lagged: false }) { + emit(&format!("DROPPED {}", old.id)); + old.close(&mut vpool, &mut apool, written); + } + emit(&format!("ATTACHED {}", id)); + } + (v, a) => { + if let Ok(mut s) = v { + s.ring.dispose(&mut vpool, written); + } + if let Ok(mut s) = a { + s.ring.dispose(&mut apool, written); + } + emit(&format!("ERROR {} open", id)); + } + } + } + Err(_) => emit("ERROR 0 id"), + }, + ["DETACH", id] => { + let id = id.parse::().unwrap_or(u32::MAX); + if pending.as_ref().map_or(false, |p| p.id == id) { + pending.take().unwrap().close(&mut vpool, &mut apool, written); + emit(&format!("DETACHED {}", id)); + } else if active.as_ref().map_or(false, |a| a.id == id) { + active.take().unwrap().close(&mut vpool, &mut apool, written); + emit(&format!("DETACHED {}", id)); + } + } + ["STATS"] => { + let ring = |s: Option<&Slot>| s.map_or("-".to_string(), |s| format!("{}:{}/{}", s.id, s.video.ring.count, s.audio.ring.count)); + emit(&format!( + "STATS ticks={} idle={} vrep={} asil={} drops={} swaps={} late_max_us={} late_over_frame={} write_max_us=0 cpu_us={} wall_us={} active={} pending={} gc_max_us=0 allocs={} backlog_max={} out_drops={} late_over_2ms={} late_over_5ms={} lagged={} behind_max={}", + stats.ticks, stats.idle, stats.vrep, stats.asil, stats.drops, stats.swaps, stats.late_max_us, + stats.late_over_frame, cpu_us() - cpu_start, start.elapsed().as_micros(), + ring(active.as_ref()), ring(pending.as_ref()), vpool.allocs + apool.allocs, stats.backlog_max, stats.out_drops, stats.late_over_2ms, stats.late_over_5ms, stats.lagged, stats.behind_max + )); + } + ["RESET"] => { + stats.late_max_us = 0; + stats.late_over_frame = 0; + stats.late_over_2ms = 0; + stats.late_over_5ms = 0; + stats.backlog_max = 0; + emit("RESET"); + } + ["QUIT"] => std::process::exit(0), + [] => {} + _ => emit("ERROR 0 command"), + } + } + } + } +} diff --git a/apps/media-worker/src/gate/sink-cli.ts b/apps/media-worker/src/gate/sink-cli.ts new file mode 100644 index 0000000..5f22ea0 --- /dev/null +++ b/apps/media-worker/src/gate/sink-cli.ts @@ -0,0 +1,30 @@ +/** + * Verifying null encoder as its own process (so its timing is not polluted by + * the harness that spawns compositors): `sink-cli.ts `. + * Opens the gate's encoder FIFOs non-blocking, prints `OPEN`; on a + * `REPORT ` line prints the JSON report and exits. + */ +import { constants, openSync } from "node:fs"; +import { Socket } from "node:net"; +import { VerifyingSink } from "./sink"; + +function main(): void { + const [videoPath, audioPath, w, h, fps] = process.argv.slice(2); + if (!videoPath || !audioPath) throw new Error("usage: sink-cli "); + const sink = new VerifyingSink({ width: Number(w), height: Number(h), fps: Number(fps) }); + const open = (path: string) => new Socket({ fd: openSync(path, constants.O_RDONLY | constants.O_NONBLOCK), readable: true, writable: false }); + sink.attach(open(videoPath), open(audioPath)); + process.stdout.write("OPEN\n"); + let buf = ""; + process.stdin.setEncoding("utf8"); + process.stdin.on("data", (chunk: string) => { + buf += chunk; + const m = /REPORT (\d+)\n/.exec(buf); + if (!m) return; + const first = Number(m[1]); + const r = sink.report; + process.stdout.write(`${JSON.stringify({ ...r, silentAt: undefined, tagChanges: r.tagChanges.slice(0, 20_000), silentAfterFirstActive: sink.silentFramesFrom(first) })}\n`, () => process.exit(0)); + }); +} + +if (require.main === module) main(); diff --git a/apps/media-worker/src/gate/sink.ts b/apps/media-worker/src/gate/sink.ts new file mode 100644 index 0000000..1277e9e --- /dev/null +++ b/apps/media-worker/src/gate/sink.ts @@ -0,0 +1,139 @@ +import type { Readable } from "node:stream"; +import { audioFrameBytes, videoFrameBytes, type FrameGeometry } from "./protocol"; + +/** + * Null encoder that verifies what an encoder would receive from the gate: + * frame counts in lockstep, arrival gaps, a luma tag per frame (Y at byte 0, + * which tests use as a monotonic frame/scene counter), and PCM continuity + * (largest sample-to-sample step, all-zero frames). + */ +export interface SinkReport { + videoFrames: number; + audioFrames: number; + /** Largest gap between two consecutive completed video frames. */ + maxGapMs: number; + /** Video frames whose arrival gap exceeded two frame intervals. */ + gapsOverTwoFrames: number; + /** Tag of each video frame is recorded only when it changes: [frameIndex, tag]. */ + tagChanges: Array<[number, number]>; + /** + * Histogram of per-frame counter steps for counter tags (luma 20 + 4·(n mod 50), decoded + * with ±1 tolerance): "1" is a normal frame, "0" a repeat, "2+" a skip, negative backwards. + */ + tagDeltas: Record; + maxPcmStep: number; + maxPcmAbs: number; + silentAudioFrames: number; + /** Frame index of each all-zero audio frame (first 100 000). */ + silentAt: number[]; +} + +export class VerifyingSink { + private readonly vBytes: number; + private readonly aBytes: number; + private vPos = 0; + private aPos = 0; + private lastFrameAt = 0n; + private prev: [number, number] | null = null; + private frameNonZero = false; + private lastTag = -1; + private readonly intervalNs: bigint; + readonly report: SinkReport = { + videoFrames: 0, audioFrames: 0, maxGapMs: 0, gapsOverTwoFrames: 0, tagChanges: [], tagDeltas: {}, + maxPcmStep: 0, maxPcmAbs: 0, silentAudioFrames: 0, silentAt: [], + }; + /** PCM steps are not measured before this frame (gate boot silence before the first compositor). */ + skipSilenceBefore = 0; + + /** All-zero audio frames at or after `frame` (e.g. the tick the first compositor went on air). */ + silentFramesFrom(frame: number): number { return this.report.silentAt.filter((i) => i >= frame).length; } + + constructor(geometry: FrameGeometry) { + this.vBytes = videoFrameBytes(geometry); + this.aBytes = audioFrameBytes(geometry); + this.intervalNs = 1_000_000_000n / BigInt(geometry.fps); + } + + attach(video: Readable, audio: Readable): void { + video.on("data", (chunk: Buffer) => this.onVideo(chunk)); + audio.on("data", (chunk: Buffer) => this.onAudio(chunk)); + } + + private onVideo(chunk: Buffer): void { + let at = 0; + while (at < chunk.length) { + if (this.vPos === 0) { + const tag = chunk[at] ?? 0; + if (this.lastTag >= 18 && this.lastTag < 220 && tag >= 18 && tag < 220) { + const decode = (y: number): number => Math.round((y - 20) / 4); + let d = (((decode(tag) - decode(this.lastTag)) % 50) + 50) % 50; + if (d > 25) d -= 50; + const key = String(d); + this.report.tagDeltas[key] = (this.report.tagDeltas[key] ?? 0) + 1; + } + if (tag !== this.lastTag) { + if (this.report.tagChanges.length < 100_000) this.report.tagChanges.push([this.report.videoFrames, tag]); + this.lastTag = tag; + } + } + const n = Math.min(chunk.length - at, this.vBytes - this.vPos); + at += n; + this.vPos += n; + if (this.vPos === this.vBytes) { + this.vPos = 0; + const now = process.hrtime.bigint(); + if (this.lastFrameAt !== 0n) { + const gap = now - this.lastFrameAt; + this.report.maxGapMs = Math.max(this.report.maxGapMs, Number(gap) / 1e6); + if (gap > 2n * this.intervalNs) this.report.gapsOverTwoFrames += 1; + } + this.lastFrameAt = now; + this.report.videoFrames += 1; + } + } + } + + private onAudio(chunk: Buffer): void { + // Chunks from a stream socket are not guaranteed to be 4-byte aligned. + let at = 0; + while (at < chunk.length) { + const sampleStart = this.aPos % 4 === 0; + if (sampleStart && at + 4 <= chunk.length) { + const l = chunk.readInt16LE(at); + const r = chunk.readInt16LE(at + 2); + this.sample(l, r); + at += 4; + this.aPos += 4; + } else { + // Rare split sample: fall back to byte stepping with a carry. + this.carry.push(chunk[at] ?? 0); + at += 1; + this.aPos += 1; + if (this.carry.length === 4) { + const b = Buffer.from(this.carry); + this.carry = []; + this.sample(b.readInt16LE(0), b.readInt16LE(2)); + } + } + if (this.aPos === this.aBytes) { + this.aPos = 0; + if (!this.frameNonZero) { + this.report.silentAudioFrames += 1; + if (this.report.silentAt.length < 100_000) this.report.silentAt.push(this.report.audioFrames); + } + this.frameNonZero = false; + this.report.audioFrames += 1; + } + } + } + private carry: number[] = []; + + private sample(l: number, r: number): void { + if (l !== 0 || r !== 0) this.frameNonZero = true; + this.report.maxPcmAbs = Math.max(this.report.maxPcmAbs, Math.abs(l), Math.abs(r)); + if (this.prev && this.report.audioFrames >= this.skipSilenceBefore) { + this.report.maxPcmStep = Math.max(this.report.maxPcmStep, Math.abs(l - this.prev[0]), Math.abs(r - this.prev[1])); + } + this.prev = [l, r]; + } +} diff --git a/apps/media-worker/src/gate/spike.ts b/apps/media-worker/src/gate/spike.ts new file mode 100644 index 0000000..39a2441 --- /dev/null +++ b/apps/media-worker/src/gate/spike.ts @@ -0,0 +1,150 @@ +/** + * §1.4 frame-gate spike harness (not part of the test suite; numbers go to + * runs/wp4/s2a-report.md). + * + * node --import tsx src/gate/spike.ts steady --impl rust --size 1920x1080 --fps 60 --seconds 20 + * node --import tsx src/gate/spike.ts swaps --impl node --size 1920x1080 --fps 30 --swaps 1000 + * + * Producers are compositor stand-ins (ffmpeg lavfi: solid luma tag = generation, + * 1500 Hz sine, so every frame holds a whole number of periods and any + * mis-spliced or inserted PCM shows up as a sample step). The sink is the + * verifying null encoder (src/gate/sink.ts). + */ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnManaged, type ManagedProcess } from "../relay/ffmpeg"; +import { FrameGate, type GateImpl } from "./frame-gate"; +import type { FrameGeometry } from "./protocol"; +import type { SinkReport } from "./sink"; + +async function waitFor(cond: () => boolean, timeoutMs = 15_000): Promise { + const t0 = Date.now(); + while (!cond()) { + if (Date.now() - t0 > timeoutMs) throw new Error("timeout"); + await new Promise((r) => setTimeout(r, 10)); + } +} + +const passthrough = { redact: (s: string) => s }; + +export function producerArgs(g: FrameGeometry, tag: number, paced = false): string[] { + return [ + "-hide_banner", "-nostdin", "-loglevel", "error", + ...(paced ? ["-re"] : []), "-f", "lavfi", "-i", `color=c=black:s=${g.width}x${g.height}:r=${g.fps},lutyuv=y=${tag}:u=128:v=128`, + ...(paced ? ["-re"] : []), "-f", "lavfi", "-i", "sine=frequency=1500:sample_rate=48000", + "-map", "0:v", "-f", "rawvideo", "-pix_fmt", "yuv420p", "-fps_mode", "passthrough", "pipe:1", + "-map", "1:a", "-ac", "2", "-f", "s16le", "-flush_packets", "1", "pipe:3", + ]; +} + +export async function startProducer(gate: FrameGate, g: FrameGeometry, tag: number, origin?: number, paced = false): Promise<{ id: number; proc: ManagedProcess }> { + const slot = gate.allocateSlot(); + const proc = spawnManaged("ffmpeg", producerArgs(g, tag, paced), ["ignore", slot.fifos.videoFd, "pipe", slot.fifos.audioFd], passthrough, () => undefined); + slot.fifos.releaseWriters(); + await proc.started; + await gate.attach(slot, origin); + return { id: slot.id, proc }; +} + +function arg(name: string, fallback: string): string { + const i = process.argv.indexOf(`--${name}`); + return i >= 0 ? (process.argv[i + 1] ?? fallback) : fallback; +} + +function percentile(values: number[], p: number): number { + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.min(sorted.length - 1, Math.floor((p / 100) * sorted.length))] ?? 0; +} + +async function main(): Promise { + const mode = process.argv[2] ?? "steady"; + const impl = arg("impl", "rust") as GateImpl; + const [w, h] = arg("size", "1920x1080").split("x").map(Number); + const g: FrameGeometry = { width: w ?? 1920, height: h ?? 1080, fps: Number(arg("fps", "30")) }; + const scratch = mkdtempSync(join(tmpdir(), "gate-spike-")); + const events: string[] = []; + const gate = new FrameGate({ + geometry: g, scratchDir: scratch, impl, copyWrites: process.argv.includes("--copy"), + redactor: passthrough, onStderr: (m) => process.stderr.write(m), + onEvent: (e) => { if (process.env.SPIKE_DEBUG) process.stderr.write(`${JSON.stringify(e)}\n`); if (e.type === "STALL" || e.type === "EOF" || e.type === "GAP") events.push(JSON.stringify(e)); }, + }); + // --sink ffmpeg: a real ffmpeg demuxer reads the encoder FIFOs (-f null), as the persistent encoder would. + const ffmpegSink = process.argv.includes("ffmpeg"); + // Default sink: the verifier in its own process (src/gate/sink-cli.ts). + const verifier = ffmpegSink ? null : spawnManaged(process.execPath, ["--import", "tsx", join(__dirname, "sink-cli.ts"), + gate.encoderFifos.videoPath, gate.encoderFifos.audioPath, String(g.width), String(g.height), String(g.fps)], + ["pipe", "pipe", "pipe"], passthrough, (m) => process.stderr.write(m), join(__dirname, "../..")); + let verifierOut = ""; + verifier?.child.stdout?.setEncoding("utf8"); + verifier?.child.stdout?.on("data", (c: string) => { verifierOut += c; }); + if (verifier) await waitFor(() => verifierOut.includes("OPEN\n")); + const inputs = ffmpegSink ? gate.encoderInputs() : null; + const nullEncoder = inputs ? spawnManaged("ffmpeg", ["-hide_banner", "-nostdin", "-loglevel", "error", + "-probesize", "32", "-analyzeduration", "0", "-f", "rawvideo", "-pix_fmt", "yuv420p", "-s", `${g.width}x${g.height}`, "-framerate", String(g.fps), "-i", "pipe:0", + "-probesize", "32", "-analyzeduration", "0", "-f", "s16le", "-ar", "48000", "-ac", "2", "-i", "pipe:3", "-map", "0:v", "-map", "1:a", "-f", "null", "-"], + [inputs.videoFd, "ignore", "pipe", inputs.audioFd], passthrough, (m) => process.stderr.write(m)) : null; + await nullEncoder?.started; + await gate.start(); + const t0 = Date.now(); + let current = await startProducer(gate, g, 20); + const firstTick = await gate.waitActive(current.id, 10_000); + await new Promise((r) => setTimeout(r, 1_000)); // encoder/producer warm-up excluded from peaks + await gate.resetPeaks(); + const base = await gate.stats(); + const swapMs: number[] = []; + + if (mode === "steady") { + await new Promise((r) => setTimeout(r, Number(arg("seconds", "20")) * 1000)); + } else { + const swaps = Number(arg("swaps", "1000")); + for (let i = 1; i <= swaps; i += 1) { + const s0 = performance.now(); + const next = await startProducer(gate, g, 20 + (i % 200)); + await gate.waitActive(next.id, 10_000); + swapMs.push(performance.now() - s0); + void current.proc.terminate(); + current = next; + } + } + const end = await gate.stats(); + await gate.stop(); + await current.proc.terminate(); + await new Promise((r) => setTimeout(r, 200)); + await nullEncoder?.terminate(); + let report: SinkReport | null = null; + if (verifier) { + verifier.child.stdin?.write(`REPORT ${firstTick}\n`); + await verifier.exited; + const line = verifierOut.split("\n").find((l) => l.startsWith("{")); + report = line ? (JSON.parse(line) as SinkReport) : null; + } + rmSync(scratch, { recursive: true, force: true }); + + const ticks = end.ticks - base.ticks; + const wallUs = end.wallUs - base.wallUs; + // Producer generation g has luma tag 20 + (g % 200); frames before the first producer are gate black (16). + const tags = (report?.tagChanges ?? []).map(([, t]) => t).filter((t, i) => !(i === 0 && t === 16)); + let nonMonotonic = 0; + for (let i = 1; i < tags.length; i += 1) { + const prev = tags[i - 1] ?? 0; + if (tags[i] !== (prev === 219 ? 20 : prev + 1)) nonMonotonic += 1; + } + const result = { + mode, impl, sink: ffmpegSink ? "ffmpeg-null" : "node-verifier", copyWrites: process.argv.includes("--copy"), geometry: g, wallS: (Date.now() - t0) / 1000, + gateCpuPctOfCore: Number((((end.cpuUs - base.cpuUs) / wallUs) * 100).toFixed(2)), + ticks, swaps: end.swaps - base.swaps, + repeatsAfterFirstActive: end.vrep - base.vrep, silenceAfterFirstActive: end.asil - base.asil, + drops: end.drops, lateMaxUs: end.lateMaxUs, lateOverFrame: end.lateOverFrame, lateOver2ms: end.lateOver2ms, lateOver5ms: end.lateOver5ms, writeMaxUs: end.writeMaxUs, + gcMaxUs: end.gcMaxUs, poolAllocs: end.allocs, encoderBacklogMax: end.backlogMax, encoderOutDrops: end.outDrops, + verifier: report ? { ...report, tagChanges: report.tagChanges.length } : null, + sinkLockstep: report ? Math.abs(report.videoFrames - report.audioFrames) <= 1 : null, + tagSequenceViolations: nonMonotonic, distinctGenerationsSeen: tags.length, + swapLatencyMs: swapMs.length ? { p50: percentile(swapMs, 50), p99: percentile(swapMs, 99), max: Math.max(...swapMs) } : null, + events: events.slice(0, 40), + }; + process.stdout.write(`${JSON.stringify(result)}\n`); + process.exit(0); +} + +if (require.main === module) void main().catch((error: unknown) => { process.stderr.write(`${String(error)}\n`); process.exit(1); }); diff --git a/apps/media-worker/src/main.ts b/apps/media-worker/src/main.ts new file mode 100644 index 0000000..90eeefb --- /dev/null +++ b/apps/media-worker/src/main.ts @@ -0,0 +1,101 @@ +import * as http from "node:http"; +import { readFileSync } from "node:fs"; +import { createHash, createPublicKey } from "node:crypto"; +import { resolve } from "node:path"; +import pino from "pino"; +import { DurableFence } from "./bus/fence"; +import { RedisBus } from "./bus/redis"; +import { FfmpegRelayJob, type EgressRelayFactory, type VaultHandleBroker } from "./relay/ffmpeg"; +import type { EndpointValidator } from "./relay/endpoints"; +import { MediaSupervisor, type RelayJob } from "./supervisor"; +import { ProductionJob, type AssetStore, type ProductionEgress } from "./compositor/production-job"; +import { resolveHelperBinary } from "./gate/binary"; +import { isDesiredChannel, type DesiredChannel } from "./types"; + +const logger = pino({ level: process.env.LOG_LEVEL ?? "info" }); +const port = Number(process.env.PORT ?? 3100); +const workerId = process.env.MEDIA_WORKER_ID ?? "media-worker-dev"; +let supervisor: MediaSupervisor | null = null; +let bus: RedisBus | null = null; +let ready = false; + +const server = http.createServer((req, res) => { + if (req.url === "/healthz") { + res.writeHead(ready ? 200 : 503, { "content-type": "application/json" }); + res.end(JSON.stringify({ status: ready ? "alive" : "starting", worker: workerId, channels: supervisor?.status() ?? [] })); + return; + } + res.writeHead(404).end(); +}); + +function readDesired(): DesiredChannel[] { + const path = process.env.MEDIA_DESIRED_CONFIG; + if (!path) return []; + const value: unknown = JSON.parse(readFileSync(path, "utf8")); + if (!Array.isArray(value) || !value.every(isDesiredChannel)) throw new Error("invalid desired channel config"); + return value; +} + +/** Deliberately fails closed until WP7 injects an egress relay and vault broker. */ +const unavailableEgress: EgressRelayFactory = { async prepare() { throw new Error("egress relay is not configured"); } }; +const unavailableValidator: EndpointValidator = { async validate() { throw new Error("endpoint validator is not configured"); } }; +const unavailableVault: VaultHandleBroker = { async deliverPushTarget() { throw new Error("vault broker is not configured"); } }; +const unavailableProductionEgress: ProductionEgress = { async prepare() { throw new Error("production egress is not configured"); } }; +const unavailableAssets: AssetStore = { async localPath() { throw new Error("overlay asset store is not configured"); } }; +const redactor = { redact: (_chunk: string) => "[media stderr redacted until a job secret-aware redactor is injected]" }; + +/** Per-channel scratch for gate FIFOs; the name is a hash so channel ids never reach a path. */ +function scratchFor(channel: DesiredChannel): string { + const id = createHash("sha256").update(`${channel.workspaceId}:${channel.channelId}`).digest("hex").slice(0, 32); + return resolve(process.env.MEDIA_SCRATCH_DIR ?? ".media-worker-state/scratch", id); +} + +/** v2 (production) snapshots run the compositor chain; v1 snapshots keep the single relay. */ +function createRelay(channel: DesiredChannel, onExit: (reason: string) => void): RelayJob { + const onStderr = (message: string): void => logger.warn({ message }, "ffmpeg stderr"); + if (channel.production) { + return new ProductionJob({ + validator: unavailableValidator, egress: unavailableProductionEgress, vault: unavailableVault, assets: unavailableAssets, + redactor, onStderr, onExit, scratchDir: scratchFor(channel), + }); + } + return new FfmpegRelayJob(unavailableValidator, unavailableEgress, unavailableVault, redactor, onStderr, onExit); +} + +/** Production channels need the packaged native helpers; say at boot which ones resolve (src/gate/binary.ts). */ +function checkHelpers(): void { + for (const name of ["frame-gate", "source-bus"] as const) { + try { logger.info({ helper: name, path: resolveHelperBinary(name) }, "native helper"); } + catch (error) { logger.warn({ helper: name, err: String(error) }, "native helper missing: production channels will fail to start"); } + } +} + +async function start(): Promise { + checkHelpers(); + const desired = readDesired(); + const keyPath = process.env.MEDIA_CONTROL_PUBLIC_KEY; + const keys = new Map>(); + if (keyPath) keys.set(process.env.MEDIA_CONTROL_KID ?? "control-v1", createPublicKey(readFileSync(keyPath))); + const fence = new DurableFence(process.env.MEDIA_FENCE_PATH ?? ".media-worker-state/fence.json"); + bus = await RedisBus.connect(process.env.REDIS_URL ?? `redis://${process.env.REDIS_HOST ?? "localhost"}:6379`); + supervisor = new MediaSupervisor(workerId, desired, bus, fence, keys, createRelay, + { transition: (event) => logger.info(event, "media state transition") }); + await supervisor.start(); + const workspaces = [...new Set(desired.map((item) => item.workspaceId))]; + for (const workspaceId of workspaces) { + // ACLs should grant only this worker's assigned command channels. + await bus.subscribe(workerId, [workspaceId], (message) => { + let command: unknown; + try { command = JSON.parse(message); } catch { logger.warn("malformed command rejected"); return; } + void supervisor?.handle(command).catch(() => logger.warn("command rejected")); + }); + } + ready = true; + logger.info({ port, workerId, channels: desired.length }, "media worker listening"); +} + +server.listen(port, () => { void start().catch((error: unknown) => { logger.error({ err: String(error) }, "media worker startup failed"); process.exitCode = 1; server.close(); }); }); +for (const signal of ["SIGINT", "SIGTERM"] as const) process.on(signal, () => { + ready = false; + void supervisor?.stop().finally(() => { bus?.close(); server.close(); }); +}); diff --git a/apps/media-worker/src/preview/argv.ts b/apps/media-worker/src/preview/argv.ts new file mode 100644 index 0000000..a59cdd5 --- /dev/null +++ b/apps/media-worker/src/preview/argv.ts @@ -0,0 +1,103 @@ +/** + * Preview tap argv (streaming-architecture.md §2.1). Typed params in, argv out, + * like MediaCommandBuilder: every tap reads raw frames from a source-bus + * subscription on inherited pipes (video fd 0, PCM fd 3) and never touches a + * remote; HLS taps write only into their own scratch directory. + * + * - JPEG: one still per `everySeconds` (the bus delivers every n-th pair), 360p, + * as an mjpeg stream on stdout (the worker splits frames and publishes). + * - video: 360p ≤ 500 kb/s H.264 + AAC 64 kb/s, fMP4 HLS with 0.5 s parts and a + * keyframe every second (each 1 s segment = 2 parts, the first independent). + * - audio: AAC 64 kb/s only, same part/segment grid. + * + * FFmpeg 5.1's hls muxer has no EXT-X-PART support, so ffmpeg writes the parts + * as its own 0.5 s "segments" (split_by_time) and `ll-hls.ts` publishes them as + * LL-HLS parts, concatenates them into segments and writes the media playlists. + */ +import { assertGeometry, GATE_CHANNELS, GATE_SAMPLE_RATE, type FrameGeometry } from "../gate/protocol"; + +export const TAP_HEIGHT = 360; +export const PART_SECONDS = 0.5; +export const VIDEO_TAP_KBPS = 400; +export const AUDIO_TAP_KBPS = 64; +/** The 500 kb/s rendition budget includes its AAC audio. */ +export const VIDEO_TAP_MAX_KBPS = 500 - AUDIO_TAP_KBPS; +/** H.264 Main@3.1 + AAC-LC (also supports the 360p60 production tap). */ +export const VIDEO_TAP_CODECS = "avc1.4d401f,mp4a.40.2"; +export const AUDIO_TAP_CODECS = "mp4a.40.2"; + +const SCRATCH = /^\/[A-Za-z0-9._/-]+$/; + +function scratch(dir: string): string { + if (!SCRATCH.test(dir) || dir.split("/").includes("..")) throw new Error("tap output must be an absolute scratch path"); + return dir; +} + +function even(n: number): number { return Math.max(2, Math.round(n / 2) * 2); } + +/** 360p (or the source height, if smaller) at the canvas aspect, even dimensions. */ +export function tapSize(g: FrameGeometry): { width: number; height: number } { + const height = Math.min(TAP_HEIGHT, g.height); + return { width: even((height * g.width) / g.height), height: even(height) }; +} + +/** Geometry of the program tap bus (the persistent encoder downscales once, §2.1 "program tap"). */ +export function programTapGeometry(canvas: FrameGeometry): FrameGeometry { + return { ...tapSize(canvas), fps: canvas.fps }; +} + +function rawVideoIn(g: FrameGeometry, rate: string): string[] { + return ["-thread_queue_size", "64", "-probesize", "32", "-analyzeduration", "0", + "-f", "rawvideo", "-pix_fmt", "yuv420p", "-s", `${g.width}x${g.height}`, "-framerate", rate, "-i", "pipe:0"]; +} + +function rawAudioIn(): string[] { + return ["-thread_queue_size", "64", "-probesize", "32", "-analyzeduration", "0", + "-f", "s16le", "-ar", String(GATE_SAMPLE_RATE), "-ac", String(GATE_CHANNELS), "-i", "pipe:3"]; +} + +function scale(g: FrameGeometry): string { + const { width, height } = tapSize(g); + return width === g.width && height === g.height ? "format=yuv420p" : `scale=${width}:${height}:flags=bilinear,format=yuv420p`; +} + +export interface JpegTapPlan { geometry: FrameGeometry; everySeconds: number } + +/** Pairs the bus skips between stills (subscription `stride`). */ +export function jpegStride(plan: JpegTapPlan): number { return plan.everySeconds * plan.geometry.fps; } + +export function buildJpegTapArgs(plan: JpegTapPlan): string[] { + assertGeometry(plan.geometry); + if (!Number.isSafeInteger(plan.everySeconds) || plan.everySeconds < 1 || plan.everySeconds > 10) throw new Error("thumbnail cadence must be 1-10 s"); + // The input runs at one frame per `everySeconds`, so every input frame becomes one still. + return ["-hide_banner", "-nostdin", "-loglevel", "error", ...rawVideoIn(plan.geometry, `1/${plan.everySeconds}`), + "-vf", scale(plan.geometry), "-c:v", "mjpeg", "-q:v", "5", "-f", "image2pipe", "pipe:1"]; +} + +export type HlsRendition = "video" | "audio"; + +export interface HlsTapPlan { geometry: FrameGeometry; rendition: HlsRendition; outDir: string } + +/** ffmpeg's own playlist of parts (read by the packager, never published). */ +export const PARTS_PLAYLIST = "parts.m3u8"; + +export function buildHlsTapArgs(plan: HlsTapPlan): string[] { + const g = plan.geometry; + assertGeometry(g); + const dir = scratch(plan.outDir); + const args = ["-hide_banner", "-nostdin", "-loglevel", "error"]; + if (plan.rendition === "video") { + args.push(...rawVideoIn(g, String(g.fps)), ...rawAudioIn(), "-map", "0:v", "-map", "1:a", "-vf", scale(g), + "-c:v", "libx264", "-preset", "veryfast", "-tune", "zerolatency", "-profile:v", "main", "-level:v", "3.1", + "-b:v", `${VIDEO_TAP_KBPS}k`, "-maxrate", `${VIDEO_TAP_MAX_KBPS}k`, "-bufsize", `${VIDEO_TAP_MAX_KBPS}k`, + // Keyframe exactly every second, never on scene cuts: parts alternate independent/dependent. + "-g", String(g.fps), "-keyint_min", String(g.fps), "-sc_threshold", "0"); + } else { + args.push(...rawAudioIn(), "-map", "0:a"); + } + args.push("-c:a", "aac", "-b:a", `${AUDIO_TAP_KBPS}k`, "-ac", "2", "-ar", String(GATE_SAMPLE_RATE), + "-f", "hls", "-hls_time", String(PART_SECONDS), "-hls_list_size", "12", "-hls_segment_type", "fmp4", + "-hls_fmp4_init_filename", "init.mp4", "-hls_flags", "split_by_time+temp_file+delete_segments+omit_endlist", + "-hls_segment_filename", `${dir}/p%d.m4s`, "-protocol_whitelist", "file", `${dir}/${PARTS_PLAYLIST}`); + return args; +} diff --git a/apps/media-worker/src/preview/contract.ts b/apps/media-worker/src/preview/contract.ts new file mode 100644 index 0000000..e99960d --- /dev/null +++ b/apps/media-worker/src/preview/contract.ts @@ -0,0 +1,85 @@ +/** + * Worker side of the preview contract (streaming-architecture.md §2.1). + * + * The API owns serving: `apps/api/src/production/preview.ts` mints signed, + * path-bound capability URLs (60 s TTL), verifies them, single-flights fills, + * keeps a 6-object LRU per tap, sets `ETag` from the body hash, rewrites HLS + * URIs to signed same-origin paths, and never redirects to S3. The worker does + * none of that. It only *publishes*: each preview file is stored as a + * content-addressed object and announced by a Redis manifest + * + * preview:::: → {"key":"sha256/.","contentType":...} + * + * which is exactly what the API's `fill()` reads. `preview.contract.test.ts` + * checks the constants below against the API source so the two cannot drift. + */ +import { createHash } from "node:crypto"; + +/** The API's preview file-name rule (also the only names its HLS rewriter accepts). */ +export const PREVIEW_FILE = /^(?:[A-Za-z0-9_-]+\.(?:m3u8|m4s|ts|aac|jpg|jpeg|mp4))$/; +/** The API's manifest object-key rule. */ +export const PREVIEW_OBJECT_KEY = /^sha256\/[0-9a-f]{64}\.[a-z0-9]+$/; +/** Content types the API accepts in a manifest (subset the worker produces). */ +export type PreviewContentType = "image/jpeg" | "application/vnd.apple.mpegurl" | "video/mp4" | "audio/mp4"; +export const PREVIEW_CONTENT_TYPES: readonly PreviewContentType[] = ["image/jpeg", "application/vnd.apple.mpegurl", "video/mp4", "audio/mp4"]; + +/** Files the API hands out capability URLs for (`PreviewService.urls`). */ +export const THUMBNAIL_FILE = "thumbnail.jpg"; +export const VIDEO_PLAYLIST = "video.m3u8"; +export const AUDIO_PLAYLIST = "audio.m3u8"; +/** Multivariant playlist over the two renditions (servable by the same route; not in `urls()` yet). */ +export const MASTER_PLAYLIST = "master.m3u8"; + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; + +/** A tap is the channel's program, or one of its sources (virtual_channel_sources.id), as the API's controller accepts. */ +export function assertTapId(tap: string): void { + if (tap !== "program" && !UUID.test(tap)) throw new Error("preview tap must be 'program' or a source UUID"); +} + +export interface PreviewChannel { workspaceId: string; channelId: string } + +export function previewManifestKey(channel: PreviewChannel, tap: string, file: string): string { + if (!UUID.test(channel.workspaceId) || !UUID.test(channel.channelId)) throw new Error("preview channel ids must be UUIDs"); + assertTapId(tap); + if (!PREVIEW_FILE.test(file)) throw new Error("invalid preview file name"); + return `preview:${channel.workspaceId}:${channel.channelId}:${tap}:${file}`; +} + +/** Content-addressed object storage (the API's S3 storage with envelope encryption, or a test double). */ +export interface PreviewObjectStore { putContent(body: Buffer, contentType: PreviewContentType): Promise<{ key: string }> } +/** The manifest store the API reads (Redis `SET key value EX ttl`). */ +export interface PreviewManifestStore { set(key: string, value: string, ttlSeconds: number): Promise } + +export interface PreviewPublisher { + /** Store `body` and point the tap's `file` at it for `ttlSeconds`. */ + publish(tap: string, file: string, body: Buffer, contentType: PreviewContentType, ttlSeconds: number): Promise; +} + +/** + * Publisher over the API's storage + manifest contract. Re-publishing an + * unchanged body (an init segment kept alive, a still thumbnail) only renews + * the manifest; the object is already stored under its hash. + */ +export class ManifestPreviewPublisher implements PreviewPublisher { + private readonly stored = new Map(); + + constructor(private readonly channel: PreviewChannel, private readonly objects: PreviewObjectStore, private readonly manifests: PreviewManifestStore) {} + + async publish(tap: string, file: string, body: Buffer, contentType: PreviewContentType, ttlSeconds: number): Promise { + const manifestKey = previewManifestKey(this.channel, tap, file); + if (!PREVIEW_CONTENT_TYPES.includes(contentType)) throw new Error("unsupported preview content type"); + if (body.length === 0 || body.length > 4_000_000) throw new Error("preview object size out of range"); + if (!Number.isSafeInteger(ttlSeconds) || ttlSeconds < 1 || ttlSeconds > 86_400) throw new Error("preview ttl out of range"); + const hash = createHash("sha256").update(body).update(contentType).digest("hex"); + let key = this.stored.get(hash); + if (!key) { + key = (await this.objects.putContent(body, contentType)).key; + if (!PREVIEW_OBJECT_KEY.test(key)) throw new Error("object store returned an invalid key"); + this.stored.set(hash, key); + // Hash → key only matters for recent bodies (renewals); bound the map. + if (this.stored.size > 512) this.stored.delete(this.stored.keys().next().value ?? ""); + } + await this.manifests.set(manifestKey, JSON.stringify({ key, contentType }), ttlSeconds); + } +} diff --git a/apps/media-worker/src/preview/ll-hls.ts b/apps/media-worker/src/preview/ll-hls.ts new file mode 100644 index 0000000..7e0f538 --- /dev/null +++ b/apps/media-worker/src/preview/ll-hls.ts @@ -0,0 +1,207 @@ +/** + * LL-HLS packaging for preview taps (streaming-architecture.md §2.1, §2.2). + * + * ffmpeg (argv.ts) writes 0.5 s fMP4 fragments and lists them in its own + * playlist. This module turns them into an LL-HLS media playlist: + * + * - every fragment is published as an `EXT-X-PART` (`INDEPENDENT=YES` when its + * first video sample is a sync sample, read from the `moof`); + * - consecutive parts from one independent part to the next form a segment, + * published as the parts' concatenation (fMP4 fragments concatenate); + * - media sequence numbers start at the wall-clock millisecond the tap started, so + * they keep increasing across tap restarts (on-demand taps stop when idle) + * and file names never collide with a previous run's still-cached manifests. + * + * There is no blocking playlist reload (`CAN-BLOCK-RELOAD`): the API serves + * stored objects with a 1 s cache, so players poll; hls.js `lowLatencyMode` + * still uses the parts and `PART-HOLD-BACK`. + */ +import { PART_SECONDS, type HlsRendition } from "./argv"; + +// --------------------------------------------------------------------------- +// fMP4 fragment inspection + +function boxes(buf: Buffer, start: number, end: number): Array<{ type: string; at: number; size: number }> { + const out: Array<{ type: string; at: number; size: number }> = []; + let at = start; + while (at + 8 <= end) { + let size = buf.readUInt32BE(at); + const type = buf.toString("latin1", at + 4, at + 8); + let header = 8; + if (size === 1) { + if (at + 16 > end) break; + size = Number(buf.readBigUInt64BE(at + 8)); + header = 16; + } else if (size === 0) { + size = end - at; + } + if (size < header || at + size > end) break; + out.push({ type, at, size }); + at += size; + } + return out; +} + +const NON_SYNC = 0x10000; + +/** First-sample flags of one `traf`: trun first_sample_flags, else trun sample[0] flags, else tfhd default flags. */ +function firstSampleFlags(buf: Buffer, traf: { at: number; size: number }): number | null { + let defaults: number | null = null; + for (const child of boxes(buf, traf.at + 8, traf.at + traf.size)) { + const flags = buf.readUInt32BE(child.at + 8) & 0xffffff; + if (child.type === "tfhd") { + let p = child.at + 16; // version/flags, track_ID + if (flags & 0x1) p += 8; // base_data_offset + if (flags & 0x2) p += 4; // sample_description_index + if (flags & 0x8) p += 4; // default_sample_duration + if (flags & 0x10) p += 4; // default_sample_size + if (flags & 0x20 && p + 4 <= child.at + child.size) defaults = buf.readUInt32BE(p); + } else if (child.type === "trun") { + let p = child.at + 16; // version/flags, sample_count + if (flags & 0x1) p += 4; // data_offset + if (flags & 0x4) return buf.readUInt32BE(p); + if (flags & 0x400) { + if (flags & 0x100) p += 4; // sample_duration + if (flags & 0x200) p += 4; // sample_size + return buf.readUInt32BE(p); + } + } + } + return defaults; +} + +/** + * Whether a fragment can start playback: every track's first sample is a sync + * sample. Unknown flags (no trun/tfhd information) count as not independent. + */ +export function fragmentIndependent(buf: Buffer): boolean { + let trafs = 0; + for (const top of boxes(buf, 0, buf.length)) { + if (top.type !== "moof") continue; + for (const traf of boxes(buf, top.at + 8, top.at + top.size)) { + if (traf.type !== "traf") continue; + trafs += 1; + const flags = firstSampleFlags(buf, traf); + if (flags === null || (flags & NON_SYNC) !== 0) return false; + } + } + return trafs > 0; +} + +// --------------------------------------------------------------------------- +// ffmpeg's part playlist + +export interface FfmpegPart { file: string; duration: number } + +/** Parts listed in ffmpeg's playlist (`#EXTINF:,` then the file name). */ +export function parseFfmpegPlaylist(text: string): FfmpegPart[] { + const parts: FfmpegPart[] = []; + let duration: number | null = null; + for (const raw of text.split(/\r?\n/)) { + const line = raw.trim(); + if (line.startsWith("#EXTINF:")) duration = Number.parseFloat(line.slice(8)); + else if (line !== "" && !line.startsWith("#") && duration !== null) { + if (/^p\d+\.m4s$/.test(line) && Number.isFinite(duration) && duration > 0) parts.push({ file: line, duration }); + duration = null; + } + } + return parts; +} + +// --------------------------------------------------------------------------- +// LL-HLS media playlist + +export interface PublishedPart { name: string; duration: number; independent: boolean; bytes: Buffer } +export interface ClosedSegment { msn: number; name: string; duration: number; parts: PublishedPart[] } + +/** Segments listed in the playlist; parts are listed for the last `PART_SEGMENTS` of them plus the open one. */ +const WINDOW_SEGMENTS = 4; +const PART_SEGMENTS = 2; + +function fmt(n: number): string { return n.toFixed(5); } + +/** + * File names: `init.mp4`, segments `s.m4s`, parts `p_.m4s` + * (all within the API's preview file-name rule). + */ +export class LlHlsPackager { + readonly prefix: string; + readonly initName: string; + private msn: number; + private open: PublishedPart[] = []; + private segments: ClosedSegment[] = []; + + constructor(rendition: HlsRendition, readonly base: number) { + if (!Number.isSafeInteger(base) || base < 0) throw new Error("media sequence base must be a non-negative integer"); + this.prefix = rendition === "video" ? "v" : "a"; + this.initName = `${this.prefix}init${base}.mp4`; + this.msn = base; + } + + /** Name the next part will be published under (it belongs to the open segment). */ + private partName(): string { return `${this.prefix}p${this.msn}_${this.open.length}.m4s`; } + + /** + * Adds the next fragment. Returns the part to publish (null for a dependent fragment before the + * first keyframe, which is dropped: a stream starts at an independent part) and, when this part + * starts a new segment, the segment it closed. Publish both before the playlist. + */ + add(fragment: { duration: number; independent: boolean; bytes: Buffer }): { part: PublishedPart | null; closed: ClosedSegment | null } { + let closed: ClosedSegment | null = null; + if (fragment.independent && this.open.length > 0) { + closed = { + msn: this.msn, name: `${this.prefix}s${this.msn}.m4s`, parts: this.open, + duration: this.open.reduce((a, p) => a + p.duration, 0), + }; + this.segments.push(closed); + while (this.segments.length > WINDOW_SEGMENTS) this.segments.shift(); + this.msn += 1; + this.open = []; + } + if (this.open.length === 0 && !fragment.independent) return { part: null, closed }; + const part: PublishedPart = { name: this.partName(), ...fragment }; + this.open.push(part); + return { part, closed }; + } + + static segmentBytes(segment: ClosedSegment): Buffer { return Buffer.concat(segment.parts.map((p) => p.bytes)); } + + /** null until the first segment has closed (a player needs at least one full segment). */ + playlist(): string | null { + if (this.segments.length === 0) return null; + const first = this.segments[0]!; + const target = Math.max(1, ...this.segments.map((s) => Math.ceil(s.duration))); + const partTarget = Math.max(PART_SECONDS, ...this.segments.flatMap((s) => s.parts.map((p) => p.duration)), ...this.open.map((p) => p.duration)); + const lines = [ + "#EXTM3U", + "#EXT-X-VERSION:7", + `#EXT-X-TARGETDURATION:${target}`, + `#EXT-X-SERVER-CONTROL:PART-HOLD-BACK=${fmt(3 * partTarget)}`, + `#EXT-X-PART-INF:PART-TARGET=${fmt(partTarget)}`, + `#EXT-X-MEDIA-SEQUENCE:${first.msn}`, + "#EXT-X-INDEPENDENT-SEGMENTS", + `#EXT-X-MAP:URI="${this.initName}"`, + ]; + const partLine = (p: PublishedPart): string => `#EXT-X-PART:DURATION=${fmt(p.duration)},URI="${p.name}"${p.independent ? ",INDEPENDENT=YES" : ""}`; + this.segments.forEach((s, i) => { + if (i >= this.segments.length - PART_SEGMENTS) for (const p of s.parts) lines.push(partLine(p)); + lines.push(`#EXTINF:${fmt(s.duration)},`, s.name); + }); + for (const p of this.open) lines.push(partLine(p)); + return `${lines.join("\n")}\n`; + } +} + +/** Multivariant playlist over a tap's two renditions (§2.1: 360p video+audio, audio-only). */ +export function masterPlaylist(video: { width: number; height: number; bandwidth: number; codecs: string }, audio: { bandwidth: number; codecs: string }): string { + return [ + "#EXTM3U", + "#EXT-X-VERSION:7", + "#EXT-X-INDEPENDENT-SEGMENTS", + `#EXT-X-STREAM-INF:BANDWIDTH=${video.bandwidth},RESOLUTION=${video.width}x${video.height},CODECS="${video.codecs}"`, + "video.m3u8", + `#EXT-X-STREAM-INF:BANDWIDTH=${audio.bandwidth},CODECS="${audio.codecs}"`, + "audio.m3u8", + "", + ].join("\n"); +} diff --git a/apps/media-worker/src/preview/preview.test.ts b/apps/media-worker/src/preview/preview.test.ts new file mode 100644 index 0000000..96a2e34 --- /dev/null +++ b/apps/media-worker/src/preview/preview.test.ts @@ -0,0 +1,305 @@ +/** + * Preview taps (streaming-architecture.md §2.1): the API contract mirror, argv + * goldens, fMP4/LL-HLS packaging, and the JPEG / video / audio tap lifecycle on + * a real source bus, plus the program tap's isolation from the encoder. + */ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import test from "node:test"; +import { buildEncoderArgs } from "../compositor/command-builder"; +import { ProgramChain } from "../compositor/program"; +import { CounterSource, ensureGateBinary, passthroughRedactor } from "../compositor/testing"; +import { buildHlsTapArgs, buildJpegTapArgs, jpegStride, programTapGeometry, tapSize } from "./argv"; +import { + ManifestPreviewPublisher, PREVIEW_CONTENT_TYPES, PREVIEW_FILE, PREVIEW_OBJECT_KEY, previewManifestKey, + type PreviewContentType, type PreviewPublisher, +} from "./contract"; +import { fragmentIndependent, LlHlsPackager, masterPlaylist, parseFfmpegPlaylist } from "./ll-hls"; +import { JpegSplitter, PreviewTaps, TTL } from "./taps"; + +const WS = "8f0c5a8e-3b1d-4c3e-9a5b-0c1d2e3f4a5b"; +const CH = "1a2b3c4d-5e6f-4a1b-8c2d-3e4f5a6b7c8d"; +const SRC = "9e8d7c6b-5a4f-4e3d-a2c1-b0a9f8e7d6c5"; +const small = { width: 320, height: 180, fps: 30 }; + +test("worker preview contract mirrors the API preview service (file rule, manifest key, object key, types)", () => { + const api = readFileSync(resolve(__dirname, "../../../api/src/production/preview.ts"), "utf8"); + const file = /const FILE = \/(.+)\/;/.exec(api)?.[1]; + assert.equal(file, PREVIEW_FILE.source, "file-name rule"); + assert.ok(api.includes("`preview:${ws}:${ch}:${tap}:${file}`"), "manifest key layout"); + assert.ok(api.includes(PREVIEW_OBJECT_KEY.source.replace(/\\\//g, "\\/")), "object key rule"); + const types = /const TYPES = new Set\(\[(.+)\]\);/.exec(api)?.[1] ?? ""; + for (const t of PREVIEW_CONTENT_TYPES) assert.ok(types.includes(`"${t}"`), `API accepts ${t}`); + // Every name the worker publishes passes the API's rule (the HLS rewriter rejects anything else). + const video = new LlHlsPackager("video", 1_700_000_000); + for (const name of ["thumbnail.jpg", "video.m3u8", "audio.m3u8", "master.m3u8", video.initName, "vs1700000000.m4s", "vp1700000000_1.m4s", "ainit1700000000.mp4"]) { + assert.ok(PREVIEW_FILE.test(name), name); + } + assert.equal(previewManifestKey({ workspaceId: WS, channelId: CH }, "program", "thumbnail.jpg"), `preview:${WS}:${CH}:program:thumbnail.jpg`); + assert.throws(() => previewManifestKey({ workspaceId: WS, channelId: CH }, "../x", "thumbnail.jpg")); + assert.throws(() => previewManifestKey({ workspaceId: WS, channelId: CH }, SRC, "../../etc.jpg")); + assert.throws(() => previewManifestKey({ workspaceId: "workspace-1", channelId: CH }, SRC, "thumbnail.jpg")); +}); + +test("ManifestPreviewPublisher stores content-addressed objects and renews manifests without re-uploading", async () => { + const puts: string[] = []; + const manifests = new Map(); + const publisher = new ManifestPreviewPublisher({ workspaceId: WS, channelId: CH }, + { async putContent(body, contentType) { puts.push(contentType); return { key: `sha256/${"a".repeat(63)}${puts.length}.jpg` }; } }, + { async set(key, value, ttl) { manifests.set(key, { value, ttl }); } }); + await publisher.publish("program", "thumbnail.jpg", Buffer.from("jpeg-1"), "image/jpeg", 30); + await publisher.publish("program", "thumbnail.jpg", Buffer.from("jpeg-1"), "image/jpeg", 30); + assert.equal(puts.length, 1, "an unchanged body is stored once"); + await publisher.publish("program", "thumbnail.jpg", Buffer.from("jpeg-2"), "image/jpeg", 30); + assert.equal(puts.length, 2); + const m = manifests.get(`preview:${WS}:${CH}:program:thumbnail.jpg`); + assert.deepEqual(JSON.parse(m?.value ?? "{}"), { key: `sha256/${"a".repeat(63)}2.jpg`, contentType: "image/jpeg" }); + assert.equal(m?.ttl, 30); + await assert.rejects(publisher.publish("program", "x.exe", Buffer.from("x"), "image/jpeg", 30)); + await assert.rejects(publisher.publish("program", "a.jpg", Buffer.alloc(0), "image/jpeg", 30)); + const bad = new ManifestPreviewPublisher({ workspaceId: WS, channelId: CH }, { async putContent() { return { key: "https://evil/x" }; } }, { async set() {} }); + await assert.rejects(bad.publish("program", "a.jpg", Buffer.from("x"), "image/jpeg", 30), /invalid key/); +}); + +test("tap argv goldens: JPEG, video and audio LL-HLS read only inherited pipes and their scratch dir", () => { + const hd = { width: 1920, height: 1080, fps: 30 }; + assert.deepEqual(tapSize(hd), { width: 640, height: 360 }); + assert.deepEqual(tapSize({ width: 1280, height: 720, fps: 30 }), { width: 640, height: 360 }); + assert.deepEqual(programTapGeometry(hd), { width: 640, height: 360, fps: 30 }); + assert.equal(jpegStride({ geometry: hd, everySeconds: 2 }), 60); + const raw = ["-thread_queue_size", "64", "-probesize", "32", "-analyzeduration", "0"]; + assert.deepEqual(buildJpegTapArgs({ geometry: hd, everySeconds: 2 }), ["-hide_banner", "-nostdin", "-loglevel", "error", + ...raw, "-f", "rawvideo", "-pix_fmt", "yuv420p", "-s", "1920x1080", "-framerate", "1/2", "-i", "pipe:0", + "-vf", "scale=640:360:flags=bilinear,format=yuv420p", "-c:v", "mjpeg", "-q:v", "5", "-f", "image2pipe", "pipe:1"]); + const hlsTail = (dir: string) => ["-c:a", "aac", "-b:a", "64k", "-ac", "2", "-ar", "48000", + "-f", "hls", "-hls_time", "0.5", "-hls_list_size", "12", "-hls_segment_type", "fmp4", "-hls_fmp4_init_filename", "init.mp4", + "-hls_flags", "split_by_time+temp_file+delete_segments+omit_endlist", "-hls_segment_filename", `${dir}/p%d.m4s`, "-protocol_whitelist", "file", `${dir}/parts.m3u8`]; + assert.deepEqual(buildHlsTapArgs({ geometry: hd, rendition: "video", outDir: "/scratch/t/v" }), ["-hide_banner", "-nostdin", "-loglevel", "error", + ...raw, "-f", "rawvideo", "-pix_fmt", "yuv420p", "-s", "1920x1080", "-framerate", "30", "-i", "pipe:0", + ...raw, "-f", "s16le", "-ar", "48000", "-ac", "2", "-i", "pipe:3", "-map", "0:v", "-map", "1:a", + "-vf", "scale=640:360:flags=bilinear,format=yuv420p", "-c:v", "libx264", "-preset", "veryfast", "-tune", "zerolatency", "-profile:v", "main", "-level:v", "3.1", + "-b:v", "400k", "-maxrate", "436k", "-bufsize", "436k", "-g", "30", "-keyint_min", "30", "-sc_threshold", "0", ...hlsTail("/scratch/t/v")]); + assert.deepEqual(buildHlsTapArgs({ geometry: programTapGeometry(hd), rendition: "audio", outDir: "/scratch/t/a" }), ["-hide_banner", "-nostdin", "-loglevel", "error", + ...raw, "-f", "s16le", "-ar", "48000", "-ac", "2", "-i", "pipe:3", "-map", "0:a", ...hlsTail("/scratch/t/a")]); + // The 360p program bus is not scaled again. + assert.ok(buildJpegTapArgs({ geometry: programTapGeometry(hd), everySeconds: 1 }).includes("format=yuv420p")); + for (const bad of ["relative/dir", "/scratch/../etc", "/scratch/t;rm"]) assert.throws(() => buildHlsTapArgs({ geometry: hd, rendition: "audio", outDir: bad }), bad); + for (const s of [0, 11, 1.5]) assert.throws(() => buildJpegTapArgs({ geometry: hd, everySeconds: s })); + const argv = [...buildJpegTapArgs({ geometry: hd, everySeconds: 2 }), ...buildHlsTapArgs({ geometry: hd, rendition: "video", outDir: "/s/v" })].join(" "); + assert.ok(!/https?:|rtmp|tcp:|udp:|concat|movie=/.test(argv), "no network or concat protocols"); +}); + +test("encoder program tap: split + 360p tee outputs with a null anchor slave, main path unchanged", () => { + const canvas = { width: 1920, height: 1080, fps: 30 }; + const plain = buildEncoderArgs({ canvas, videoBitrateKbps: 6000, audioBitrateKbps: 128, output: { kind: "flvPipe", fd: 4 } }); + const tapped = buildEncoderArgs({ canvas, videoBitrateKbps: 6000, audioBitrateKbps: 128, output: { kind: "flvPipe", fd: 4 }, programTap: { videoFd: 5, audioFd: 6, size: { width: 640, height: 360 } } }); + assert.ok(!plain.includes("-filter_complex") && plain.includes("0:v")); + assert.equal(tapped[tapped.indexOf("-filter_complex") + 1], "[0:v]split=2[enc][tap];[tap]scale=640:360:flags=bilinear,format=yuv420p[tapv];[1:a]asplit=2[enca][tapa]"); + assert.deepEqual(tapped.slice(-16), ["-f", "flv", "pipe:4", + "-map", "[tapv]", "-c:v", "rawvideo", "-f", "tee", "[f=rawvideo:onfail=ignore]pipe:5|[f=null]-", + "-map", "[tapa]", "-c:a", "pcm_s16le", "-f", "tee", "[f=s16le:onfail=ignore]pipe:6|[f=null]-"].slice(-16)); + // Encoding options of the main output are identical with and without the tap. + const opts = (a: string[]) => a.slice(a.indexOf("-c:v"), a.indexOf("-f", a.indexOf("-c:v"))); + assert.deepEqual(opts(tapped), opts(plain)); + assert.throws(() => buildEncoderArgs({ canvas, videoBitrateKbps: 6000, audioBitrateKbps: 128, output: { kind: "null" }, programTap: { videoFd: 5, audioFd: 6, size: { width: 641, height: 360 } } })); +}); + +test("fMP4 independence from moof sample flags on real ffmpeg parts (keyframe every 2nd part)", () => { + const dir = mkdtempSync(join(tmpdir(), "fmp4-")); + try { + const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=s=320x180:r=30:d=3", "-f", "lavfi", "-i", "sine=d=3", + // Same GOP shape as the video tap (zerolatency: no B-frames, so 0.5 s splits are exact). + "-c:v", "libx264", "-preset", "veryfast", "-tune", "zerolatency", "-g", "30", "-keyint_min", "30", "-sc_threshold", "0", "-c:a", "aac", "-b:a", "64k", + "-f", "hls", "-hls_time", "0.5", "-hls_list_size", "0", "-hls_segment_type", "fmp4", "-hls_flags", "split_by_time+temp_file", + "-hls_segment_filename", `${dir}/p%d.m4s`, `${dir}/parts.m3u8`]); + assert.equal(r.status, 0); + const parts = parseFfmpegPlaylist(readFileSync(join(dir, "parts.m3u8"), "utf8")); + assert.ok(parts.length >= 5, `parts ${parts.length}`); + assert.ok(parts.slice(0, -1).every((p) => Math.abs(p.duration - 0.5) < 0.01), "0.5 s parts"); + const flags = parts.map((p) => fragmentIndependent(readFileSync(join(dir, p.file)))); + assert.deepEqual(flags.slice(0, 6), [true, false, true, false, true, false]); + assert.equal(fragmentIndependent(Buffer.from("not an mp4")), false); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); + +test("LL-HLS packager: parts, segments at independent parts, monotonic sequence, bounded window", () => { + const p = new LlHlsPackager("video", 100); + const frag = (independent: boolean) => ({ duration: 0.5, independent, bytes: Buffer.from(independent ? "K" : "d") }); + assert.equal(p.add(frag(false)).part, null, "a stream starts at an independent part"); + assert.equal(p.playlist(), null, "no playlist before the first full segment"); + const first = p.add(frag(true)); + assert.equal(first.part?.name, "vp100_0.m4s"); + assert.equal(p.add(frag(false)).part?.name, "vp100_1.m4s"); + const third = p.add(frag(true)); + assert.equal(third.closed?.name, "vs100.m4s"); + assert.equal(third.part?.name, "vp101_0.m4s"); + assert.equal(LlHlsPackager.segmentBytes(third.closed!).toString(), "Kd"); + assert.equal(p.playlist(), ["#EXTM3U", "#EXT-X-VERSION:7", "#EXT-X-TARGETDURATION:1", "#EXT-X-SERVER-CONTROL:PART-HOLD-BACK=1.50000", + "#EXT-X-PART-INF:PART-TARGET=0.50000", "#EXT-X-MEDIA-SEQUENCE:100", "#EXT-X-INDEPENDENT-SEGMENTS", '#EXT-X-MAP:URI="vinit100.mp4"', + '#EXT-X-PART:DURATION=0.50000,URI="vp100_0.m4s",INDEPENDENT=YES', '#EXT-X-PART:DURATION=0.50000,URI="vp100_1.m4s"', + "#EXTINF:1.00000,", "vs100.m4s", '#EXT-X-PART:DURATION=0.50000,URI="vp101_0.m4s",INDEPENDENT=YES', ""].join("\n")); + for (let i = 0; i < 20; i += 1) p.add(frag(i % 2 === 1)); + const text = p.playlist() ?? ""; + assert.equal((text.match(/#EXTINF/g) ?? []).length, 4, "four segments in the window"); + assert.match(text, /#EXT-X-MEDIA-SEQUENCE:107/); + assert.equal((text.match(/#EXT-X-PART:/g) ?? []).length, 2 * 2 + 1, "parts only for the last two segments + the open one"); + assert.match(new LlHlsPackager("audio", 5).initName, /^ainit5\.mp4$/); + assert.equal(masterPlaylist({ width: 640, height: 360, bandwidth: 564000, codecs: "avc1.4d401e,mp4a.40.2" }, { bandwidth: 64000, codecs: "mp4a.40.2" }), + '#EXTM3U\n#EXT-X-VERSION:7\n#EXT-X-INDEPENDENT-SEGMENTS\n#EXT-X-STREAM-INF:BANDWIDTH=564000,RESOLUTION=640x360,CODECS="avc1.4d401e,mp4a.40.2"\nvideo.m3u8\n#EXT-X-STREAM-INF:BANDWIDTH=64000,CODECS="mp4a.40.2"\naudio.m3u8\n'); +}); + +test("JPEG splitter yields whole stills across chunk boundaries", () => { + const a = Buffer.from([0xff, 0xd8, 1, 2, 0xff, 0x00, 3, 0xff, 0xd9]); + const b = Buffer.from([0xff, 0xd8, 9, 0xff, 0xd9]); + const s = new JpegSplitter(); + const all = Buffer.concat([Buffer.from([7]), a, b]); + const got = [...s.push(all.subarray(0, 4)), ...s.push(all.subarray(4, 11)), ...s.push(all.subarray(11))]; + assert.deepEqual(got, [a, b]); +}); + +class MemoryPublisher implements PreviewPublisher { + readonly files = new Map(); + async publish(tap: string, file: string, body: Buffer, contentType: PreviewContentType, ttlSeconds: number): Promise { + assert.ok(PREVIEW_FILE.test(file), file); + this.files.set(`${tap}/${file}`, { body, contentType, ttl: ttlSeconds, at: Date.now() }); + } +} + +async function until(cond: () => boolean, ms: number, what: string): Promise { + const t0 = Date.now(); + while (!cond()) { + if (Date.now() - t0 > ms) throw new Error(`timeout: ${what}`); + await new Promise((r) => setTimeout(r, 50)); + } +} + +test("tap lifecycle on a live bus: JPEG always on, video/audio LL-HLS on demand with lease expiry", { timeout: 120_000 }, async (t) => { + if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return; } + const scratch = mkdtempSync(join(tmpdir(), "taps-")); + const source = new CounterSource(small, scratch); + const publisher = new MemoryPublisher(); + const errors: string[] = []; + const taps = new PreviewTaps({ publisher, scratchDir: scratch, redactor: passthroughRedactor, onStderr: () => undefined, onError: (tap, r) => errors.push(`${tap}: ${r}`) }, 5_000); + try { + await source.start(); + await taps.add(SRC, source.bus); + await until(() => publisher.files.has(`${SRC}/thumbnail.jpg`), 15_000, "thumbnail"); + const thumb = publisher.files.get(`${SRC}/thumbnail.jpg`)!; + assert.equal(thumb.contentType, "image/jpeg"); + assert.equal(thumb.ttl, TTL.thumbnail); + assert.deepEqual([...thumb.body.subarray(0, 2)], [0xff, 0xd8]); + // Nothing else runs until demanded. + assert.equal(taps.hls(SRC, "video"), undefined); + + await Promise.all([taps.demand(SRC, "video"), taps.demand(SRC, "video")]); + assert.equal((await source.bus.stats()).subscribers.length, 2, "concurrent demands share one video process plus JPEG"); + await taps.demand(SRC, "audio"); + await until(() => publisher.files.has(`${SRC}/video.m3u8`) && publisher.files.has(`${SRC}/audio.m3u8`), 30_000, "playlists"); + await until(() => (taps.hls(SRC, "video")?.parts ?? 0) >= 4, 30_000, "video parts"); + const video = publisher.files.get(`${SRC}/video.m3u8`)!.body.toString(); + assert.match(video, /#EXT-X-PART-INF:PART-TARGET=0\.50000/); + assert.match(video, /INDEPENDENT=YES/); + // Every URI the playlist names has been published before it. + for (const uri of [...video.matchAll(/URI="([^"]+)"/g)].map((m) => m[1]!).concat(video.split("\n").filter((l) => l.endsWith(".m4s")))) { + assert.ok(publisher.files.has(`${SRC}/${uri}`), `published ${uri}`); + } + const audio = publisher.files.get(`${SRC}/audio.m3u8`)!.body.toString(); + assert.match(audio, /ainit\d+\.mp4/); + assert.ok(publisher.files.has(`${SRC}/master.m3u8`)); + // A video tap reads both streams of the bus and decodes to 360p (here: source size). + const seg = [...publisher.files.keys()].find((k) => /\/vs\d+\.m4s$/.test(k)); + assert.ok(seg, "a closed video segment"); + + // A dead HLS rendition is replaced without retaining its watcher or scratch directory. + const dead = taps.hls(SRC, "video")!; + const oldInit = /URI="(vinit\d+\.mp4)"/.exec(video)?.[1]; + process.kill(dead.pid ?? 0, "SIGKILL"); + await until(() => !dead.running, 5_000, "video exit"); + await Promise.all([taps.demand(SRC, "video"), taps.demand(SRC, "video")]); + await until(() => (taps.hls(SRC, "video")?.parts ?? 0) >= 4, 30_000, "restarted video parts"); + assert.equal(readdirSync(join(scratch, "preview")).length, 2, "only current video and audio scratch directories"); + const newVideo = publisher.files.get(`${SRC}/video.m3u8`)!.body.toString(); + assert.notEqual(/URI="(vinit\d+\.mp4)"/.exec(newVideo)?.[1], oldInit, "restart never aliases a cached init"); + + // Re-demand extends the lease; expiry stops the process and removes its scratch files. + const vtap = taps.hls(SRC, "video")!; + const pid = vtap.pid; + await taps.demand(SRC, "video"); + assert.equal(taps.hls(SRC, "video")?.pid, pid, "a live rendition is reused"); + await taps.sweep(Date.now() + 10_000); + assert.equal(taps.hls(SRC, "video"), undefined); + assert.equal(taps.hls(SRC, "audio"), undefined); + assert.ok(!vtap.running); + assert.deepEqual(existsSync(join(scratch, "preview")) ? readdirSync(join(scratch, "preview")) : [], [], "rendition scratch removed"); + + // A killed JPEG tap only loses its own pairs; the bus keeps serving, and the sweep restarts it. + const jpeg = taps.jpeg(SRC)!; + const before = jpeg.stills; + process.kill(jpeg.pid ?? 0, "SIGKILL"); + await until(() => !jpeg.running, 5_000, "jpeg exit"); + const pairs = (await source.bus.stats()).pairs; + await taps.sweep(); + await until(() => jpeg.stills > before, 15_000, "jpeg restarted"); + assert.ok((await source.bus.stats()).pairs > pairs, "bus kept running"); + assert.ok(errors.some((e) => e.includes("tap exited")), "the kill was reported"); + + await taps.setThumbnailCadence(SRC, 1); + assert.equal(taps.jpeg(SRC)?.cadence, 1); + await assert.rejects(taps.setThumbnailCadence(SRC, 0)); + assert.ok(taps.jpeg(SRC)?.running, "invalid cadence leaves the live thumbnail running"); + + await taps.remove(SRC); + await taps.add(SRC, source.bus, { alwaysOn: ["video"] }); + const masterAt = publisher.files.get(`${SRC}/master.m3u8`)!.at; + await new Promise((r) => setTimeout(r, 10)); + await taps.sweep(Date.now() + TTL.init * 1000); + assert.ok(publisher.files.get(`${SRC}/master.m3u8`)!.at > masterAt, "active master manifest renewed before expiry"); + assert.ok(taps.hls(SRC, "video")?.running, "always-on rendition never expires"); + } finally { + await taps.stop(); + await source.stop(); + rmSync(scratch, { recursive: true, force: true }); + } +}); + +test("program tap: the encoder feeds a 360p bus, and losing that bus never takes the encoder down", { timeout: 120_000 }, async (t) => { + if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return; } + const { composeOver, makeAssets, scriptedGraph, ZERO_GAP_CANVAS: canvas } = await import("../compositor/testing"); + const scratch = mkdtempSync(join(tmpdir(), "pgm-tap-")); + const sources = Array.from({ length: 4 }, () => new CounterSource(canvas, scratch)); + const fatal: string[] = []; + const chain = new ProgramChain({ canvas, scratchDir: scratch, videoBitrateKbps: 500, output: { kind: "null" }, programTap: true, + redactor: passthroughRedactor, onStderr: () => undefined, onFatal: (r) => fatal.push(r) }); + const publisher = new MemoryPublisher(); + const taps = new PreviewTaps({ publisher, scratchDir: scratch, redactor: passthroughRedactor, onStderr: () => undefined }); + try { + for (const s of sources) await s.start(); + const assets = makeAssets(scratch); + await chain.start(composeOver(sources.map((s) => s.bus), assets, scriptedGraph(0))); + const bus = chain.programBus!; + assert.deepEqual(bus.geometry, programTapGeometry(canvas)); + await taps.add("program", bus); + await until(() => publisher.files.has("program/thumbnail.jpg"), 15_000, "program thumbnail"); + const p0 = (await bus.stats()).pairs; + await new Promise((r) => setTimeout(r, 1_000)); + const p1 = (await bus.stats()).pairs; + assert.ok(p1 - p0 >= canvas.fps * 0.7, `program bus at frame rate (${p1 - p0}/s)`); + // Kill the tap bus helper: the encoder's tee drops that slave and keeps encoding. + process.kill(bus.pid ?? 0, "SIGKILL"); + const s0 = await chain.stats(); + await new Promise((r) => setTimeout(r, 2_000)); + const s1 = await chain.stats(); + assert.ok(chain.encoderPid !== undefined && spawnSync("kill", ["-0", String(chain.encoderPid)]).status === 0, "encoder alive"); + assert.ok(s1.ticks - s0.ticks >= canvas.fps, "gate kept ticking"); + assert.equal(s1.outDrops - s0.outDrops, 0, "encoder kept reading the gate"); + assert.deepEqual(fatal, []); + } finally { + await taps.stop(); + await chain.stop(); + for (const s of sources) await s.stop(); + rmSync(scratch, { recursive: true, force: true }); + } +}); diff --git a/apps/media-worker/src/preview/taps.ts b/apps/media-worker/src/preview/taps.ts new file mode 100644 index 0000000..7519a5b --- /dev/null +++ b/apps/media-worker/src/preview/taps.ts @@ -0,0 +1,352 @@ +/** + * Preview tap lifecycle (streaming-architecture.md §2.1). + * + * Every tap is a source-bus subscriber, so taps never sit in the gate path: a + * slow or dead tap loses only its own pairs (the bus drops whole pairs for a + * subscriber that falls behind; its EPIPE removes it), and the compositors and + * the encoder never wait for it. The program tap's bus is fed by the persistent + * encoder (360p, `tee` with a null anchor), so the PGM tile shows exactly what + * the encoder receives. + * + * - JPEG: always on per tap, one still every 2 s by default (1–10 s; 1 s for the + * PST tile), published as `thumbnail.jpg`. + * - video / audio LL-HLS: on demand. `demand()` starts a rendition and extends + * its lease; an expired lease stops it. The program tap may be made always-on. + */ +import { mkdirSync, readFileSync, rmSync, watch, type FSWatcher } from "node:fs"; +import { join } from "node:path"; +import type { Readable } from "node:stream"; +import type { BusSubscription, SourceBus } from "../compositor/source-bus"; +import { spawnManaged, type ManagedProcess, type StderrRedactor } from "../relay/ffmpeg"; +import { + AUDIO_TAP_CODECS, AUDIO_TAP_KBPS, buildHlsTapArgs, buildJpegTapArgs, jpegStride, PARTS_PLAYLIST, tapSize, + VIDEO_TAP_CODECS, VIDEO_TAP_MAX_KBPS, type HlsRendition, +} from "./argv"; +import { assertTapId, AUDIO_PLAYLIST, MASTER_PLAYLIST, THUMBNAIL_FILE, VIDEO_PLAYLIST, type PreviewPublisher } from "./contract"; +import { fragmentIndependent, LlHlsPackager, masterPlaylist, parseFfmpegPlaylist } from "./ll-hls"; + +/** Manifest TTLs: a stopped tap's files expire on their own. */ +export const TTL = { thumbnail: 30, playlist: 10, media: 60, init: 600 } as const; + +let lastSequenceBase = 0; +function nextSequenceBase(): number { + lastSequenceBase = Math.max(Date.now(), lastSequenceBase + 1); + return lastSequenceBase; +} + +export interface TapDeps { + publisher: PreviewPublisher; + scratchDir: string; + redactor: StderrRedactor; + onStderr(message: string): void; + /** A tap process failed (it is restarted on the next demand/cadence change; on air is unaffected). */ + onError?(tap: string, reason: string): void; +} + +/** + * Splits an mjpeg byte stream into whole JPEGs (SOI … EOI). Inside entropy-coded + * data 0xFF is always stuffed (FF 00), so FF D9 occurs only as the end marker. + */ +export class JpegSplitter { + private buf: Buffer = Buffer.alloc(0); + + push(chunk: Buffer): Buffer[] { + this.buf = this.buf.length === 0 ? chunk : Buffer.concat([this.buf, chunk]); + const out: Buffer[] = []; + for (;;) { + const start = this.buf.indexOf(Buffer.from([0xff, 0xd8])); + if (start < 0) { + // Keep a trailing 0xFF: it may be the first byte of the next SOI. + this.buf = this.buf.length > 0 && this.buf[this.buf.length - 1] === 0xff ? this.buf.subarray(-1) : Buffer.alloc(0); + break; + } + const end = this.buf.indexOf(Buffer.from([0xff, 0xd9]), start + 2); + if (end < 0) { this.buf = this.buf.subarray(start); break; } + out.push(Buffer.from(this.buf.subarray(start, end + 2))); + this.buf = this.buf.subarray(end + 2); + } + if (this.buf.length > 8_000_000) this.buf = Buffer.alloc(0); // never a valid preview still + return out; + } +} + +/** One tap process reading a bus subscription. */ +abstract class BusTap { + protected proc: ManagedProcess | null = null; + protected sub: BusSubscription | null = null; + protected stopping = false; + + constructor(readonly tap: string, protected readonly bus: SourceBus, protected readonly deps: TapDeps) { assertTapId(tap); } + + get running(): boolean { return this.proc !== null; } + get pid(): number | undefined { return this.proc?.child.pid; } + + protected async spawn(args: string[], opts: { video: boolean; audio: boolean; stride?: number }, stdout: "pipe" | "ignore"): Promise { + this.stopping = false; + const sub = await this.bus.subscribe(opts); + this.sub = sub; + let proc: ManagedProcess; + try { + proc = spawnManaged("ffmpeg", args, [sub.videoFd ?? "ignore", stdout, "pipe", sub.audioFd ?? "ignore"], this.deps.redactor, this.deps.onStderr); + } finally { + sub.releaseReaders(); + } + this.proc = proc; + void proc.exited.then(({ code, signal }) => { + if (this.proc === proc) { this.proc = null; this.sub?.unsubscribe(); this.sub = null; } + if (!this.stopping) this.deps.onError?.(this.tap, `tap exited code=${code} signal=${signal}`); + }); + await proc.started; + return proc; + } + + async stop(): Promise { + this.stopping = true; + const proc = this.proc; + this.proc = null; + this.sub?.unsubscribe(); + this.sub = null; + await proc?.terminate(1_000); + } +} + +export class JpegTap extends BusTap { + private everySeconds = 2; + stills = 0; + + get cadence(): number { return this.everySeconds; } + + async start(everySeconds = this.everySeconds): Promise { + this.everySeconds = everySeconds; + const plan = { geometry: this.bus.geometry, everySeconds }; + const proc = await this.spawn(buildJpegTapArgs(plan), { video: true, audio: false, stride: jpegStride(plan) }, "pipe"); + const splitter = new JpegSplitter(); + (proc.child.stdout as Readable | null)?.on("data", (chunk: Buffer) => { + for (const jpeg of splitter.push(chunk)) { + this.stills += 1; + void this.deps.publisher.publish(this.tap, THUMBNAIL_FILE, jpeg, "image/jpeg", TTL.thumbnail) + .catch((error: unknown) => this.deps.onError?.(this.tap, `thumbnail publish failed: ${String(error)}`)); + } + }); + } + + /** 1–10 s (§2.2: 2 s default, 1 s for the PST tile). Restarts the tap process. */ + async setCadence(everySeconds: number): Promise { + if (everySeconds === this.everySeconds && this.running) return; + await this.stop(); + await this.start(everySeconds); + } +} + +/** + * One LL-HLS rendition. ffmpeg writes 0.5 s fMP4 parts into a private scratch + * dir; each time its playlist is renamed into place, new parts are published + * (parts, closed segments, then the playlist), and the local files are removed. + */ +export class HlsTap extends BusTap { + private dir = ""; + private watcher: FSWatcher | null = null; + private packager: LlHlsPackager | null = null; + private seen = new Set(); + private initPublished = ""; + private initRenewedAt = 0; + private chain: Promise = Promise.resolve(); + parts = 0; + + constructor(tap: string, bus: SourceBus, deps: TapDeps, readonly rendition: HlsRendition) { super(tap, bus, deps); } + + get playlistFile(): string { return this.rendition === "video" ? VIDEO_PLAYLIST : AUDIO_PLAYLIST; } + + async start(): Promise { + const name = `${this.tap === "program" ? "program" : this.tap.slice(0, 8)}-${this.rendition}-${Date.now()}`; + this.dir = join(this.deps.scratchDir, "preview", name); + mkdirSync(this.dir, { recursive: true, mode: 0o700 }); + this.packager = new LlHlsPackager(this.rendition, nextSequenceBase()); + this.seen = new Set(); + this.initPublished = ""; + this.watcher = watch(this.dir, (_event, file) => { if (file === PARTS_PLAYLIST) this.schedule(); }); + await this.spawn(buildHlsTapArgs({ geometry: this.bus.geometry, rendition: this.rendition, outDir: this.dir }), + { video: this.rendition === "video", audio: true }, "ignore"); + } + + private schedule(): void { + this.chain = this.chain.then(() => this.collect()).catch((error: unknown) => this.deps.onError?.(this.tap, `hls publish failed: ${String(error)}`)); + } + + /** Waits for in-flight publishing (tests). */ + settled(): Promise { return this.chain; } + + private async collect(): Promise { + const packager = this.packager; + if (!packager) return; + let text: string; + try { text = readFileSync(join(this.dir, PARTS_PLAYLIST), "utf8"); } catch { return; } + const contentType = this.rendition === "video" ? "video/mp4" : "audio/mp4"; + for (const entry of parseFfmpegPlaylist(text)) { + if (this.seen.has(entry.file)) continue; + let bytes: Buffer; + try { bytes = readFileSync(join(this.dir, entry.file)); } catch { continue; } + this.seen.add(entry.file); + if (this.seen.size > 64) this.seen.delete(this.seen.values().next().value ?? ""); + rmSync(join(this.dir, entry.file), { force: true }); + await this.publishInit(packager, contentType); + const { part, closed } = packager.add({ duration: entry.duration, independent: fragmentIndependent(bytes), bytes }); + if (closed) await this.deps.publisher.publish(this.tap, closed.name, LlHlsPackager.segmentBytes(closed), contentType, TTL.media); + if (!part) continue; + await this.deps.publisher.publish(this.tap, part.name, part.bytes, contentType, TTL.media); + this.parts += 1; + const playlist = packager.playlist(); + if (playlist) await this.deps.publisher.publish(this.tap, this.playlistFile, Buffer.from(playlist), "application/vnd.apple.mpegurl", TTL.playlist); + } + } + + /** The init segment is published once and its manifest renewed well inside its TTL. */ + private async publishInit(packager: LlHlsPackager, contentType: "video/mp4" | "audio/mp4"): Promise { + const due = Date.now() - this.initRenewedAt > (TTL.init / 2) * 1000; + if (this.initPublished === packager.initName && !due) return; + let init: Buffer; + try { init = readFileSync(join(this.dir, "init.mp4")); } catch { return; } + await this.deps.publisher.publish(this.tap, packager.initName, init, contentType, TTL.init); + this.initPublished = packager.initName; + this.initRenewedAt = Date.now(); + } + + override async stop(): Promise { + this.watcher?.close(); + this.watcher = null; + await super.stop(); + await this.chain; + this.packager = null; + if (this.dir) rmSync(this.dir, { recursive: true, force: true }); + } +} + +interface TapSet { bus: SourceBus; jpeg: JpegTap; hls: Partial>; masterRenewedAt: number } + +/** + * All preview taps of one channel: a JPEG tap per bus (always on), LL-HLS + * renditions on demand with a lease. `alwaysOn` renditions (the program tap's, + * per §2.2 "PGM always live") never expire. + */ +export class PreviewTaps { + private readonly taps = new Map(); + private readonly alwaysOn = new Set(); + private sweeper: ReturnType | null = null; + private operations: Promise = Promise.resolve(); + + constructor(private readonly deps: TapDeps, private readonly leaseMs = 30_000) {} + + get tapIds(): string[] { return [...this.taps.keys()]; } + jpeg(tap: string): JpegTap | undefined { return this.taps.get(tap)?.jpeg; } + hls(tap: string, rendition: HlsRendition): HlsTap | undefined { return this.taps.get(tap)?.hls[rendition]?.tap; } + + private serialize(operation: () => Promise): Promise { + const next = this.operations.catch(() => undefined).then(operation); + this.operations = next; + return next; + } + + add(tap: string, bus: SourceBus, opts: { alwaysOn?: HlsRendition[] } = {}): Promise { + return this.serialize(() => this.addNow(tap, bus, opts)); + } + + private async addNow(tap: string, bus: SourceBus, opts: { alwaysOn?: HlsRendition[] }): Promise { + assertTapId(tap); + if (this.taps.has(tap)) throw new Error(`preview tap ${tap} already exists`); + const jpeg = new JpegTap(tap, bus, this.deps); + this.taps.set(tap, { bus, jpeg, hls: {}, masterRenewedAt: 0 }); + try { + await jpeg.start(); + for (const r of opts.alwaysOn ?? []) { + this.alwaysOn.add(`${tap}:${r}`); + await this.demandNow(tap, r); + } + } catch (error) { + await this.removeNow(tap); + throw error; + } + this.sweeper ??= setInterval(() => { + void this.sweep().catch((error: unknown) => this.deps.onError?.("program", `preview sweep failed: ${String(error)}`)); + }, 1_000); + } + + /** Starts (or keeps) a rendition for `leaseMs` more. */ + demand(tap: string, rendition: HlsRendition): Promise { + return this.serialize(() => this.demandNow(tap, rendition)); + } + + private async demandNow(tap: string, rendition: HlsRendition): Promise { + const set = this.taps.get(tap); + if (!set) throw new Error(`no preview tap ${tap}`); + const until = this.alwaysOn.has(`${tap}:${rendition}`) ? Infinity : Date.now() + this.leaseMs; + const current = set.hls[rendition]; + if (current?.tap.running) { current.until = Math.max(current.until, until); return; } + await current?.tap.stop(); + const hls = new HlsTap(tap, set.bus, this.deps, rendition); + set.hls[rendition] = { tap: hls, until }; + try { await hls.start(); } catch (error) { + delete set.hls[rendition]; + await hls.stop(); + throw error; + } + await this.publishMaster(tap, set.bus); + } + + setThumbnailCadence(tap: string, everySeconds: number): Promise { + return this.serialize(async () => { + const set = this.taps.get(tap); + if (!set) throw new Error(`no preview tap ${tap}`); + buildJpegTapArgs({ geometry: set.bus.geometry, everySeconds }); + await set.jpeg.setCadence(everySeconds); + }); + } + + private async publishMaster(tap: string, bus: SourceBus): Promise { + const size = tapSize(bus.geometry); + const body = masterPlaylist({ ...size, bandwidth: (VIDEO_TAP_MAX_KBPS + AUDIO_TAP_KBPS) * 1000, codecs: VIDEO_TAP_CODECS }, { bandwidth: AUDIO_TAP_KBPS * 1000, codecs: AUDIO_TAP_CODECS }); + await this.deps.publisher.publish(tap, MASTER_PLAYLIST, Buffer.from(body), "application/vnd.apple.mpegurl", TTL.init); + const set = this.taps.get(tap); + if (set) set.masterRenewedAt = Date.now(); + } + + /** Stops expired renditions; restarts a JPEG tap whose process died. */ + sweep(now = Date.now()): Promise { + return this.serialize(() => this.sweepNow(now)); + } + + private async sweepNow(now: number): Promise { + for (const set of this.taps.values()) { + for (const r of ["video", "audio"] as const) { + const entry = set.hls[r]; + if (entry && entry.until <= now) { delete set.hls[r]; await entry.tap.stop(); } + else if (entry && !entry.tap.running) { + await this.demandNow(set.jpeg.tap, r).catch((error: unknown) => this.deps.onError?.(set.jpeg.tap, `hls restart failed: ${String(error)}`)); + } + } + if (!set.jpeg.running) await set.jpeg.start().catch((error: unknown) => this.deps.onError?.(set.jpeg.tap, `thumbnail restart failed: ${String(error)}`)); + if (Object.keys(set.hls).length && now - set.masterRenewedAt >= TTL.init * 500) await this.publishMaster(set.jpeg.tap, set.bus); + } + } + + remove(tap: string): Promise { return this.serialize(() => this.removeNow(tap)); } + + private async removeNow(tap: string): Promise { + const set = this.taps.get(tap); + if (!set) return; + this.taps.delete(tap); + this.alwaysOn.delete(`${tap}:video`); + this.alwaysOn.delete(`${tap}:audio`); + await set.jpeg.stop(); + for (const entry of Object.values(set.hls)) await entry?.tap.stop(); + } + + async stop(): Promise { + if (this.sweeper) clearInterval(this.sweeper); + this.sweeper = null; + await this.serialize(async () => { + if (this.sweeper) clearInterval(this.sweeper); + this.sweeper = null; + for (const tap of [...this.taps.keys()]) await this.removeNow(tap); + }); + } +} diff --git a/apps/media-worker/src/production.ts b/apps/media-worker/src/production.ts new file mode 100644 index 0000000..e8206e7 --- /dev/null +++ b/apps/media-worker/src/production.ts @@ -0,0 +1,358 @@ +/** + * Desired-state v2: the production part of a channel snapshot + * (docs/design/streaming-architecture.md §2, §4, §6 S1). + * + * This file is mirrored byte-for-byte at apps/media-worker/src/production.ts + * (a test asserts it). It has no imports so both builds compile it as-is. + * + * Everything that can reach a filtergraph or argv is typed: ids are UUIDs, + * assets are sha256 content ids, geometry is integer basis points, gains are + * 3 dB steps, enums are closed. Labels are display text for operators and logs + * only; they are still held to a strict allowlist so a filtergraph payload + * (`;`, `[`, `,`, `'`, `movie=`) can never ride along in a snapshot. + */ + +export type SceneLayout = "single" | "pip" | "2x1" | "2x2" | "3x3" | "4x4" | "custom"; +export type SlotFit = "contain" | "cover" | "crop"; +export type AudioMode = "off" | "on" | "primary"; +export type OverlayKind = "lower_third" | "chyron" | "logo" | "clock"; +export type ClockFormat = "HH:MM" | "HH:MM:SS"; +export type FadeMs = 300 | 500 | 1000 | 2000; + +export const SCENE_LAYOUTS: readonly SceneLayout[] = ["single", "pip", "2x1", "2x2", "3x3", "4x4", "custom"]; +export const SLOT_FITS: readonly SlotFit[] = ["contain", "cover", "crop"]; +export const AUDIO_MODES: readonly AudioMode[] = ["off", "on", "primary"]; +export const OVERLAY_KINDS: readonly OverlayKind[] = ["lower_third", "chyron", "logo", "clock"]; +export const CLOCK_FORMATS: readonly ClockFormat[] = ["HH:MM", "HH:MM:SS"]; +export const FADE_MS: readonly FadeMs[] = [300, 500, 1000, 2000]; + +/** Slot capacity per layout. `custom` places up to 16 explicit rectangles. */ +export const LAYOUT_CAPACITY: Readonly> = { + single: 1, pip: 2, "2x1": 2, "2x2": 4, "3x3": 9, "4x4": 16, custom: 16, +}; + +/** Geometry is integer basis points of the output frame (0..10000), resolution independent. */ +export const GEOMETRY_SCALE = 10_000; +export const GAIN_DB_MIN = -24; +export const GAIN_DB_MAX = 0; +export const GAIN_DB_STEP = 3; +export const CHYRON_SPEED_MIN = 40; +export const CHYRON_SPEED_MAX = 400; +export const CHYRON_STRIP_MAX_PX = 16_384; +export const MAX_SOURCES = 32; +export const MAX_SCENES = 32; +export const MAX_OVERLAYS = 64; +export const MAX_PLAYLIST_ITEMS = 500; +export const MAX_OVERLAY_LAYER = 7; + +export interface Rect { x: number; y: number; w: number; h: number } +export interface Point { x: number; y: number } + +export interface HlsProductionSource { id: string; label: string; kind: "hls"; url: string } +/** Ordered, pre-normalized video asset ids (video_assets.id), fed by the worker's asset feeder. */ +export interface PlaylistProductionSource { id: string; label: string; kind: "playlist"; assetIds: string[]; loop: boolean } +export type ProductionSource = HlsProductionSource | PlaylistProductionSource; + +export interface SceneSlot { + index: number; + /** null = empty slot (renders the fallback). */ + sourceId: string | null; + /** Required for `custom` and `pip` layouts; null means the layout's own grid cell. */ + rect: Rect | null; + fit: SlotFit; + audioMode: AudioMode; + gainDb: number; +} + +export interface ProductionScene { id: string; label: string; layout: SceneLayout; slots: SceneSlot[] } + +export interface ProductionBuses { preview: string | null; program: string | null } + +export interface ProductionOverlay { + /** overlay_slots.id */ + id: string; + sceneId: string; + kind: OverlayKind; + enabled: boolean; + layer: number; + position: Point; + /** overlay_revisions.id; null only for the server-generated clock. */ + revisionId: string | null; + /** sha256 content id of the rendered PNG; null only for the clock. */ + assetId: string | null; + chyron: { speedPxPerSec: number; stripWidthPx: number } | null; + clock: { format: ClockFormat } | null; +} + +export type Transition = { type: "cut" } | { type: "fade"; ms: FadeMs }; + +export interface ProgramAudio { masterGainDb: number } + +export interface ProductionSpec { + scenes: ProductionScene[]; + buses: ProductionBuses; + sources: ProductionSource[]; + overlays: ProductionOverlay[]; + transition: Transition; + audio: ProgramAudio; +} + +export class ProductionSpecError extends Error { + constructor(message: string) { + super(message); + this.name = "ProductionSpecError"; + } +} + +const SECRETISH = /key|secret|token|password|passwd|credential(?!HandleId)|auth/i; +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +const SHA256 = /^[0-9a-f]{64}$/; +/** Letters, digits, space and `_ . -` only: no filtergraph or shell metacharacters, no `=`. */ +const LABEL = /^[\p{L}\p{N}][\p{L}\p{N} _.-]{0,63}$/u; + +function fail(message: string): never { throw new ProductionSpecError(message); } + +function record(value: unknown, field: string, allowed: readonly string[]): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${field} must be an object`); + for (const key of Object.keys(value)) { + if (allowed.includes(key)) continue; + if (SECRETISH.test(key)) fail(`${field}.${key}: secrets are never accepted; reference a vault credentialHandleId`); + fail(`${field}.${key} is not allowed`); + } + for (const key of allowed) if (!(key in value)) fail(`${field}.${key} is required`); + return value as Record; +} + +function list(value: unknown, field: string, max: number): unknown[] { + if (!Array.isArray(value)) fail(`${field} must be an array`); + if (value.length > max) fail(`${field} allows at most ${max} entries`); + return value; +} + +function uuid(value: unknown, field: string): string { + if (typeof value !== "string" || !UUID.test(value)) fail(`${field} must be a lowercase UUID`); + return value; +} + +function nullableUuid(value: unknown, field: string): string | null { + return value === null ? null : uuid(value, field); +} + +function label(value: unknown, field: string): string { + if (typeof value !== "string" || !LABEL.test(value)) { + fail(`${field} must be 1-64 letters, digits, spaces, '_', '.', '-'`); + } + return value; +} + +function int(value: unknown, field: string, min: number, max: number): number { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < min || value > max) { + fail(`${field} must be an integer in ${min}..${max}`); + } + return value; +} + +function bool(value: unknown, field: string): boolean { + if (typeof value !== "boolean") fail(`${field} must be a boolean`); + return value; +} + +function oneOf(value: unknown, field: string, options: readonly T[]): T { + const found = options.find((option) => option === value); + if (found === undefined) fail(`${field} must be one of ${options.join(", ")}`); + return found; +} + +export function gainDb(value: unknown, field: string): number { + const gain = int(value, field, GAIN_DB_MIN, GAIN_DB_MAX); + if (gain % GAIN_DB_STEP !== 0) fail(`${field} must be a multiple of ${GAIN_DB_STEP} dB`); + return gain === 0 ? 0 : gain; +} + +function rect(value: unknown, field: string): Rect { + const r = record(value, field, ["x", "y", "w", "h"]); + const x = int(r.x, `${field}.x`, 0, GEOMETRY_SCALE - 1); + const y = int(r.y, `${field}.y`, 0, GEOMETRY_SCALE - 1); + const w = int(r.w, `${field}.w`, 1, GEOMETRY_SCALE); + const h = int(r.h, `${field}.h`, 1, GEOMETRY_SCALE); + if (x + w > GEOMETRY_SCALE || y + h > GEOMETRY_SCALE) fail(`${field} must lie inside the frame`); + return { x, y, w, h }; +} + +function point(value: unknown, field: string): Point { + const p = record(value, field, ["x", "y"]); + return { x: int(p.x, `${field}.x`, 0, GEOMETRY_SCALE), y: int(p.y, `${field}.y`, 0, GEOMETRY_SCALE) }; +} + +function unique(ids: string[], field: string): void { + if (new Set(ids).size !== ids.length) fail(`${field} ids must be unique`); +} + +/** + * HLS URLs are shape-checked by the caller with its own endpoint validator + * (API and worker differ on loopback strictness); here only bounds are checked. + */ +function source(value: unknown, field: string): ProductionSource { + if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${field} must be an object`); + const kind = (value as Record).kind; + if (kind === "hls") { + const s = record(value, field, ["id", "label", "kind", "url"]); + if (typeof s.url !== "string" || s.url.trim() === "" || s.url.length > 2048) fail(`${field}.url must be a non-empty string`); + return { id: uuid(s.id, `${field}.id`), label: label(s.label, `${field}.label`), kind: "hls", url: s.url }; + } + if (kind === "playlist") { + const s = record(value, field, ["id", "label", "kind", "assetIds", "loop"]); + const assetIds = list(s.assetIds, `${field}.assetIds`, MAX_PLAYLIST_ITEMS).map((item, i) => uuid(item, `${field}.assetIds[${i}]`)); + if (assetIds.length === 0) fail(`${field}.assetIds must not be empty`); + return { id: uuid(s.id, `${field}.id`), label: label(s.label, `${field}.label`), kind: "playlist", assetIds, loop: bool(s.loop, `${field}.loop`) }; + } + return fail(`${field}.kind must be hls or playlist`); +} + +function slot(value: unknown, field: string, layout: SceneLayout, sources: ReadonlySet): SceneSlot { + const s = record(value, field, ["index", "sourceId", "rect", "fit", "audioMode", "gainDb"]); + const index = int(s.index, `${field}.index`, 0, LAYOUT_CAPACITY[layout] - 1); + const sourceId = nullableUuid(s.sourceId, `${field}.sourceId`); + if (sourceId !== null && !sources.has(sourceId)) fail(`${field}.sourceId must name a source of this channel`); + const needsRect = layout === "custom" || (layout === "pip" && index > 0); + const slotRect = s.rect === null ? null : rect(s.rect, `${field}.rect`); + if (needsRect && slotRect === null) fail(`${field}.rect is required for the ${layout} layout`); + if (!needsRect && slotRect !== null) fail(`${field}.rect is only allowed for custom layouts and PIP insets`); + return { + index, sourceId, rect: slotRect, + fit: oneOf(s.fit, `${field}.fit`, SLOT_FITS), + audioMode: oneOf(s.audioMode, `${field}.audioMode`, AUDIO_MODES), + gainDb: gainDb(s.gainDb, `${field}.gainDb`), + }; +} + +function scene(value: unknown, field: string, sources: ReadonlySet): ProductionScene { + const s = record(value, field, ["id", "label", "layout", "slots"]); + const layout = oneOf(s.layout, `${field}.layout`, SCENE_LAYOUTS); + const slots = list(s.slots, `${field}.slots`, LAYOUT_CAPACITY[layout]).map((item, i) => slot(item, `${field}.slots[${i}]`, layout, sources)); + if (new Set(slots.map((x) => x.index)).size !== slots.length) fail(`${field}.slots indexes must be unique`); + if (slots.filter((x) => x.audioMode === "primary").length > 1) fail(`${field}.slots: at most one primary audio slot per scene`); + return { id: uuid(s.id, `${field}.id`), label: label(s.label, `${field}.label`), layout, slots }; +} + +function overlay(value: unknown, field: string, scenes: ReadonlySet): ProductionOverlay { + const o = record(value, field, ["id", "sceneId", "kind", "enabled", "layer", "position", "revisionId", "assetId", "chyron", "clock"]); + const sceneId = uuid(o.sceneId, `${field}.sceneId`); + if (!scenes.has(sceneId)) fail(`${field}.sceneId must name a scene of this channel`); + const kind = oneOf(o.kind, `${field}.kind`, OVERLAY_KINDS); + const rendered = kind !== "clock"; + let revisionId: string | null = null; + let assetId: string | null = null; + if (rendered) { + revisionId = uuid(o.revisionId, `${field}.revisionId`); + if (typeof o.assetId !== "string" || !SHA256.test(o.assetId)) fail(`${field}.assetId must be a sha256 content id`); + assetId = o.assetId; + } else if (o.revisionId !== null || o.assetId !== null) fail(`${field}: a clock overlay has no rendered asset`); + + let chyron: ProductionOverlay["chyron"] = null; + if (kind === "chyron") { + const c = record(o.chyron, `${field}.chyron`, ["speedPxPerSec", "stripWidthPx"]); + chyron = { + speedPxPerSec: int(c.speedPxPerSec, `${field}.chyron.speedPxPerSec`, CHYRON_SPEED_MIN, CHYRON_SPEED_MAX), + stripWidthPx: int(c.stripWidthPx, `${field}.chyron.stripWidthPx`, 1, CHYRON_STRIP_MAX_PX), + }; + } else if (o.chyron !== null) fail(`${field}.chyron is only allowed for chyron overlays`); + + let clock: ProductionOverlay["clock"] = null; + if (kind === "clock") { + const c = record(o.clock, `${field}.clock`, ["format"]); + clock = { format: oneOf(c.format, `${field}.clock.format`, CLOCK_FORMATS) }; + } else if (o.clock !== null) fail(`${field}.clock is only allowed for clock overlays`); + + return { + id: uuid(o.id, `${field}.id`), sceneId, kind, + enabled: bool(o.enabled, `${field}.enabled`), + layer: int(o.layer, `${field}.layer`, 0, MAX_OVERLAY_LAYER), + position: point(o.position, `${field}.position`), + revisionId, assetId, chyron, clock, + }; +} + +function transition(value: unknown, field: string): Transition { + if (typeof value !== "object" || value === null || Array.isArray(value)) fail(`${field} must be an object`); + const type = (value as Record).type; + if (type === "cut") { + record(value, field, ["type"]); + return { type: "cut" }; + } + if (type === "fade") { + const t = record(value, field, ["type", "ms"]); + return { type: "fade", ms: oneOf(t.ms, `${field}.ms`, FADE_MS) }; + } + return fail(`${field}.type must be cut or fade`); +} + +/** Parse and normalize; throws ProductionSpecError with a field path on the first problem. */ +export function parseProductionSpec(value: unknown, field = "production"): ProductionSpec { + const p = record(value, field, ["scenes", "buses", "sources", "overlays", "transition", "audio"]); + const sources = list(p.sources, `${field}.sources`, MAX_SOURCES).map((item, i) => source(item, `${field}.sources[${i}]`)); + unique(sources.map((s) => s.id), `${field}.sources`); + const sourceIds = new Set(sources.map((s) => s.id)); + const scenes = list(p.scenes, `${field}.scenes`, MAX_SCENES).map((item, i) => scene(item, `${field}.scenes[${i}]`, sourceIds)); + unique(scenes.map((s) => s.id), `${field}.scenes`); + const sceneIds = new Set(scenes.map((s) => s.id)); + const overlays = list(p.overlays, `${field}.overlays`, MAX_OVERLAYS).map((item, i) => overlay(item, `${field}.overlays[${i}]`, sceneIds)); + unique(overlays.map((o) => o.id), `${field}.overlays`); + + const b = record(p.buses, `${field}.buses`, ["preview", "program"]); + const buses: ProductionBuses = { + preview: nullableUuid(b.preview, `${field}.buses.preview`), + program: nullableUuid(b.program, `${field}.buses.program`), + }; + for (const bus of ["preview", "program"] as const) { + const id = buses[bus]; + if (id !== null && !sceneIds.has(id)) fail(`${field}.buses.${bus} must name a scene of this channel`); + } + const a = record(p.audio, `${field}.audio`, ["masterGainDb"]); + return { + scenes, buses, sources, overlays, + transition: transition(p.transition, `${field}.transition`), + audio: { masterGainDb: gainDb(a.masterGainDb, `${field}.audio.masterGainDb`) }, + }; +} + +export function isProductionSpec(value: unknown): value is ProductionSpec { + try { + parseProductionSpec(value); + return true; + } catch { + return false; + } +} + +/** Every remote URL a production snapshot can make the worker fetch. */ +export function productionHlsUrls(spec: ProductionSpec): string[] { + return spec.sources.flatMap((s) => (s.kind === "hls" ? [s.url] : [])); +} + +/** Declared key order for signing: insertion order of the input can never drift the signed bytes. */ +export function canonicalProduction(spec: ProductionSpec): ProductionSpec { + return { + scenes: spec.scenes.map((s) => ({ + id: s.id, label: s.label, layout: s.layout, + slots: s.slots.map((x) => ({ + index: x.index, sourceId: x.sourceId, + rect: x.rect === null ? null : { x: x.rect.x, y: x.rect.y, w: x.rect.w, h: x.rect.h }, + fit: x.fit, audioMode: x.audioMode, gainDb: x.gainDb, + })), + })), + buses: { preview: spec.buses.preview, program: spec.buses.program }, + sources: spec.sources.map((s): ProductionSource => (s.kind === "hls" + ? { id: s.id, label: s.label, kind: "hls", url: s.url } + : { id: s.id, label: s.label, kind: "playlist", assetIds: [...s.assetIds], loop: s.loop })), + overlays: spec.overlays.map((o) => ({ + id: o.id, sceneId: o.sceneId, kind: o.kind, enabled: o.enabled, layer: o.layer, + position: { x: o.position.x, y: o.position.y }, + revisionId: o.revisionId, assetId: o.assetId, + chyron: o.chyron === null ? null : { speedPxPerSec: o.chyron.speedPxPerSec, stripWidthPx: o.chyron.stripWidthPx }, + clock: o.clock === null ? null : { format: o.clock.format }, + })), + transition: spec.transition.type === "cut" ? { type: "cut" } : { type: "fade", ms: spec.transition.ms }, + audio: { masterGainDb: spec.audio.masterGainDb }, + }; +} diff --git a/apps/media-worker/src/relay/endpoints.ts b/apps/media-worker/src/relay/endpoints.ts new file mode 100644 index 0000000..40df0a6 --- /dev/null +++ b/apps/media-worker/src/relay/endpoints.ts @@ -0,0 +1,39 @@ +import { BlockList, isIP } from "node:net"; + +export type EndpointUse = "hls" | "output"; +export interface ValidatedEndpoint { url: URL; pinnedAddress: string } + +/** WP7 supplies DNS pinning, redirect and HLS child-URI validation. */ +export interface EndpointValidator { + validate(raw: string, use: EndpointUse): Promise; +} + +const blocked = new BlockList(); +for (const [subnet, prefix] of [ + ["0.0.0.0", 8], ["10.0.0.0", 8], ["100.64.0.0", 10], ["127.0.0.0", 8], + ["169.254.0.0", 16], ["172.16.0.0", 12], ["192.0.0.0", 24], ["192.168.0.0", 16], + ["192.0.2.0", 24], ["198.18.0.0", 15], ["198.51.100.0", 24], + ["203.0.113.0", 24], ["224.0.0.0", 4], ["240.0.0.0", 4], +] as const) blocked.addSubnet(subnet, prefix, "ipv4"); +for (const [subnet, prefix] of [ + ["::", 128], ["::1", 128], ["fc00::", 7], ["fe80::", 10], ["ff00::", 8], + ["2001::", 32], ["2001:db8::", 32], ["2002::", 16], ["64:ff9b::", 96], +] as const) blocked.addSubnet(subnet, prefix, "ipv6"); + +export function assertPublicAddress(address: string): void { + const normalized = address.replace(/^\[|\]$/g, "").toLowerCase(); + const family = isIP(normalized); + if (family === 0) throw new Error("unresolved endpoint address"); + // Node's BlockList normalizes IPv4 to mapped IPv6 internally; reject mapped literals outright. + if (normalized.startsWith("::ffff:")) throw new Error("mapped endpoint rejected"); + if (blocked.check(normalized, family === 4 ? "ipv4" : "ipv6")) throw new Error("private endpoint rejected"); +} + +export function validateEndpointShape(raw: string, use: EndpointUse): URL { + let url: URL; + try { url = new URL(raw); } catch { throw new Error("invalid endpoint URL"); } + const schemes = use === "hls" ? ["https:"] : ["rtmp:", "rtmps:"]; + if (!schemes.includes(url.protocol) || !url.hostname || url.username || url.password || url.search || url.hash) throw new Error("endpoint scheme or credentials rejected"); + if (isIP(url.hostname.replace(/^\[|\]$/g, ""))) assertPublicAddress(url.hostname); + return url; +} diff --git a/apps/media-worker/src/relay/ffmpeg.ts b/apps/media-worker/src/relay/ffmpeg.ts new file mode 100644 index 0000000..1abeb82 --- /dev/null +++ b/apps/media-worker/src/relay/ffmpeg.ts @@ -0,0 +1,122 @@ +import { spawn, type ChildProcess, type StdioOptions } from "node:child_process"; +import { OUTPUT_PROFILES, type DesiredChannel } from "../types"; +import { assertPublicAddress, validateEndpointShape, type EndpointValidator } from "./endpoints"; + +/** The vault broker delivers a key directly to the push relay's inherited fd. */ +export interface VaultHandleBroker { + deliverPushTarget(handleId: string, context: { workspaceId: string; channelId: string; assignmentEpoch: number }, relayFd: number): Promise; +} +export interface LocalRelayPipes { ingestFd: number; pushFd: number; close(): Promise } +/** WP7 implementation fetches HLS through pinned egress and pushes RTMP with the vault key. */ +export interface EgressRelayFactory { + prepare(config: DesiredChannel, validator: EndpointValidator, vault: VaultHandleBroker): Promise; +} +export interface StderrRedactor { redact(chunk: string): string } + +/** Media children get a minimal environment: no worker secrets, no inherited config. */ +export function mediaEnv(): NodeJS.ProcessEnv { return { PATH: process.env.PATH ?? "/usr/bin:/bin" }; } + +export interface ManagedProcess { + readonly child: ChildProcess; + /** Resolves on spawn, rejects if the binary cannot start. */ + readonly started: Promise; + /** Resolves once the process has exited (never rejects). */ + readonly exited: Promise<{ code: number | null; signal: NodeJS.Signals | null }>; + kill(signal?: NodeJS.Signals): void; + /** SIGTERM, then SIGKILL after `graceMs`; resolves on exit. */ + terminate(graceMs?: number): Promise; +} + +/** + * Shared spawn path for every media child (relay, ingest, compositor, encoder, + * frame gate): argv only, never a shell; minimal env; stderr is redacted and + * truncated before it reaches the logger. + */ +export function spawnManaged( + command: string, args: readonly string[], stdio: StdioOptions, + redactor: StderrRedactor, onStderr: (message: string) => void, cwd?: string, +): ManagedProcess { + const child = spawn(command, args, { stdio, shell: false, env: mediaEnv(), ...(cwd === undefined ? {} : { cwd }) }); + child.stderr?.on("data", (chunk: Buffer) => onStderr(redactor.redact(chunk.toString("utf8").slice(0, 4096)))); + const started = new Promise((resolve, reject) => { + child.once("spawn", () => resolve()); + child.once("error", reject); + }); + started.catch(() => undefined); + const exited = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve) => { + child.once("exit", (code, signal) => resolve({ code, signal })); + child.once("error", () => { if (child.pid === undefined) resolve({ code: null, signal: null }); }); + }); + const kill = (signal: NodeJS.Signals = "SIGTERM"): void => { if (child.exitCode === null && child.signalCode === null) child.kill(signal); }; + return { + child, started, exited, kill, + async terminate(graceMs = 2_000) { + kill("SIGTERM"); + const timer = setTimeout(() => kill("SIGKILL"), graceMs); + await exited; + clearTimeout(timer); + }, + }; +} + +export function ffmpegArgs(config: DesiredChannel): string[] { + const args = ["-hide_banner", "-nostdin", "-loglevel", "warning", "-f", "mpegts", "-i", "pipe:3"]; + if (config.mode === "passthrough") args.push("-c", "copy"); + else { + const profile = OUTPUT_PROFILES[config.profile]; + args.push("-vf", `scale=${profile.width}:${profile.height},fps=${profile.fps}`, + "-c:v", "libx264", "-preset", "veryfast", "-b:v", profile.videoBitrate, + "-c:a", "aac", "-b:a", "128k"); + } + args.push("-f", "flv", "pipe:4"); + return args; +} + +export class FfmpegRelayJob { + private child: ChildProcess | null = null; + private pipes: LocalRelayPipes | null = null; + private stopping = false; + constructor( + private readonly validator: EndpointValidator, + private readonly egress: EgressRelayFactory, + private readonly vault: VaultHandleBroker, + private readonly redactor: StderrRedactor, + private readonly onStderr: (message: string) => void, + private readonly onExit: (reason: string) => void, + ) {} + + async start(config: DesiredChannel): Promise { + if (!config.source || !config.output) throw new Error("running channel needs source and output"); + validateEndpointShape(config.source.url, "hls"); + validateEndpointShape(config.output.endpointUrl, "output"); + const source = await this.validator.validate(config.source.url, "hls"); + const output = await this.validator.validate(config.output.endpointUrl, "output"); + assertPublicAddress(source.pinnedAddress); + assertPublicAddress(output.pinnedAddress); + this.pipes = await this.egress.prepare(config, this.validator, this.vault); + this.stopping = false; + const managed = spawnManaged("ffmpeg", ffmpegArgs(config), + ["ignore", "ignore", "pipe", this.pipes.ingestFd, this.pipes.pushFd], this.redactor, this.onStderr); + const child = managed.child; + this.child = child; + let spawned = false; + void managed.started.then(() => { spawned = true; }, () => undefined); + child.on("error", (error) => { if (spawned) this.onExit(this.redactor.redact(error.message)); }); + const started = managed.started; + child.once("exit", (code, signal) => { + void this.pipes?.close(); + this.pipes = null; + this.child = null; + if (!this.stopping) this.onExit(`ffmpeg exited code=${code} signal=${signal}`); + }); + await started; + } + + async stop(): Promise { + this.stopping = true; + this.child?.kill("SIGTERM"); + await this.pipes?.close(); + this.pipes = null; + this.child = null; + } +} diff --git a/apps/media-worker/src/relay/relay.test.ts b/apps/media-worker/src/relay/relay.test.ts new file mode 100644 index 0000000..753fbaf --- /dev/null +++ b/apps/media-worker/src/relay/relay.test.ts @@ -0,0 +1,56 @@ +import assert from "node:assert/strict"; +import { resolve } from "node:path"; +import test from "node:test"; +import { pathToFileURL } from "node:url"; +import { assertPublicAddress, validateEndpointShape } from "./endpoints"; +import { ffmpegArgs } from "./ffmpeg"; +import type { DesiredChannel } from "../types"; + +const channel: DesiredChannel = { + workspaceId: "workspace-1", channelId: "channel-1", workerId: "worker-1", + assignmentEpoch: 1, desiredVersion: 1, desiredStatus: "RUNNING", + source: { kind: "hls", url: "https://mock.local/hls/live-1/master.m3u8" }, + output: { endpointUrl: "rtmp://mock.local/live", credentialHandleId: "vault-handle" }, + mode: "passthrough", profile: "720p30", +}; + +test("relay argv uses only local inherited pipes and mode profile", () => { + const passthrough = ffmpegArgs(channel); + assert.deepEqual(passthrough.slice(-3), ["-f", "flv", "pipe:4"]); + assert.ok(passthrough.includes("copy")); + assert.ok(!passthrough.join(" ").includes("mock.local")); + assert.ok(!passthrough.join(" ").includes("vault-handle")); + const normalized = ffmpegArgs({ ...channel, mode: "normalize", profile: "1080p60" }); + assert.ok(normalized.includes("scale=1920:1080,fps=60")); +}); + +test("endpoint shape and literal/private pin denial", () => { + assert.equal(validateEndpointShape(channel.source?.url ?? "", "hls").protocol, "https:"); + assert.throws(() => validateEndpointShape("file:///etc/passwd", "hls")); + assert.throws(() => validateEndpointShape("https://127.0.0.1/live.m3u8", "hls")); + assert.throws(() => validateEndpointShape("rtmp://169.254.169.254/live", "output")); + assert.throws(() => validateEndpointShape("rtmp://user:secret@example.com/live", "output")); + for (const address of ["10.0.0.1", "172.20.1.2", "192.168.1.1", "100.64.0.1", "::1", "fc00::1", "::ffff:127.0.0.1", "::ffff:7f00:1"]) { + assert.throws(() => assertPublicAddress(address), address); + } + assert.doesNotThrow(() => assertPublicAddress("8.8.8.8")); +}); + +test("mock livestream source and single-live simulator fixture feed the relay contract", async () => { + const mockPath = pathToFileURL(resolve(__dirname, "../../../../packages/connector-mock-livestream/src/index.ts")).href; + const fixturePath = pathToFileURL(resolve(__dirname, "../../../../packages/testing/src/scenarios.ts")).href; + const mockModule: unknown = await import(mockPath); + const fixtures: unknown = await import(fixturePath); + const mock = mockModule as { createMockLivestreamConnector(): { + createLiveStream(connection: null, request: { channelId: string; title: string; permanent: boolean; saveReplay: boolean }): Promise<{ remoteId: string }>; + getStreamPlaybackSources(connection: null, request: { remoteId: string }): Promise>; + } }; + const scenarios = fixtures as { SCENARIOS: Record }> }; + assert.deepEqual(scenarios.SCENARIOS["single-live"]?.steps.map((step) => [step.source, step.action]), [["A", "go-live"]]); + const connector = mock.createMockLivestreamConnector(); + const live = await connector.createLiveStream(null, { channelId: "A", title: "test", permanent: true, saveReplay: false }); + const sources = await connector.getStreamPlaybackSources(null, { remoteId: live.remoteId }); + assert.equal(sources[0]?.kind, "hls"); + assert.equal(validateEndpointShape(sources[0]?.url ?? "", "hls").hostname, "mock.local"); + assert.ok(!ffmpegArgs({ ...channel, source: { kind: "hls", url: sources[0]?.url ?? "" } }).join(" ").includes("mock.local")); +}); diff --git a/apps/media-worker/src/supervisor/index.ts b/apps/media-worker/src/supervisor/index.ts new file mode 100644 index 0000000..1b9ee81 --- /dev/null +++ b/apps/media-worker/src/supervisor/index.ts @@ -0,0 +1,249 @@ +import type { KeyObject } from "node:crypto"; +import { sandboxRelay, supervisorSandboxEnabled } from "./sandbox"; +import { verifyCommand } from "../bus/envelope"; +import { DurableFence } from "../bus/fence"; +import type { ObservedStore } from "../bus/redis"; +import type { ChannelState, DesiredChannel, ObservedChannel } from "../types"; + +export interface TransitionLogger { + transition(event: { workspaceId: string; channelId: string; from: ChannelState; to: ChannelState; reason: string }): void; +} +interface ChannelRuntime { + desired: DesiredChannel; + state: ChannelState; + reason: string; + realizedVersion: number | null; + relay: RelayJob | null; + retries: number; + retryTimer: { cancel(): void } | null; + /** Consecutive failed swaps for the current desired version (the previous version stays on air). */ + swapFailures: number; +} +export interface RelayJob { + start(channel: DesiredChannel): Promise; + stop(): Promise; + /** + * Optional in-place change (production jobs: an A/B compositor swap behind the + * frame gate). Resolves true once the new version is on air; false means the + * change needs a restart (sources, output or profile changed). + */ + reconfigure?(channel: DesiredChannel): Promise; +} +export interface RetryScheduler { schedule(action: () => Promise, delayMs: number): { cancel(): void } } + +/** + * A swap that fails (B never went on air; A is still on air) is retried this many times, 1 s then + * 2 s later, before the supervisor falls back to a full restart. The full restart converges too, + * but it takes the encoder and the push session down (WP9 "output that does not stop"). + */ +export const SWAP_RETRIES = 2; +const defaultRetryScheduler: RetryScheduler = { + schedule(action, delayMs) { + const timer = setTimeout(() => { void action(); }, delayMs); + return { cancel: () => clearTimeout(timer) }; + }, +}; + +/** Accepted versions are durable; realized versions describe only this process lifetime. */ +export class MediaSupervisor { + private readonly channels = new Map(); + private readonly operations = new Map>(); + private heartbeatTimer: ReturnType | null = null; + /** Set by `stop()`: late retries and relaunches become no-ops. */ + private stopped = false; + constructor( + private readonly workerId: string, + desired: DesiredChannel[], + private readonly store: ObservedStore, + private readonly fence: DurableFence, + private readonly controlKeys: ReadonlyMap, + private readonly createRelay: (channel: DesiredChannel, onExit: (reason: string) => void) => RelayJob, + private readonly logger: TransitionLogger, + private readonly retryScheduler: RetryScheduler = defaultRetryScheduler, + ) { + supervisorSandboxEnabled(); // Production cannot silently opt out of confinement. + for (const channel of desired) { + if (channel.workerId !== workerId) throw new Error("config worker mismatch"); + this.channels.set(this.key(channel.workspaceId, channel.channelId), { + desired: channel, state: "STOPPED", reason: "worker boot", realizedVersion: null, + relay: null, retries: 0, retryTimer: null, swapFailures: 0, + }); + } + } + + private key(workspaceId: string, channelId: string): string { return `${workspaceId}:${channelId}`; } + status(): ObservedChannel[] { + return [...this.channels.values()].map((runtime) => ({ + workspaceId: runtime.desired.workspaceId, channelId: runtime.desired.channelId, + workerId: this.workerId, assignmentEpoch: runtime.desired.assignmentEpoch, + state: runtime.state, realizedVersion: runtime.realizedVersion, + updatedAt: new Date().toISOString(), reason: runtime.reason, + })); + } + private async transition(runtime: ChannelRuntime, to: ChannelState, reason: string): Promise { + const from = runtime.state; + runtime.state = to; + runtime.reason = reason; + this.logger.transition({ workspaceId: runtime.desired.workspaceId, channelId: runtime.desired.channelId, from, to, reason }); + await this.store.put(this.status().find((item) => item.workspaceId === runtime.desired.workspaceId && item.channelId === runtime.desired.channelId)!); + } + + async start(): Promise { + for (const runtime of this.channels.values()) { + const old = await this.store.get(runtime.desired.workspaceId, runtime.desired.channelId); + await this.transition(runtime, "STOPPED", old + ? `restart reconcile: prior ${old.state}; awaiting fresh signed snapshot` + : "start reconcile: awaiting signed snapshot"); + } + await this.beat(); + this.heartbeatTimer = setInterval(() => { void this.beat().catch((error: unknown) => this.logger.transition({ workspaceId: "", channelId: "", from: "DEGRADED", to: "DEGRADED", reason: `heartbeat failed: ${String(error)}` })); }, 5_000); + } + + async beat(): Promise { await this.store.heartbeat(this.workerId, this.status()); } + + async handle(raw: unknown, now = Date.now()): Promise<"accepted" | "replay" | "stale"> { + const command = verifyCommand(raw, this.controlKeys, this.workerId, now); + const body = command.body; + return this.serialize(this.key(body.workspaceId, body.channelId), () => this.apply(body, now)); + } + + /** One operation at a time per channel (commands and swap retries). */ + private serialize(key: string, operation: () => Promise): Promise { + const previous = this.operations.get(key) ?? Promise.resolve(); + const task = previous.catch(() => undefined).then(operation); + this.operations.set(key, task); + void task.then(() => { if (this.operations.get(key) === task) this.operations.delete(key); }, + () => { if (this.operations.get(key) === task) this.operations.delete(key); }); + return task; + } + + private async apply(body: ReturnType["body"], now: number): Promise<"accepted" | "replay" | "stale"> { + const runtime = this.channels.get(this.key(body.workspaceId, body.channelId)); + if (!runtime || runtime.desired.assignmentEpoch !== body.assignmentEpoch) throw new Error("unassigned channel or epoch"); + const result = this.fence.accept(body, now); + if (result !== "accepted") return result; + // RUNNING → RUNNING on a job that can swap: the encoder and push session stay up, + // and the version is realized only once the new compositor is on air. + const relay = runtime.relay; + if (runtime.state === "RUNNING" && body.snapshot.desiredStatus === "RUNNING" && relay?.reconfigure) { + runtime.desired = body.snapshot; + runtime.retryTimer?.cancel(); + runtime.retryTimer = null; + runtime.swapFailures = 0; + if (await this.swap(runtime, relay) !== "restart") return result; + } + runtime.desired = body.snapshot; + runtime.swapFailures = 0; + runtime.realizedVersion = null; + runtime.retryTimer?.cancel(); + runtime.retryTimer = null; + await runtime.relay?.stop(); + runtime.relay = null; + if (body.snapshot.desiredStatus === "STOPPED") { + await this.transition(runtime, "STOPPED", "signed desired state stopped"); + runtime.realizedVersion = body.desiredVersion; + await this.beat(); + return result; + } + runtime.retries = 0; + await this.launch(runtime); + return result; + } + + /** + * One swap attempt for `runtime.desired`. "done": on air (or the job is gone and its own exit + * path owns the channel); "retrying": B failed, A stays on air (`realizedVersion` keeps naming + * what viewers see) and a retry is scheduled; "restart": not swappable, or out of retries. + */ + private async swap(runtime: ChannelRuntime, relay: RelayJob): Promise<"done" | "retrying" | "restart"> { + const version = runtime.desired.desiredVersion; + try { + if (!(await relay.reconfigure?.(runtime.desired))) return "restart"; + if (runtime.relay !== relay) return "done"; + runtime.swapFailures = 0; + runtime.realizedVersion = version; + await this.transition(runtime, "RUNNING", "compositor swapped at a frame boundary"); + await this.beat(); + return "done"; + } catch (error: unknown) { + if (runtime.relay !== relay) return "done"; + const name = error instanceof Error ? error.name : "unknown error"; + runtime.swapFailures += 1; + if (runtime.swapFailures > SWAP_RETRIES) { + runtime.reason = `swap failed: ${name}`; + return "restart"; + } + const delay = 1_000 * 2 ** (runtime.swapFailures - 1); + await this.transition(runtime, "RUNNING", `swap to v${version} failed (${name}); v${runtime.realizedVersion ?? "?"} stays on air; retry ${runtime.swapFailures}/${SWAP_RETRIES} in ${delay}ms`); + const key = this.key(runtime.desired.workspaceId, runtime.desired.channelId); + runtime.retryTimer = this.retryScheduler.schedule(() => this.serialize(key, async () => { + runtime.retryTimer = null; + if (this.stopped) return; + // A newer command or an exit has taken over since. + if (runtime.desired.desiredVersion !== version || runtime.relay !== relay || runtime.state !== "RUNNING") return; + if (await this.swap(runtime, relay) === "restart") await this.restart(runtime); + }), delay); + return "retrying"; + } + } + + /** Full restart for the current desired snapshot (the swap path gave up). */ + private async restart(runtime: ChannelRuntime): Promise { + runtime.realizedVersion = null; + runtime.swapFailures = 0; + await runtime.relay?.stop(); + runtime.relay = null; + runtime.retries = 0; + await this.launch(runtime); + } + + private async launch(runtime: ChannelRuntime): Promise { + await this.transition(runtime, runtime.retries ? "RESTARTING" : "STARTING", runtime.retries ? "relay retry" : "signed desired state running"); + try { + const relay = sandboxRelay(this.createRelay(runtime.desired, (reason) => { + if (runtime.relay === relay) void this.relayExited(runtime, reason); + }), runtime.desired); + runtime.relay = relay; + await relay.start(runtime.desired); + if (runtime.relay !== relay) return; + runtime.realizedVersion = runtime.desired.desiredVersion; + await this.transition(runtime, "RUNNING", "ffmpeg spawned with local relay pipes"); + await this.beat(); + } catch (error: unknown) { + runtime.relay = null; + await this.relayExited(runtime, `start failed: ${error instanceof Error ? error.name : "unknown error"}`); + } + } + + private async relayExited(runtime: ChannelRuntime, reason: string): Promise { + if (this.stopped || runtime.desired.desiredStatus !== "RUNNING") return; + runtime.realizedVersion = null; + runtime.relay = null; + await this.transition(runtime, "DEGRADED", reason); + if (runtime.retries >= 5) { await this.transition(runtime, "FAILED", "relay retry budget exhausted"); return; } + const delay = Math.min(30_000, 1_000 * 2 ** runtime.retries); + runtime.retries += 1; + await this.transition(runtime, "RESTARTING", `retry in ${delay}ms`); + runtime.retryTimer = this.retryScheduler.schedule(() => { runtime.retryTimer = null; return this.stopped ? Promise.resolve() : this.launch(runtime); }, delay); + } + + /** + * Worker shutdown. In-flight per-channel operations (a command, a swap retry, a relaunch) finish + * first, so none of them can start a job after the channels were stopped. + */ + async stop(): Promise { + this.stopped = true; + if (this.heartbeatTimer) clearInterval(this.heartbeatTimer); + this.heartbeatTimer = null; + for (const runtime of this.channels.values()) runtime.retryTimer?.cancel(); + await Promise.allSettled([...this.operations.values()]); + for (const runtime of this.channels.values()) { + runtime.retryTimer?.cancel(); + runtime.retryTimer = null; + runtime.desired = { ...runtime.desired, desiredStatus: "STOPPED" }; + await runtime.relay?.stop(); + runtime.realizedVersion = null; + await this.transition(runtime, "STOPPED", "worker shutdown"); + } + } +} diff --git a/apps/media-worker/src/supervisor/production.e2e.test.ts b/apps/media-worker/src/supervisor/production.e2e.test.ts new file mode 100644 index 0000000..a788d87 --- /dev/null +++ b/apps/media-worker/src/supervisor/production.e2e.test.ts @@ -0,0 +1,215 @@ +/** + * A/B lifecycle end to end (streaming-architecture.md §6 S2): signed snapshots + * through the real MediaSupervisor into a real ProductionJob (synthetic + * ingests standing in for the WP7 relay, a playlist source through the asset + * feeder, the Rust gate, an x264 encoder into `-f null`, preview taps). + * + * - a TAKE (program scene change) is a swap: the encoder process is the same + * one, and the version is realized only once the gate reports B on air; + * - a playlist edit is adopted by its feeder in place (still the same encoder); + * - a changed source URL is a restart (a new encoder); + * - STOPPED tears everything down. + */ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { generateKeyPairSync, randomUUID } from "node:crypto"; +import { createReadStream, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { signCommand, type CommandBody } from "../bus/envelope"; +import { DurableFence } from "../bus/fence"; +import type { ObservedStore } from "../bus/redis"; +import { ProductionJob } from "../compositor/production-job"; +import { ensureGateBinary, makeAssets, passthroughRedactor } from "../compositor/testing"; +import { parseProductionSpec, type ProductionSpec } from "../production"; +import type { PreviewContentType } from "../preview/contract"; +import type { DesiredChannel, ObservedChannel } from "../types"; +import { MediaSupervisor, type RelayJob } from "./index"; + +const id = (n: number): string => `00000000-0000-4000-8000-${String(n).padStart(12, "0")}`; +const WS = "8f0c5a8e-3b1d-4c3e-9a5b-0c1d2e3f4a5b"; +const CH = "1a2b3c4d-5e6f-4a1b-8c2d-3e4f5a6b7c8d"; + +function spec(opts: { take: 10 | 11; playlist: string[]; urlB?: string }): ProductionSpec { + return parseProductionSpec({ + sources: [ + { id: id(1), label: "Cam A", kind: "hls", url: "https://a.example/live.m3u8" }, + { id: id(2), label: "Cam B", kind: "hls", url: opts.urlB ?? "https://b.example/live.m3u8" }, + { id: id(3), label: "Loop", kind: "playlist", assetIds: opts.playlist, loop: true }, + ], + scenes: [ + { id: id(10), label: "Two up", layout: "2x1", slots: [ + { index: 0, sourceId: id(1), rect: null, fit: "cover", audioMode: "primary", gainDb: 0 }, + { index: 1, sourceId: id(2), rect: null, fit: "cover", audioMode: "on", gainDb: -6 }, + ] }, + { id: id(11), label: "Playlist", layout: "pip", slots: [ + { index: 0, sourceId: id(3), rect: null, fit: "contain", audioMode: "primary", gainDb: 0 }, + { index: 1, sourceId: id(1), rect: { x: 6500, y: 6500, w: 3000, h: 3000 }, fit: "cover", audioMode: "off", gainDb: 0 }, + ] }, + ], + buses: { preview: id(10), program: id(opts.take) }, + overlays: [ + { id: id(20), sceneId: id(10), kind: "lower_third", enabled: true, layer: 1, position: { x: 500, y: 7500 }, revisionId: id(30), assetId: "a".repeat(64), chyron: null, clock: null }, + ], + transition: { type: "cut" }, + audio: { masterGainDb: 0 }, + }); +} + +class MemoryObserved implements ObservedStore { + rows = new Map(); + async get(workspaceId: string, channelId: string): Promise { return this.rows.get(`${workspaceId}:${channelId}`) ?? null; } + async put(row: ObservedChannel): Promise { this.rows.set(`${row.workspaceId}:${row.channelId}`, row); } + async heartbeat(_workerId: string, channels: ObservedChannel[]): Promise { for (const row of channels) await this.put(row); } +} + +const alive = (pid: number | undefined): boolean => pid !== undefined && spawnSync("kill", ["-0", String(pid)]).status === 0; + +test("signed TAKE / playlist edit / source change / stop through the supervisor into a real production job", { timeout: 300_000 }, async (t) => { + if (!ensureGateBinary()) { t.skip("native helpers unavailable"); return; } + const dir = mkdtempSync(join(tmpdir(), "prod-e2e-")); + const [overlay] = makeAssets(dir); + // Two 2 s playlist assets (pre-normalized MP4, faststart), served as byte streams like a presigned GET. + const clips = new Map(); + for (const [n, hz] of [[40, 300], [41, 600], [42, 900]] as const) { + const path = join(dir, `asset-${n}.mp4`); + const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-f", "lavfi", "-i", "testsrc2=s=1280x720:r=30:d=2", "-f", "lavfi", "-i", `sine=frequency=${hz}:d=2`, + "-c:v", "libx264", "-preset", "ultrafast", "-g", "30", "-c:a", "aac", "-movflags", "+faststart", path]); + assert.equal(r.status, 0, "asset encode"); + clips.set(id(n), path); + } + const opened: string[] = []; + const store = new MemoryObserved(); + const keys = generateKeyPairSync("ed25519"); + const published = new Map(); + const jobs: ProductionJob[] = []; + const exits: string[] = []; + const createRelay = (_channel: DesiredChannel, onExit: (reason: string) => void): RelayJob => { + const job = new ProductionJob({ + validator: { async validate() { throw new Error("not used: synthetic egress"); } }, + vault: { async deliverPushTarget() { throw new Error("not used: null output"); } }, + egress: { + // Stand-in for the WP7 relay: each HLS source becomes a synthetic, real-time ingest. + async prepare(config) { + const sources = new Map((config.production?.sources ?? []).filter((s) => s.kind === "hls") + .map((s, i) => [s.id, { input: { kind: "synthetic" as const, pattern: "smptehdbars" as const, seed: i + 1, toneHz: 440 + 220 * i }, fd: null }])); + return { sources, output: { kind: "null" as const }, async close() {} }; + }, + }, + assets: { async localPath() { return overlay!; } }, + playlists: { + async open(assetId) { + opened.push(assetId); + const path = clips.get(assetId); + if (!path) throw new Error("unknown asset"); + return createReadStream(path); + }, + }, + preview: { + objects: { async putContent(body: Buffer, contentType: PreviewContentType) { return { key: `sha256/${"0".repeat(64)}.${contentType === "image/jpeg" ? "jpg" : "bin"}` }; } }, + manifests: { async set(key: string) { published.set(key, (published.get(key) ?? 0) + 1); } }, + }, + redactor: passthroughRedactor, onStderr: () => undefined, + onExit: (reason) => { exits.push(reason); onExit(reason); }, + scratchDir: join(dir, `scratch-${jobs.length}`), + }); + jobs.push(job); + return job; + }; + const base: DesiredChannel = { + workspaceId: WS, channelId: CH, workerId: "worker-1", assignmentEpoch: 1, desiredVersion: 0, desiredStatus: "RUNNING", + source: null, output: { endpointUrl: "rtmp://push.example/live", credentialHandleId: "keyref-1" }, profile: "720p30", mode: "normalize", + production: spec({ take: 10, playlist: [id(40), id(41)] }), + }; + const worker = new MediaSupervisor("worker-1", [base], store, new DurableFence(join(dir, "fence.json")), + new Map([["control-v1", keys.publicKey]]), createRelay, { transition() {} }); + const send = (version: number, production: ProductionSpec, status: "RUNNING" | "STOPPED" = "RUNNING") => { + const body: CommandBody = { + workspaceId: WS, channelId: CH, workerId: "worker-1", assignmentEpoch: 1, commandId: randomUUID(), desiredVersion: version, + issuedAt: new Date(Date.now() - 1_000).toISOString(), expiresAt: new Date(Date.now() + 120_000).toISOString(), + snapshot: { ...base, desiredVersion: version, desiredStatus: status, production }, + }; + return worker.handle(signCommand(body, "control-v1", keys.privateKey)); + }; + const status = () => worker.status()[0]!; + try { + await worker.start(); + assert.equal(await send(1, spec({ take: 10, playlist: [id(40), id(41)] })), "accepted"); + assert.equal(status().state, "RUNNING"); + assert.equal(status().realizedVersion, 1); + const job1 = jobs[0]!; + const encoder = job1.program?.encoderPid; + assert.ok(alive(encoder), "encoder up"); + // Baseline after the startup transient (compositor and x264 warming up repeat a few frames on a + // loaded host, as cpu-smoke reports): wait until the gate has not repeated for 1 s (at most 15 s). + let s1 = await job1.program!.stats(); + for (let calm = 0, waited = 0; calm < 2 && waited < 30; waited += 1) { + await new Promise((r) => setTimeout(r, 500)); + const now = await job1.program!.stats(); + calm = now.vrep + now.asil === s1.vrep + s1.asil ? calm + 1 : 0; + s1 = now; + } + + // TAKE: program scene 10 → 11 (playlist PIP). Same encoder; realized once B is on air. + let realizedDuringSwap: number | null = -1; + const swapping = send(2, spec({ take: 11, playlist: [id(40), id(41)] })); + await new Promise((r) => setTimeout(r, 50)); + realizedDuringSwap = status().realizedVersion; + await swapping; + // On an oversubscribed host B can lag the on-air source (CompositorLagError): A stays on air, + // v1 stays realized, and the supervisor retries the swap (1 s, then 2 s). That is the contract too. + const retried = status().realizedVersion === 1 && /stays on air; retry/.test(status().reason); + const tSwap = Date.now(); + while (status().realizedVersion !== 2 && jobs.length === 1 && Date.now() - tSwap < 30_000) await new Promise((r) => setTimeout(r, 100)); + if (retried) t.diagnostic("TAKE went on air through a supervisor swap retry (B lagged on the first attempt)"); + assert.equal(realizedDuringSwap, 1, "not realized before the new compositor is on air"); + assert.equal(jobs.length, 1, `no restart (${status().reason})`); + assert.equal(status().realizedVersion, 2, `TAKE realized (${status().reason})`); + assert.match(status().reason, /frame boundary/); + const s2 = await job1.program!.stats(); + assert.equal(job1.program?.encoderPid, encoder, "same encoder process: the push session never dropped"); + assert.equal(s2.swaps, s1.swaps + 1); + // A fill here would be the gate itself starved on an oversubscribed host, never the splice; keep it rare. + const fills = s2.vrep + s2.asil - (s1.vrep + s1.asil); + assert.ok(fills <= 2, `no repeat or silence fill across the swap (${fills})`); + assert.deepEqual(job1.lastSkipped, []); + + // Playlist edit: adopted in place by the feeder (no restart), and the program keeps running. + await send(3, spec({ take: 11, playlist: [id(42), id(40)] })); + assert.equal(status().realizedVersion, 3); + assert.equal(jobs.length, 1); + assert.equal(job1.program?.encoderPid, encoder); + const feeder = job1.feeder(id(3))!; + const t0 = Date.now(); + while (!opened.includes(id(42)) && Date.now() - t0 < 15_000) await new Promise((r) => setTimeout(r, 100)); + assert.ok(opened.includes(id(42)), `the edited list is played (opened ${opened.join(",")})`); + assert.equal(feeder.stats.failures, 0); + + // Previews: the program tap's thumbnail and always-on LL-HLS reached the manifest store. + const key = (tap: string, file: string): string => `preview:${WS}:${CH}:${tap}:${file}`; + const t1 = Date.now(); + while (!published.has(key("program", "video.m3u8")) && Date.now() - t1 < 30_000) await new Promise((r) => setTimeout(r, 200)); + assert.ok(published.has(key("program", "thumbnail.jpg")), "program thumbnail"); + assert.ok(published.has(key("program", "video.m3u8")), "program LL-HLS"); + assert.ok(published.has(key(id(1), "thumbnail.jpg")) || published.has(key(id(2), "thumbnail.jpg")), "source thumbnails"); + + // A changed source URL cannot be a swap: full restart, new encoder. + await send(4, spec({ take: 11, playlist: [id(42), id(40)], urlB: "https://b2.example/live.m3u8" })); + assert.equal(status().realizedVersion, 4); + assert.equal(jobs.length, 2, "restarted"); + assert.ok(!alive(encoder), "the old encoder is gone"); + assert.ok(alive(jobs[1]!.program?.encoderPid)); + + const last = jobs[1]!.program?.encoderPid; + await send(5, spec({ take: 11, playlist: [id(42)] }), "STOPPED"); + assert.equal(status().state, "STOPPED"); + assert.ok(!alive(last), "stopped"); + assert.deepEqual(exits, [], "no job exited on its own"); + } finally { + // Also on failure: the heartbeat interval and any pending swap retry would keep the process alive. + await worker.stop(); + for (const job of jobs) await job.stop(); + rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/apps/media-worker/src/supervisor/reconfigure.test.ts b/apps/media-worker/src/supervisor/reconfigure.test.ts new file mode 100644 index 0000000..b1b6794 --- /dev/null +++ b/apps/media-worker/src/supervisor/reconfigure.test.ts @@ -0,0 +1,155 @@ +import assert from "node:assert/strict"; +import { generateKeyPairSync, randomUUID } from "node:crypto"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { signCommand, type CommandBody } from "../bus/envelope"; +import { DurableFence } from "../bus/fence"; +import type { ObservedStore } from "../bus/redis"; +import { MediaSupervisor, type RelayJob, type RetryScheduler } from "./index"; +import type { DesiredChannel, ObservedChannel } from "../types"; + +const base: DesiredChannel = { + workspaceId: "workspace-1", channelId: "channel-1", workerId: "worker-1", + assignmentEpoch: 2, desiredVersion: 1, desiredStatus: "RUNNING", + source: { kind: "hls", url: "https://mock.local/hls/live-1/master.m3u8" }, + output: { endpointUrl: "rtmp://mock.local/live", credentialHandleId: "keyref-1" }, + profile: "720p30", mode: "normalize", +}; + +const store: ObservedStore = { + async get() { return null; }, + async put(_row: ObservedChannel) {}, + async heartbeat() {}, +}; + +test("RUNNING → RUNNING on a swappable job is an in-place reconfigure, realized only when on air", async () => { + const dir = mkdtempSync(join(tmpdir(), "media-reconf-")); + const keys = generateKeyPairSync("ed25519"); + const calls: string[] = []; + let accept = true; + let failures = 0; + const job = (): RelayJob => ({ + async start(c) { calls.push(`start v${c.desiredVersion}`); }, + async stop() { calls.push("stop"); }, + async reconfigure(c) { + calls.push(`swap v${c.desiredVersion}`); + if (failures > 0) { failures -= 1; throw new Error("compositor B never went on air"); } + return accept; + }, + }); + // Manual retry scheduler: the test fires retries itself. + const pending: Array<{ action: () => Promise; delayMs: number; cancelled: boolean }> = []; + const scheduler: RetryScheduler = { + schedule(action, delayMs) { const entry = { action, delayMs, cancelled: false }; pending.push(entry); return { cancel: () => { entry.cancelled = true; } }; }, + }; + const fire = async (): Promise => { const entry = pending.shift(); assert.ok(entry && !entry.cancelled, "a retry is scheduled"); await entry.action(); return entry.delayMs; }; + const worker = new MediaSupervisor("worker-1", [base], store, new DurableFence(join(dir, "fence.json")), + new Map([["control-v1", keys.publicKey]]), job, { transition() {} }, scheduler); + const send = (version: number, status: "RUNNING" | "STOPPED" = "RUNNING") => { + const body: CommandBody = { + workspaceId: base.workspaceId, channelId: base.channelId, workerId: base.workerId, assignmentEpoch: base.assignmentEpoch, + commandId: randomUUID(), desiredVersion: version, issuedAt: new Date(Date.now() - 1_000).toISOString(), + expiresAt: new Date(Date.now() + 60_000).toISOString(), snapshot: { ...base, desiredVersion: version, desiredStatus: status }, + }; + return worker.handle(signCommand(body, "control-v1", keys.privateKey)); + }; + const status = () => worker.status()[0]!; + try { + await worker.start(); + await send(1); + assert.deepEqual(calls, ["start v1"]); + await send(2); + assert.deepEqual(calls, ["start v1", "swap v2"], "a TAKE never stops the job (encoder/RTMP stay up)"); + assert.equal(status().state, "RUNNING"); + assert.equal(status().realizedVersion, 2); + assert.match(status().reason, /frame boundary/); + accept = false; + await send(3); + assert.deepEqual(calls.slice(2), ["swap v3", "stop", "start v3"], "an unswappable change falls back to restart"); + accept = true; + + // A failed swap keeps the old compositor on air: still RUNNING, realized = what viewers see, retried. + failures = 1; + await send(4); + assert.deepEqual(calls.slice(5), ["swap v4"]); + assert.equal(status().state, "RUNNING"); + assert.equal(status().realizedVersion, 3, "v3 is still on air"); + assert.match(status().reason, /swap to v4 failed .*v3 stays on air; retry 1\/2/); + assert.equal(await fire(), 1_000); + assert.deepEqual(calls.slice(5), ["swap v4", "swap v4"], "the retry swaps; nothing restarts"); + assert.equal(status().realizedVersion, 4); + + // A newer command supersedes a pending retry. + failures = 1; + await send(5); + await send(6); + assert.deepEqual(calls.slice(7), ["swap v5", "swap v6"]); + assert.equal(pending.shift()?.cancelled, true, "the v5 retry was cancelled by v6"); + assert.equal(status().realizedVersion, 6); + + // Out of retries: the full restart converges (and is the only path that drops the push session). + failures = 3; + await send(7); + assert.equal(await fire(), 1_000); + assert.equal(await fire(), 2_000); + assert.deepEqual(calls.slice(9), ["swap v7", "swap v7", "swap v7", "stop", "start v7"]); + assert.equal(status().realizedVersion, 7); + assert.equal(status().state, "RUNNING"); + + await send(8, "STOPPED"); + assert.deepEqual(calls.slice(14), ["stop"]); + assert.equal(pending.length, 0); + await worker.stop(); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); + +test("worker stop waits for an in-flight swap and no retry or relaunch runs after it", async () => { + const dir = mkdtempSync(join(tmpdir(), "media-reconf-stop-")); + const keys = generateKeyPairSync("ed25519"); + const calls: string[] = []; + let releaseSwap: (() => void) | null = null; + const job = (): RelayJob => ({ + async start(c) { calls.push(`start v${c.desiredVersion}`); }, + async stop() { calls.push("stop"); }, + async reconfigure(c) { + calls.push(`swap v${c.desiredVersion}`); + // B is still trying to go on air when the worker shuts down; then it lags. + await new Promise((resolve) => { releaseSwap = resolve; }); + throw new Error("compositor B never went on air"); + }, + }); + const pending: Array<{ action: () => Promise; cancelled: boolean }> = []; + const scheduler: RetryScheduler = { + schedule(action) { const entry = { action, cancelled: false }; pending.push(entry); return { cancel: () => { entry.cancelled = true; } }; }, + }; + const worker = new MediaSupervisor("worker-1", [base], store, new DurableFence(join(dir, "fence.json")), + new Map([["control-v1", keys.publicKey]]), job, { transition() {} }, scheduler); + const send = (version: number) => { + const body: CommandBody = { + workspaceId: base.workspaceId, channelId: base.channelId, workerId: base.workerId, assignmentEpoch: base.assignmentEpoch, + commandId: randomUUID(), desiredVersion: version, issuedAt: new Date(Date.now() - 1_000).toISOString(), + expiresAt: new Date(Date.now() + 60_000).toISOString(), snapshot: { ...base, desiredVersion: version }, + }; + return worker.handle(signCommand(body, "control-v1", keys.privateKey)); + }; + try { + await worker.start(); + await send(1); + const swapping = send(2); + while (!releaseSwap) await new Promise((r) => setImmediate(r)); + let stopped = false; + const stopping = worker.stop().then(() => { stopped = true; }); + await new Promise((r) => setImmediate(r)); + assert.equal(stopped, false, "stop waits for the in-flight swap"); + (releaseSwap as () => void)(); + await swapping; + await stopping; + assert.deepEqual(calls, ["start v1", "swap v2", "stop"]); + assert.equal(worker.status()[0]?.state, "STOPPED"); + // The retry the failed swap scheduled is cancelled, and running it anyway does nothing. + for (const entry of pending) { assert.equal(entry.cancelled, true); await entry.action(); } + assert.deepEqual(calls, ["start v1", "swap v2", "stop"]); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); diff --git a/apps/media-worker/src/supervisor/sandbox.test.ts b/apps/media-worker/src/supervisor/sandbox.test.ts new file mode 100644 index 0000000..86acb90 --- /dev/null +++ b/apps/media-worker/src/supervisor/sandbox.test.ts @@ -0,0 +1,109 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { generateKeyPairSync, randomUUID } from "node:crypto"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import test from "node:test"; +import { signCommand } from "../bus/envelope"; +import { DurableFence } from "../bus/fence"; +import type { ObservedStore } from "../bus/redis"; +import { buildCompositorArgs, buildEncoderArgs, buildIngestArgs, buildPlaylistRemuxArgs, buildSlateArgs } from "../compositor/command-builder"; +import { buildHlsTapArgs } from "../preview/argv"; +import { ffmpegArgs, spawnManaged, type ManagedProcess } from "../relay/ffmpeg"; +import type { DesiredChannel } from "../types"; +import { MediaSupervisor, type RelayJob } from "./index"; +import { supervisorSandboxEnabled } from "./sandbox"; +const pkg = (): { + sandboxCapability(options: { rootfs: string; cgroupRoot: string }): { available: boolean; reason?: string }; + MediaCommandBuilder: new () => { supervisor(binary: "ffmpeg", args: readonly string[], scratch: string): { argv: readonly string[] } }; +} => require(resolve(__dirname, "../../../../packages/media-sandbox/dist")); +const channel: DesiredChannel = { + workspaceId: "sandbox-workspace", channelId: "sandbox-channel", workerId: "sandbox-worker", assignmentEpoch: 1, + desiredVersion: 1, desiredStatus: "RUNNING", profile: "720p30", mode: "passthrough", + source: { kind: "hls", url: "https://source.example/live.m3u8" }, + output: { endpointUrl: "rtmp://output.example/live", credentialHandleId: "opaque-handle" }, +}; +const store: ObservedStore = { async get() { return null; }, async put() {}, async heartbeat() {} }; +async function env(values: Record, action: () => Promise): Promise { + const previous = Object.fromEntries(Object.keys(values).map(key => [key, process.env[key]])); + for (const [key, value] of Object.entries(values)) { if (value === undefined) delete process.env[key]; else process.env[key] = value; } + try { return await action(); } + finally { for (const [key, value] of Object.entries(previous)) { if (value === undefined) delete process.env[key]; else process.env[key] = value; } } +} +function command(keys: ReturnType) { + return signCommand({ workspaceId: channel.workspaceId, channelId: channel.channelId, workerId: channel.workerId, + assignmentEpoch: 1, commandId: randomUUID(), desiredVersion: 1, + issuedAt: new Date(Date.now() - 1000).toISOString(), expiresAt: new Date(Date.now() + 60_000).toISOString(), snapshot: channel }, "control", keys.privateKey); +} +test("sandbox flag defaults locally off and on in production; production off is rejected", async () => { + await env({ NODE_ENV: "development", MEDIA_SANDBOX: undefined }, async () => assert.equal(supervisorSandboxEnabled(), false)); + await env({ NODE_ENV: "production", MEDIA_SANDBOX: undefined }, async () => assert.equal(supervisorSandboxEnabled(), true)); + await env({ NODE_ENV: "production", MEDIA_SANDBOX: "off" }, async () => assert.throws(supervisorSandboxEnabled, /requires/)); +}); +test("sandbox builder accepts existing relay/ingest/compositor/encoder/playlist/preview plans", () => { + const builder = new (pkg().MediaCommandBuilder)(); const canvas = { width: 1280, height: 720, fps: 30 }; + const plans = [ffmpegArgs(channel), buildPlaylistRemuxArgs(0), + buildIngestArgs({ canvas, input: { kind: "mpegtsPipe" }, audio: "source" }), + buildIngestArgs({ canvas, input: { kind: "synthetic", pattern: "testsrc2", seed: 1, toneHz: 440 }, audio: "source" }), + buildEncoderArgs({ canvas, videoBitrateKbps: 2800, audioBitrateKbps: 128, output: { kind: "null" }, programTap: { videoFd: 5, audioFd: 6, size: { width: 320, height: 180 } } }), + buildSlateArgs({ canvas, seconds: 5, image: `/job/${"a".repeat(64)}.png`, offsetUs: 0 }), + buildHlsTapArgs({ geometry: canvas, rendition: "video", outDir: "/job/taps/video" }), + buildCompositorArgs({ sources: [{ video: { kind: "rawPipe", fd: 3 }, audio: { kind: "rawPipe", fd: 4 } }], assets: [`/job/${"a".repeat(64)}.png`], + graph: { canvas, layout: "single", tiles: [{ source: 0, rect: { x: 0, y: 0, w: 1280, h: 720 }, fit: "cover" }], + overlays: [{ kind: "chyron", asset: 0, y: 10, speedPxPerSec: 40 }], audio: { inputs: [{ source: 0, gainDb: 0 }], masterGainDb: 0 } }, output: { kind: "gate", videoFd: 1, audioFd: 3 } })]; + for (const args of plans) assert.ok(builder.supervisor("ffmpeg", args, "/job").argv.includes("-protocol_whitelist")); +}); +test("supervisor fails closed if sandbox is enabled without configuration", async () => { + const dir = mkdtempSync(join(tmpdir(), "media-sandbox-closed-")); const keys = generateKeyPairSync("ed25519"); + const lines: string[] = []; + await env({ NODE_ENV: "development", MEDIA_SANDBOX: "on", MEDIA_SANDBOX_ROOTFS: undefined, MEDIA_SANDBOX_CGROUP_ROOT: undefined, MEDIA_SCRATCH_DIR: join(dir, "scratch") }, async () => { + const worker = new MediaSupervisor(channel.workerId, [channel], store, new DurableFence(join(dir, "fence")), new Map([["control", keys.publicKey]]), + (): RelayJob => ({ async start() { spawnManaged("ffmpeg", ffmpegArgs(channel), ["ignore", "ignore", "pipe"], { redact: v => v }, line => lines.push(line)); }, async stop() {} }), + { transition: event => lines.push(JSON.stringify(event)) }, { schedule() { return { cancel() {} }; } }); + try { await worker.start(); await worker.handle(command(keys)); assert.equal(worker.status()[0]?.state, "RESTARTING"); assert.ok(lines.join("\n").includes("SandboxUnavailable")); } + finally { await worker.stop(); } + }); + rmSync(dir, { recursive: true, force: true }); +}); + +for (const enabled of [false, true]) test(`secret-leak canaries through signed supervisor command (sandbox ${enabled ? "on" : "off"})`, { timeout: 20_000 }, async t => { + const dir = mkdtempSync(join(tmpdir(), "media-supervisor-canary-")); + const rootfs = join(dir, "rootfs"); + for (const d of ["upload", "scratch", "tmp", "proc", "dev", "usr/bin"]) mkdirSync(join(rootfs, d), { recursive: true }); + writeFileSync(join(rootfs, "dev/null"), ""); + try { + if (enabled) { + const compiled = spawnSync("cc", ["-static", "-O0", resolve(__dirname, "../../../../packages/media-sandbox/native/test-child.c"), "-o", join(rootfs, "usr/bin/ffmpeg")]); + const cap = compiled.status === 0 ? pkg().sandboxCapability({ rootfs, cgroupRoot: process.env.MEDIA_SANDBOX_CGROUP_ROOT ?? "/sys/fs/cgroup" }) : { available: false, reason: "static compiler unavailable" }; + if (!cap.available) { t.skip(`rootless sandbox canary skipped: ${cap.reason}`); return; } + } + const canaries = ["stream-key-canary-supervisor-531", "oauth-token-canary-supervisor-794", "srt-passphrase-canary-supervisor-652"]; + await env({ NODE_ENV: "development", MEDIA_SANDBOX: enabled ? "on" : "off", MEDIA_SANDBOX_ROOTFS: rootfs, + MEDIA_SANDBOX_CGROUP_ROOT: process.env.MEDIA_SANDBOX_CGROUP_ROOT ?? "/sys/fs/cgroup", MEDIA_SCRATCH_DIR: join(dir, "scratch"), + CANARY_STREAM_KEY: canaries[0], CANARY_OAUTH_TOKEN: canaries[1], CANARY_SRT_PASSPHRASE: canaries[2] }, async () => { + const keys = generateKeyPairSync("ed25519"); const lines: string[] = []; let proc: ManagedProcess | undefined; + const worker = new MediaSupervisor(channel.workerId, [channel], store, new DurableFence(join(dir, "fence")), new Map([["control", keys.publicKey]]), + (_channel, onExit): RelayJob => ({ + async start() { + proc = spawnManaged("ffmpeg", ["-hide_banner", "-nostdin", "-loglevel", "error", "-f", "mpegts", "-i", "pipe:0", "-f", "null", "pipe:1"], + ["pipe", "ignore", "pipe"], { redact: value => value }, line => lines.push(line)); + void proc.exited.then(({ code }) => onExit(`ffmpeg exited code=${code}`)); await proc.started; + }, + async stop() { await proc?.terminate(100); }, + }), { transition: event => lines.push(JSON.stringify(event)) }, { schedule() { return { cancel() {} }; } }); + try { + await worker.start(); await worker.handle(command(keys)); + assert.equal(worker.status()[0]?.state, "RUNNING"); assert.ok(proc?.child.pid); + assert.equal(proc.child.spawnfile.endsWith("media-confine"), enabled); + const cmdline = readFileSync(`/proc/${proc.child.pid}/cmdline`, "utf8"); + const environment = readFileSync(`/proc/${proc.child.pid}/environ`, "utf8"); + proc.child.stdin!.end("invalid-media-to-force-ffmpeg-error"); + const result = await proc.exited; assert.notEqual(result.code, 0); + await worker.stop(); + assert.ok(lines.some(line => enabled ? line.includes("stderr suppressed") : /Invalid data|could not find codec|End of file/i.test(line))); + for (const canary of canaries) for (const output of [cmdline, environment, ...lines]) assert.ok(!output.includes(canary)); + } finally { await worker.stop(); } + }); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); diff --git a/apps/media-worker/src/supervisor/sandbox.ts b/apps/media-worker/src/supervisor/sandbox.ts new file mode 100644 index 0000000..877e2e2 --- /dev/null +++ b/apps/media-worker/src/supervisor/sandbox.ts @@ -0,0 +1,99 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import { createHash } from "node:crypto"; +import type * as ChildProcessApi from "node:child_process"; +import { chmodSync, mkdirSync, constants, openSync, closeSync, fchmodSync } from "node:fs"; +import { resolve } from "node:path"; +import { Transform } from "node:stream"; +import type { DesiredChannel } from "../types"; +import type { RelayJob } from "./index"; + +const childProcess: { spawn: typeof ChildProcessApi.spawn } = require("node:child_process"); + +// Resolve the CommonJS package from either src/supervisor or dist/supervisor. +// A small structural boundary keeps worker typechecking independent of package build order. +interface MediaCommand { binary: "ffmpeg" | "ffprobe"; argv: readonly string[] } +interface SandboxModule { + MediaCommandBuilder: new () => { supervisor(binary: MediaCommand["binary"], argv: readonly string[], scratch: string): MediaCommand }; + SandboxUnavailable: new () => Error; + spawnSandbox(command: MediaCommand, options: { rootfs: string; cgroupRoot: string; scratchDir: string; limits: { wallTimeMs: number } }, spawnOptions: ChildProcessApi.SpawnOptions | undefined, spawner: typeof ChildProcessApi.spawn): { child: ChildProcessApi.ChildProcess }; +} +const sandbox = (): SandboxModule => require(resolve(__dirname, "../../../../packages/media-sandbox/dist")) as SandboxModule; +interface Context { scratch: string } +const context = new AsyncLocalStorage(); +let installed = false; + +/** + * The only common spawn path lives in relay/, outside this change's boundary. + * A scoped Node spawn adapter covers existing typed FFmpeg builders, including + * asynchronous swaps/playlist producers, without altering their process API. + * Non-media children and calls outside a supervisor job retain normal spawning. + */ +function installAdapter(): void { + if (installed) return; + const original = childProcess.spawn; + childProcess.spawn = ((command: string, args?: readonly string[] | ChildProcessApi.SpawnOptions, options?: ChildProcessApi.SpawnOptions) => { + const job = context.getStore(); + if (!job || !/(?:^|\/)(?:ffmpeg|ffprobe)$/.test(command)) { + return typeof args === "object" && !Array.isArray(args) + ? original(command, args as ChildProcessApi.SpawnOptions) + : original(command, (args as readonly string[] | undefined) ?? [], options ?? {}); + } + if (!Array.isArray(args)) throw new Error("media argv required"); + const runtime = sandbox(); + const rootfs = process.env.MEDIA_SANDBOX_ROOTFS; + const cgroupRoot = process.env.MEDIA_SANDBOX_CGROUP_ROOT; + if (!rootfs || !cgroupRoot) throw new runtime.SandboxUnavailable(); + const media = new runtime.MediaCommandBuilder().supervisor(command.endsWith("ffprobe") ? "ffprobe" : "ffmpeg", args, job.scratch); + // Make only generated output directories writable by the mapped UID. + // Traverse through directory fds: a compromised sibling cannot race a + // symlink replacement into chmod of a host file. + for (const path of media.argv.filter(value => value.startsWith("file:/scratch/"))) { + const parts = path.slice("file:/scratch/".length).split("/").slice(0, -1).filter(part => part && part !== "."); + if (parts.length > 32 || parts.some(part => part === "..")) throw new runtime.SandboxUnavailable(); + let fd = openSync(job.scratch, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + try { + fchmodSync(fd, 0o777); + for (const part of parts) { + const next = openSync(`/proc/self/fd/${fd}/${part}`, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + closeSync(fd); fd = next; fchmodSync(fd, 0o777); + } + } finally { closeSync(fd); } + } + const child = runtime.spawnSandbox(media, { + rootfs, cgroupRoot, scratchDir: job.scratch, + // Long-running channels still have a finite job lease; uploads use the package defaults. + limits: { wallTimeMs: Number(process.env.MEDIA_SANDBOX_WALL_TIME_MS ?? 86_400_000) }, + }, options, original).child; + // Replace the public stderr stream BEFORE spawnManaged attaches its logger. + // Arbitrary media bytes cannot bypass a caller-supplied passthrough redactor. + if (child.stderr) { + const safe = new Transform({ transform(_chunk, _encoding, done) { done(null, Buffer.from("[media stderr suppressed]\n")); } }); + child.stderr.pipe(safe); child.stderr = safe; child.stdio[2] = safe; + } + return child; + }) as typeof childProcess.spawn; + installed = true; +} +function enabled(): boolean { + const flag = process.env.MEDIA_SANDBOX; + if (flag !== undefined && flag !== "on" && flag !== "off") throw new Error("MEDIA_SANDBOX must be on or off"); + if (process.env.NODE_ENV === "production" && flag === "off") throw new Error("production requires media sandbox"); + return flag === "on" || process.env.NODE_ENV === "production"; +} +export function supervisorSandboxEnabled(): boolean { return enabled(); } + +export function sandboxRelay(job: RelayJob, channel: DesiredChannel): RelayJob { + if (!enabled()) return job; + installAdapter(); + const id = createHash("sha256").update(`${channel.workspaceId}:${channel.channelId}`).digest("hex").slice(0, 32); + const parent = resolve(process.env.MEDIA_SCRATCH_DIR ?? ".media-worker-state/scratch"); + const scratch = resolve(parent, id); + mkdirSync(parent, { recursive: true }); chmodSync(parent, 0o700); + mkdirSync(scratch, { recursive: true }); chmodSync(scratch, 0o777); + const run = (action: () => T): T => context.run({ scratch }, action); + return { + start: config => run(() => job.start(config)), + stop: () => run(() => job.stop()), + ...(job.reconfigure ? { reconfigure: (config: DesiredChannel) => run(() => job.reconfigure!(config)) } : {}), + }; +} diff --git a/apps/media-worker/src/supervisor/supervisor.test.ts b/apps/media-worker/src/supervisor/supervisor.test.ts new file mode 100644 index 0000000..adde96f --- /dev/null +++ b/apps/media-worker/src/supervisor/supervisor.test.ts @@ -0,0 +1,135 @@ +import assert from "node:assert/strict"; +import { generateKeyPairSync, randomUUID } from "node:crypto"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { signCommand, type CommandBody } from "../bus/envelope"; +import { DurableFence } from "../bus/fence"; +import type { ObservedStore } from "../bus/redis"; +import { MediaSupervisor, type RelayJob, type RetryScheduler } from "./index"; +import type { DesiredChannel, ObservedChannel } from "../types"; + +const source: DesiredChannel = { + workspaceId: "workspace-1", channelId: "channel-1", workerId: "worker-1", + assignmentEpoch: 2, desiredVersion: 4, desiredStatus: "RUNNING", + // Same HLS/RTMP endpoint shape supplied by connector-mock-livestream and the single-live simulator. + source: { kind: "hls", url: "https://mock.local/hls/live-1/master.m3u8" }, + output: { endpointUrl: "rtmp://mock.local/live", credentialHandleId: "keyref-1" }, + profile: "720p30", mode: "passthrough", +}; + +class MemoryObserved implements ObservedStore { + rows = new Map(); + heartbeats: Array<{ workerId: string; channels: ObservedChannel[] }> = []; + async get(workspaceId: string, channelId: string): Promise { return this.rows.get(`${workspaceId}:${channelId}`) ?? null; } + async put(row: ObservedChannel): Promise { this.rows.set(`${row.workspaceId}:${row.channelId}`, row); } + async heartbeat(workerId: string, channels: ObservedChannel[]): Promise { + this.heartbeats.push({ workerId, channels }); + for (const row of channels) await this.put(row); + } +} + +test("durable replay fence, stale version, and restart reconcile", async () => { + const dir = mkdtempSync(join(tmpdir(), "media-fence-")); + const store = new MemoryObserved(); + const keys = generateKeyPairSync("ed25519"); + const pinned = new Map([["control-v1", keys.publicKey]]); + let starts = 0; + const makeSupervisor = () => new MediaSupervisor("worker-1", [source], store, + new DurableFence(join(dir, "fence.json")), pinned, + (): RelayJob => ({ async start() { starts += 1; }, async stop() {} }), + { transition() {} }); + const makeBody = (commandId: string, version: number): CommandBody => ({ + workspaceId: source.workspaceId, channelId: source.channelId, workerId: source.workerId, + assignmentEpoch: source.assignmentEpoch, commandId, desiredVersion: version, + issuedAt: new Date(Date.now() - 1_000).toISOString(), + expiresAt: new Date(Date.now() + 60_000).toISOString(), + snapshot: { ...source, desiredVersion: version }, + }); + try { + const first = signCommand(makeBody(randomUUID(), 4), "control-v1", keys.privateKey); + const worker = makeSupervisor(); + await worker.start(); + assert.equal(worker.status()[0]?.realizedVersion, null); + assert.equal(await worker.handle(first), "accepted"); + assert.equal(worker.status()[0]?.state, "RUNNING"); + assert.equal(worker.status()[0]?.realizedVersion, 4); + assert.equal(starts, 1); + await worker.stop(); + + const restarted = makeSupervisor(); + await restarted.start(); + assert.equal(restarted.status()[0]?.realizedVersion, null); + assert.match(restarted.status()[0]?.reason ?? "", /restart reconcile/); + assert.equal(await restarted.handle(first), "replay"); + assert.equal(starts, 1); + const stale = signCommand(makeBody(randomUUID(), 3), "control-v1", keys.privateKey); + assert.equal(await restarted.handle(stale), "stale"); + const freshSnapshot = signCommand(makeBody(randomUUID(), 4), "control-v1", keys.privateKey); + assert.equal(await restarted.handle(freshSnapshot), "accepted"); + assert.equal(restarted.status()[0]?.realizedVersion, 4); + assert.equal(starts, 2); + await restarted.stop(); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); + +test("unsigned, expired, wrong target and tampered snapshots are rejected", async () => { + const dir = mkdtempSync(join(tmpdir(), "media-command-")); + const keys = generateKeyPairSync("ed25519"); + const worker = new MediaSupervisor("worker-1", [source], new MemoryObserved(), + new DurableFence(join(dir, "fence.json")), new Map([["control-v1", keys.publicKey]]), + (): RelayJob => ({ async start() {}, async stop() {} }), { transition() {} }); + const body: CommandBody = { + workspaceId: source.workspaceId, channelId: source.channelId, workerId: source.workerId, + assignmentEpoch: source.assignmentEpoch, commandId: randomUUID(), desiredVersion: 4, + issuedAt: new Date(Date.now() - 2_000).toISOString(), expiresAt: new Date(Date.now() + 30_000).toISOString(), snapshot: source, + }; + try { + await worker.start(); + await assert.rejects(worker.handle({ kid: "control-v1", body, signature: "" }), /signature/); + await assert.rejects(worker.handle(signCommand({ ...body, workerId: "other" }, "control-v1", keys.privateKey)), /fields/); + const expired = { ...body, expiresAt: new Date(Date.now() - 1).toISOString() }; + await assert.rejects(worker.handle(signCommand(expired, "control-v1", keys.privateKey)), /expired/); + const tampered = signCommand(body, "control-v1", keys.privateKey); + tampered.body.snapshot = { ...source, desiredStatus: "STOPPED" }; + await assert.rejects(worker.handle(tampered), /signature/); + await worker.stop(); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); + +test("relay failure follows DEGRADED, RESTARTING and bounded FAILED states", async () => { + const dir = mkdtempSync(join(tmpdir(), "media-retry-")); + const keys = generateKeyPairSync("ed25519"); + const queued: Array<{ action: () => Promise; delay: number }> = []; + const scheduler: RetryScheduler = { + schedule(action, delay) { queued.push({ action, delay }); return { cancel() {} }; }, + }; + const transitions: string[] = []; + const worker = new MediaSupervisor("worker-1", [source], new MemoryObserved(), + new DurableFence(join(dir, "fence.json")), new Map([["control-v1", keys.publicKey]]), + (): RelayJob => ({ async start() { throw new Error("ingest unavailable"); }, async stop() {} }), + { transition(event) { transitions.push(event.to); } }, scheduler); + try { + await worker.start(); + const body: CommandBody = { + workspaceId: source.workspaceId, channelId: source.channelId, workerId: source.workerId, + assignmentEpoch: source.assignmentEpoch, commandId: randomUUID(), desiredVersion: 4, + issuedAt: new Date(Date.now() - 1_000).toISOString(), expiresAt: new Date(Date.now() + 60_000).toISOString(), snapshot: source, + }; + assert.equal(await worker.handle(signCommand(body, "control-v1", keys.privateKey)), "accepted"); + assert.equal(worker.status()[0]?.state, "RESTARTING"); + const delays: number[] = []; + while (queued.length) { + const next = queued.shift(); + assert.ok(next); + delays.push(next.delay); + await next.action(); + } + assert.deepEqual(delays, [1_000, 2_000, 4_000, 8_000, 16_000]); + assert.equal(worker.status()[0]?.state, "FAILED"); + assert.equal(worker.status()[0]?.realizedVersion, null); + assert.ok(transitions.includes("DEGRADED")); + await worker.stop(); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); diff --git a/apps/media-worker/src/types.ts b/apps/media-worker/src/types.ts new file mode 100644 index 0000000..32ce8d4 --- /dev/null +++ b/apps/media-worker/src/types.ts @@ -0,0 +1,69 @@ +import { isProductionSpec, productionHlsUrls, type ProductionSpec } from "./production"; + +export type ChannelState = "STOPPED" | "STARTING" | "RUNNING" | "DEGRADED" | "RESTARTING" | "FAILED"; +export type DesiredState = "STOPPED" | "RUNNING"; +export type OutputPreset = "720p30" | "1080p30" | "1080p60"; +export type OutputMode = "passthrough" | "normalize"; + +export const OUTPUT_PROFILES: Record = { + "720p30": { width: 1280, height: 720, fps: 30, videoBitrate: "3000k" }, + "1080p30": { width: 1920, height: 1080, fps: 30, videoBitrate: "6000k" }, + "1080p60": { width: 1920, height: 1080, fps: 60, videoBitrate: "9000k" }, +}; + +/** A complete, secret-free channel snapshot. The control plane owns this record. */ +export interface DesiredChannel { + workspaceId: string; + channelId: string; + workerId: string; + assignmentEpoch: number; + desiredVersion: number; + desiredStatus: DesiredState; + source: { kind: "hls"; url: string } | null; + output: { endpointUrl: string; credentialHandleId: string } | null; + profile: OutputPreset; + mode: OutputMode; + /** + * Desired-state v2 (production: scenes, buses, sources, overlays, + * transition, audio). Absent on v1 snapshots; when present, `source` is null. + */ + production?: ProductionSpec; +} + +export interface ObservedChannel { + workspaceId: string; + channelId: string; + workerId: string; + assignmentEpoch: number; + state: ChannelState; + realizedVersion: number | null; + updatedAt: string; + reason: string; +} + +export function isDesiredChannel(value: unknown): value is DesiredChannel { + if (typeof value !== "object" || value === null) return false; + const x = value as Partial; + if (!onlyKeys(value, ["workspaceId", "channelId", "workerId", "assignmentEpoch", "desiredVersion", "desiredStatus", "source", "output", "profile", "mode", "production"])) return false; + if ("production" in value && (x.source !== null || !isProductionSpec(x.production))) return false; + return typeof x.workspaceId === "string" && x.workspaceId.length > 0 && + typeof x.channelId === "string" && x.channelId.length > 0 && + typeof x.workerId === "string" && x.workerId.length > 0 && + Number.isSafeInteger(x.assignmentEpoch) && (x.assignmentEpoch ?? -1) >= 0 && + Number.isSafeInteger(x.desiredVersion) && (x.desiredVersion ?? -1) >= 0 && + (x.desiredStatus === "STOPPED" || x.desiredStatus === "RUNNING") && + (x.mode === "passthrough" || x.mode === "normalize") && + (x.profile === "720p30" || x.profile === "1080p30" || x.profile === "1080p60") && + (x.source === null || (typeof x.source === "object" && x.source?.kind === "hls" && typeof x.source.url === "string" && x.source.url.length > 0 && onlyKeys(x.source, ["kind", "url"]))) && + (x.output === null || (typeof x.output === "object" && typeof x.output?.endpointUrl === "string" && x.output.endpointUrl.length > 0 && typeof x.output.credentialHandleId === "string" && x.output.credentialHandleId.length > 0 && onlyKeys(x.output, ["endpointUrl", "credentialHandleId"]))) && + (x.desiredStatus !== "RUNNING" || (x.output !== null && (x.production === undefined ? x.source !== null : x.production.buses.program !== null))); +} + +/** Every remote HLS URL a snapshot can make the worker fetch (v1 source or v2 production sources). */ +export function snapshotHlsUrls(s: DesiredChannel): string[] { + return [...(s.source === null ? [] : [s.source.url]), ...(s.production === undefined ? [] : productionHlsUrls(s.production))]; +} + +function onlyKeys(value: object, allowed: string[]): boolean { + return Object.keys(value).every((key) => allowed.includes(key)); +} diff --git a/apps/media-worker/tsconfig.json b/apps/media-worker/tsconfig.json new file mode 100644 index 0000000..acb1540 --- /dev/null +++ b/apps/media-worker/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "module": "CommonJS", + "moduleResolution": "node" + }, + "include": [ + "src" + ], + "exclude": [ + "src/compositor/wp12-hardening.test.ts" + ] +} \ No newline at end of file diff --git a/apps/scheduler/package.json b/apps/scheduler/package.json new file mode 100644 index 0000000..7fdb4e3 --- /dev/null +++ b/apps/scheduler/package.json @@ -0,0 +1,39 @@ +{ + "name": "@tactical-media-broadcast-hub/scheduler", + "private": true, + "version": "0.1.0", + "scripts": { + "dev": "tsx watch src/worker.ts", + "build": "tsc -p tsconfig.json", + "lint": "eslint src", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "vitest run", + "test:integration": "node scripts/wp3-integration.mjs" + }, + "dependencies": { + "@temporalio/client": "1.24.0", + "@temporalio/worker": "1.24.0", + "@temporalio/workflow": "1.24.0", + "pino": "^9.3.2", + "@tactical-media-broadcast-hub/publishing-execution": "workspace:*", + "@tactical-media-broadcast-hub/connector-runtime": "workspace:*", + "@tactical-media-broadcast-hub/connector-mastodon": "workspace:*", + "@tactical-media-broadcast-hub/core": "workspace:*", + "@tactical-media-broadcast-hub/connector-sdk": "workspace:*", + "@tactical-media-broadcast-hub/connector-telegram": "workspace:*", + "@temporalio/activity": "1.24.0", + "@temporalio/common": "1.24.0", + "pg": "^8.23.1" + }, + "devDependencies": { + "@types/node": "^22.5.0", + "tsx": "^4.17.0", + "vitest": "^2.0.5", + "@types/pg": "^8.23.1", + "typescript": "^5.5.4" + }, + "engines": { + "node": ">=22.13.0" + }, + "type": "module" +} diff --git a/apps/scheduler/scripts/wp3-integration.mjs b/apps/scheduler/scripts/wp3-integration.mjs new file mode 100644 index 0000000..12e0a51 --- /dev/null +++ b/apps/scheduler/scripts/wp3-integration.mjs @@ -0,0 +1,57 @@ +// Local compiled-worker smoke. Uses a dedicated queue and only safe missing-job input. +import { spawn } from "node:child_process"; +import { createServer } from "node:net"; +import { resolve } from "node:path"; +import { Client, Connection, WorkflowExecutionAlreadyStartedError } from "@temporalio/client"; +import { randomUUID } from "node:crypto"; +if (!process.env.PUBLISHING_IT_DATABASE_URL) throw new Error("PUBLISHING_IT_DATABASE_URL is required (migrate dedicated local DB first)"); +const probe = createServer(); +await new Promise((done) => probe.listen(0, "127.0.0.1", done)); +const port = probe.address().port; +await new Promise((done) => probe.close(done)); +const queue = `wp3-sol-${randomUUID()}`; +const child = spawn(process.execPath, [resolve("dist/worker.js")], { env: { ...process.env, SCHEDULER_DATABASE_URL: process.env.PUBLISHING_IT_DATABASE_URL, SCHEDULER_HEALTH_PORT: String(port), TEMPORAL_TASK_QUEUE: queue, TEMPORAL_ADDRESS: "127.0.0.1:7233" }, stdio: ["ignore", "pipe", "pipe"] }); +let output = ""; +child.stdout.on("data", (chunk) => { output += chunk.toString(); }); +child.stderr.on("data", (chunk) => { output += chunk.toString(); }); +const exit = new Promise((done) => child.once("exit", (code) => done(code))); +let connection; +try { + let ready = false; + for (let count = 0; count < 100; count++) { + if (child.exitCode !== null) throw new Error("compiled worker exited before readiness"); + try { const response = await fetch(`http://127.0.0.1:${port}/ready`, { signal: AbortSignal.timeout(1000) }); ready = response.status === 200; } catch {} + if (ready) break; + await new Promise((done) => setTimeout(done, 300)); + } + if (!ready) throw new Error("compiled worker readiness timed out"); + connection = await Connection.connect({ address: "127.0.0.1:7233" }); + const client = new Client({ connection, namespace: process.env.TEMPORAL_NAMESPACE ?? "default" }); + const ref = { workspaceId: randomUUID(), scheduleId: randomUUID(), generation: "1", idempotencyKey: `wp3-smoke-${randomUUID()}` }; + // Use an existing, empty fixture workspace so claim returns a safe missing-fence + // outcome. This smoke does not acquire any credential or call a provider. + const { Pool } = await import("pg"); + const pool = new Pool({ connectionString: process.env.PUBLISHING_IT_SUPER_DATABASE_URL }); + const workspace = await pool.query("INSERT INTO workspaces(name,slug) VALUES('Temporal Smoke',$1) RETURNING id", [queue]); + ref.workspaceId = workspace.rows[0].id; + try { + const workflowId = `scheduled-publication-${ref.idempotencyKey}`; + const handle = await client.workflow.start("scheduledPublish", { taskQueue: queue, workflowId, args: [ref] }); + const result = await handle.result(); + if (result.kind !== "failed" || result.code !== "EXECUTION_FENCE_MISSING") throw new Error("unexpected workflow result"); + let duplicate = false; + try { await client.workflow.start("scheduledPublish", { taskQueue: queue, workflowId, args: [ref], workflowIdReusePolicy: "REJECT_DUPLICATE" }); } + catch (error) { if (!(error instanceof WorkflowExecutionAlreadyStartedError)) throw error; duplicate = true; } + if (!duplicate) throw new Error("duplicate start was not fenced"); + console.log("PASS: compiled worker readiness, registry/workflow bundle, real Temporal activity, structured outcome, duplicate start, graceful shutdown. No provider sends."); + } finally { await pool.query("DELETE FROM workspaces WHERE id=$1", [ref.workspaceId]); await pool.end(); } +} catch (error) { + console.error(output.slice(-6000)); + throw error; +} finally { + child.kill("SIGTERM"); + const code = await Promise.race([exit, new Promise((done) => setTimeout(() => done("timeout"), 15_000))]); + if (code === "timeout") { child.kill("SIGKILL"); throw new Error("worker did not drain"); } + if (code !== 0) throw new Error(`worker shutdown exit code ${code}`); + await connection?.close(); +} diff --git a/apps/scheduler/src/activities/dispatch-gate.test.ts b/apps/scheduler/src/activities/dispatch-gate.test.ts new file mode 100644 index 0000000..6c2e08d --- /dev/null +++ b/apps/scheduler/src/activities/dispatch-gate.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from "vitest"; +import { evaluateE3, E3_REASON_CODES, type E3Outcome } from "./dispatch-gate.js"; + +/** + * E3 pre-dispatch re-check (chatgov-design.md §4 step 5) — pure decision table. + * Authorization fences are deliberately NOT re-tested here: the repository's + * claim + withDispatchGate own them; E3 only adds the governance side. + */ + +type Row = Parameters[0]; + +function row(overrides: Partial> = {}): NonNullable { + return { + state: "approved", + approved_hash: "abc123", + approved_revision: "1", + reviewer_user_id: "20000000-0000-4000-8000-000000000002", + effective: { mode: "immediate" }, + policy_version: "pv1", + poll_state: null, + ...overrides, + }; +} + +describe("E3 pre-dispatch evaluation", () => { + it("fires an approved draft under a live snapshot", () => { + expect(evaluateE3(row())).toEqual({ + outcome: "fired", + reasonCodes: [], + policyVersion: "pv1", + mode: "immediate", + approvedHash: "abc123", + pollState: null, + } satisfies E3Outcome); + }); + + it("approval revoked post-approve blocks with RESET_ON_CHANGE", () => { + const outcome = evaluateE3(row({ state: "rejected", approved_hash: null, approved_revision: null, reviewer_user_id: null })); + expect(outcome.outcome).toBe("blocked"); + expect(outcome.reasonCodes).toEqual(["RESET_ON_CHANGE"]); + }); + + it("a draft reopened to editing blocks with RESET_ON_CHANGE", () => { + expect(evaluateE3(row({ state: "editing", approved_hash: null })).reasonCodes).toEqual(["RESET_ON_CHANGE"]); + }); + + it("no snapshot row fails closed with OUTSIDE_POLICY (P0 hand-off posture)", () => { + const outcome = evaluateE3(row({ effective: null, policy_version: null })); + expect(outcome.outcome).toBe("blocked"); + expect(outcome.reasonCodes).toEqual(["OUTSIDE_POLICY"]); + expect(outcome.policyVersion).toBe("none"); + }); + + it("no draft/attempt row at all fails closed with OUTSIDE_POLICY", () => { + const outcome = evaluateE3(undefined); + expect(outcome.outcome).toBe("blocked"); + expect(outcome.reasonCodes).toEqual(["OUTSIDE_POLICY"]); + }); + + it("quorum mode blocks until the linked poll passed (P2 seam)", () => { + const blocked = evaluateE3(row({ effective: { mode: "quorum" }, poll_state: "open" })); + expect(blocked.outcome).toBe("blocked"); + expect(blocked.reasonCodes).toEqual(["INSUFFICIENT_QUORUM", "PENDING_POLL"]); + const fired = evaluateE3(row({ effective: { mode: "quorum" }, poll_state: "passed" })); + expect(fired.outcome).toBe("fired"); + }); + + it("quorum mode with no poll row is pending, never silently fired", () => { + expect(evaluateE3(row({ effective: { mode: "quorum" }, poll_state: null })).reasonCodes).toEqual(["INSUFFICIENT_QUORUM", "PENDING_POLL"]); + }); + + it("admin_only does not re-gate at E3 (its E1 verdict already fired)", () => { + expect(evaluateE3(row({ effective: { mode: "admin_only" } })).outcome).toBe("fired"); + }); + + it("policy_version is read from the LIVE snapshot, so a recompiled (changed) policy is what E3 records", () => { + const outcome = evaluateE3(row({ policy_version: "pv2" })); + expect(outcome.policyVersion).toBe("pv2"); + }); +}); + +describe("reason-code vocabulary mirror", () => { + it("matches the §4 closed vocabulary the gate_decisions CHECK enforces", () => { + expect(E3_REASON_CODES).toEqual([ + "INSUFFICIENT_QUORUM", + "INELIGIBLE_ACTOR", + "SELF_APPROVAL_DENIED", + "TIMEBOX_EXPIRED_AUTO_REJECT", + "TIMEBOX_EXPIRED_AUTO_APPROVE", + "RESET_ON_CHANGE", + "PENDING_POLL", + "VETOED", + "OUTSIDE_POLICY", + "POLICY_DRY_RUN", + "OVERRIDDEN_BY_OWNER", + "LOCKED_CHANNEL", + "NO_CHANNEL_BOUND", + "CONFIRM_EXPIRED", + ]); + }); +}); diff --git a/apps/scheduler/src/activities/dispatch-gate.ts b/apps/scheduler/src/activities/dispatch-gate.ts new file mode 100644 index 0000000..f00eb38 --- /dev/null +++ b/apps/scheduler/src/activities/dispatch-gate.ts @@ -0,0 +1,228 @@ +import type { Pool } from "pg"; + +/** + * §4 reason-code vocabulary (chatgov-design.md §4). Mirrored from the P0 + * spine (apps/api/src/chatgov/policy.ts REASON_CODES) rather than imported + * across the app package boundary; gate_decisions CHECKs values, so this list + * must stay in sync — the unit tests assert the mirror. + */ +export const E3_REASON_CODES = [ + "INSUFFICIENT_QUORUM", + "INELIGIBLE_ACTOR", + "SELF_APPROVAL_DENIED", + "TIMEBOX_EXPIRED_AUTO_REJECT", + "TIMEBOX_EXPIRED_AUTO_APPROVE", + "RESET_ON_CHANGE", + "PENDING_POLL", + "VETOED", + "OUTSIDE_POLICY", + "POLICY_DRY_RUN", + "OVERRIDDEN_BY_OWNER", + "LOCKED_CHANNEL", + "NO_CHANNEL_BOUND", + "CONFIRM_EXPIRED", +] as const; +export type ReasonCode = (typeof E3_REASON_CODES)[number]; + +/** + * E3 pre-dispatch re-check (chatgov-design.md §4 step 5, P1). + * + * Re-evaluated IMMEDIATELY BEFORE each platform dispatch, per destination: + * - approval still holds (the draft is still `approved`/`scheduled` with its + * pinned approved_hash/revision — contentFenceHolds owns the content side + * in the execution claim; E3 owns the *governance* side and re-reads the + * draft row so a revocation that landed after the claim is caught here); + * - the destination's compiled policy_version is still current (a rule + * change after approval recompiles the snapshot ⇒ RESET_ON_CHANGE); + * - a compiled snapshot exists at all — none is OUTSIDE_POLICY, fail closed + * (P0 hand-off: "OUTSIDE_POLICY is the evaluation posture when no + * snapshot row exists"). + * + * Authorization fences (principal/approver/credential) are NOT duplicated + * here: the repository's claim + withDispatchGate already re-read every live + * authz input; E3 integrates by never weakening them. + * + * Every evaluation writes a gate_decisions row (fired or blocked) with the + * §4 reason codes, so partial fan-out ("A fired, B blocked") is observable + * from the blocked queue without touching the publishing spine. + */ + +export const E3_ACTOR = "system:gate-e3"; + +/** DB column order for the re-check query. */ +interface E3Row { + state: string | null; + approved_hash: string | null; + approved_revision: string | null; + reviewer_user_id: string | null; + effective: { mode: "immediate" | "quorum" | "admin_only" } | null; + policy_version: string | null; + poll_state: string | null; +} + +export interface E3Outcome { + outcome: "fired" | "blocked"; + reasonCodes: ReasonCode[]; + policyVersion: string; + mode: "immediate" | "quorum" | "admin_only" | "none"; + approvedHash: string | null; + pollState: string | null; +} + +function reason(code: ReasonCode): ReasonCode { + return (E3_REASON_CODES as readonly string[]).includes(code) ? code : "OUTSIDE_POLICY"; +} + +export function evaluateE3(row: E3Row | undefined): E3Outcome { + // Fail closed: no attempt/draft/snapshot row at all. + if (row === undefined) { + return { outcome: "blocked", reasonCodes: [reason("OUTSIDE_POLICY")], policyVersion: "none", mode: "none", approvedHash: null, pollState: null }; + } + const approvedHash = row.approved_hash === null ? null : String(row.approved_hash); + + // 1) Approval still holds. `approved`/`scheduled` with a pinned hash is the + // approved posture; anything else (rejected, editing, revoked approval) + // is RESET_ON_CHANGE — the approval changed after it landed. + if (row.state !== "approved" && row.state !== "scheduled") { + return { outcome: "blocked", reasonCodes: [reason("RESET_ON_CHANGE")], policyVersion: row.policy_version ?? "none", mode: row.effective?.mode ?? "none", approvedHash, pollState: null }; + } + + // 2) A compiled snapshot exists for this destination. None => fail closed + // (P0 hand-off: OUTSIDE_POLICY posture, never a silent ungated send). + if (row.effective === null || row.policy_version === null) { + return { outcome: "blocked", reasonCodes: [reason("OUTSIDE_POLICY")], policyVersion: "none", mode: "none", approvedHash, pollState: null }; + } + + // 3) Policy version still current is implied by reading the LIVE snapshot + // row: a rule change recompiled it in place (whole-block replacement), + // so the row we read here IS the current version — recorded below. A + // version pinned in an E1/E2 decision that no longer matches is the + // RESET_ON_CHANGE case the UI surfaces from gate_decisions history. + + // 4) Mode: immediate/admin_only gate nothing at E3 (their E1 verdict already + // fired; §4 "immediate mode = E1 minimal + E3 always" — E3 re-checks the + // fences above and fires). quorum still requires the linked poll passed. + if (row.effective.mode === "quorum") { + const pollState = row.poll_state === null ? "none" : String(row.poll_state); + if (pollState !== "passed") { + return { + outcome: "blocked", + reasonCodes: [reason("INSUFFICIENT_QUORUM"), reason("PENDING_POLL")], + policyVersion: String(row.policy_version), + mode: "quorum", + approvedHash, + pollState, + }; + } + } + return { + outcome: "fired", + reasonCodes: [], + policyVersion: String(row.policy_version), + mode: row.effective.mode, + approvedHash, + pollState: row.poll_state === null ? null : String(row.poll_state), + }; +} + +export class DispatchGateService { + constructor(private readonly pool: Pool) {} + + /** + * E3 re-check + decision record for one destination. One round-trip: the + * CTE reads draft + live snapshot + latest poll, evaluates, and inserts the + * gate_decisions row (and its audit event) in the same transaction. + * Returns the outcome the dispatch path must obey. + */ + async check(args: { + workspaceId: string; + draftId: string; + idempotencyKey: string; + connectionId: string; + platform: string; + topicBindingId?: string | null; + }): Promise { + let outcome: E3Outcome; + let decisionId: string | null = null; + const client = await this.pool.connect(); + try { + await client.query("BEGIN"); + await client.query("SELECT set_config('app.workspace_id', $1, true)", [args.workspaceId]); + // NOTE: parameter numbering must be dense from $1 — PostgreSQL cannot + // type an unused placeholder, and the workspace id rides in set_config. + const result = await client.query( + `WITH attempt AS ( + SELECT a.id AS attempt_id, a.draft_id, a.connection_id + FROM publishing_attempts a + WHERE a.idempotency_key = $1::text + ), target AS ( + SELECT d.state, d.approved_hash, d.approved_revision, d.reviewer_user_id, + s.effective, s.policy_version, + (SELECT p.state FROM polls p WHERE p.draft_id = d.id AND p.state <> 'cancelled' + ORDER BY p.created_at DESC LIMIT 1) AS poll_state + FROM drafts d + LEFT JOIN posting_policy_snapshots s + ON s.workspace_id = d.workspace_id + AND s.account_id = $2::uuid + AND s.content_type = '' + AND s.topic_binding_id IS NOT DISTINCT FROM NULLIF($3::text, '')::uuid + WHERE d.id = (SELECT draft_id FROM attempt) + ) + SELECT state, approved_hash, approved_revision, reviewer_user_id, effective, policy_version, poll_state FROM target`, + [args.idempotencyKey, args.connectionId, args.topicBindingId ?? null], + ); + const row = result.rows[0] as E3Row | undefined; + outcome = evaluateE3(row); + const inserted = await client.query<{ id: string }>( + `INSERT INTO gate_decisions + (workspace_id, draft_id, destination, gate, policy_version, outcome, inputs, reason_codes, actor) + SELECT d.workspace_id, d.id, + jsonb_build_object('platform', $1::text, 'accountId', $2::uuid, + 'topicBindingId', $3::uuid, 'idempotencyKey', $4::text), + 'E3', $5::text, $6::text, + jsonb_build_object('approvals', 1, 'approvedHash', d.approved_hash, + 'policyMode', $7::text, 'pollState', $8::text), + $9::text[], 'system:gate-e3' + FROM drafts d WHERE d.id = $10::uuid + RETURNING id`, + [ + args.platform, + args.connectionId, + args.topicBindingId ?? null, + args.idempotencyKey, + outcome.policyVersion, + outcome.outcome, + outcome.mode, + outcome.pollState ?? "none", + outcome.reasonCodes, + args.draftId, + ], + ); + decisionId = inserted.rows[0]?.id ?? null; + if (decisionId !== null) { + await client.query("SELECT audit_append($1::uuid, NULL, 'sp_scheduler', 'policy.gate', 'draft', $2::text, $3::jsonb)", [ + args.workspaceId, + args.draftId, + JSON.stringify({ + gate: "E3", + decisionId, + idempotencyKey: args.idempotencyKey, + destination: { platform: args.platform, accountId: args.connectionId }, + outcome: outcome.outcome, + reasonCodes: outcome.reasonCodes, + policyVersion: outcome.policyVersion, + mode: outcome.mode, + pollState: outcome.pollState ?? "none", + }), + ]); + } + await client.query("COMMIT"); + } catch (error) { + await client.query("ROLLBACK").catch(() => undefined); + throw error; + } finally { + client.release(); + } + return outcome; + } +} diff --git a/apps/scheduler/src/activities/publish.activity.gate.test.ts b/apps/scheduler/src/activities/publish.activity.gate.test.ts new file mode 100644 index 0000000..1d4f9e0 --- /dev/null +++ b/apps/scheduler/src/activities/publish.activity.gate.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it, vi } from "vitest"; +import { RateLimitedHttpClient, type PlatformConnector } from "@tactical-media-broadcast-hub/connector-sdk"; +import type { AttemptRef, ClaimedJob, ClaimOutcome, GateOutcome, Lease, StoredPublication } from "@tactical-media-broadcast-hub/publishing-execution"; +import { PublishActivity, type ActivityRepository } from "./publish.activity.js"; +import type { DispatchGateService, E3Outcome } from "./dispatch-gate.js"; + +/** + * E3 integration into the dispatch path (chatgov-design.md §4 step 5): the + * gate runs before any provider call, settles a blocked destination as a + * known no-send, and is skipped (ungated, dispatch as today) when absent. + */ + +const ref: AttemptRef = { workspaceId: "workspace", scheduleId: "schedule", generation: "1", idempotencyKey: "key" }; +const job: ClaimedJob = { + lease: { ...ref, attemptId: "attempt", draftId: "draft", ownerToken: "owner", claimEpoch: "1", leaseUntil: new Date(Date.now() + 90_000) }, + attemptNo: "1", retryCount: 0, post: { text: "hello", mediaAssetIds: [], visibility: "public", providerMetadata: {} }, + connection: { id: "connection", provider: "telegram", channelId: "channel", fields: {}, createdAt: new Date() }, + credential: { id: "credential-uuid", generation: "3" }, priorDispatch: undefined, +}; +const publication: StoredPublication = { id: "publication", workspaceId: ref.workspaceId, draftId: "draft", connectionId: "connection", idempotencyKey: "key", remoteId: "remote", publishedAt: new Date(), providerMetadata: {} }; + +function harness(claim: ClaimOutcome = { kind: "claimed", job }, gate?: DispatchGateService | null) { + let stored = false; + const repository: ActivityRepository = { + claim: vi.fn(async () => stored ? { kind: "done" as const, publication } : claim), + heartbeat: vi.fn(async () => true), + withDispatchGate: vi.fn(async (_lease: Lease, _policy: { allowRedispatch?: boolean; dedupeWindowMs?: number }, send: () => Promise): Promise> => ({ kind: "sent", value: await send() })), + recordReceipt: vi.fn(async () => ({ inserted: true, receipt: { id: "receipt", workspaceId: ref.workspaceId, attemptId: "attempt", remoteId: "remote", receivedAt: new Date(), providerMetadata: {} } })), + finalize: vi.fn(async () => { stored = true; return { kind: "done" as const, publication }; }), + retryLater: vi.fn(async () => ({ kind: "settled" as const })), retryDeduplicated: vi.fn(async () => ({ kind: "settled" as const })), + pauseForReauthorization: vi.fn(async () => ({ kind: "settled" as const })), fail: vi.fn(async () => ({ kind: "settled" as const })), markUnknown: vi.fn(async () => ({ kind: "settled" as const })), + }; + const fetchImpl = vi.fn(async () => new Response("{}")); + const http = new RateLimitedHttpClient({ fetchImpl }); + const publish = vi.fn(async () => { await http.request("telegram", "https://api.telegram.org/sendMessage", { method: "POST" }, { retry: false }); return publication; }); + const connector = { publish } as unknown as PlatformConnector; + const activity = new PublishActivity(repository, { get: () => connector }, () => ({ heartbeat: vi.fn() }), gate); + return { activity, repository, publish, fetchImpl }; +} + +function gateReturning(outcome: E3Outcome): DispatchGateService { + return { + check: vi.fn(async () => outcome), + } as unknown as DispatchGateService; +} + +describe("E3 gate in the dispatch path", () => { + it("no gate configured: dispatches exactly as today (existing drafts unchanged)", async () => { + const h = harness(); + await expect(h.activity.publishScheduled(ref)).resolves.toMatchObject({ kind: "done", publicationId: "publication" }); + expect(h.publish).toHaveBeenCalledTimes(1); + expect(h.repository.fail).not.toHaveBeenCalled(); + }); + + it("gate fires: send proceeds and no policy settlement occurs", async () => { + const gate = gateReturning({ outcome: "fired", reasonCodes: [], policyVersion: "pv1", mode: "immediate", approvedHash: "abc", pollState: null }); + const h = harness({ kind: "claimed", job }, gate); + await expect(h.activity.publishScheduled(ref)).resolves.toMatchObject({ kind: "done" }); + expect(h.publish).toHaveBeenCalledTimes(1); + expect(h.repository.fail).not.toHaveBeenCalled(); + }); + + it("approval revoked post-approve: blocks with RESET_ON_CHANGE before any send", async () => { + const gate = gateReturning({ outcome: "blocked", reasonCodes: ["RESET_ON_CHANGE"], policyVersion: "pv1", mode: "immediate", approvedHash: null, pollState: null }); + const h = harness({ kind: "claimed", job }, gate); + await expect(h.activity.publishScheduled(ref)).resolves.toEqual({ kind: "failed", code: "POLICY_RESET_ON_CHANGE" }); + expect(h.fetchImpl).not.toHaveBeenCalled(); + expect(h.publish).not.toHaveBeenCalled(); + expect(h.repository.fail).toHaveBeenCalledWith(job.lease, "POLICY_RESET_ON_CHANGE"); + }); + + it("no snapshot: fails closed with POLICY_OUTSIDE_POLICY and zero HTTP requests", async () => { + const gate = gateReturning({ outcome: "blocked", reasonCodes: ["OUTSIDE_POLICY"], policyVersion: "none", mode: "none", approvedHash: null, pollState: null }); + const h = harness({ kind: "claimed", job }, gate); + await expect(h.activity.publishScheduled(ref)).resolves.toEqual({ kind: "failed", code: "POLICY_OUTSIDE_POLICY" }); + expect(h.fetchImpl).not.toHaveBeenCalled(); + }); + + it("stale settle (lease lost) yields wait, never a second send", async () => { + const gate = gateReturning({ outcome: "blocked", reasonCodes: ["RESET_ON_CHANGE"], policyVersion: "pv1", mode: "immediate", approvedHash: null, pollState: null }); + const h = harness({ kind: "claimed", job }, gate); + vi.mocked(h.repository.fail).mockResolvedValue({ kind: "stale" }); + const outcome = await h.activity.publishScheduled(ref); + expect(outcome).toEqual({ kind: "wait", until: expect.any(String) }); + expect(h.publish).not.toHaveBeenCalled(); + }); + + it("gate evaluated per destination with the lease's idempotency key", async () => { + const gate = gateReturning({ outcome: "fired", reasonCodes: [], policyVersion: "pv1", mode: "immediate", approvedHash: "abc", pollState: null }); + const h = harness({ kind: "claimed", job }, gate); + await h.activity.publishScheduled(ref); + expect(gate.check).toHaveBeenCalledWith(expect.objectContaining({ + workspaceId: job.lease.workspaceId, + draftId: job.lease.draftId, + idempotencyKey: job.lease.idempotencyKey, + connectionId: job.connection.id, + platform: job.connection.provider, + })); + }); +}); diff --git a/apps/scheduler/src/activities/publish.activity.test.ts b/apps/scheduler/src/activities/publish.activity.test.ts new file mode 100644 index 0000000..aab3b6c --- /dev/null +++ b/apps/scheduler/src/activities/publish.activity.test.ts @@ -0,0 +1,85 @@ +import { ApplicationFailure } from "@temporalio/common"; +import { describe, expect, it, vi } from "vitest"; +import { RateLimitedHttpClient, type PlatformConnector } from "@tactical-media-broadcast-hub/connector-sdk"; +import type { AttemptRef, ClaimedJob, ClaimOutcome, StoredPublication, Lease, GateOutcome } from "@tactical-media-broadcast-hub/publishing-execution"; +import { PublishActivity, type ActivityRepository } from "./publish.activity.js"; +const ref: AttemptRef = { workspaceId: "workspace", scheduleId: "schedule", generation: "1", idempotencyKey: "key" }; +const job: ClaimedJob = { + lease: { ...ref, attemptId: "attempt", draftId: "draft", ownerToken: "owner", claimEpoch: "1", leaseUntil: new Date(Date.now() + 90_000) }, + attemptNo: "1", retryCount: 0, post: { text: "hello", mediaAssetIds: [], visibility: "public", providerMetadata: {} }, + connection: { id: "connection", provider: "telegram", channelId: "channel", fields: {}, createdAt: new Date() }, + credential: { id: "credential-uuid", generation: "3" }, priorDispatch: undefined, +}; +const publication: StoredPublication = { id: "publication", workspaceId: ref.workspaceId, draftId: "draft", connectionId: "connection", idempotencyKey: "key", remoteId: "remote", publishedAt: new Date(), providerMetadata: {} }; +function harness(claim: ClaimOutcome = { kind: "claimed", job }) { + let stored = false; + const repository: ActivityRepository = { + claim: vi.fn(async () => stored ? { kind: "done" as const, publication } : claim), + heartbeat: vi.fn(async () => true), + withDispatchGate: vi.fn(async (_lease: Lease, _policy: { allowRedispatch?: boolean; dedupeWindowMs?: number }, send: () => Promise): Promise> => ({ kind: "sent", value: await send() })), + recordReceipt: vi.fn(async () => ({ inserted: true, receipt: { id: "receipt", workspaceId: ref.workspaceId, attemptId: "attempt", remoteId: "remote", receivedAt: new Date(), providerMetadata: {} } })), + finalize: vi.fn(async () => { stored = true; return { kind: "done" as const, publication }; }), + retryLater: vi.fn(async () => ({ kind: "settled" as const })), retryDeduplicated: vi.fn(async () => ({ kind: "settled" as const })), + pauseForReauthorization: vi.fn(async () => ({ kind: "settled" as const })), fail: vi.fn(async () => ({ kind: "settled" as const })), markUnknown: vi.fn(async () => ({ kind: "settled" as const })), + }; + const fetchImpl = vi.fn(async () => new Response("{}")); + const http = new RateLimitedHttpClient({ fetchImpl }); + const publish = vi.fn(async () => { await http.request("telegram", "https://api.telegram.org/sendMessage", { method: "POST" }, { retry: false }); return publication; }); + // Only publish is used by this test composition; production registers complete connectors. + const connector = { publish } as unknown as PlatformConnector; + const activity = new PublishActivity(repository, { get: () => connector }, () => ({ heartbeat: vi.fn() })); + return { activity, repository, publish, fetchImpl }; +} +describe("publishing execution wiring", () => { + it("propagates a non-retryable claim refusal without publishing or failing a job", async () => { + const h = harness(); + const claimError = ApplicationFailure.nonRetryable("draft is not approved"); + vi.mocked(h.repository.claim).mockRejectedValue(claimError); + await expect(h.activity.publishScheduled(ref)).rejects.toBe(claimError); + expect(claimError.nonRetryable).toBe(true); + expect(h.publish).not.toHaveBeenCalled(); expect(h.repository.fail).not.toHaveBeenCalled(); + }); + it("returns stored publication without publishing twice", async () => { + const h = harness(); + const first = await h.activity.publishScheduled(ref); const second = await h.activity.publishScheduled(ref); + expect(second).toEqual(first); expect(h.publish).toHaveBeenCalledTimes(1); + expect(h.repository.recordReceipt).toHaveBeenCalledTimes(1); expect(h.repository.finalize).toHaveBeenCalledTimes(1); + expect(h.repository.withDispatchGate).toHaveBeenCalledTimes(1); expect(h.fetchImpl).toHaveBeenCalledTimes(1); + }); + it("finalizes durable evidence with zero provider calls", async () => { + const h = harness({ kind: "finalize", job, receipt: { id: "receipt", workspaceId: ref.workspaceId, attemptId: "attempt", remoteId: "remote", providerMetadata: {}, receivedAt: new Date() } }); + await expect(h.activity.publishScheduled(ref)).resolves.toEqual({ kind: "done", publicationId: "publication" }); + expect(h.publish).not.toHaveBeenCalled(); expect(h.repository.recordReceipt).not.toHaveBeenCalled(); + }); + it("never routes a receipt failure into a fresh send or safe retry", async () => { + const h = harness(); const error = new Error("database unavailable"); vi.mocked(h.repository.recordReceipt).mockRejectedValue(error); + await expect(h.activity.publishScheduled(ref)).rejects.toBe(error); + expect(h.publish).toHaveBeenCalledTimes(1); expect(h.repository.retryLater).not.toHaveBeenCalled(); expect(h.repository.fail).not.toHaveBeenCalled(); expect(h.repository.finalize).not.toHaveBeenCalled(); + }); + it("refuses a fenced send with zero HTTP requests", async () => { + const h = harness(); vi.mocked(h.repository.withDispatchGate).mockResolvedValue({ kind: "paused", code: "AUTHZ_REVOKED" }); + await expect(h.activity.publishScheduled(ref)).resolves.toEqual({ kind: "paused", code: "AUTHZ_REVOKED" }); + expect(h.fetchImpl).not.toHaveBeenCalled(); expect(h.repository.retryLater).not.toHaveBeenCalled(); + }); + it("does not resend ambiguous Telegram work", async () => { + const h = harness({ kind: "claimed", job: { ...job, priorDispatch: { startedAt: new Date(), count: 1 } } }); + await expect(h.activity.publishScheduled(ref)).resolves.toEqual({ kind: "unknown", code: "PUBLISH_OUTCOME_UNKNOWN" }); + expect(h.fetchImpl).not.toHaveBeenCalled(); expect(h.repository.markUnknown).toHaveBeenCalledTimes(1); + }); + it("pauses an authentication rejection", async () => { + const h = harness(); const { AuthError } = await import("@tactical-media-broadcast-hub/connector-sdk"); h.publish.mockRejectedValue(new AuthError("rejected", "telegram", false)); + await expect(h.activity.publishScheduled(ref)).resolves.toEqual({ kind: "paused", code: "CREDENTIAL_REJECTED" }); + expect(h.repository.pauseForReauthorization).toHaveBeenCalledTimes(1); expect(h.repository.retryLater).not.toHaveBeenCalled(); + }); + it("retains unexpected provider scheduling evidence as unknown without a fabricated Publication", async () => { + const h = harness(); + const http = new RateLimitedHttpClient({ fetchImpl: h.fetchImpl }); + h.publish.mockImplementation(async () => { + await http.request("telegram", "https://api.telegram.org/sendMessage", { method: "POST" }, { retry: false }); + return { ...publication, remoteId: "scheduled:42", providerMetadata: { outcome: "provider_scheduled", scheduledAt: "2030-01-01T00:00:00.000Z" } }; + }); + await expect(h.activity.publishScheduled(ref)).resolves.toEqual({ kind: "unknown", code: "UNEXPECTED_PROVIDER_SCHEDULED" }); + expect(h.repository.markUnknown).toHaveBeenCalledWith(job.lease, { remoteId: "scheduled:42", scheduledAt: "2030-01-01T00:00:00.000Z" }); + expect(h.repository.recordReceipt).not.toHaveBeenCalled(); expect(h.repository.finalize).not.toHaveBeenCalled(); expect(h.repository.retryLater).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/scheduler/src/activities/publish.activity.ts b/apps/scheduler/src/activities/publish.activity.ts new file mode 100644 index 0000000..3a35e83 --- /dev/null +++ b/apps/scheduler/src/activities/publish.activity.ts @@ -0,0 +1,164 @@ +import { Context } from "@temporalio/activity"; +import type { ConnectorConnection, NormalizedPost } from "@tactical-media-broadcast-hub/core"; +import type { ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk"; +import { DispatchFenceError, publishWithPolicy, publishingDispatchPolicy, runWithPublishingContext } from "@tactical-media-broadcast-hub/connector-runtime"; +import { DispatchGateService, type E3Outcome } from "./dispatch-gate.js"; +import type { AttemptRef, ClaimedJob, GateOutcome, PublishingExecutionRepository, SettleOutcome, Lease } from "@tactical-media-broadcast-hub/publishing-execution"; + +export type ActivityOutcome = + | { kind: "done"; publicationId: string } + | { kind: "wait"; until: string } + | { kind: "paused" | "failed" | "superseded" | "unknown"; code: string }; +export type ActivityRepository = Pick & { withDispatchGate(lease: Lease, policy: { allowRedispatch?: boolean; dedupeWindowMs?: number }, send: () => Promise): Promise> }; +export interface ActivityLifecycle { heartbeat(): void; signal?: AbortSignal } +function temporalLifecycle(): ActivityLifecycle { + const context = Context.current(); + return { heartbeat: () => context.heartbeat(), signal: context.cancellationSignal }; +} +function connectionOf(job: ClaimedJob): ConnectorConnection { + const fields: Record = {}; + for (const [key, value] of Object.entries(job.connection.fields)) if (typeof value === "string") fields[key] = value; + return { ...job.connection, fields, workspaceId: job.lease.workspaceId, credentialId: { kind: "credential-ref", id: job.connection.id } }; +} +function postOf(job: ClaimedJob): NormalizedPost { + const visibility = job.post.visibility; + if (!["public", "unlisted", "private", "followers", "provider_default"].includes(visibility)) throw new Error("INVALID_POST_VISIBILITY"); + return { ...job.post, visibility: visibility as NormalizedPost["visibility"] }; +} +/** Persist only documented remote evidence; unknown fields never enter DB/history. */ +export function safeMetadata(metadata: Record): Record { + const result: Record = {}; + for (const key of ["messageId", "chatId", "channelId", "outcome", "scheduledAt"]) { + const value = metadata[key]; + if ((typeof value === "string" && value.length <= 512) || (typeof value === "number" && Number.isSafeInteger(value))) result[key] = value; + } + return result; +} +class GateRefusal extends Error { + constructor(readonly outcome: Exclude, { kind: "sent" }>) { super("DISPATCH_FENCE_REFUSED"); } +} +export class PublishActivity { + /** + * E3 chatgov pre-dispatch re-check (chatgov-design.md §4 step 5). Optional: + * `undefined` preserves today's ungated dispatch (used by suites whose DB + * predates the chatgov spine); `null` explicitly disables it. + */ + constructor( + private readonly repository: ActivityRepository, + private readonly registry: Pick, + private readonly lifecycle: () => ActivityLifecycle = temporalLifecycle, + private readonly dispatchGate?: DispatchGateService | null, + ) {} + + /** E3 re-check before any send. Authorization fences are NOT duplicated: + * the repository's claim + withDispatchGate already re-read every live + * authz input; this gate adds the governance side (approval + policy + * currency) and blocks closed when it cannot prove a destination is + * inside policy. Absent gate or absent chatgov tables => dispatch as today. */ + private async e3Check(lease: Lease, job: ClaimedJob): Promise { + const gate = this.dispatchGate ?? null; + if (gate === null) return undefined; + try { + return await gate.check({ + workspaceId: lease.workspaceId, + draftId: lease.draftId, + idempotencyKey: lease.idempotencyKey, + connectionId: job.connection.id, + platform: job.connection.provider, + }); + } catch (error) { + if (error instanceof Error && error.message.includes("does not exist")) return undefined; // chatgov tables absent: ungated + throw error; + } + } + + async publishScheduled(ref: AttemptRef): Promise { + const claim = await this.repository.claim(ref); + if (claim.kind === "done") return { kind: "done", publicationId: claim.publication.id }; + if (claim.kind === "wait") return { kind: "wait", until: claim.until.toISOString() }; + if (claim.kind !== "claimed" && claim.kind !== "finalize") return claim; + const job = claim.job; + const lease = job.lease; + const finalize = async (): Promise => { + const result = await this.repository.finalize(lease); + return result.kind === "done" ? { kind: "done", publicationId: result.publication.id } : { kind: "wait", until: new Date(Date.now() + 1000).toISOString() }; + }; + // Evidence/finalization errors propagate to infrastructure retries. They + // never pass through provider failure classification or trigger a send. + if (claim.kind === "finalize") return finalize(); + const lifecycle = this.lifecycle(); + const abort = new AbortController(); + const signal = AbortSignal.any([abort.signal, ...(lifecycle.signal ? [lifecycle.signal] : [])]); + const heartbeatTimer = setInterval(() => { try { lifecycle.heartbeat(); } catch { abort.abort(); } }, 5000); + let renewal: Promise | undefined; + const timer = setInterval(() => { + if (renewal) return; + renewal = this.repository.heartbeat(lease).then((owned) => { if (!owned) abort.abort(); else lifecycle.heartbeat(); }).catch(() => abort.abort()).finally(() => { renewal = undefined; }); + }, 10_000); + const settle = (result: SettleOutcome, outcome: ActivityOutcome): ActivityOutcome => result.kind === "stale" ? { kind: "wait", until: new Date(Date.now() + 1000).toISOString() } : outcome; + try { + lifecycle.heartbeat(); + const connection = connectionOf(job); + // E3: chatgov pre-dispatch re-check, per destination, before any send. + const gate = await this.e3Check(lease, job); + if (gate !== undefined && gate.outcome === "blocked") { + // Known no-send: settle the attempt failed (governance refusal) so the + // blocked destination surfaces in the failed schedule + gate_decisions. + const settled = await this.repository.fail(lease, `POLICY_${gate.reasonCodes[0] ?? "OUTSIDE_POLICY"}`); + return settled.kind === "stale" + ? { kind: "wait", until: new Date(Date.now() + 1000).toISOString() } + : { kind: "failed", code: `POLICY_${gate.reasonCodes[0] ?? "OUTSIDE_POLICY"}` }; + } + const connector = this.registry.get(connection.provider); + const { policy, origin, dedupeVerified } = publishingDispatchPolicy(connection); + let outcome; + try { + outcome = await runWithPublishingContext({ workspaceId: lease.workspaceId, connectionId: connection.id, credentialId: job.credential.id, credentialGeneration: job.credential.generation, signal }, () => publishWithPolicy(connector, connection, postOf(job), { + idempotencyKey: lease.idempotencyKey, policy, dedupeVerified, + firstDispatchAt: job.firstDispatchStartedAt, + unsettledPriorDispatch: job.priorDispatch !== undefined, + aroundMutation: async (event, send) => { + signal.throwIfAborted(); + const gate = await this.repository.withDispatchGate(lease, { allowRedispatch: dedupeVerified && event.origin === origin, dedupeWindowMs: dedupeVerified && event.origin === origin ? policy.dedupeWindowMs : 0 }, send); + if (gate.kind === "sent") return gate.value; + throw new DispatchFenceError(job.priorDispatch !== undefined, new GateRefusal(gate)); + }, + })); + } catch (error) { + if (error instanceof DispatchFenceError && error.cause instanceof GateRefusal) { + const gate = error.cause.outcome; + if (gate.kind === "finalize") return finalize(); + if (gate.kind === "lost") return { kind: "wait", until: new Date(Date.now() + 1000).toISOString() }; + return gate; + } + // Infrastructure faults are retried by Temporal and re-claim the same journal. + throw error; + } + if (outcome.kind === "published") { + const receipt = outcome.receipt; + await this.repository.recordReceipt(lease, { remoteId: receipt.remoteId, remoteUrl: receipt.remoteUrl, publishedAt: receipt.publishedAt, providerMetadata: safeMetadata(receipt.providerMetadata) }); + return finalize(); + } + if (outcome.kind === "provider_scheduled") { + // This is remote evidence, not a published receipt. Keep it in the + // unknown journal via a safe audit; never manufacture a Publication. + return settle(await this.repository.markUnknown(lease, { remoteId: outcome.scheduledRemoteId, scheduledAt: outcome.scheduledAt.toISOString() }), { kind: "unknown", code: "UNEXPECTED_PROVIDER_SCHEDULED" }); + } + if (outcome.kind === "unknown") return settle(await this.repository.markUnknown(lease), { kind: "unknown", code: outcome.code }); + if (outcome.kind === "rejected_permanent") { + // An earlier ambiguous send cannot be disproved by this rejection. + if (job.priorDispatch) return settle(await this.repository.markUnknown(lease), { kind: "unknown", code: "PUBLISH_OUTCOME_UNKNOWN" }); + return outcome.needsReauthorization + ? settle(await this.repository.pauseForReauthorization(lease, outcome.code), { kind: "paused", code: outcome.code }) + : settle(await this.repository.fail(lease, outcome.code), { kind: "failed", code: outcome.code }); + } + const delay = Math.max(outcome.retryAfterMs ?? 0, Math.min(300_000, 1000 * 2 ** job.retryCount)); + const until = new Date(Date.now() + delay); + const result = outcome.resend === "deduplicated" + ? await this.repository.retryDeduplicated(lease, outcome.code, until) + : await this.repository.retryLater(lease, outcome.code, until); + // Settlement may exhaust the budget: re-claim reads the authoritative terminal state. + return settle(result, { kind: "wait", until: (result.kind === "settled" ? result.until ?? until : until).toISOString() }); + } finally { clearInterval(timer); clearInterval(heartbeatTimer); await renewal; } + } +} diff --git a/apps/scheduler/src/director/adapters.ts b/apps/scheduler/src/director/adapters.ts new file mode 100644 index 0000000..0290e15 --- /dev/null +++ b/apps/scheduler/src/director/adapters.ts @@ -0,0 +1,75 @@ +/** + * Port adapters for the scheduler side of the director wiring (WP10). + * + * The engine's ports (director.activity.ts) speak engine types; the API + * answers in transport shapes (ports.ts). These adapters bridge them: + * sources are converted with the same semantics engine.sourceInput uses, and + * the desired-state calls are forwarded to the API's internal endpoints. + */ +import type { DirectorSourceInput } from "./engine.js"; +import type { ChannelRef, ChannelSnapshot, DesiredFence, DesiredStatePort, DirectorStatePort, ProgramSwitchCommand } from "./director.activity.js"; +import type { DirectorApi, DirectorApplyRequest, DirectorChannelSource, DirectorStateRead } from "./ports.js"; + +/** Engine source input from the transport shape: the shapes agree field-for-field. */ +export function engineSource(source: DirectorChannelSource): DirectorSourceInput { + return { ...source }; +} + +/** Transport snapshot -> engine snapshot. */ +export function engineSnapshot(read: DirectorStateRead & { kind: "current" }): ChannelSnapshot { + return { + kind: "current", + desiredVersion: read.desiredVersion, + authzEpoch: read.authzEpoch, + sources: read.sources.map(engineSource), + rules: read.rules, + policy: read.policy, + fallbackSceneId: read.fallbackSceneId, + override: read.override, + principalPermissions: read.principalPermissions, + }; +} + +/** DirectorStatePort backed by the API's internal state endpoint. */ +export class HttpDirectorStatePort implements DirectorStatePort { + constructor(private readonly api: DirectorApi) {} + + async read(ref: ChannelRef): Promise { + const read = await this.api.channelState(ref); + return read.kind === "superseded" ? { kind: "superseded", code: "SUPERSEDED" } : engineSnapshot(read); + } +} + +/** DesiredStatePort backed by the API's apply/audit endpoints. */ +export class HttpDesiredStatePort implements DesiredStatePort { + constructor(private readonly api: DirectorApi) {} + + async fence(ref: ChannelRef): Promise { + return this.api.fence(ref); + } + + async findCommand(_ref: ChannelRef, _commandId: string): Promise<{ desiredVersion: number } | undefined> { + // Idempotency is the API's job: the apply endpoint records the commandId in + // the production_command audit chain and replays the stored receipt. A + // pre-check here would double the round trips for no extra safety. + return undefined; + } + + async writeProgram(command: ProgramSwitchCommand): Promise<{ kind: "applied"; desiredVersion: number } | { kind: "conflict" }> { + const body: DirectorApplyRequest = { + commandId: command.commandId, + expectedDesiredVersion: command.expectedDesiredVersion, + authzEpoch: command.authzEpoch, + program: command.program, + layout: command.layout, + reasons: command.reasons, + ...(command.releaseOverride === undefined ? {} : { releaseOverride: command.releaseOverride }), + }; + const result = await this.api.applyProgramSwitch(command.ref, body); + return result.kind === "conflict" ? { kind: "conflict" } : { kind: "applied", desiredVersion: result.desiredVersion }; + } + + async audit(ref: ChannelRef, event: { commandId: string; outcome: "SUPERSEDED" | "AUTHZ_REVOKED"; reason: string }): Promise { + await this.api.audit(ref, event); + } +} diff --git a/apps/scheduler/src/director/config.ts b/apps/scheduler/src/director/config.ts new file mode 100644 index 0000000..a178ecf --- /dev/null +++ b/apps/scheduler/src/director/config.ts @@ -0,0 +1,144 @@ +import type { AutoDirectorRule, DirectorAction, DirectorCondition, SceneLayout, SwitchPolicy } from "@tactical-media-broadcast-hub/core"; + +/** + * Director channel configuration (WP10 wiring slice). + * + * `DIRECTOR_CHANNELS` is a server-controlled JSON array — the same pattern as + * MEDIA_POLLER_SOURCES / MEDIA_CONTROL_ASSIGNMENTS. Rule CRUD is a later + * work package; this slice makes the configured set live, so the array is the + * authority for which channels get an evaluator, which rules it evaluates and + * which generation fences its writes. + * + * The API side parses the same document (apps/api/src/director/config.ts) to + * answer snapshot/fence reads, so the two parsers must stay shape-compatible: + * both are tested against the same fixtures. + */ +export interface DirectorChannelConfig { + readonly workspaceId: string; + readonly channelId: string; + /** Assignment generation; a newer value supersedes running evaluators. */ + readonly generation: string; + readonly rules: readonly AutoDirectorRule[]; + readonly policy: SwitchPolicy | undefined; + readonly fallbackSceneId: string | null; +} + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +const GENERATION = /^[\x21-\x7e]{1,64}$/; +const POLICY_NAMES: readonly SwitchPolicy["name"][] = [ + "highest_priority", "oldest_live", "newest_live", "round_robin", "operator_required", + "picture_in_picture", "two_up", "split_screen", "grid", +]; +const LAYOUTS: readonly SceneLayout[] = ["single", "pip", "two_up", "split_horizontal", "split_vertical", "grid", "custom"]; +const CONDITION_FIELDS: readonly DirectorCondition["field"][] = ["live_source_count", "max_priority", "source_state"]; +const CONDITION_OPS: readonly DirectorCondition["op"][] = [">=", "<=", "==", ">"]; +const SELECTIONS: readonly DirectorAction["selection"][] = ["top_priority", "all_live", "oldest_live", "explicit"]; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function uuid(value: unknown, field: string): string { + if (typeof value !== "string" || !UUID.test(value)) throw new Error(`${field} must be a UUID`); + return value; +} + +function nonNegativeInt(value: unknown, field: string): number { + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) throw new Error(`${field} must be a non-negative integer`); + return value; +} + +function parseCondition(value: unknown, field: string): DirectorCondition { + if (!isRecord(value)) throw new Error(`${field} must be an object`); + const conditionField = value.field; + const op = value.op; + if (typeof conditionField !== "string" || !CONDITION_FIELDS.includes(conditionField as DirectorCondition["field"])) { + throw new Error(`${field}.field must be one of ${CONDITION_FIELDS.join(", ")}`); + } + if (typeof op !== "string" || !CONDITION_OPS.includes(op as DirectorCondition["op"])) { + throw new Error(`${field}.op must be one of ${CONDITION_OPS.join(", ")}`); + } + const ruleValue = value.value; + if (typeof ruleValue !== "number" && typeof ruleValue !== "string") throw new Error(`${field}.value must be a number or string`); + return { field: conditionField as DirectorCondition["field"], op: op as DirectorCondition["op"], value: ruleValue }; +} + +function parseAction(value: unknown, field: string): DirectorAction { + if (!isRecord(value)) throw new Error(`${field} must be an object`); + const selection = value.selection; + if (typeof selection !== "string" || !SELECTIONS.includes(selection as DirectorAction["selection"])) { + throw new Error(`${field}.selection must be one of ${SELECTIONS.join(", ")}`); + } + const layout = value.layout; + if (typeof layout !== "string" || !LAYOUTS.includes(layout as SceneLayout)) { + throw new Error(`${field}.layout must be one of ${LAYOUTS.join(", ")}`); + } + let sourceIds: string[] | undefined; + if (value.sourceIds !== undefined) { + if (!Array.isArray(value.sourceIds)) throw new Error(`${field}.sourceIds must be an array`); + sourceIds = value.sourceIds.map((id, i) => uuid(id, `${field}.sourceIds[${i}]`)); + if (new Set(sourceIds).size !== sourceIds.length) throw new Error(`${field}.sourceIds must be unique`); + } + if (selection === "explicit" && (sourceIds === undefined || sourceIds.length === 0)) { + throw new Error(`${field}.selection explicit requires sourceIds`); + } + return { layout: layout as SceneLayout, selection: selection as DirectorAction["selection"], ...(sourceIds === undefined ? {} : { sourceIds }) }; +} + +function parseRule(value: unknown, channelId: string, index: number): AutoDirectorRule { + const field = `DIRECTOR_CHANNELS[${index}].rules[${index}]`; + if (!isRecord(value)) throw new Error(`${field} must be an object`); + if (value.virtualChannelId !== undefined && value.virtualChannelId !== channelId) { + throw new Error(`${field}.virtualChannelId must match the channel it is configured on`); + } + return { + id: uuid(value.id, `${field}.id`), + virtualChannelId: channelId, + priority: nonNegativeInt(value.priority ?? 0, `${field}.priority`), + condition: parseCondition(value.condition, `${field}.condition`), + action: parseAction(value.action, `${field}.action`), + }; +} + +function parsePolicy(value: unknown, field: string): SwitchPolicy { + if (!isRecord(value)) throw new Error(`${field} must be an object`); + const name = value.name; + if (typeof name !== "string" || !POLICY_NAMES.includes(name as SwitchPolicy["name"])) { + throw new Error(`${field}.name must be one of ${POLICY_NAMES.join(", ")}`); + } + return { + name: name as SwitchPolicy["name"], + liveConfirmations: nonNegativeInt(value.liveConfirmations ?? 2, `${field}.liveConfirmations`), + offlineConfirmations: nonNegativeInt(value.offlineConfirmations ?? 3, `${field}.offlineConfirmations`), + noMediaSeconds: nonNegativeInt(value.noMediaSeconds ?? 10, `${field}.noMediaSeconds`), + minHoldMs: nonNegativeInt(value.minHoldMs ?? 2_000, `${field}.minHoldMs`), + switchCooldownMs: nonNegativeInt(value.switchCooldownMs ?? 10_000, `${field}.switchCooldownMs`), + }; +} + +/** Parse and validate DIRECTOR_CHANNELS; throws with a field path on the first problem. */ +export function parseDirectorChannels(raw: string | undefined): DirectorChannelConfig[] { + if (raw === undefined || raw.trim() === "") return []; + const parsed: unknown = JSON.parse(raw); + if (!Array.isArray(parsed)) throw new Error("DIRECTOR_CHANNELS must be a JSON array"); + return parsed.map((entry, index) => { + const field = `DIRECTOR_CHANNELS[${index}]`; + if (!isRecord(entry)) throw new Error(`${field} must be an object`); + const channelId = uuid(entry.channelId, `${field}.channelId`); + const generation = entry.generation; + if (typeof generation !== "string" || !GENERATION.test(generation)) { + throw new Error(`${field}.generation must be 1-64 printable characters`); + } + const rules = Array.isArray(entry.rules) ? entry.rules.map((rule, i) => parseRule(rule, channelId, i)) : []; + const fallbackSceneId = entry.fallbackSceneId === undefined || entry.fallbackSceneId === null + ? null : uuid(entry.fallbackSceneId, `${field}.fallbackSceneId`); + return { + workspaceId: uuid(entry.workspaceId, `${field}.workspaceId`), + channelId, + generation, + rules, + policy: entry.policy === undefined ? undefined : parsePolicy(entry.policy, `${field}.policy`), + fallbackSceneId, + }; + }); +} diff --git a/apps/scheduler/src/director/director.activity.test.ts b/apps/scheduler/src/director/director.activity.test.ts new file mode 100644 index 0000000..a28c9ed --- /dev/null +++ b/apps/scheduler/src/director/director.activity.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it, vi } from "vitest"; +import { Client, WorkflowExecutionAlreadyStartedError } from "@temporalio/client"; +import { DirectorActivity, unwiredDesiredPort, type DesiredFence, type DesiredStatePort, type ProgramSwitchCommand } from "./director.activity.js"; +import { ensureDirector, evaluatorWorkflowId } from "./starter.js"; + +const ref = { workspaceId: "ws", channelId: "vc", generation: "g2" }; +const command: ProgramSwitchCommand = { ref, commandId: "vc:g2:1", expectedDesiredVersion: 7, authzEpoch: 3, program: { kind: "sources", sourceIds: ["a"] }, layout: "single", reasons: [] }; +function port(fence: DesiredFence | undefined, over: Partial = {}): DesiredStatePort & { writeProgram: ReturnType; audit: ReturnType } { + return { fence: async () => fence, findCommand: async () => undefined, writeProgram: vi.fn(async () => ({ kind: "applied" as const, desiredVersion: 8 })), audit: vi.fn(async () => undefined), ...over } as never; +} +const current: DesiredFence = { generation: "g2", desiredVersion: 7, authzEpoch: 3, directorAuthorized: true }; +const activity = (desired: DesiredStatePort): DirectorActivity => new DirectorActivity(undefined, desired); + +describe("applyProgramSwitch generation fences", () => { + it("applies under a current fence", async () => { + const desired = port(current); + await expect(activity(desired).applyProgramSwitch(command)).resolves.toEqual({ kind: "applied", desiredVersion: 8 }); + expect(desired.writeProgram).toHaveBeenCalledWith(command); + }); + it("a retried command with a recorded commandId is a no-op duplicate", async () => { + const desired = port(current, { findCommand: async () => ({ desiredVersion: 8 }) }); + await expect(activity(desired).applyProgramSwitch(command)).resolves.toEqual({ kind: "duplicate", desiredVersion: 8 }); + expect(desired.writeProgram).not.toHaveBeenCalled(); + }); + it("a superseded channel generation is audited SUPERSEDED and never writes", async () => { + for (const fence of [{ ...current, generation: "g3" }, undefined]) { + const desired = port(fence); + await expect(activity(desired).applyProgramSwitch(command)).resolves.toEqual({ kind: "generation_superseded", code: "SUPERSEDED" }); + expect(desired.writeProgram).not.toHaveBeenCalled(); + expect(desired.audit).toHaveBeenCalledWith(ref, expect.objectContaining({ outcome: "SUPERSEDED" })); + } + }); + it("a stale authzEpoch whose principal lost director.* is audited AUTHZ_REVOKED", async () => { + const desired = port({ ...current, authzEpoch: 4, directorAuthorized: false }); + await expect(activity(desired).applyProgramSwitch(command)).resolves.toEqual({ kind: "authz_revoked", code: "AUTHZ_REVOKED" }); + expect(desired.writeProgram).not.toHaveBeenCalled(); + expect(desired.audit).toHaveBeenCalledWith(ref, expect.objectContaining({ outcome: "AUTHZ_REVOKED" })); + }); + it("an unrelated epoch bump with the principal still authorized proceeds", async () => { + await expect(activity(port({ ...current, authzEpoch: 4 })).applyProgramSwitch(command)).resolves.toMatchObject({ kind: "applied" }); + }); + it("a superseded desiredVersion or lost CAS is SUPERSEDED", async () => { + const stale = port({ ...current, desiredVersion: 9 }); + await expect(activity(stale).applyProgramSwitch(command)).resolves.toEqual({ kind: "superseded", code: "SUPERSEDED" }); + expect(stale.writeProgram).not.toHaveBeenCalled(); + const raced = port(current, { writeProgram: vi.fn(async () => ({ kind: "conflict" as const })) }); + await expect(activity(raced).applyProgramSwitch(command)).resolves.toEqual({ kind: "superseded", code: "SUPERSEDED" }); + }); + it("is inert until the go-live slice wires the port", async () => { + await expect(activity(unwiredDesiredPort).applyProgramSwitch(command)).rejects.toMatchObject({ type: "DIRECTOR_NOT_WIRED", nonRetryable: true }); + }); +}); + +describe("ensureDirector", () => { + const already = (): WorkflowExecutionAlreadyStartedError => new WorkflowExecutionAlreadyStartedError("existing", evaluatorWorkflowId(ref), "autoDirectorEvaluator"); + it("starts one evaluator per channel", async () => { + const start = vi.fn(async () => ({})); + await expect(ensureDirector({ workflow: { start } } as unknown as Client, ref, "q")).resolves.toBe("started"); + expect(start).toHaveBeenCalledWith("autoDirectorEvaluator", expect.objectContaining({ workflowId: "auto-director-ws-vc", args: [{ ref }] })); + }); + it("nudges a running evaluator instead of starting a duplicate", async () => { + const start = vi.fn().mockRejectedValue(already()); + const signal = vi.fn(async () => undefined); + const client = { workflow: { start, getHandle: () => ({ describe: async () => ({ status: { name: "RUNNING" } }), signal }) } } as unknown as Client; + await expect(ensureDirector(client, ref, "q")).resolves.toBe("running"); + expect(start).toHaveBeenCalledTimes(1); + expect(signal).toHaveBeenCalledWith("sourceChanged", { sourceId: "*", version: 0 }); + }); + it("restarts once when the evaluator closed between start and describe", async () => { + const start = vi.fn().mockRejectedValueOnce(already()).mockResolvedValue({}); + const client = { workflow: { start, getHandle: () => ({ describe: async () => ({ status: { name: "COMPLETED" } }), signal: vi.fn() }) } } as unknown as Client; + await expect(ensureDirector(client, ref, "q")).resolves.toBe("restarted"); + expect(start).toHaveBeenCalledTimes(2); + }); +}); diff --git a/apps/scheduler/src/director/director.activity.ts b/apps/scheduler/src/director/director.activity.ts new file mode 100644 index 0000000..9bf0cae --- /dev/null +++ b/apps/scheduler/src/director/director.activity.ts @@ -0,0 +1,103 @@ +import { ApplicationFailure } from "@temporalio/common"; +import type { AutoDirectorRule, DirectorOverrideReleaseReason, SceneLayout, SwitchPolicy } from "@tactical-media-broadcast-hub/core"; +import type { DirectorOverride, DirectorProgram, DirectorSourceInput } from "./engine.js"; + +/** Workflow identity. `generation` is the channel assignment generation; a newer one supersedes this evaluator. */ +export interface ChannelRef { workspaceId: string; channelId: string; generation: string } + +/** Fresh monitor + desired-state snapshot; signals carry only hints, every evaluation re-reads this. */ +export type ChannelSnapshot = + | { + kind: "current"; + desiredVersion: number; + authzEpoch: number; + sources: DirectorSourceInput[]; + rules: AutoDirectorRule[]; + policy: SwitchPolicy; + fallbackSceneId: string | null; + override: DirectorOverride | null; + principalPermissions: Record; + } + | { kind: "superseded"; code: "SUPERSEDED" }; + +export interface ProgramSwitchCommand { + ref: ChannelRef; + /** Deterministic per workflow decision: retries and replays reuse it (idempotency key). */ + commandId: string; + expectedDesiredVersion: number; + authzEpoch: number; + program: DirectorProgram; + layout: SceneLayout; + reasons: string[]; + releaseOverride?: DirectorOverrideReleaseReason; +} + +export type SwitchOutcome = + | { kind: "applied" | "duplicate"; desiredVersion: number } + | { kind: "superseded"; code: "SUPERSEDED" } + | { kind: "generation_superseded"; code: "SUPERSEDED" } + | { kind: "authz_revoked"; code: "AUTHZ_REVOKED" }; + +export interface DesiredFence { + generation: string; + desiredVersion: number; + authzEpoch: number; + /** The evaluator's recorded principal still holds `director.*` under the current epoch. */ + directorAuthorized: boolean; +} + +/** + * Ports owned by the media-control go-live slice. `writeProgram` must be one + * transaction: CAS on `expectedDesiredVersion`, desired row + `desiredVersion` + * bump + outbox row keyed by `commandId`. + */ +export interface DirectorStatePort { read(ref: ChannelRef): Promise } +export interface DesiredStatePort { + fence(ref: ChannelRef): Promise; + findCommand(ref: ChannelRef, commandId: string): Promise<{ desiredVersion: number } | undefined>; + writeProgram(command: ProgramSwitchCommand): Promise<{ kind: "applied"; desiredVersion: number } | { kind: "conflict" }>; + audit(ref: ChannelRef, event: { commandId: string; outcome: "SUPERSEDED" | "AUTHZ_REVOKED"; reason: string }): Promise; +} + +function notWired(): never { + throw ApplicationFailure.nonRetryable("director desired-state port is not wired", "DIRECTOR_NOT_WIRED"); +} +/** Placeholder until the go-live slice provides the media-control desired-state adapter. */ +export const unwiredStatePort: DirectorStatePort = { read: async () => notWired() }; +export const unwiredDesiredPort: DesiredStatePort = { + fence: async () => notWired(), findCommand: async () => notWired(), writeProgram: async () => notWired(), audit: async () => notWired(), +}; + +export class DirectorActivity { + constructor(private readonly state: DirectorStatePort = unwiredStatePort, private readonly desired: DesiredStatePort = unwiredDesiredPort) {} + + async readChannelState(ref: ChannelRef): Promise { + return this.state.read(ref); + } + + /** Generation fences are re-checked here, at emission time, never trusted from the workflow. */ + async applyProgramSwitch(command: ProgramSwitchCommand): Promise { + const { ref, commandId } = command; + const prior = await this.desired.findCommand(ref, commandId); + if (prior) return { kind: "duplicate", desiredVersion: prior.desiredVersion }; + const fence = await this.desired.fence(ref); + if (!fence || fence.generation !== ref.generation) { + await this.desired.audit(ref, { commandId, outcome: "SUPERSEDED", reason: "channel generation superseded" }); + return { kind: "generation_superseded", code: "SUPERSEDED" }; + } + if (fence.authzEpoch !== command.authzEpoch && !fence.directorAuthorized) { + await this.desired.audit(ref, { commandId, outcome: "AUTHZ_REVOKED", reason: "director principal lost director.*" }); + return { kind: "authz_revoked", code: "AUTHZ_REVOKED" }; + } + if (fence.desiredVersion !== command.expectedDesiredVersion) { + await this.desired.audit(ref, { commandId, outcome: "SUPERSEDED", reason: "desiredVersion advanced since snapshot" }); + return { kind: "superseded", code: "SUPERSEDED" }; + } + const written = await this.desired.writeProgram(command); + if (written.kind === "conflict") { + await this.desired.audit(ref, { commandId, outcome: "SUPERSEDED", reason: "desiredVersion CAS lost" }); + return { kind: "superseded", code: "SUPERSEDED" }; + } + return written; + } +} diff --git a/apps/scheduler/src/director/discovery.ts b/apps/scheduler/src/director/discovery.ts new file mode 100644 index 0000000..fce5855 --- /dev/null +++ b/apps/scheduler/src/director/discovery.ts @@ -0,0 +1,54 @@ +/** + * Director evaluator discovery (WP10 wiring slice). + * + * Mirrors sweepDue: starts are repair hints, and ensureDirector handles + * AlreadyStarted itself (nudge a running evaluator, restart a closed one). + * A discovery pass that fails must never take the scheduler down, so every + * channel error is logged and skipped. + */ +import type { Client } from "@temporalio/client"; +import { ensureDirector } from "./starter.js"; +import { parseDirectorChannels, type DirectorChannelConfig } from "./config.js"; + +export interface DirectorSweepOutcome { + started: number; + running: number; + restarted: number; +} + +/** Configured channels whose generation (and therefore evaluator identity) is still current. */ +export function currentChannels(channels: readonly DirectorChannelConfig[]): DirectorChannelConfig[] { + const latest = new Map(); + for (const channel of channels) { + const key = `${channel.workspaceId}:${channel.channelId}`; + const existing = latest.get(key); + // Generations are opaque tokens; the highest wins only when comparable, so + // keep the last-declared entry for equal tokens (config is authoritative). + if (existing === undefined || existing.generation !== channel.generation) latest.set(key, channel); + } + return [...latest.values()]; +} + +/** Start (or nudge) one evaluator per configured channel. */ +export async function sweepDirectors( + client: Client, + options: { taskQueue: string; channels: readonly DirectorChannelConfig[] }, +): Promise { + const outcome: DirectorSweepOutcome = { started: 0, running: 0, restarted: 0 }; + for (const channel of currentChannels(options.channels)) { + const ref = { workspaceId: channel.workspaceId, channelId: channel.channelId, generation: channel.generation }; + try { + const result = await ensureDirector(client, ref, options.taskQueue); + outcome[result] += 1; + } catch { + // Temporal unavailable or the start refused: the next sweep retries. + continue; + } + } + return outcome; +} + +/** Parse DIRECTOR_CHANNELS, failing startup loudly (a malformed doc must not silently disable the director). */ +export function directorChannelsFromEnv(env: NodeJS.ProcessEnv = process.env): DirectorChannelConfig[] { + return parseDirectorChannels(env.DIRECTOR_CHANNELS); +} diff --git a/apps/scheduler/src/director/engine.test.ts b/apps/scheduler/src/director/engine.test.ts new file mode 100644 index 0000000..19d021a --- /dev/null +++ b/apps/scheduler/src/director/engine.test.ts @@ -0,0 +1,279 @@ +import { describe, expect, it } from "vitest"; +import type { AutoDirectorRule, DirectorCondition, DirectorAction, SwitchPolicy } from "@tactical-media-broadcast-hub/core"; +import { decide, type DecideInput, type Decision, type DirectorMemory, type DirectorOverride, type DirectorSourceInput, type LastSwitch } from "./engine.js"; + +const policy = (over: Partial = {}): SwitchPolicy => ({ name: "highest_priority", liveConfirmations: 2, offlineConfirmations: 3, noMediaSeconds: 10, minHoldMs: 2_000, switchCooldownMs: 10_000, ...over }); +const src = (id: string, priority: number, state: DirectorSourceInput["state"] = "LIVE", extra: Partial = {}): DirectorSourceInput => ({ id, priority, enabled: true, autoSelect: true, state, ...extra }); +const confirmed = (...ids: string[]): DirectorMemory => ({ sources: Object.fromEntries(ids.map((id) => [id, { liveStreak: 2, offlineStreak: 0, confirmed: true, confirmedAtMs: 0 }])) }); +const rule = (id: string, priority: number, condition: DirectorCondition, action: DirectorAction): AutoDirectorRule => ({ id, virtualChannelId: "vc", priority, condition, action }); +const on = (ids: string[], atMs = 0, layout: LastSwitch["layout"] = "single", heldSinceMs = atMs): LastSwitch => ({ atMs, program: { kind: "sources", sourceIds: ids }, layout, heldSinceMs }); +const program = (decision: Decision): string[] => decision.program.kind === "sources" ? decision.program.sourceIds : [`FALLBACK:${decision.program.sceneId ?? ""}`]; +const base = (over: Partial): DecideInput => ({ sources: [], rules: [], policy: policy(), now: 100_000, ...over }); + +/** Feed successive snapshots through decide, carrying memory and lastSwitch like the workflow does. */ +function run(steps: Array<{ now: number; sources: DirectorSourceInput[] }>, over: Partial = {}): Decision[] { + let memory: DirectorMemory | undefined = over.memory; + let lastSwitch: LastSwitch | undefined = over.lastSwitch; + return steps.map(({ now, sources }) => { + const decision = decide({ ...base(over), now, sources, ...(memory ? { memory } : {}), ...(lastSwitch ? { lastSwitch } : {}) }); + memory = decision.memory; lastSwitch = decision.lastSwitch; + return decision; + }); +} + +describe("rule evaluation", () => { + const sources = [src("a", 10), src("b", 50), src("c", 30)]; + it("evaluates by priority desc and the first matching rule wins regardless of list order", () => { + const rules = [ + rule("low", 1, { field: "live_source_count", op: ">=", value: 1 }, { layout: "single", selection: "top_priority" }), + rule("high", 9, { field: "live_source_count", op: ">=", value: 3 }, { layout: "grid", selection: "all_live" }), + rule("mid", 5, { field: "live_source_count", op: ">=", value: 2 }, { layout: "two_up", selection: "top_priority" }), + ]; + const decision = decide(base({ sources, rules, memory: confirmed("a", "b", "c") })); + expect(decision.ruleId).toBe("high"); + expect(decision.layout).toBe("grid"); + expect(program(decision)).toEqual(["b", "c", "a"]); + }); + it("breaks priority ties by rule id and skips non-matching rules", () => { + const rules = [ + rule("z", 5, { field: "live_source_count", op: ">=", value: 1 }, { layout: "single", selection: "top_priority" }), + rule("y", 5, { field: "live_source_count", op: ">=", value: 1 }, { layout: "pip", selection: "top_priority" }), + rule("never", 99, { field: "live_source_count", op: ">", value: 3 }, { layout: "grid", selection: "all_live" }), + ]; + expect(decide(base({ sources, rules, memory: confirmed("a", "b", "c") })).ruleId).toBe("y"); + }); + it("supports every condition op on live_source_count and max_priority", () => { + const pick = (condition: DirectorCondition): string | undefined => decide(base({ sources, memory: confirmed("a", "b", "c"), + rules: [rule("r", 1, condition, { layout: "single", selection: "oldest_live" })] })).ruleId; + expect(pick({ field: "live_source_count", op: "==", value: 3 })).toBe("r"); + expect(pick({ field: "live_source_count", op: "<=", value: 2 })).toBeUndefined(); + expect(pick({ field: "max_priority", op: ">", value: 49 })).toBe("r"); + expect(pick({ field: "max_priority", op: ">", value: 50 })).toBeUndefined(); + expect(pick({ field: "max_priority", op: ">=", value: "50" })).toBe("r"); + expect(pick({ field: "max_priority", op: "<=", value: 10 })).toBeUndefined(); + }); + it("matches source_state on the debounced state, globally or per source", () => { + const action: DirectorAction = { layout: "single", selection: "explicit", sourceIds: ["c"] }; + const memory = confirmed("a", "c"); + const pick = (value: string, op: DirectorCondition["op"] = "=="): string | undefined => + decide(base({ sources, memory, rules: [rule("r", 1, { field: "source_state", op, value }, action)] })).ruleId; + expect(pick("c:LIVE")).toBe("r"); + // b reports LIVE but is unconfirmed: debounced state is PROBING. + expect(pick("b:LIVE")).toBeUndefined(); + expect(pick("b:PROBING")).toBe("r"); + expect(pick("LIVE")).toBe("r"); + expect(pick("LIVE", ">=")).toBeUndefined(); + }); + it("ignores rules for other channels when channelId is given", () => { + const foreign = { ...rule("x", 9, { field: "live_source_count", op: ">=", value: 1 }, { layout: "pip", selection: "top_priority" }), virtualChannelId: "other" }; + expect(decide(base({ channelId: "vc", sources, rules: [foreign], memory: confirmed("a", "b") })).ruleId).toBeUndefined(); + }); + it("explicit sourceIds pick the listed sources, even lower priority or non-auto ones", () => { + const decision = decide(base({ sources: [src("a", 10, "LIVE", { autoSelect: false }), src("b", 50)], memory: confirmed("a", "b"), + rules: [rule("r", 1, { field: "live_source_count", op: ">=", value: 1 }, { layout: "single", selection: "explicit", sourceIds: ["a"] })] })); + expect(program(decision)).toEqual(["a"]); + }); + it("falls through an explicit rule whose sources are all dead", () => { + const decision = decide(base({ sources: [src("a", 10, "OFFLINE"), src("b", 50)], memory: confirmed("b"), + rules: [rule("r", 9, { field: "live_source_count", op: ">=", value: 1 }, { layout: "single", selection: "explicit", sourceIds: ["a"] })] })); + expect(decision.ruleId).toBeUndefined(); + expect(program(decision)).toEqual(["b"]); + expect(decision.reasons.join("\n")).toContain("falling through"); + }); + it("oldest_live rule selection orders by liveSince", () => { + const decision = decide(base({ sources: [src("a", 99, "LIVE", { liveSinceMs: 5 }), src("b", 1, "LIVE", { liveSinceMs: 1 })], memory: confirmed("a", "b"), + rules: [rule("r", 1, { field: "live_source_count", op: ">=", value: 1 }, { layout: "single", selection: "oldest_live" })] })); + expect(program(decision)).toEqual(["b"]); + }); +}); + +describe("selection policies", () => { + const sources = [src("a", 10, "LIVE", { liveSinceMs: 300 }), src("b", 50, "LIVE", { liveSinceMs: 100 }), src("c", 30, "LIVE", { liveSinceMs: 200 }), src("d", 20, "LIVE", { liveSinceMs: 400 }), src("e", 5, "LIVE", { liveSinceMs: 500 })]; + const memory = confirmed("a", "b", "c", "d", "e"); + const pick = (name: SwitchPolicy["name"], over: Partial = {}): Decision => decide(base({ sources, memory, policy: policy({ name }), ...over })); + it("highest_priority", () => { expect(program(pick("highest_priority"))).toEqual(["b"]); }); + it("oldest_live", () => { expect(program(pick("oldest_live"))).toEqual(["b"]); expect(program(pick("oldest_live", { sources: sources.filter((s) => s.id !== "b") }))).toEqual(["c"]); }); + it("newest_live", () => { expect(program(pick("newest_live"))).toEqual(["e"]); }); + it("round_robin holds for minHold then rotates in a stable order", () => { + const rr = policy({ name: "round_robin", minHoldMs: 30_000, switchCooldownMs: 0 }); + const decisions = run([0, 10_000, 30_000, 60_000, 90_000, 120_000, 150_000].map((now) => ({ now, sources })), { policy: rr, memory }); + expect(decisions.map((d) => program(d)[0])).toEqual(["a", "a", "b", "c", "d", "e", "a"]); + }); + it("operator_required never auto-selects; keeps a healthy program; dead program -> fallback", () => { + expect(program(pick("operator_required", { fallbackSceneId: "slate" }))).toEqual(["FALLBACK:slate"]); + expect(pick("operator_required").reasons).toContain("operator_required: awaiting operator TAKE"); + expect(program(pick("operator_required", { lastSwitch: on(["e"]) }))).toEqual(["e"]); + const dead = pick("operator_required", { lastSwitch: on(["e"]), sources: sources.map((s) => s.id === "e" ? { ...s, state: "ERROR" as const } : s) }); + expect(program(dead)).toEqual(["FALLBACK:"]); + expect(dead.forced).toBe(true); + }); + it("picture_in_picture / two_up / split_screen take the top two", () => { + expect([pick("picture_in_picture").layout, program(pick("picture_in_picture"))]).toEqual(["pip", ["b", "c"]]); + expect([pick("two_up").layout, program(pick("two_up"))]).toEqual(["two_up", ["b", "c"]]); + expect([pick("split_screen").layout, program(pick("split_screen"))]).toEqual(["split_horizontal", ["b", "c"]]); + }); + it("grid caps at four slots", () => { + expect([pick("grid").layout, program(pick("grid"))]).toEqual(["grid", ["b", "c", "d", "a"]]); + }); + it("composites degrade to single with one live source", () => { + for (const name of ["picture_in_picture", "two_up", "split_screen", "grid"] as const) { + const decision = decide(base({ sources: [src("a", 1)], memory: confirmed("a"), policy: policy({ name }) })); + expect([decision.layout, program(decision)]).toEqual(["single", ["a"]]); + } + }); + it("skips disabled and non-autoSelect sources", () => { + const decision = decide(base({ sources: [src("a", 99, "LIVE", { enabled: false }), src("b", 50, "LIVE", { autoSelect: false }), src("c", 1)], memory: confirmed("a", "b", "c") })); + expect(program(decision)).toEqual(["c"]); + }); +}); + +describe("anti-flapping", () => { + it("needs liveConfirmations samples before a source is eligible", () => { + const decisions = run([{ now: 1_000, sources: [src("a", 1, "LIVE", { version: 1 })] }, { now: 2_000, sources: [src("a", 1, "LIVE", { version: 2 })] }]); + expect(program(decisions[0]!)).toEqual(["FALLBACK:"]); + expect(program(decisions[1]!)).toEqual(["a"]); + }); + it("does not count a re-read of the same monitor version as a confirmation", () => { + const decisions = run([1_000, 2_000, 3_000].map((now) => ({ now, sources: [src("a", 1, "LIVE", { version: 7 })] }))); + expect(decisions.map((d) => program(d)[0])).toEqual(["FALLBACK:", "FALLBACK:", "FALLBACK:"]); + }); + it("keeps program through offlineConfirmations - 1 failures (UNSTABLE counts) then fails over", () => { + const a = (state: DirectorSourceInput["state"], version: number): DirectorSourceInput => src("a", 10, state, { version }); + const decisions = run([ + { now: 1_000, sources: [a("UNSTABLE", 1), src("b", 1)] }, + { now: 2_000, sources: [a("OFFLINE", 2), src("b", 1)] }, + { now: 3_000, sources: [a("OFFLINE", 3), src("b", 1)] }, + ], { memory: confirmed("a", "b"), lastSwitch: on(["a"], 0) }); + expect(decisions.map((d) => program(d)[0])).toEqual(["a", "a", "b"]); + expect(decisions[2]!.forced).toBe(true); + expect(decisions[2]!.reasons.join("\n")).toContain("failover: a offline_confirmed"); + }); + it("a LIVE sample resets the offline streak", () => { + const decisions = run([["OFFLINE", 1], ["OFFLINE", 2], ["LIVE", 3], ["OFFLINE", 4], ["OFFLINE", 5]].map(([state, version], i) => ({ now: 1_000 * (i + 1), sources: [src("a", 10, state as "LIVE", { version: version as number }), src("b", 1)] })), + { memory: confirmed("a", "b"), lastSwitch: on(["a"], 0) }); + expect(decisions.every((d) => program(d)[0] === "a")).toBe(true); + }); + it("fails over after noMediaSeconds without media and wakes exactly at the timeout", () => { + const sources = (now: number): DirectorSourceInput[] => [src("a", 10, "LIVE", { lastMediaAtMs: 50_000 }), src("b", 1, "LIVE", { lastMediaAtMs: now })]; + const before = decide(base({ now: 59_999, sources: sources(59_999), memory: confirmed("a", "b"), lastSwitch: on(["a"], 0) })); + expect(program(before)).toEqual(["a"]); + expect(before.nextWakeMs).toBe(60_000); + const after = decide(base({ now: 60_000, sources: sources(60_000), memory: before.memory, lastSwitch: before.lastSwitch })); + expect(program(after)).toEqual(["b"]); + expect(after.memory.sources.a).toMatchObject({ confirmed: false, degraded: "no_media" }); + }); + it("media-health failure bypasses confirmations", () => { + const decision = decide(base({ sources: [src("a", 10, "LIVE", { mediaFailed: true }), src("b", 1)], memory: confirmed("a", "b"), lastSwitch: on(["a"], 99_000) })); + expect(program(decision)).toEqual(["b"]); + expect(decision.forced).toBe(true); + }); + it("switch cooldown blocks a non-forced preemption until it lapses", () => { + const sources = [src("a", 10), src("hi", 99)]; + const opts = { memory: confirmed("a", "hi"), lastSwitch: on(["a"], 95_000, "single", 0) }; + const held = decide(base({ now: 100_000, sources, ...opts })); + expect(program(held)).toEqual(["a"]); + expect(held.switched).toBe(false); + expect(held.nextWakeMs).toBe(105_000); + expect(program(decide(base({ now: 105_000, sources, ...opts })))).toEqual(["hi"]); + }); + it("minHold keeps the main source but cooldown alone does not apply when zero", () => { + const sources = [src("a", 10), src("hi", 99)]; + const opts = { memory: confirmed("a", "hi"), policy: policy({ switchCooldownMs: 0, minHoldMs: 15_000 }), lastSwitch: on(["a"], 90_000) }; + const held = decide(base({ now: 100_000, sources, ...opts })); + expect(program(held)).toEqual(["a"]); + expect(held.reasons.join("\n")).toContain("min hold"); + const moved = decide(base({ now: 105_000, sources, ...opts })); + expect(program(moved)).toEqual(["hi"]); + expect(moved.lastSwitch).toEqual({ atMs: 105_000, program: { kind: "sources", sourceIds: ["hi"] }, layout: "single", heldSinceMs: 105_000 }); + }); + it("high-priority interrupt happens after hold and cooldown", () => { + const decisions = run([0, 2_000, 4_000].map((now) => ({ now, sources: [src("a", 10), src("hi", 99, now === 0 ? "OFFLINE" : "LIVE", { version: now })] })), + { memory: confirmed("a"), policy: policy({ minHoldMs: 1_000, switchCooldownMs: 3_000 }) }); + expect(decisions.map((d) => program(d)[0])).toEqual(["a", "a", "hi"]); + }); +}); + +describe("failover ladder and fallback", () => { + it("degrades down the priority ladder then to the fallback scene, then recovers", () => { + const step = (now: number, a: DirectorSourceInput["state"], b: DirectorSourceInput["state"], c: DirectorSourceInput["state"]): { now: number; sources: DirectorSourceInput[] } => + ({ now, sources: [src("a", 30, a), src("b", 20, b), src("c", 10, c)] }); + const decisions = run([ + step(1_000, "LIVE", "LIVE", "LIVE"), + step(1_100, "ERROR", "LIVE", "LIVE"), + step(1_200, "ERROR", "ERROR", "LIVE"), + step(1_300, "ERROR", "ERROR", "ERROR"), + step(1_400, "LIVE", "ERROR", "ERROR"), + step(1_500, "LIVE", "ERROR", "ERROR"), + ], { memory: confirmed("a", "b", "c"), fallbackSceneId: "slate" }); + expect(decisions.map((d) => program(d)[0])).toEqual(["a", "b", "c", "FALLBACK:slate", "FALLBACK:slate", "a"]); + expect(decisions.slice(1, 4).every((d) => d.forced && d.switched)).toBe(true); + expect(decisions[3]!.mode).toBe("FALLBACK"); + expect(decisions[5]!.reasons).toContain("leaving fallback scene"); + }); + it("a composite with one dead slot re-forms immediately", () => { + const decision = decide(base({ policy: policy({ name: "two_up" }), sources: [src("a", 30), src("b", 20, "ERROR"), src("c", 10)], memory: confirmed("a", "b", "c"), lastSwitch: on(["a", "b"], 99_000, "two_up") })); + expect([decision.layout, program(decision), decision.forced]).toEqual(["two_up", ["a", "c"], true]); + }); + it("all sources OFFLINE from a cold start -> fallback with null scene", () => { + const decision = decide(base({ sources: [src("a", 1, "OFFLINE"), src("b", 2, "OFFLINE")] })); + expect(decision.program).toEqual({ kind: "fallback", sceneId: null }); + expect(decision.reasons).toContain("all sources dead: fallback scene"); + }); + it("no sources at all -> fallback", () => { + expect(decide(base({ fallbackSceneId: "loop" })).program).toEqual({ kind: "fallback", sceneId: "loop" }); + }); + it("single source stays on program and is not re-switched", () => { + const first = decide(base({ sources: [src("only", 1)], memory: confirmed("only") })); + expect([program(first), first.switched]).toEqual([["only"], true]); + const second = decide(base({ now: 200_000, sources: [src("only", 1)], memory: first.memory, lastSwitch: first.lastSwitch })); + expect([program(second), second.switched]).toEqual([["only"], false]); + }); +}); + +describe("operator override", () => { + const take = (over: Partial = {}): DirectorOverride => ({ kind: "take", release: "until_released", program: { kind: "sources", sourceIds: ["low"] }, layout: "single", principalId: "op", takenAtMs: 90_000, ...over }); + const sources = [src("low", 1), src("hi", 99)]; + const permitted = { op: ["director.switch"] }; + const rules = [rule("r", 9, { field: "live_source_count", op: ">=", value: 1 }, { layout: "single", selection: "top_priority" })]; + it("operator TAKE wins over AUTO rules and is not undone by automation", () => { + const decision = decide(base({ sources, rules, override: take(), principalPermissions: permitted, memory: confirmed("low", "hi"), lastSwitch: on(["low"], 90_000) })); + expect([program(decision), decision.mode, decision.switched, decision.override]).toEqual([["low"], "MANUAL", false, take()]); + }); + it("program lock holds even when its source is dead", () => { + const decision = decide(base({ sources: [src("low", 1, "ERROR"), src("hi", 99)], override: take({ kind: "lock" }), principalPermissions: permitted, memory: confirmed("low", "hi"), lastSwitch: on(["low"], 0) })); + expect([program(decision), decision.mode]).toEqual([["low"], "LOCKED"]); + expect(decision.reasons).toContain("program lock holds an unhealthy source"); + }); + it("revoking director.* releases the override back to AUTO immediately (revocation matrix)", () => { + for (const [kind, permissions] of [["take", {}], ["lock", { op: ["posts.read"] }], ["take", { other: ["director.switch"] }]] as const) { + const decision = decide(base({ sources, override: take({ kind }), principalPermissions: permissions, memory: confirmed("low", "hi"), lastSwitch: on(["low"], 99_999) })); + expect([decision.overrideReleased, decision.override, decision.mode, program(decision), decision.forced]).toEqual(["AUTHZ_REVOKED", null, "AUTO", ["hi"], true]); + } + }); + it("timed override expires; until_source_offline and until_next_event release on their triggers", () => { + const opts = { sources, principalPermissions: permitted, lastSwitch: on(["low"], 99_000) }; + const expired = decide(base({ ...opts, override: take({ release: "timed", expiresAtMs: 100_000 }), memory: confirmed("low", "hi") })); + expect([expired.overrideReleased, program(expired)]).toEqual(["OVERRIDE_EXPIRED", ["hi"]]); + const pending = decide(base({ ...opts, now: 99_000, override: take({ release: "timed", expiresAtMs: 100_000 }), memory: confirmed("low", "hi") })); + expect([pending.override?.release, pending.nextWakeMs]).toEqual(["timed", 100_000]); + + const offline = decide(base({ ...opts, sources: [src("low", 1, "ERROR"), src("hi", 99)], override: take({ release: "until_source_offline" }), memory: confirmed("low", "hi") })); + expect([offline.overrideReleased, program(offline)]).toEqual(["SOURCE_OFFLINE", ["hi"]]); + + const armed = decide(base({ ...opts, override: take({ release: "until_next_event" }), memory: confirmed("low", "hi") })); + expect(armed.override?.eligibleAtTake).toEqual(["hi", "low"]); + const same = decide(base({ ...opts, override: armed.override, memory: armed.memory })); + expect(same.override).not.toBeNull(); + const event = decide(base({ ...opts, sources: [...sources, src("new", 50)], override: armed.override, memory: confirmed("low", "hi", "new") })); + expect([event.overrideReleased, program(event)]).toEqual(["NEXT_EVENT", ["hi"]]); + }); + it("a lock ignores source-offline release modes", () => { + const decision = decide(base({ sources: [src("low", 1, "ERROR")], override: take({ kind: "lock", release: "until_source_offline" }), principalPermissions: permitted, memory: confirmed("low") })); + expect([decision.override?.kind, program(decision)]).toEqual(["lock", ["low"]]); + }); + it("override applies with switch and keeps counting confirmations underneath", () => { + const decision = decide(base({ sources: [src("low", 1), src("hi", 99, "LIVE", { version: 1 })], override: take(), principalPermissions: permitted, memory: confirmed("low"), lastSwitch: on(["hi"], 0) })); + expect([decision.switched, program(decision)]).toEqual([true, ["low"]]); + expect(decision.memory.sources.hi).toMatchObject({ liveStreak: 1, confirmed: false }); + }); +}); diff --git a/apps/scheduler/src/director/engine.ts b/apps/scheduler/src/director/engine.ts new file mode 100644 index 0000000..68806dd --- /dev/null +++ b/apps/scheduler/src/director/engine.ts @@ -0,0 +1,344 @@ +/** + * WP10 Auto Director decision engine. Pure and deterministic: no I/O, no clock, + * no Temporal. Every timestamp is epoch milliseconds so inputs and outputs stay + * JSON-serializable across activity boundaries and continue-as-new. + * Imports from core are type-only so the workflow bundle stays sandbox-safe. + */ +import type { + AutoDirectorRule, DirectorCondition, DirectorMode, DirectorOverrideKind, DirectorOverrideRelease, + DirectorOverrideReleaseReason, LiveSourceState, LiveSourceStateType, SceneLayout, SwitchPolicy, VirtualChannelSource, +} from "@tactical-media-broadcast-hub/core"; + +export interface DirectorSourceInput { + id: string; + priority: number; + enabled: boolean; + autoSelect: boolean; + state: LiveSourceStateType; + /** Monitor row version; a re-read of the same version is not a new confirmation sample. */ + version?: number; + liveSinceMs?: number; + /** Last media-health "media present" instant; drives the no-media timeout. */ + lastMediaAtMs?: number; + /** Media-health tier reports failure: bypasses confirmations (proposal §5.2 ladder step 2). */ + mediaFailed?: boolean; +} + +export interface SourceTrack { + liveStreak: number; + offlineStreak: number; + confirmed: boolean; + confirmedAtMs?: number; + lastVersion?: number; + /** Why the source last lost (or never had) eligibility. */ + degraded?: string; +} + +export interface DirectorMemory { sources: Record } + +export type DirectorProgram = + | { kind: "sources"; sourceIds: string[] } + | { kind: "fallback"; sceneId: string | null }; + +export interface LastSwitch { + atMs: number; + program: DirectorProgram; + layout: SceneLayout; + /** When the program's main (first-slot) source went on air; minHold is measured from here. */ + heldSinceMs: number; +} + +export interface DirectorOverride { + kind: DirectorOverrideKind; + release: DirectorOverrideRelease; + program: DirectorProgram; + layout: SceneLayout; + principalId: string; + takenAtMs: number; + expiresAtMs?: number; + /** Eligible set at TAKE time, filled by the engine for `until_next_event`. */ + eligibleAtTake?: string[]; +} + +export interface DecideInput { + sources: readonly DirectorSourceInput[]; + rules: readonly AutoDirectorRule[]; + policy: SwitchPolicy; + now: number; + lastSwitch?: LastSwitch; + memory?: DirectorMemory; + channelId?: string; + fallbackSceneId?: string | null; + override?: DirectorOverride | null; + /** Current permissions per principal (re-read every evaluation, never the TAKE-time snapshot). */ + principalPermissions?: Readonly>; +} + +export interface Decision { + program: DirectorProgram; + layout: SceneLayout; + reasons: string[]; + mode: DirectorMode; + /** Program or layout differs from `lastSwitch`; the wrapper must emit a desired-state write. */ + switched: boolean; + /** Switch bypassed minHold/cooldown (failover, fallback exit, override end). */ + forced: boolean; + ruleId?: string; + memory: DirectorMemory; + override: DirectorOverride | null; + overrideReleased?: DirectorOverrideReleaseReason; + lastSwitch: LastSwitch; + /** Earliest instant a purely time-based change could alter the decision. */ + nextWakeMs?: number; +} + +interface Candidate { id: string; priority: number; liveSinceMs: number } +interface Selection { program: DirectorProgram; layout: SceneLayout; ruleId?: string } + +const CAPACITY: Record = { single: 1, pip: 2, two_up: 2, split_horizontal: 2, split_vertical: 2, grid: 4, custom: Number.POSITIVE_INFINITY }; +const COMPOSITE_POLICY: Partial> = { picture_in_picture: "pip", two_up: "two_up", split_screen: "split_horizontal", grid: "grid" }; + +/** Adapt the persisted monitor/domain types into engine input. */ +export function sourceInput(source: Pick, state: LiveSourceState, extra: { version?: number; mediaFailed?: boolean } = {}): DirectorSourceInput { + return { + id: source.id, priority: source.priority, enabled: source.enabled, autoSelect: source.autoSelect, state: state.state, + ...(extra.version === undefined ? {} : { version: extra.version }), + ...(state.liveSince ? { liveSinceMs: state.liveSince.getTime() } : {}), + ...(state.lastHealthyAt ? { lastMediaAtMs: state.lastHealthyAt.getTime() } : {}), + ...(extra.mediaFailed === undefined ? {} : { mediaFailed: extra.mediaFailed }), + }; +} + +/** Fold one snapshot into per-source confirmation tracking. */ +function track(previous: SourceTrack | undefined, source: DirectorSourceInput, policy: SwitchPolicy, now: number): SourceTrack { + let next: SourceTrack = previous ? { ...previous } : { liveStreak: 0, offlineStreak: 0, confirmed: false }; + const fresh = source.version === undefined || previous?.lastVersion !== source.version; + if (fresh) { + if (source.version !== undefined) next.lastVersion = source.version; + switch (source.state) { + case "LIVE": + next.liveStreak += 1; next.offlineStreak = 0; + if (!next.confirmed && next.liveStreak >= policy.liveConfirmations) next = { ...next, confirmed: true, confirmedAtMs: now, degraded: undefined }; + break; + case "OFFLINE": case "ENDING": case "UNSTABLE": + next.offlineStreak += 1; next.liveStreak = 0; + if (next.confirmed && next.offlineStreak >= policy.offlineConfirmations) next = { ...next, confirmed: false, degraded: "offline_confirmed" }; + break; + case "ERROR": + next = { ...next, liveStreak: 0, offlineStreak: 0, confirmed: false, degraded: next.confirmed ? "error" : next.degraded }; + break; + default: + next.liveStreak = 0; + } + } + // Hard media failures override confirmations and force re-confirmation. + if (next.confirmed && source.mediaFailed === true) next = { ...next, confirmed: false, liveStreak: 0, degraded: "media_failed" }; + if (next.confirmed && source.lastMediaAtMs !== undefined && now - source.lastMediaAtMs >= policy.noMediaSeconds * 1000) { + next = { ...next, confirmed: false, liveStreak: 0, degraded: "no_media" }; + } + if (!source.enabled && next.confirmed) next = { ...next, confirmed: false, degraded: "disabled" }; + if (next.degraded === undefined) delete next.degraded; + return next; +} + +function byPriority(a: Candidate, b: Candidate): number { + return b.priority - a.priority || a.liveSinceMs - b.liveSinceMs || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0); +} +function byAge(a: Candidate, b: Candidate): number { + return a.liveSinceMs - b.liveSinceMs || byPriority(a, b); +} +function compare(actual: number, op: DirectorCondition["op"], expected: number): boolean { + switch (op) { + case ">=": return actual >= expected; + case "<=": return actual <= expected; + case "==": return actual === expected; + case ">": return actual > expected; + } +} +function programKey(program: DirectorProgram, layout: SceneLayout): string { + return program.kind === "fallback" ? `fallback:${program.sceneId ?? ""}` : `${layout}:${program.sourceIds.join(",")}`; +} +function mainSource(program: DirectorProgram): string | undefined { + return program.kind === "sources" ? program.sourceIds[0] : undefined; +} +/** Composite layouts collapse to `single` when only one source can fill them. */ +function fill(ids: string[], layout: SceneLayout, reasons: string[]): { program: DirectorProgram; layout: SceneLayout } | undefined { + const picked = ids.slice(0, CAPACITY[layout]); + if (picked.length === 0) return undefined; + if (layout !== "single" && layout !== "custom" && picked.length < 2) { + reasons.push(`layout ${layout} degraded to single: one eligible source`); + return { program: { kind: "sources", sourceIds: picked }, layout: "single" }; + } + return { program: { kind: "sources", sourceIds: picked }, layout }; +} + +export function decide(input: DecideInput): Decision { + const { policy, now } = input; + const reasons: string[] = []; + const memory: DirectorMemory = { sources: {} }; + for (const source of input.sources) memory.sources[source.id] = track(input.memory?.sources[source.id], source, policy, now); + + const live: Candidate[] = input.sources + .filter((source) => source.enabled && memory.sources[source.id]?.confirmed === true) + .map((source) => ({ id: source.id, priority: source.priority, liveSinceMs: source.liveSinceMs ?? memory.sources[source.id]?.confirmedAtMs ?? now })) + .sort(byPriority); + const liveIds = new Set(live.map((candidate) => candidate.id)); + const autoPool = live.filter((candidate) => input.sources.find((source) => source.id === candidate.id)?.autoSelect !== false); + const healthy = (program: DirectorProgram): boolean => program.kind === "sources" && program.sourceIds.length > 0 && program.sourceIds.every((id) => liveIds.has(id)); + const fallback: DirectorProgram = { kind: "fallback", sceneId: input.fallbackSceneId ?? null }; + + // Wake for the earliest no-media timeout among eligible sources. + const wakes: number[] = []; + for (const source of input.sources) { + if (liveIds.has(source.id) && source.lastMediaAtMs !== undefined) wakes.push(source.lastMediaAtMs + policy.noMediaSeconds * 1000); + } + const nextWake = (): number | undefined => { + const future = wakes.filter((at) => at > now); + return future.length === 0 ? undefined : Math.min(...future); + }; + + // Ladder step 1: operator override / program lock. + let override = input.override ? { ...input.override } : null; + let overrideReleased: DirectorOverrideReleaseReason | undefined; + if (override) { + const permissions = input.principalPermissions?.[override.principalId] ?? []; + const liveSet = [...liveIds].sort(); + if (!permissions.some((permission) => permission.startsWith("director."))) overrideReleased = "AUTHZ_REVOKED"; + else if (override.expiresAtMs !== undefined && now >= override.expiresAtMs) overrideReleased = "OVERRIDE_EXPIRED"; + else if (override.kind === "take" && override.release === "until_source_offline" && !healthy(override.program)) overrideReleased = "SOURCE_OFFLINE"; + else if (override.kind === "take" && override.release === "until_next_event") { + if (override.eligibleAtTake === undefined) override.eligibleAtTake = liveSet; + else if (override.eligibleAtTake.join(",") !== liveSet.join(",")) overrideReleased = "NEXT_EVENT"; + } + if (overrideReleased) { + reasons.push(`${override.kind} by ${override.principalId} released: ${overrideReleased}; returning to AUTO`); + override = null; + } else { + if (override.expiresAtMs !== undefined) wakes.push(override.expiresAtMs); + const switched = !input.lastSwitch || programKey(input.lastSwitch.program, input.lastSwitch.layout) !== programKey(override.program, override.layout); + if (override.kind === "lock" && !healthy(override.program) && override.program.kind === "sources") reasons.push("program lock holds an unhealthy source"); + reasons.push(`${override.kind === "lock" ? "program lock" : "operator TAKE"} by ${override.principalId} wins over AUTO`); + const lastSwitch: LastSwitch = switched + ? { atMs: now, program: override.program, layout: override.layout, heldSinceMs: mainSource(override.program) === mainSource(input.lastSwitch?.program ?? fallback) ? input.lastSwitch?.heldSinceMs ?? now : now } + : input.lastSwitch!; + return { + program: override.program, layout: override.layout, reasons, mode: override.kind === "lock" ? "LOCKED" : "MANUAL", + switched, forced: switched, memory, override, lastSwitch, ...(nextWake() === undefined ? {} : { nextWakeMs: nextWake()! }), + }; + } + } + + // Ladder: ordered rules (priority desc, id asc for determinism), first match wins. + let selection: Selection | undefined; + const rules = input.rules.filter((rule) => input.channelId === undefined || rule.virtualChannelId === input.channelId) + .sort((a, b) => b.priority - a.priority || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)); + for (const rule of rules) { + if (!matches(rule, input.sources, live, memory, reasons)) continue; + const pool = rule.action.selection === "explicit" + ? (rule.action.sourceIds ?? []).map((id) => live.find((candidate) => candidate.id === id)).filter((candidate): candidate is Candidate => candidate !== undefined) + : autoPool; + const ordered = rule.action.selection === "oldest_live" ? [...pool].sort(byAge) : pool; + const filled = fill(ordered.map((candidate) => candidate.id), rule.action.layout, reasons); + if (!filled) { reasons.push(`rule ${rule.id} matched but has no eligible source; falling through`); continue; } + reasons.push(`rule ${rule.id} matched (${rule.condition.field} ${rule.condition.op} ${String(rule.condition.value)})`); + selection = { ...filled, ruleId: rule.id }; + break; + } + + // No rule matched: the channel's selection policy. + if (!selection) { + const ids = autoPool.map((candidate) => candidate.id); + const current = input.lastSwitch; + const currentMain = current ? mainSource(current.program) : undefined; + let picked: { program: DirectorProgram; layout: SceneLayout } | undefined; + switch (policy.name) { + case "highest_priority": picked = fill(ids, "single", reasons); break; + case "oldest_live": picked = fill([...autoPool].sort(byAge).map((candidate) => candidate.id), "single", reasons); break; + case "newest_live": picked = fill([...autoPool].sort((a, b) => byAge(b, a)).map((candidate) => candidate.id), "single", reasons); break; + case "round_robin": { + const rotation = [...ids].sort(); + const index = currentMain === undefined ? -1 : rotation.indexOf(currentMain); + if (index >= 0 && current && now - current.heldSinceMs < policy.minHoldMs) { + picked = fill([rotation[index]!], "single", reasons); + wakes.push(current.heldSinceMs + policy.minHoldMs); + } else { + picked = fill(rotation.length === 0 ? [] : [rotation[(index + 1) % rotation.length]!], "single", reasons); + if (index >= 0 && rotation.length > 1) reasons.push("round_robin rotation"); + } + break; + } + case "operator_required": + if (current && healthy(current.program)) picked = { program: current.program, layout: current.layout }; + else if (live.length > 0) reasons.push("operator_required: awaiting operator TAKE"); + break; + default: picked = fill(ids, COMPOSITE_POLICY[policy.name] ?? "single", reasons); + } + if (picked) { selection = picked; reasons.push(`policy ${policy.name} selected ${programKey(picked.program, picked.layout)}`); } + } + + // Ladder terminal: fallback scene keeps output alive. + if (!selection) { + selection = { program: fallback, layout: "single" }; + reasons.push(live.length === 0 ? "all sources dead: fallback scene" : "no selection: fallback scene"); + } + + // Anti-flapping against the current program. + const current = input.lastSwitch; + const desiredKey = programKey(selection.program, selection.layout); + let outcome: Selection = selection; + let switched = false; + let forced = false; + if (!current) { + switched = true; forced = true; + reasons.push("initial program"); + } else if (programKey(current.program, current.layout) !== desiredKey) { + if (overrideReleased || !healthy(current.program)) { + switched = true; forced = true; + if (current.program.kind === "sources") { + const lost = current.program.sourceIds.filter((id) => !liveIds.has(id)); + reasons.push(`failover: ${lost.map((id) => `${id} ${memory.sources[id]?.degraded ?? "not live"}`).join(", ") || "override released"}`); + } else if (!overrideReleased) reasons.push("leaving fallback scene"); + } else if (now - current.atMs < policy.switchCooldownMs) { + outcome = { program: current.program, layout: current.layout }; + wakes.push(current.atMs + policy.switchCooldownMs); + reasons.push(`switch cooldown holds program until ${current.atMs + policy.switchCooldownMs}`); + } else if (mainSource(current.program) !== mainSource(selection.program) && now - current.heldSinceMs < policy.minHoldMs) { + outcome = { program: current.program, layout: current.layout }; + wakes.push(current.heldSinceMs + policy.minHoldMs); + reasons.push(`min hold keeps ${mainSource(current.program) ?? "program"} until ${current.heldSinceMs + policy.minHoldMs}`); + } else { + switched = true; + } + } + const lastSwitch: LastSwitch = switched + ? { atMs: now, program: outcome.program, layout: outcome.layout, heldSinceMs: current && mainSource(current.program) === mainSource(outcome.program) && mainSource(outcome.program) !== undefined ? current.heldSinceMs : now } + : current!; + const wake = nextWake(); + return { + program: outcome.program, layout: outcome.layout, reasons, mode: outcome.program.kind === "fallback" ? "FALLBACK" : "AUTO", + switched, forced, ...(outcome.ruleId === undefined || outcome !== selection ? {} : { ruleId: outcome.ruleId }), + memory, override: null, ...(overrideReleased ? { overrideReleased } : {}), lastSwitch, ...(wake === undefined ? {} : { nextWakeMs: wake }), + }; +} + +/** Effective (debounced) state: LIVE only once confirmed and healthy. */ +function effectiveState(source: DirectorSourceInput, memory: DirectorMemory): LiveSourceStateType { + if (source.enabled && memory.sources[source.id]?.confirmed === true) return "LIVE"; + return source.state === "LIVE" ? "PROBING" : source.state; +} + +function matches(rule: AutoDirectorRule, sources: readonly DirectorSourceInput[], live: readonly Candidate[], memory: DirectorMemory, reasons: string[]): boolean { + const { field, op, value } = rule.condition; + if (field === "source_state") { + if (op !== "==" || typeof value !== "string") { reasons.push(`rule ${rule.id} skipped: source_state supports only == "" or ":"`); return false; } + const split = value.lastIndexOf(":"); + const wanted = split < 0 ? value : value.slice(split + 1); + const scope = split >= 0 ? [value.slice(0, split)] : rule.action.selection === "explicit" ? rule.action.sourceIds ?? [] : undefined; + return sources.some((source) => (scope === undefined ? source.enabled : scope.includes(source.id)) && effectiveState(source, memory) === wanted); + } + const expected = typeof value === "number" ? value : Number(value); + if (!Number.isFinite(expected)) { reasons.push(`rule ${rule.id} skipped: non-numeric ${field} value`); return false; } + if (field === "live_source_count") return compare(live.length, op, expected); + if (live.length === 0) return false; + return compare(Math.max(...live.map((candidate) => candidate.priority)), op, expected); +} diff --git a/apps/scheduler/src/director/evaluator.workflow.ts b/apps/scheduler/src/director/evaluator.workflow.ts new file mode 100644 index 0000000..18c90a9 --- /dev/null +++ b/apps/scheduler/src/director/evaluator.workflow.ts @@ -0,0 +1,100 @@ +import { condition, continueAsNew, defineQuery, defineSignal, proxyActivities, setHandler, workflowInfo } from "@temporalio/workflow"; +import type { DirectorMode } from "@tactical-media-broadcast-hub/core"; +import { decide, type DirectorMemory, type LastSwitch } from "./engine.js"; +import type { ChannelRef, DirectorActivity } from "./director.activity.js"; + +const { readChannelState } = proxyActivities>({ + startToCloseTimeout: "10 seconds", + retry: { initialInterval: "500 milliseconds", maximumInterval: "5 seconds", nonRetryableErrorTypes: ["DIRECTOR_NOT_WIRED"] }, +}); +const { applyProgramSwitch } = proxyActivities>({ + startToCloseTimeout: "15 seconds", + retry: { initialInterval: "500 milliseconds", maximumInterval: "5 seconds", maximumAttempts: 5, nonRetryableErrorTypes: ["DIRECTOR_NOT_WIRED"] }, +}); + +/** Hint only: `(sourceId, version)`. Evaluations always re-read a fresh snapshot. */ +export interface SourceHint { sourceId: string; version: number; urgent?: boolean } +export interface EvaluatorCarry { memory: DirectorMemory; lastSwitch?: LastSwitch; seq: number } +export interface EvaluatorInput { ref: ChannelRef; carry?: EvaluatorCarry } +export type EvaluatorResult = { kind: "stopped" } | { kind: "superseded"; code: "SUPERSEDED" } | { kind: "needs_reauthorization"; code: "AUTHZ_REVOKED" }; +export interface EvaluatorStatus { mode?: DirectorMode; lastSwitch?: LastSwitch; reasons: string[]; evaluations: number } + +export const sourceChanged = defineSignal<[SourceHint]>("sourceChanged"); +export const stopDirector = defineSignal("stopDirector"); +export const directorStatus = defineQuery("directorStatus"); + +export const DEBOUNCE_MS = 250; +export const MAX_WAIT_MS = 1000; +export const POLL_MS = 5000; +export const CONTINUE_AS_NEW_EVENTS = 2000; + +/** One evaluator per virtual channel: debounce hints, decide, emit fenced desired-state writes. */ +export async function autoDirectorEvaluator({ ref, carry }: EvaluatorInput): Promise { + let memory: DirectorMemory = carry?.memory ?? { sources: {} }; + let lastSwitch = carry?.lastSwitch; + let seq = carry?.seq ?? 0; + let dirty = true; + let urgent = false; + let stopping = false; + let firstPendingAt: number | undefined; + let lastSignalAt = 0; + let events = 0; + let evaluations = 0; + let nextWakeMs: number | undefined; + let status: EvaluatorStatus = { reasons: [], evaluations: 0, ...(lastSwitch ? { lastSwitch } : {}) }; + + setHandler(sourceChanged, (hint) => { + events += 1; dirty = true; + if (hint.urgent === true) urgent = true; + lastSignalAt = Date.now(); + firstPendingAt ??= lastSignalAt; + }); + setHandler(stopDirector, () => { stopping = true; }); + setHandler(directorStatus, () => status); + + for (;;) { + const idle = Math.min(POLL_MS, nextWakeMs === undefined ? POLL_MS : Math.max(0, nextWakeMs - Date.now())); + await condition(() => dirty || stopping, idle); + if (stopping) return { kind: "stopped" }; + // Trailing debounce: 250 ms quiet window, 1 s max wait; urgent program-source failure bypasses. + while (dirty && !urgent && firstPendingAt !== undefined) { + const deadline = Math.min(lastSignalAt + DEBOUNCE_MS, firstPendingAt + MAX_WAIT_MS); + if (Date.now() >= deadline) break; + const seen = lastSignalAt; + if (!(await condition(() => urgent || stopping || lastSignalAt !== seen, deadline - Date.now()))) break; + if (stopping) return { kind: "stopped" }; + } + dirty = false; urgent = false; firstPendingAt = undefined; + + const snapshot = await readChannelState(ref); + if (snapshot.kind === "superseded") return { kind: "superseded", code: "SUPERSEDED" }; + const decision = decide({ + sources: snapshot.sources, rules: snapshot.rules, policy: snapshot.policy, now: Date.now(), channelId: ref.channelId, + fallbackSceneId: snapshot.fallbackSceneId, override: snapshot.override, principalPermissions: snapshot.principalPermissions, + memory, ...(lastSwitch ? { lastSwitch } : {}), + }); + memory = decision.memory; + evaluations += 1; + nextWakeMs = decision.nextWakeMs; + if (decision.switched || decision.overrideReleased) { + seq += 1; + const outcome = await applyProgramSwitch({ + ref, commandId: `${ref.channelId}:${ref.generation}:${seq}`, expectedDesiredVersion: snapshot.desiredVersion, + authzEpoch: snapshot.authzEpoch, program: decision.program, layout: decision.layout, reasons: decision.reasons, + ...(decision.overrideReleased ? { releaseOverride: decision.overrideReleased } : {}), + }); + if (outcome.kind === "generation_superseded") return { kind: "superseded", code: "SUPERSEDED" }; + if (outcome.kind === "authz_revoked") return { kind: "needs_reauthorization", code: "AUTHZ_REVOKED" }; + // A concurrent writer (e.g. operator TAKE) won the CAS: re-read rather than re-apply. + if (outcome.kind === "superseded") dirty = true; + else lastSwitch = decision.lastSwitch; + } else { + lastSwitch = decision.lastSwitch; + } + status = { mode: decision.mode, reasons: decision.reasons, evaluations, ...(lastSwitch ? { lastSwitch } : {}) }; + + if (events + evaluations >= CONTINUE_AS_NEW_EVENTS || workflowInfo().continueAsNewSuggested) { + return continueAsNew({ ref, carry: { memory, seq, ...(lastSwitch ? { lastSwitch } : {}) } }); + } + } +} diff --git a/apps/scheduler/src/director/http.client.ts b/apps/scheduler/src/director/http.client.ts new file mode 100644 index 0000000..e4206f8 --- /dev/null +++ b/apps/scheduler/src/director/http.client.ts @@ -0,0 +1,128 @@ +/** + * HTTP implementation of the director ports (WP10 wiring slice). + * + * The scheduler worker dials the API's internal director endpoints. Every + * response is validated before it reaches the engine: the API is a different + * process, so a malformed or truncated reply must surface as a transport + * error (retryable) rather than poison the evaluator's memory. + * + * Endpoints (apps/api/src/director/director.controller.ts): + * GET /internal/director/channels/:workspaceId/:channelId/state + * GET /internal/director/channels/:workspaceId/:channelId/fence + * POST /internal/director/channels/:workspaceId/:channelId/apply + * POST /internal/director/channels/:workspaceId/:channelId/audit + */ +import type { ChannelRef, DesiredFence } from "./director.activity.js"; +import type { DirectorApplyRequest, DirectorApplyResult, DirectorApi, DirectorStateRead } from "./ports.js"; + +export interface DirectorHttpClientDependencies { + /** Base URL of the API (default http://127.0.0.1:3000). */ + baseUrl?: string; + /** Shared-secret bearer for the internal route group. */ + token?: string; + fetch?: typeof fetch; + /** Default 8s; short so an unresponsive API fails the activity fast. */ + timeoutMs?: number; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function requiredString(value: unknown, path: string): string { + if (typeof value !== "string" || value === "") throw new Error(`director api: ${path} must be a non-empty string`); + return value; +} + +/** Shape-check an apply response without trusting the status code alone. */ +export function parseApplyResult(value: unknown): DirectorApplyResult { + if (!isRecord(value)) throw new Error("director api: apply response must be an object"); + const kind = value.kind; + if (kind === "conflict") return { kind: "conflict" }; + if (kind !== "applied" && kind !== "duplicate") throw new Error("director api: apply response kind is invalid"); + const desiredVersion = value.desiredVersion; + if (typeof desiredVersion !== "number" || !Number.isSafeInteger(desiredVersion) || desiredVersion < 0) { + throw new Error("director api: apply response desiredVersion is invalid"); + } + return { kind, desiredVersion }; +} + +/** Shape-check a fence response (fence may legitimately be absent). */ +export function parseFence(value: unknown): DesiredFence | undefined { + if (value === null || value === undefined) return undefined; + if (!isRecord(value)) throw new Error("director api: fence response must be an object or null"); + const generation = requiredString(value.generation, "fence.generation"); + const desiredVersion = value.desiredVersion; + const authzEpoch = value.authzEpoch; + if (typeof desiredVersion !== "number" || !Number.isSafeInteger(desiredVersion) || desiredVersion < 0) { + throw new Error("director api: fence.desiredVersion is invalid"); + } + if (typeof authzEpoch !== "number" || !Number.isSafeInteger(authzEpoch) || authzEpoch < 0) { + throw new Error("director api: fence.authzEpoch is invalid"); + } + if (typeof value.directorAuthorized !== "boolean") throw new Error("director api: fence.directorAuthorized is invalid"); + return { generation, desiredVersion, authzEpoch, directorAuthorized: value.directorAuthorized }; +} + +export class DirectorHttpClient implements DirectorApi { + private readonly baseUrl: string; + private readonly token: string | undefined; + private readonly fetchImpl: typeof fetch; + private readonly timeoutMs: number; + + constructor(dependencies: DirectorHttpClientDependencies = {}) { + this.baseUrl = (dependencies.baseUrl ?? process.env.DIRECTOR_API_URL ?? "http://127.0.0.1:3000").replace(/\/+$/, ""); + this.token = dependencies.token ?? process.env.DIRECTOR_API_TOKEN; + this.fetchImpl = dependencies.fetch ?? fetch; + this.timeoutMs = dependencies.timeoutMs ?? 8_000; + } + + private path(ref: ChannelRef, suffix: string): string { + return `${this.baseUrl}/internal/director/channels/${encodeURIComponent(ref.workspaceId)}/${encodeURIComponent(ref.channelId)}/${suffix}`; + } + + private async request(path: string, init?: RequestInit): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), this.timeoutMs); + try { + const response = await this.fetchImpl(path, { + ...init, + signal: controller.signal, + headers: { + "content-type": "application/json", + ...(this.token === undefined ? {} : { authorization: `Bearer ${this.token}` }), + ...init?.headers, + }, + }); + if (!response.ok) { + // Transport/server errors are retryable; the evaluator treats 4xx-shaped + // outcomes as fenced results, which the callers parse explicitly. + throw new Error(`director api: ${response.status} ${response.statusText} for ${path}`); + } + const text = await response.text(); + return text === "" ? null : JSON.parse(text) as unknown; + } finally { + clearTimeout(timeout); + } + } + + async channelState(ref: ChannelRef): Promise { + const value = await this.request(this.path(ref, "state")); + if (!isRecord(value)) throw new Error("director api: state response must be an object"); + if (value.kind === "superseded") return { kind: "superseded", code: "SUPERSEDED" }; + if (value.kind !== "current") throw new Error("director api: state response kind is invalid"); + return value as DirectorStateRead; + } + + async fence(ref: ChannelRef): Promise { + return parseFence(await this.request(this.path(ref, "fence"))); + } + + async applyProgramSwitch(ref: ChannelRef, body: DirectorApplyRequest): Promise { + return parseApplyResult(await this.request(this.path(ref, "apply"), { method: "POST", body: JSON.stringify(body) })); + } + + async audit(ref: ChannelRef, event: { commandId: string; outcome: string; reason: string }): Promise { + await this.request(this.path(ref, "audit"), { method: "POST", body: JSON.stringify(event) }); + } +} diff --git a/apps/scheduler/src/director/ports.ts b/apps/scheduler/src/director/ports.ts new file mode 100644 index 0000000..413947a --- /dev/null +++ b/apps/scheduler/src/director/ports.ts @@ -0,0 +1,104 @@ +/** + * Transport-agnostic port contracts for the auto-director evaluator (WP10 + * wiring slice). + * + * The Temporal worker hosts the evaluator workflow and proxies these ports; + * the API process implements them. Because the worker runs in the scheduler + * package and the implementations in apps/api, the interfaces are declared + * with JSON-serializable shapes only (the scheduler cannot compile apps/api + * sources), and both sides share the engine's type definitions via relative + * imports — the same cross-package pattern apps/api/src/live-monitor/poller + * uses for packages/connector-mock-livestream. + * + * `read`/`fence` return `kind: "superseded"` / `undefined` rather than throwing + * so a dead generation never becomes a retryable activity failure loop. + */ +import type { AutoDirectorRule, SceneLayout, SwitchPolicy } from "@tactical-media-broadcast-hub/core"; +import type { DirectorOverride, DirectorProgram } from "./engine.js"; +import type { ChannelRef, DesiredFence } from "./director.activity.js"; + +/** + * A channel source with its monitor state resolved for engine consumption. + * + * Optional fields mirror the engine's DirectorSourceInput exactly: ABSENT, not + * null — the engine treats `lastMediaAtMs !== undefined` as "known", so a null + * would read as epoch 0 and de-confirm every source as no-media. JSON drops + * undefined keys, which keeps the wire form and the engine form identical. + */ +export interface DirectorChannelSource { + id: string; + priority: number; + enabled: boolean; + autoSelect: boolean; + /** Debounced monitor state (LIVE only once confirmed). */ + state: + | "OFFLINE" | "UNKNOWN" | "PROBING" | "STARTING" | "LIVE" | "UNSTABLE" | "ENDING" | "ERROR"; + /** Monitor row version; absent when no monitor row exists (every read is then a fresh sample). */ + version?: number; + /** `source_monitor_states.state_entered_at` — when the current state was entered. */ + liveSinceMs?: number; + /** Last media-health "media present" instant, for the no-media timeout. */ + lastMediaAtMs?: number; + /** Media-health tier reports failure: de-confirms immediately (§5.2 step 2). */ + mediaFailed?: boolean; +} + +/** Result of a DirectorStatePort.read: the engine's full input for one channel. */ +export interface DirectorChannelState { + kind: "current"; + desiredVersion: number; + authzEpoch: number; + generation: string; + sources: DirectorChannelSource[]; + rules: AutoDirectorRule[]; + policy: SwitchPolicy; + fallbackSceneId: string | null; + override: DirectorOverride | null; + principalPermissions: Record; +} + +export type DirectorStateRead = DirectorChannelState | { kind: "superseded"; code: "SUPERSEDED" }; + +/** The read port: everything the evaluator needs for one decision. */ +export interface DirectorStatePort { + read(ref: ChannelRef): Promise; +} + +/** The desired-state port contract, transport-agnostic twin of DesiredStatePort. */ +export interface DesiredStateOperations { + fence(ref: ChannelRef): Promise; + /** The engine's decision applied as a desired-state write; the director's commandId is the idempotency reference. */ + apply(command: { + ref: ChannelRef; + commandId: string; + expectedDesiredVersion: number; + authzEpoch: number; + program: DirectorProgram; + layout: SceneLayout; + reasons: string[]; + releaseOverride?: string; + }): Promise<{ kind: "applied" | "duplicate"; desiredVersion: number } | { kind: "conflict" }>; + audit(ref: ChannelRef, event: { commandId: string; outcome: "SUPERSEDED" | "AUTHZ_REVOKED"; reason: string }): Promise; +} + +/** An HTTP implementation of the API's /director endpoints (see api-side director.service.ts). */ +export interface DirectorApi { + channelState(ref: ChannelRef): Promise; + fence(ref: ChannelRef): Promise; + applyProgramSwitch(ref: ChannelRef, body: DirectorApplyRequest): Promise; + audit(ref: ChannelRef, event: { commandId: string; outcome: string; reason: string }): Promise; +} + +export interface DirectorApplyRequest { + commandId: string; + expectedDesiredVersion: number; + authzEpoch: number; + program: DirectorProgram; + layout: SceneLayout; + reasons: string[]; + releaseOverride?: string; +} + +export type DirectorApplyResult = + | { kind: "applied" | "duplicate"; desiredVersion: number } + | { kind: "conflict" }; diff --git a/apps/scheduler/src/director/starter.ts b/apps/scheduler/src/director/starter.ts new file mode 100644 index 0000000..f4cbf3b --- /dev/null +++ b/apps/scheduler/src/director/starter.ts @@ -0,0 +1,33 @@ +import { WorkflowExecutionAlreadyStartedError, type Client } from "@temporalio/client"; +import { WorkflowIdReusePolicy } from "@temporalio/common"; +import type { ChannelRef } from "./director.activity.js"; + +export function evaluatorWorkflowId(ref: Pick): string { + return `auto-director-${ref.workspaceId}-${ref.channelId}`; +} + +export type EnsureOutcome = "started" | "running" | "restarted"; + +/** + * Starts are idempotent hints. A running evaluator is only nudged (it re-reads + * state and fences its own generation); a closed one may be started afresh. + */ +export async function ensureDirector(client: Client, ref: ChannelRef, taskQueue: string): Promise { + const workflowId = evaluatorWorkflowId(ref); + const options = { taskQueue, workflowId, args: [{ ref }] as const, workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE }; + try { + await client.workflow.start("autoDirectorEvaluator", options); + return "started"; + } catch (error) { + if (!(error instanceof WorkflowExecutionAlreadyStartedError)) throw error; + const handle = client.workflow.getHandle(workflowId); + const description = await handle.describe().catch(() => undefined); + if (description?.status.name === "RUNNING") { + await handle.signal("sourceChanged", { sourceId: "*", version: 0 }); + return "running"; + } + // Closed between start and describe: one retry; a concurrent starter winning is fine. + try { await client.workflow.start("autoDirectorEvaluator", options); return "restarted"; } + catch (retryError) { if (retryError instanceof WorkflowExecutionAlreadyStartedError) return "running"; throw retryError; } + } +} diff --git a/apps/scheduler/src/director/wiring.test.ts b/apps/scheduler/src/director/wiring.test.ts new file mode 100644 index 0000000..b0d7b08 --- /dev/null +++ b/apps/scheduler/src/director/wiring.test.ts @@ -0,0 +1,223 @@ +import { describe, expect, it } from "vitest"; +import { parseDirectorChannels } from "./config.js"; +import { currentChannels, sweepDirectors, directorChannelsFromEnv } from "./discovery.js"; +import { DirectorHttpClient, parseApplyResult, parseFence } from "./http.client.js"; +import { HttpDirectorStatePort, HttpDesiredStatePort, engineSource, engineSnapshot } from "./adapters.js"; +import type { DirectorApi, DirectorChannelSource, DirectorStateRead } from "./ports.js"; +import type { ChannelRef } from "./director.activity.js"; +import { decide } from "./engine.js"; + +/** + * WP10 wiring slice, scheduler side: config parsing, the HTTP port adapters + * against a fake API, and the evaluator's discovery sweep. The port-pair + * behaviour (state read -> decide -> desired applied) lives in + * apps/api/src/director/director.service.test.ts, which imports this + * package's engine directly. + */ + +const WS = "0b7c6d50-93a4-4a1e-8e94-93b7e0000001"; +const CH = "0b7c6d50-93a4-4a1e-8e94-93b7e0000002"; +const SRC = "0b7c6d50-93a4-4a1e-8e94-93b7e0000003"; +const RULE = "0b7c6d50-93a4-4a1e-8e94-93b7e0000004"; + +const rule = { + id: RULE, virtualChannelId: CH, priority: 1, + condition: { field: "live_source_count" as const, op: ">=" as const, value: 1 }, + action: { layout: "single" as const, selection: "top_priority" as const }, +}; +const policy = { name: "highest_priority" as const, liveConfirmations: 1, offlineConfirmations: 1, noMediaSeconds: 600, minHoldMs: 0, switchCooldownMs: 0 }; +const document = JSON.stringify([{ workspaceId: WS, channelId: CH, generation: "g1", rules: [rule], policy, fallbackSceneId: null }]); + +describe("parseDirectorChannels", () => { + it("parses the full document", () => { + const channels = parseDirectorChannels(document); + expect(channels).toHaveLength(1); + expect(channels[0]).toMatchObject({ workspaceId: WS, channelId: CH, generation: "g1", fallbackSceneId: null }); + expect(channels[0]?.rules).toEqual([rule]); + expect(channels[0]?.policy).toEqual(policy); + }); + it("rejects malformed entries with a field path", () => { + expect(parseDirectorChannels(undefined)).toEqual([]); + expect(parseDirectorChannels("")).toEqual([]); + expect(() => parseDirectorChannels("[]{}")).toThrow(); + expect(() => parseDirectorChannels('{"a":1}')).toThrow(/must be a JSON array/); + expect(() => parseDirectorChannels('[{"channelId":"x"}]')).toThrow(/channelId must be a UUID/); + expect(() => parseDirectorChannels(`[{"workspaceId":"${WS}","channelId":"x","generation":"g"}]`)).toThrow(/channelId must be a UUID/); + expect(() => parseDirectorChannels(`[{"workspaceId":"${WS}","channelId":"${CH}","generation":""}]`)).toThrow(/generation/); + expect(() => parseDirectorChannels(`[{"workspaceId":"${WS}","channelId":"${CH}","generation":"g","rules":[{"id":"bad"}]}]`)).toThrow(/rules\[0\]\.id/); + expect(() => parseDirectorChannels(`[{"workspaceId":"${WS}","channelId":"${CH}","generation":"g","policy":{"name":"nope"}}]`)).toThrow(/policy\.name/); + }); + it("reads from env and defaults to none", () => { + expect(directorChannelsFromEnv({})).toEqual([]); + expect(directorChannelsFromEnv({ DIRECTOR_CHANNELS: document })).toHaveLength(1); + }); +}); + +describe("discovery", () => { + it("keeps the latest declared entry per channel", () => { + const channels = parseDirectorChannels(`[${document.slice(1, -1)},${document.slice(1, -1).replace('"g1"', '"g2"')}]`); + expect(channels).toHaveLength(2); + expect(currentChannels(channels)).toHaveLength(1); + expect(currentChannels(channels)[0]?.generation).toBe("g2"); + }); + it("ensures one evaluator per configured channel and swallows start failures", async () => { + const channels = parseDirectorChannels(document); + const workflowIds: string[] = []; + const clientFor = (behaviour: () => Promise<"started" | "running" | "restarted" | "failed">): Parameters[0] => + ({ + workflow: { + start: async (_name: string, options: { workflowId: string }) => { + workflowIds.push(options.workflowId); + return behaviour(); + }, + getHandle: () => ({ describe: async () => ({ status: { name: "RUNNING" } }), signal: async () => undefined }), + }, + }) as unknown as Parameters[0]; + // start succeeds: one evaluator ensured + const started = await sweepDirectors(clientFor(async () => "started"), { taskQueue: "tmbh-publishing", channels }); + expect(started).toEqual({ started: 1, running: 0, restarted: 0 }); + // every Temporal failure is contained: the sweep reports zero and lives on + const failing = await sweepDirectors(clientFor(async () => { throw new Error("temporal down"); }), { taskQueue: "q", channels }); + expect(failing).toEqual({ started: 0, running: 0, restarted: 0 }); + expect(workflowIds).toEqual([`auto-director-${WS}-${CH}`, `auto-director-${WS}-${CH}`]); + }); +}); + +/** In-memory DirectorApi double used by the adapter tests. */ +function fakeApi(over: Partial = {}): DirectorApi & { requests: string[] } { + const requests: string[] = []; + const current: DirectorStateRead = { + kind: "current", desiredVersion: 4, authzEpoch: 7, generation: "g1", + sources: [{ id: SRC, priority: 50, enabled: true, autoSelect: true, state: "LIVE", version: 3 }], + rules: [rule], policy, fallbackSceneId: null, override: null, principalPermissions: {}, + }; + return { + requests, + channelState: async (ref) => { requests.push(`state:${ref.channelId}`); return current; }, + fence: async (ref) => { requests.push(`fence:${ref.channelId}`); return { generation: "g1", desiredVersion: 4, authzEpoch: 7, directorAuthorized: true }; }, + applyProgramSwitch: async (ref, body) => { requests.push(`apply:${body.commandId}`); return { kind: "applied", desiredVersion: 5 }; }, + audit: async (ref, event) => { requests.push(`audit:${event.outcome}`); }, + ...over, + }; +} + +describe("port adapters", () => { + const ref: ChannelRef = { workspaceId: WS, channelId: CH, generation: "g1" }; + + it("converts transport sources without inventing optional engine fields", () => { + const source: DirectorChannelSource = { id: SRC, priority: 50, enabled: true, autoSelect: true, state: "LIVE" }; + const input = engineSource(source); + expect(input).toEqual({ id: SRC, priority: 50, enabled: true, autoSelect: true, state: "LIVE" }); + expect("version" in input).toBe(false); + expect("lastMediaAtMs" in input).toBe(false); + expect("mediaFailed" in input).toBe(false); + }); + + it("state read feeds decide() unchanged (engine input round-trip)", async () => { + const api = fakeApi(); + const port = new HttpDirectorStatePort(api); + const snapshot = await port.read(ref); + if (snapshot.kind !== "current") throw new Error("expected current"); + expect(snapshot.sources[0]).toEqual({ id: SRC, priority: 50, enabled: true, autoSelect: true, state: "LIVE", version: 3 }); + const decision = decide({ sources: snapshot.sources, rules: snapshot.rules, policy: snapshot.policy, now: 1_000 }); + expect(decision.program).toEqual({ kind: "sources", sourceIds: [SRC] }); + }); + + it("engineSnapshot maps a transport read to the engine snapshot", () => { + const read = { kind: "current" as const, desiredVersion: 4, authzEpoch: 7, generation: "g1", + sources: [{ id: SRC, priority: 50, enabled: true, autoSelect: true, state: "LIVE", version: 3 } as DirectorChannelSource], + rules: [rule], policy, fallbackSceneId: null, override: null, principalPermissions: {} }; + expect(engineSnapshot(read)).toMatchObject({ kind: "current", desiredVersion: 4, authzEpoch: 7 }); + }); + + it("desired-state port forwards fence/apply/audit with the commandId as idempotency key", async () => { + const api = fakeApi(); + const port = new HttpDesiredStatePort(api); + expect(await port.fence(ref)).toMatchObject({ generation: "g1", desiredVersion: 4 }); + expect(await port.findCommand(ref, "cmd")).toBeUndefined(); + const outcome = await port.writeProgram({ + ref, commandId: `${CH}:1`, expectedDesiredVersion: 4, authzEpoch: 7, + program: { kind: "sources", sourceIds: [SRC] }, layout: "single", reasons: ["rule"], + }); + expect(outcome).toEqual({ kind: "applied", desiredVersion: 5 }); + await port.audit(ref, { commandId: "c", outcome: "SUPERSEDED", reason: "generation" }); + expect(api.requests).toEqual([`fence:${CH}`, `apply:${CH}:1`, "audit:SUPERSEDED"]); + }); +}); + +describe("response parsers (fail closed on malformed API replies)", () => { + it("parseFence accepts a fence or absence and rejects junk", () => { + expect(parseFence(null)).toBeUndefined(); + expect(parseFence({ generation: "g1", desiredVersion: 4, authzEpoch: 7, directorAuthorized: true })).toEqual({ generation: "g1", desiredVersion: 4, authzEpoch: 7, directorAuthorized: true }); + expect(() => parseFence({})).toThrow(/generation/); + expect(() => parseFence({ generation: "g1", desiredVersion: "4", authzEpoch: 7, directorAuthorized: true })).toThrow(/desiredVersion/); + expect(() => parseFence({ generation: "g1", desiredVersion: 4, authzEpoch: 7, directorAuthorized: "yes" })).toThrow(/directorAuthorized/); + }); + it("parseApplyResult accepts applied/duplicate/conflict and rejects junk", () => { + expect(parseApplyResult({ kind: "applied", desiredVersion: 5 })).toEqual({ kind: "applied", desiredVersion: 5 }); + expect(parseApplyResult({ kind: "duplicate", desiredVersion: 5 })).toEqual({ kind: "duplicate", desiredVersion: 5 }); + expect(parseApplyResult({ kind: "conflict" })).toEqual({ kind: "conflict" }); + expect(() => parseApplyResult({ kind: "applied" })).toThrow(/desiredVersion/); + expect(() => parseApplyResult({ kind: "nope" })).toThrow(/kind/); + expect(() => parseApplyResult("x")).toThrow(); + }); +}); + +describe("DirectorHttpClient", () => { + const ref: ChannelRef = { workspaceId: WS, channelId: CH, generation: "g1" }; + + function harness(): { client: DirectorHttpClient; seen: Array<{ path: string; init: RequestInit | undefined }> } { + const seen: Array<{ path: string; init: RequestInit | undefined }> = []; + const client = new DirectorHttpClient({ + baseUrl: "http://api.test/", + token: "sekrit", + fetch: async (input, init) => { + seen.push({ path: String(input), init }); + const path = String(input); + if (path.endsWith("/state")) return new Response(JSON.stringify({ kind: "superseded", code: "SUPERSEDED" }), { status: 200 }); + if (path.endsWith("/fence")) return new Response(JSON.stringify({ generation: "g1", desiredVersion: 4, authzEpoch: 7, directorAuthorized: true }), { status: 200 }); + if (path.endsWith("/apply")) return new Response(JSON.stringify({ kind: "applied", desiredVersion: 5 }), { status: 200 }); + return new Response(null, { status: 204 }); + }, + }); + return { client, seen }; + } + + it("routes to the internal endpoints with the bearer token and validates replies", async () => { + const { client, seen } = harness(); + expect(await client.channelState(ref)).toEqual({ kind: "superseded", code: "SUPERSEDED" }); + expect(await client.fence(ref)).toMatchObject({ generation: "g1" }); + expect(await client.applyProgramSwitch(ref, { + commandId: "c1", expectedDesiredVersion: 4, authzEpoch: 7, + program: { kind: "sources", sourceIds: [SRC] }, layout: "single", reasons: ["r"], + })).toEqual({ kind: "applied", desiredVersion: 5 }); + await client.audit(ref, { commandId: "c1", outcome: "SUPERSEDED", reason: "gen" }); + expect(seen.map((entry) => entry.path)).toEqual([ + `http://api.test/internal/director/channels/${WS}/${CH}/state`, + `http://api.test/internal/director/channels/${WS}/${CH}/fence`, + `http://api.test/internal/director/channels/${WS}/${CH}/apply`, + `http://api.test/internal/director/channels/${WS}/${CH}/audit`, + ]); + for (const entry of seen) { + expect((entry.init?.headers as Record).authorization).toBe("Bearer sekrit"); + expect((entry.init?.headers as Record)["content-type"]).toBe("application/json"); + } + }); + + it("surfaces non-2xx as transport errors (retryable by Temporal)", async () => { + const client = new DirectorHttpClient({ + baseUrl: "http://api.test", fetch: async () => new Response("nope", { status: 503, statusText: "Service Unavailable" }), + }); + await expect(client.channelState(ref)).rejects.toThrow(/503/); + }); + + it("aborts a slow API within the timeout budget", async () => { + const client = new DirectorHttpClient({ + baseUrl: "http://api.test", timeoutMs: 20, + fetch: async (_input, init) => new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => reject(new Error("aborted"))); + }), + }); + await expect(client.fence(ref)).rejects.toThrow(); + }); +}); diff --git a/apps/scheduler/src/publishing.integration.test.ts b/apps/scheduler/src/publishing.integration.test.ts new file mode 100644 index 0000000..d454e3c --- /dev/null +++ b/apps/scheduler/src/publishing.integration.test.ts @@ -0,0 +1,80 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { Pool } from "pg"; +import { ConnectorRegistry } from "@tactical-media-broadcast-hub/connector-sdk"; +import { createMastodonConnector } from "@tactical-media-broadcast-hub/connector-mastodon"; +import { createTelegramConnector } from "@tactical-media-broadcast-hub/connector-telegram"; +import { Vault, createPublishingCredentialAccessor } from "@tactical-media-broadcast-hub/connector-runtime"; +import { PublishingExecutionRepository, canonicalPostHash, assertPublishingSchema, type AttemptRef } from "@tactical-media-broadcast-hub/publishing-execution"; +import { PublishActivity } from "./activities/publish.activity.js"; +const appUrl = process.env.PUBLISHING_IT_DATABASE_URL; +const ownerUrl = process.env.PUBLISHING_IT_SUPER_DATABASE_URL; +describe.skipIf(!appUrl || !ownerUrl)("TEST MODE real connectors with fenced DB activity", () => { + let pool: Pool; let owner: Pool; let repository: PublishingExecutionRepository; let workspaceId: string; let userId: string; + beforeAll(async () => { + pool = new Pool({ connectionString: appUrl, max: 12 }); owner = new Pool({ connectionString: ownerUrl }); + await assertPublishingSchema(pool); repository = new PublishingExecutionRepository(pool); + workspaceId = String((await owner.query("INSERT INTO workspaces(name,slug) VALUES('Scheduler Test',$1) RETURNING id", [`scheduler-sol-${Date.now()}`])).rows[0].id); + userId = String((await owner.query("INSERT INTO users(external_id,username,display_name) VALUES($1,$1,'Scheduler Test') RETURNING id", [`scheduler-sol-${Date.now()}`])).rows[0].id); + await owner.query("INSERT INTO memberships(workspace_id,user_id,role,status) VALUES($1,$2,'Owner','active')", [workspaceId, userId]); + }); + afterAll(async () => { + if (workspaceId) await owner.query("DELETE FROM workspaces WHERE id=$1", [workspaceId]); + if (userId) await owner.query("DELETE FROM users WHERE id=$1", [userId]); + await pool?.end(); await owner?.end(); + }); + async function fixture(provider: "mastodon" | "telegram"): Promise { + const fields = provider === "mastodon" ? { instanceUrl: "https://social.test" } : { channelId: "-10042" }; + const connectionId = String((await owner.query("INSERT INTO connections(workspace_id,provider,channel_id,fields) VALUES($1,$2,$3,$4) RETURNING id", [workspaceId, provider, `channel-${provider}`, fields])).rows[0].id); + const sealed = await new Vault().seal(workspaceId, connectionId, Buffer.from(JSON.stringify(provider === "mastodon" ? { access_token: "canary-mastodon-secret" } : { botToken: "canary-telegram-secret" }))); + await owner.query("INSERT INTO credentials(workspace_id,connection_id,kind,ciphertext,key_version) VALUES($1,$2,'connector-oauth',$3,$4)", [workspaceId, connectionId, sealed.ciphertext, sealed.keyVersion]); + const post = { text: "[TMBH TEST MODE] hello", mediaAssetIds: [], visibility: "public", providerMetadata: { connectionId } }; + const draftId = String((await owner.query("INSERT INTO drafts(workspace_id,author_user_id,post,state,reviewer_user_id,approved_revision,approved_hash,approved_at) VALUES($1,$2,$3,'approved',$2,1,$4,now()) RETURNING id", [workspaceId, userId, post, canonicalPostHash(post)])).rows[0].id); + const client = await pool.connect(); + try { + await client.query("BEGIN"); await client.query("SELECT set_config('app.workspace_id',$1,true)", [workspaceId]); + const attempt = await repository.scheduleAttempt(client, { workspaceId, draftId, connectionId, scheduledByUserId: userId, requiredPermission: "publication.publish", publishAt: new Date(Date.now() - 1000) }); + await client.query("COMMIT"); return { workspaceId, scheduleId: attempt.scheduleId, generation: attempt.generation, idempotencyKey: attempt.idempotencyKey }; + } catch (error) { await client.query("ROLLBACK"); throw error; } finally { client.release(); } + } + for (const provider of ["mastodon", "telegram"] as const) it(`publishes ${provider} through exact APV1 credentials, gate, receipt and one canonical Publication`, async () => { + const ref = await fixture(provider); let sends = 0; let observedKey: string | null = null; + const fetchImpl: typeof fetch = async (input, init) => { + const url = new URL(String(input)); + if (url.pathname.endsWith("/getChat")) return new Response(JSON.stringify({ ok: true, result: { id: -10042, type: "channel" } }), { headers: { "content-type": "application/json" } }); + sends += 1; + expect(init?.method).toBe("POST"); + if (provider === "mastodon") { + expect(url.pathname).toBe("/api/v1/statuses"); expect(new Headers(init?.headers).get("authorization")).toBe("Bearer canary-mastodon-secret"); + observedKey = new Headers(init?.headers).get("idempotency-key"); + return new Response(JSON.stringify({ id: "10942", created_at: new Date().toISOString(), url: "https://social.test/@test/10942", visibility: "public" }), { headers: { "content-type": "application/json" } }); + } + expect(url.pathname).toBe("/botcanary-telegram-secret/sendMessage"); + expect(JSON.parse(String(init?.body))).toMatchObject({ chat_id: "-10042" }); + return new Response(JSON.stringify({ ok: true, result: { message_id: 42, date: Math.floor(Date.now()/1000), chat: { id: -10042 } } }), { headers: { "content-type": "application/json" } }); + }; + const registry = new ConnectorRegistry(); const options = { credentials: createPublishingCredentialAccessor(pool), fetchImpl, requestsPerSecond: 1000 }; + registry.register(createMastodonConnector(options)); registry.register(createTelegramConnector(options)); + const activity = new PublishActivity(repository, registry, () => ({ heartbeat: () => undefined })); + const first = await activity.publishScheduled(ref); const second = await activity.publishScheduled(ref); + expect(first.kind).toBe("done"); expect(second).toEqual(first); expect(sends).toBe(1); + if (provider === "mastodon") expect(observedKey).toBe(ref.idempotencyKey); + const evidence = await owner.query("SELECT p.provider_metadata, a.dispatch_count, (SELECT count(*)::int FROM publications WHERE idempotency_key=$1) AS publications, (SELECT count(*)::int FROM publishing_receipts WHERE attempt_id=a.id) AS receipts FROM publishing_attempts a JOIN publications p ON p.idempotency_key=a.idempotency_key WHERE a.idempotency_key=$1", [ref.idempotencyKey]); + expect(evidence.rows[0]).toMatchObject({ dispatch_count: 1, publications: 1, receipts: 1 }); + expect(JSON.stringify({ first, second, evidence: evidence.rows })).not.toContain("canary-"); + }); + it("retains accepted-response-loss as unknown with no Telegram resend", async () => { + const ref = await fixture("telegram"); let sends = 0; + const fetchImpl: typeof fetch = async (input) => { + const url = new URL(String(input)); + if (url.pathname.endsWith("/getChat")) return new Response(JSON.stringify({ ok: true, result: { id: -10042, type: "channel" } })); + expect(url.pathname).toBe("/botcanary-telegram-secret/sendMessage"); sends += 1; + throw new Error("TEST_RESPONSE_LOST_AFTER_ACCEPT"); + }; + const registry = new ConnectorRegistry(); registry.register(createTelegramConnector({ credentials: createPublishingCredentialAccessor(pool), fetchImpl, requestsPerSecond: 1000 })); + const activity = new PublishActivity(repository, registry, () => ({ heartbeat: () => undefined })); + await expect(activity.publishScheduled(ref)).resolves.toMatchObject({ kind: "unknown", code: "PUBLISH_OUTCOME_UNKNOWN" }); + await expect(activity.publishScheduled(ref)).resolves.toMatchObject({ kind: "unknown" }); + expect(sends).toBe(1); + expect((await owner.query("SELECT count(*)::int AS n FROM publications WHERE idempotency_key=$1", [ref.idempotencyKey])).rows[0]?.n).toBe(0); + }); +}); diff --git a/apps/scheduler/src/worker.test.ts b/apps/scheduler/src/worker.test.ts new file mode 100644 index 0000000..6ece4b7 --- /dev/null +++ b/apps/scheduler/src/worker.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it, vi } from "vitest"; +import { Client, WorkflowExecutionAlreadyStartedError } from "@temporalio/client"; +import { WorkflowIdReusePolicy } from "@temporalio/common"; +import { sweepDue } from "./worker.js"; +const due = { workspaceId: "workspace", scheduleId: "schedule", generation: "1", idempotencyKey: "key", publishAt: new Date(), evidenceOnly: false }; +describe("durable discovery", () => { + it("passes only fenced identifiers and uses a separate receipt recovery workflow", async () => { + const start = vi.fn(async () => ({})); + await sweepDue({ listDue: async () => [{ ...due, evidenceOnly: true }] }, { workflow: { start } } as unknown as Client); + expect(start).toHaveBeenCalledWith("scheduledPublish", expect.objectContaining({ workflowId: "scheduled-publication-key-receipt", args: [{ workspaceId: "workspace", scheduleId: "schedule", generation: "1", idempotencyKey: "key" }] })); + }); + for (const status of ["COMPLETED", "RUNNING", "FAILED"]) it(`describes ${status} before allowing recovery`, async () => { + const start = vi.fn().mockRejectedValueOnce(new WorkflowExecutionAlreadyStartedError("existing", "scheduled-publication-key", "scheduledPublish")).mockResolvedValue({}); + const describeWorkflow = vi.fn(async () => ({ status: { name: status } })); + await sweepDue({ listDue: async () => [due] }, { workflow: { start, getHandle: () => ({ describe: describeWorkflow }) } } as unknown as Client); + expect(describeWorkflow).toHaveBeenCalledTimes(1); + expect(start).toHaveBeenCalledTimes(status === "FAILED" ? 2 : 1); + if (status === "FAILED") expect(start.mock.calls[1]?.[1]).toMatchObject({ workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY }); + }); +}); diff --git a/apps/scheduler/src/worker.ts b/apps/scheduler/src/worker.ts new file mode 100644 index 0000000..77e2512 --- /dev/null +++ b/apps/scheduler/src/worker.ts @@ -0,0 +1,106 @@ +import { Client, Connection, WorkflowExecutionAlreadyStartedError } from "@temporalio/client"; +import { WorkflowIdReusePolicy } from "@temporalio/common"; +import { NativeConnection, Worker } from "@temporalio/worker"; +import { Pool } from "pg"; +import { createServer } from "node:http"; +import { fileURLToPath } from "node:url"; +import { resolve } from "node:path"; +import pino from "pino"; +import { createPublishingRegistry } from "@tactical-media-broadcast-hub/connector-runtime"; +import { PublishingExecutionRepository, assertPublishingSchema, type AttemptRef } from "@tactical-media-broadcast-hub/publishing-execution"; +import { PublishActivity } from "./activities/publish.activity.js"; +import { DirectorActivity } from "./director/director.activity.js"; +import { DirectorHttpClient } from "./director/http.client.js"; +import { HttpDirectorStatePort, HttpDesiredStatePort } from "./director/adapters.js"; +import { directorChannelsFromEnv, sweepDirectors } from "./director/discovery.js"; +import { DispatchGateService } from "./activities/dispatch-gate.js"; + +const logger = pino({ level: process.env.LOG_LEVEL ?? "info" }); +const address = process.env.TEMPORAL_ADDRESS ?? "localhost:7233"; +const namespace = process.env.TEMPORAL_NAMESPACE ?? "default"; +const taskQueue = process.env.TEMPORAL_TASK_QUEUE ?? "tmbh-publishing"; + +/** Starts are repair hints. Only repository claim/gate can authorize a send. */ +export async function sweepDue(repository: Pick, client: Client): Promise { + const due = await repository.listDue(100); + for (let offset = 0; offset < due.length; offset += 4) { + await Promise.all(due.slice(offset, offset + 4).map(async (item) => { + const ref: AttemptRef = { workspaceId: item.workspaceId, scheduleId: item.scheduleId, generation: item.generation, idempotencyKey: item.idempotencyKey }; + const workflowId = `scheduled-publication-${ref.idempotencyKey}${item.evidenceOnly ? "-receipt" : ""}`; + try { + await client.workflow.start("scheduledPublish", { taskQueue, workflowId, args: [ref], workflowIdReusePolicy: WorkflowIdReusePolicy.REJECT_DUPLICATE }); + } catch (error) { + if (error instanceof WorkflowExecutionAlreadyStartedError) { + // Closed terminal workflows stay closed. A failed workflow can be + // repaired on the next sweep only while DB discovery still allows it. + const description = await client.workflow.getHandle(workflowId).describe().catch(() => undefined); + if (description?.status.name === "FAILED") { + try { await client.workflow.start("scheduledPublish", { taskQueue, workflowId, args: [ref], workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY }); } + catch (retryError) { if (!(retryError instanceof WorkflowExecutionAlreadyStartedError)) logger.warn({ code: "WORKFLOW_RECOVERY_FAILED", scheduleId: item.scheduleId }, "scheduler recovery failed"); } + } + return; + } + logger.warn({ code: "WORKFLOW_START_FAILED", scheduleId: item.scheduleId }, "scheduler sweep failed"); + } + })); + } +} +export async function main(): Promise { + if (!process.env.SCHEDULER_DATABASE_URL) throw new Error("SCHEDULER_DATABASE_URL is required"); + const pool = new Pool({ connectionString: process.env.SCHEDULER_DATABASE_URL, max: 20 }); + let native: NativeConnection | undefined; + let connection: Connection | undefined; + let worker: Worker | undefined; + let timer: ReturnType | undefined; + let sweeping: Promise | undefined; + let closing = false; + let ready = false; + const health = createServer((_request, response) => { response.writeHead(ready ? 200 : 503, { "content-type": "application/json" }); response.end(JSON.stringify({ ready })); }); + const stop = (): void => { closing = true; ready = false; clearTimeout(timer); if (worker?.getState() === "RUNNING") worker.shutdown(); }; + process.once("SIGTERM", stop); process.once("SIGINT", stop); + try { + await assertPublishingSchema(pool); + const repository = new PublishingExecutionRepository(pool); + const registry = createPublishingRegistry(pool); + native = await NativeConnection.connect({ address }); + connection = await Connection.connect({ address }); + const temporal = new Client({ connection, namespace }); + const activity = new PublishActivity(repository, registry, undefined, new DispatchGateService(pool)); + const directorApi = new DirectorHttpClient(); + const director = new DirectorActivity(new HttpDirectorStatePort(directorApi), new HttpDesiredStatePort(directorApi)); + const channels = directorChannelsFromEnv(); + worker = await Worker.create({ connection: native, namespace, taskQueue, + workflowsPath: fileURLToPath(new URL("./workflows/index.js", import.meta.url)), + activities: { + publishScheduled: activity.publishScheduled.bind(activity), + readChannelState: director.readChannelState.bind(director), + applyProgramSwitch: director.applyProgramSwitch.bind(director), + }, maxConcurrentActivityTaskExecutions: 4 }); + await new Promise((resolveListen, reject) => { health.once("error", reject); health.listen(Number(process.env.SCHEDULER_HEALTH_PORT ?? 8081), resolveListen); }); + const sweep = async (): Promise => { + if (closing) return; + try { + await assertPublishingSchema(pool); await connection?.workflowService.getSystemInfo({}); await sweepDue(repository, temporal); + const directors = await sweepDirectors(temporal, { taskQueue, channels }); + if (directors.started + directors.restarted > 0) logger.info(directors, "director evaluators ensured"); + ready = !closing; + } + catch { ready = false; logger.warn({ code: "DISCOVERY_UNAVAILABLE" }, "scheduler discovery unavailable"); } + if (!closing) timer = setTimeout(() => { sweeping = sweep(); }, 5000 + Math.floor(Math.random() * 500)); + }; + sweeping = sweep(); + await sweeping; + if (ready) logger.info({ namespace, taskQueue }, "scheduler sweep started"); + const running = worker.run(); + if (closing) stop(); + await running; + } finally { + stop(); await sweeping; + process.off("SIGTERM", stop); process.off("SIGINT", stop); + if (health.listening) await new Promise((done) => health.close(() => done())); + await connection?.close(); await native?.close(); await pool.end(); + } +} +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + void main().catch(() => { logger.error({ code: "SCHEDULER_STARTUP_FAILED" }, "scheduler failed readiness"); process.exitCode = 1; }); +} diff --git a/apps/scheduler/src/workflows/index.ts b/apps/scheduler/src/workflows/index.ts new file mode 100644 index 0000000..9026997 --- /dev/null +++ b/apps/scheduler/src/workflows/index.ts @@ -0,0 +1,8 @@ +/** + * Workflow entrypoints the scheduler worker registers. One file keeps the + * worker's workflow bundle small: publish.workflow.js stays the only publishing + * entrypoint, and the director evaluator joins it without a second Worker + * instance. Both are pure re-exports so bundling stays stable. + */ +export * from "./publish.workflow.js"; +export * from "../director/evaluator.workflow.js"; diff --git a/apps/scheduler/src/workflows/publish.workflow.ts b/apps/scheduler/src/workflows/publish.workflow.ts new file mode 100644 index 0000000..3c27b18 --- /dev/null +++ b/apps/scheduler/src/workflows/publish.workflow.ts @@ -0,0 +1,14 @@ +import { proxyActivities, sleep } from "@temporalio/workflow"; +import type { AttemptRef } from "@tactical-media-broadcast-hub/publishing-execution"; +import type { PublishActivity, ActivityOutcome } from "../activities/publish.activity.js"; +const { publishScheduled } = proxyActivities>({ + startToCloseTimeout: "60 seconds", heartbeatTimeout: "10 seconds", + retry: { initialInterval: "1 second", maximumInterval: "30 seconds", maximumAttempts: 5 }, +}); +export async function scheduledPublish(ref: AttemptRef): Promise { + for (;;) { + const outcome = await publishScheduled(ref); + if (outcome.kind !== "wait") return outcome; + await sleep(Math.max(1000, Date.parse(outcome.until) - Date.now())); + } +} diff --git a/apps/scheduler/tsconfig.json b/apps/scheduler/tsconfig.json new file mode 100644 index 0000000..bd8a252 --- /dev/null +++ b/apps/scheduler/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "rootDir": "src" + }, + "include": [ + "src" + ] +} diff --git a/apps/web/next-env.d.ts b/apps/web/next-env.d.ts new file mode 100644 index 0000000..40c3d68 --- /dev/null +++ b/apps/web/next-env.d.ts @@ -0,0 +1,5 @@ +/// +/// + +// NOTE: This file should not be edited +// see https://nextjs.org/docs/app/building-your-application/configuring/typescript for more information. diff --git a/apps/web/next.config.mjs b/apps/web/next.config.mjs new file mode 100644 index 0000000..fbb13fe --- /dev/null +++ b/apps/web/next.config.mjs @@ -0,0 +1,13 @@ +/** @type {import('next').NextConfig} */ +const nextConfig = { + reactStrictMode: true, + async redirects() { + return [{ source: "/", destination: "/streams", permanent: false }]; + }, + async rewrites() { + const api = (process.env.NEXT_PUBLIC_API_URL ?? "http://localhost:3000").replace(/\/$/, ""); + return [{ source: "/media-control/preview/:path*", destination: `${api}/media-control/preview/:path*` }]; + }, +}; + +export default nextConfig; diff --git a/apps/web/package.json b/apps/web/package.json new file mode 100644 index 0000000..0472028 --- /dev/null +++ b/apps/web/package.json @@ -0,0 +1,33 @@ +{ + "name": "@tactical-media-broadcast-hub/web", + "private": true, + "version": "0.1.0", + "scripts": { + "dev": "next dev", + "build": "next build", + "lint": "eslint src", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@tactical-media-broadcast-hub/connector-sdk": "workspace:^", + "@tailwindcss/postcss": "^4.3.3", + "@tanstack/react-virtual": "^3.14.13", + "@tiptap/extension-character-count": "^3.31.4", + "@tiptap/extension-placeholder": "^3.31.4", + "@tiptap/react": "^3.31.4", + "@tiptap/starter-kit": "^3.31.4", + "cmdk": "^1.1.1", + "next": "^14.2.5", + "postcss": "^8.5.29", + "react": "^18.3.1", + "react-dom": "^18.3.1", + "tailwindcss": "^4.3.3" + }, + "devDependencies": { + "@types/node": "^22.5.0", + "@types/react": "^18.3.3", + "@types/react-dom": "^18.3.0", + "playwright": "^1.63.0", + "typescript": "^5.5.4" + } +} diff --git a/apps/web/postcss.config.mjs b/apps/web/postcss.config.mjs new file mode 100644 index 0000000..a9b5c43 --- /dev/null +++ b/apps/web/postcss.config.mjs @@ -0,0 +1 @@ +export default { plugins: { "@tailwindcss/postcss": {} } }; diff --git a/apps/web/public/brand/empty-state.svg b/apps/web/public/brand/empty-state.svg new file mode 100644 index 0000000..6dc6b6f --- /dev/null +++ b/apps/web/public/brand/empty-state.svg @@ -0,0 +1,11 @@ + + + + + + + + + + \ No newline at end of file diff --git a/apps/web/public/brand/favicon.svg b/apps/web/public/brand/favicon.svg new file mode 100644 index 0000000..cef6a31 --- /dev/null +++ b/apps/web/public/brand/favicon.svg @@ -0,0 +1,7 @@ + + + + + + \ No newline at end of file diff --git a/apps/web/public/brand/hero.png b/apps/web/public/brand/hero.png new file mode 100644 index 0000000..521a713 Binary files /dev/null and b/apps/web/public/brand/hero.png differ diff --git a/apps/web/public/brand/icon-bluesky.svg b/apps/web/public/brand/icon-bluesky.svg new file mode 100644 index 0000000..7295511 --- /dev/null +++ b/apps/web/public/brand/icon-bluesky.svg @@ -0,0 +1,6 @@ + + Bluesky + + + \ No newline at end of file diff --git a/apps/web/public/brand/icon-discord.svg b/apps/web/public/brand/icon-discord.svg new file mode 100644 index 0000000..cf2d999 --- /dev/null +++ b/apps/web/public/brand/icon-discord.svg @@ -0,0 +1,6 @@ + + Discord + + + \ No newline at end of file diff --git a/apps/web/public/brand/icon-facebook.svg b/apps/web/public/brand/icon-facebook.svg new file mode 100644 index 0000000..2359872 --- /dev/null +++ b/apps/web/public/brand/icon-facebook.svg @@ -0,0 +1,6 @@ + + Facebook + + + \ No newline at end of file diff --git a/apps/web/public/brand/icon-instagram.svg b/apps/web/public/brand/icon-instagram.svg new file mode 100644 index 0000000..edfe6bd --- /dev/null +++ b/apps/web/public/brand/icon-instagram.svg @@ -0,0 +1,6 @@ + + Instagram + + + \ No newline at end of file diff --git a/apps/web/public/brand/icon-linkedin.svg b/apps/web/public/brand/icon-linkedin.svg new file mode 100644 index 0000000..7797174 --- /dev/null +++ b/apps/web/public/brand/icon-linkedin.svg @@ -0,0 +1,6 @@ + + Linkedin + + + \ No newline at end of file diff --git a/apps/web/public/brand/icon-mastodon.svg b/apps/web/public/brand/icon-mastodon.svg new file mode 100644 index 0000000..2d92c11 --- /dev/null +++ b/apps/web/public/brand/icon-mastodon.svg @@ -0,0 +1,6 @@ + + Mastodon + + + \ No newline at end of file diff --git a/apps/web/public/brand/icon-peertube.svg b/apps/web/public/brand/icon-peertube.svg new file mode 100644 index 0000000..787cdb0 --- /dev/null +++ b/apps/web/public/brand/icon-peertube.svg @@ -0,0 +1,6 @@ + + Peertube + + + \ No newline at end of file diff --git a/apps/web/public/brand/icon-reddit.svg b/apps/web/public/brand/icon-reddit.svg new file mode 100644 index 0000000..497ad38 --- /dev/null +++ b/apps/web/public/brand/icon-reddit.svg @@ -0,0 +1,6 @@ + + Reddit + + + \ No newline at end of file diff --git a/apps/web/public/brand/icon-slack.svg b/apps/web/public/brand/icon-slack.svg new file mode 100644 index 0000000..21dbdbc --- /dev/null +++ b/apps/web/public/brand/icon-slack.svg @@ -0,0 +1,6 @@ + + Slack + + + \ No newline at end of file diff --git a/apps/web/public/brand/icon-telegram.svg b/apps/web/public/brand/icon-telegram.svg new file mode 100644 index 0000000..2c04c3b --- /dev/null +++ b/apps/web/public/brand/icon-telegram.svg @@ -0,0 +1,6 @@ + + Telegram + + + \ No newline at end of file diff --git a/apps/web/public/brand/icon-threads.svg b/apps/web/public/brand/icon-threads.svg new file mode 100644 index 0000000..87aa902 --- /dev/null +++ b/apps/web/public/brand/icon-threads.svg @@ -0,0 +1,6 @@ + + Threads + + + \ No newline at end of file diff --git a/apps/web/public/brand/icon-twitch.svg b/apps/web/public/brand/icon-twitch.svg new file mode 100644 index 0000000..dd99360 --- /dev/null +++ b/apps/web/public/brand/icon-twitch.svg @@ -0,0 +1,6 @@ + + Twitch + + + \ No newline at end of file diff --git a/apps/web/public/brand/logo-new.svg b/apps/web/public/brand/logo-new.svg new file mode 100644 index 0000000..64d429b --- /dev/null +++ b/apps/web/public/brand/logo-new.svg @@ -0,0 +1,6 @@ + + Tactical Media Broadcast Hub — broadcast logo — circle with signal arrows + + + + \ No newline at end of file diff --git a/apps/web/public/brand/logo.svg b/apps/web/public/brand/logo.svg new file mode 100644 index 0000000..310c380 --- /dev/null +++ b/apps/web/public/brand/logo.svg @@ -0,0 +1,44 @@ + + Tactical Media Broadcast Hub — broadcast mark + Red anarchist arrow A rotated 17.5 degrees counterclockwise so its top arrow points upper-left. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/apps/web/public/brand/nav-analytics.svg b/apps/web/public/brand/nav-analytics.svg new file mode 100644 index 0000000..96c623d --- /dev/null +++ b/apps/web/public/brand/nav-analytics.svg @@ -0,0 +1,4 @@ + + Nav Analytics + + \ No newline at end of file diff --git a/apps/web/public/brand/nav-audit.svg b/apps/web/public/brand/nav-audit.svg new file mode 100644 index 0000000..82dd8ef --- /dev/null +++ b/apps/web/public/brand/nav-audit.svg @@ -0,0 +1,4 @@ + + Nav Audit + + \ No newline at end of file diff --git a/apps/web/public/brand/nav-calendar.svg b/apps/web/public/brand/nav-calendar.svg new file mode 100644 index 0000000..1398cca --- /dev/null +++ b/apps/web/public/brand/nav-calendar.svg @@ -0,0 +1,4 @@ + + Nav Calendar + + \ No newline at end of file diff --git a/apps/web/public/brand/nav-compose.svg b/apps/web/public/brand/nav-compose.svg new file mode 100644 index 0000000..eb9228c --- /dev/null +++ b/apps/web/public/brand/nav-compose.svg @@ -0,0 +1,4 @@ + + Nav Compose + + \ No newline at end of file diff --git a/apps/web/public/brand/nav-connections.svg b/apps/web/public/brand/nav-connections.svg new file mode 100644 index 0000000..a6f759c --- /dev/null +++ b/apps/web/public/brand/nav-connections.svg @@ -0,0 +1,4 @@ + + Nav Connections + + \ No newline at end of file diff --git a/apps/web/public/brand/nav-home.svg b/apps/web/public/brand/nav-home.svg new file mode 100644 index 0000000..c324549 --- /dev/null +++ b/apps/web/public/brand/nav-home.svg @@ -0,0 +1,4 @@ + + Nav Home + + \ No newline at end of file diff --git a/apps/web/public/brand/nav-inbox.svg b/apps/web/public/brand/nav-inbox.svg new file mode 100644 index 0000000..fb4dde6 --- /dev/null +++ b/apps/web/public/brand/nav-inbox.svg @@ -0,0 +1,4 @@ + + Nav Inbox + + \ No newline at end of file diff --git a/apps/web/public/brand/nav-more.svg b/apps/web/public/brand/nav-more.svg new file mode 100644 index 0000000..86d7e8e --- /dev/null +++ b/apps/web/public/brand/nav-more.svg @@ -0,0 +1,4 @@ + + Nav More + + \ No newline at end of file diff --git a/apps/web/public/brand/nav-search.svg b/apps/web/public/brand/nav-search.svg new file mode 100644 index 0000000..424d7fe --- /dev/null +++ b/apps/web/public/brand/nav-search.svg @@ -0,0 +1,4 @@ + + Nav Search + + \ No newline at end of file diff --git a/apps/web/public/brand/nav-settings.svg b/apps/web/public/brand/nav-settings.svg new file mode 100644 index 0000000..6ab306a --- /dev/null +++ b/apps/web/public/brand/nav-settings.svg @@ -0,0 +1,4 @@ + + Nav Settings + + \ No newline at end of file diff --git a/apps/web/public/brand/nav-team.svg b/apps/web/public/brand/nav-team.svg new file mode 100644 index 0000000..0a5fe68 --- /dev/null +++ b/apps/web/public/brand/nav-team.svg @@ -0,0 +1,4 @@ + + Nav Team + + \ No newline at end of file diff --git a/apps/web/public/brand/nav-video.svg b/apps/web/public/brand/nav-video.svg new file mode 100644 index 0000000..74e982a --- /dev/null +++ b/apps/web/public/brand/nav-video.svg @@ -0,0 +1,4 @@ + + Nav Video + + diff --git a/apps/web/public/brand/pattern.svg b/apps/web/public/brand/pattern.svg new file mode 100644 index 0000000..8a38621 --- /dev/null +++ b/apps/web/public/brand/pattern.svg @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/apps/web/public/brand/spinner.svg b/apps/web/public/brand/spinner.svg new file mode 100644 index 0000000..ca6de79 --- /dev/null +++ b/apps/web/public/brand/spinner.svg @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/apps/web/scripts/control-room-audit.cjs b/apps/web/scripts/control-room-audit.cjs new file mode 100644 index 0000000..bfe1567 --- /dev/null +++ b/apps/web/scripts/control-room-audit.cjs @@ -0,0 +1,152 @@ +const { chromium } = require("playwright"); +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const path = require("node:path"); +require("./control-room-register.cjs"); +const { fixtureChannel } = require("../src/components/control-room/fixtures.ts"); +const web = path.resolve(__dirname, ".."); +const origin = "http://audit.local"; +const workspaceId = "50000000-0000-4000-8000-000000000001"; +const mime = { ".js": "application/javascript", ".css": "text/css", ".svg": "image/svg+xml", ".png": "image/png", ".html": "text/html", ".rsc": "text/x-component" }; +const results = []; +const cors = { "Access-Control-Allow-Origin": origin, "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Headers": "Authorization,Content-Type,Idempotency-Key", "Access-Control-Allow-Methods": "GET,POST,OPTIONS" }; +const json = (route, body, status = 200) => route.fulfill({ status, headers: cors, json: body }); +async function auditViewport(browser, viewport) { + const context = await browser.newContext({ viewport, isMobile: viewport.width < 1024, hasTouch: viewport.width < 1024 }); + const page = await context.newPage(); + const errors = []; page.on("pageerror", error => errors.push(error.message)); + let channel = fixtureChannel(), mode = "live", seq = 0, socket; + const requests = [], keys = new Set(); + const observation = version => ({ workspaceId, channelId: channel.channelId, workerId: channel.workerId, assignmentEpoch: channel.assignmentEpoch, state: "RUNNING", realizedVersion: version, updatedAt: new Date().toISOString(), reason: "Audit worker" }); + channel.observed = observation(1); + const inject = (kind, data) => socket.send(JSON.stringify({ type: "state", workspaceId, channelId: channel.channelId, seq: ++seq, kind, data })); + await context.addInitScript(() => localStorage.setItem("tmbh.session", "audit-token")); + await page.routeWebSocket(/\/realtime$/, ws => { + socket = ws; + ws.onMessage(raw => { + const message = JSON.parse(String(raw)); + if (message.type === "auth") { assert.equal(message.token, "audit-token"); assert.equal(message.workspaceId, workspaceId); ws.send(JSON.stringify({ type: "auth_ack", workspaceId })); } + if (message.type === "subscribe") { + ws.send(JSON.stringify({ type: "subscribe_ack", channelId: channel.channelId, seq })); + inject("desired", channel.desired); inject("observed", channel.observed); + } + }); + }); + await page.route("**/*", async route => { + const request = route.request(), url = new URL(request.url()); + if (request.method() === "OPTIONS") return route.fulfill({ status: 204, headers: cors }); + if (url.pathname === "/auth/me") return json(route, { user: { id: workspaceId, email: "operator@audit.local", displayName: "Operator" }, workspace: { id: workspaceId, name: "Audit studio", slug: "audit" }, role: "owner", permissions: ["director.take", "director.view", "director.lock_program", "director.override_auto", "director.change_layout", "media.view", "stream.view"], expiresAt: Date.now() + 3600000 }); + if (url.pathname === "/media-control/status") { + if (mode === "offline") return route.abort(); + if (mode === "forbidden") return json(route, { message: "Missing media.view" }, 403); + return json(route, { channels: mode === "empty" ? [] : [channel], workers: [] }); + } + const action = /\/media-control\/channels\/[^/]+\/(pst-select|cut|take|lock|override)$/.exec(url.pathname)?.[1]; + if (action && request.method() === "POST" || url.pathname.startsWith("/production/overlays/") && request.method() === "POST") { + const key = request.headers()["idempotency-key"]; assert.match(key, /^[\x20-\x7e]{1,128}$/); assert.match(key, /^[0-9a-f-]{36}$/); assert.ok(!keys.has(key)); keys.add(key); + const body = request.postDataJSON(); requests.push({ action: action ?? "overlay", body, key }); + if (action === "pst-select") channel.desired.production.buses.preview = body.sceneId; + if (action === "cut" || action === "take") channel.desired.production.buses.program = channel.desired.production.buses.preview; + if (action === "lock") channel.desired.director.locked = body.enabled; + if (action === "override") channel.desired.director.override = body.mode ? { mode: body.mode, expiresAt: null } : null; + if (!action) channel.desired.production.overlays.find(item => url.pathname.endsWith(item.id)).revisionId = "60000000-0000-4000-8000-000000000001"; + channel.desired.desiredVersion += 1; + await json(route, { desiredVersion: channel.desired.desiredVersion, commandId: key }); + inject("desired", channel.desired); return; + } + if (url.pathname.includes("/preview/")) return json(route, { message: "No audit media" }, 404); + if (url.hostname !== "audit.local") return route.abort(); + let file; + if (url.pathname.startsWith("/_next/")) file = path.join(web, ".next", url.pathname.slice(7)); + else if (url.pathname.startsWith("/brand/")) file = path.join(web, "public", url.pathname); + else { + const name = url.pathname === "/" ? "index" : url.pathname.slice(1); + const rsc = request.headers().rsc === "1" || url.searchParams.has("_rsc"); + file = path.join(web, ".next/server/app", `${name}.${rsc ? "rsc" : "html"}`); + } + if (!fs.existsSync(file)) return route.fulfill({ status: 404, body: "Not found" }); + return route.fulfill({ contentType: mime[path.extname(file)] || "application/octet-stream", body: fs.readFileSync(file) }); + }); + const first = channel.desired.production.sources[0].id, second = channel.desired.production.sources[1].id; + const sceneB = channel.desired.production.scenes[1].id; + const tally = id => page.locator(`[data-source='${id}']`).getAttribute("data-tally"); + const waitTally = (id, value) => page.waitForFunction(({ id, value }) => document.querySelector(`[data-source='${id}']`)?.getAttribute("data-tally") === value, { id, value }); + const checkLayout = async stage => { + const layout = await page.evaluate(() => { + const root = document.querySelector(".control-room"); + const undersized = Array.from(root.querySelectorAll("button, input, select, textarea, a[href]")).filter(el => { const r = el.getBoundingClientRect(); return r.width && r.height && (r.width < 43.5 || r.height < 43.5); }).map(el => ({ text: el.getAttribute("aria-label") || el.textContent.trim().slice(0, 50), width: el.getBoundingClientRect().width, height: el.getBoundingClientRect().height })); + return { width: document.documentElement.clientWidth, scrollWidth: document.documentElement.scrollWidth, rootWidth: root.clientWidth, rootScrollWidth: root.scrollWidth, undersized, liveDecodes: document.querySelectorAll("[data-live-decode]").length }; + }); + assert.ok(layout.scrollWidth <= layout.width, `${stage}: document overflow`); + assert.ok(layout.rootScrollWidth <= layout.rootWidth, `${stage}: control room overflow`); + assert.deepEqual(layout.undersized, [], `${stage}: tap targets`); + assert.ok(layout.liveDecodes <= (viewport.width < 1024 ? 1 : 4)); + results.push({ viewport, stage, ...layout }); + }; + await page.goto(`${origin}/control-room`, { waitUntil: "networkidle" }); + await waitTally(first, "program"); await checkLayout("initial"); + await page.getByRole("combobox", { name: "PST scene", exact: true }).selectOption(sceneB); + await page.getByRole("status").filter({ hasText: "Request accepted" }).waitFor(); + assert.equal(await tally(first), "program"); assert.equal(await tally(second), "none"); + inject("observed", observation(channel.desired.desiredVersion)); + await waitTally(second, "preview"); + await page.getByRole("button", { name: "CUT", exact: true }).click(); + await page.waitForFunction(() => document.querySelector("[role='status']")?.textContent.includes("version 3")); + assert.equal(await tally(first), "program"); assert.equal(await tally(second), "preview"); + const injectedAt = Date.now(); + inject("observed", observation(3)); await waitTally(second, "program"); assert.equal(await tally(first), "none"); + const observedToTallyMs = Date.now() - injectedAt; + assert.ok(observedToTallyMs < 300, "observed frame to tally must take under 300 ms"); + await page.getByRole("button", { name: "TAKE", exact: true }).click(); + await page.getByRole("button", { name: "Lock program", exact: true }).click(); + await page.getByRole("button", { name: "Unlock program", exact: true }).waitFor(); + await page.getByRole("combobox", { name: "Operator override" }).selectOption("five_minutes"); + await page.getByRole("combobox", { name: "Operator override" }).selectOption("auto"); + for (const template of ["lower_third", "chyron", "logo"]) { + const slot = channel.desired.production.overlays.find(item => item.kind === template); + await page.getByRole("combobox", { name: "Overlay slot" }).selectOption(slot.id); + if (template === "logo") await page.getByRole("textbox", { name: "Logo SVG" }).fill(''); + else await page.getByRole("textbox", { name: template === "chyron" ? /Chyron text/ : /Title/ }).fill("Audit title"); + await page.getByRole("button", { name: "Render overlay" }).click(); + await page.getByRole("status").filter({ hasText: "Render accepted" }).waitFor(); + assert.equal(requests.at(-1).body.templateId, template); + } + for (const tier of ["2x1", "2x2", "3x3", "4x4", "custom"]) { + await page.getByRole("combobox", { name: "Multiview layout" }).selectOption(tier); + await checkLayout(tier); + } + if (viewport.width >= 1024) { + await page.getByRole("combobox", { name: "Multiview layout" }).selectOption("4x4"); + await page.getByRole("button", { name: "Focus Sample source 1", exact: true }).click(); + const upgrades = page.getByRole("button", { name: "Upgrade live", exact: true }); + await upgrades.nth(0).click(); await upgrades.nth(0).click(); + assert.ok(await upgrades.nth(0).isDisabled(), "5th decode must be refused"); + } + const resyncBefore = requests.length; + socket.send(JSON.stringify({ type: "RESYNC_REQUIRED", channelId: channel.channelId })); + await page.waitForTimeout(100); assert.equal(requests.length, resyncBefore); + mode = "empty"; await page.getByRole("button", { name: "Reconnect", exact: true }).click(); + await page.getByRole("heading", { name: "No production scenes configured" }).waitFor(); + assert.equal(await page.locator("[data-tally='program']").count(), 0); assert.equal(await page.getByText("FIXTURE · sample data", { exact: true }).count(), 0); + await checkLayout("empty"); + mode = "forbidden"; await page.getByRole("button", { name: "Reconnect", exact: true }).click(); + await page.getByRole("status").filter({ hasText: "Missing media.view" }).waitFor(); + assert.equal(await page.getByText("FIXTURE · sample data", { exact: true }).count(), 0); + mode = "offline"; await page.getByRole("button", { name: "Reconnect", exact: true }).click(); + await page.getByText("FIXTURE · sample data", { exact: true }).waitFor(); + assert.equal(await page.locator("[data-tally='program']").count(), 0); + const before = requests.length; + await page.getByRole("button", { name: "CUT", exact: true }).click(); + assert.equal(requests.length, before); assert.equal(await page.locator("[data-tally='program']").count(), 0); + await checkLayout("offline-fixture"); + assert.deepEqual(errors, [], "browser runtime errors"); + results.push({ viewport, flow: "PST→observed preview→CUT→observed program", actions: requests.map(item => item.action), uniqueKeys: keys.size, observedToTallyMs, runtimeErrors: errors }); + await context.close(); +} +async function run() { + const browser = await chromium.launch({ headless: true }); + try { for (const viewport of [{ width: 1440, height: 900 }, { width: 390, height: 844 }]) await auditViewport(browser, viewport); } + finally { await browser.close(); } + console.log(JSON.stringify(results, null, 2)); +} +run().catch(error => { console.error(error); process.exitCode = 1; }); diff --git a/apps/web/scripts/control-room-register.cjs b/apps/web/scripts/control-room-register.cjs new file mode 100644 index 0000000..d7768f5 --- /dev/null +++ b/apps/web/scripts/control-room-register.cjs @@ -0,0 +1,11 @@ +// Existing TypeScript compiler only; node:test remains the test runner. No runtime dependency. +const ts = require("typescript"); +const fs = require("node:fs"); +function compile(module, filename) { + const result = ts.transpileModule(fs.readFileSync(filename, "utf8"), { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022, jsx: ts.JsxEmit.ReactJSX, esModuleInterop: true }, fileName: filename, + }); + module._compile(result.outputText, filename); +} +require.extensions[".ts"] = compile; +require.extensions[".tsx"] = compile; diff --git a/apps/web/scripts/manual-shots.mjs b/apps/web/scripts/manual-shots.mjs new file mode 100644 index 0000000..9056cbc --- /dev/null +++ b/apps/web/scripts/manual-shots.mjs @@ -0,0 +1,85 @@ +import { chromium } from "playwright"; +import { mkdirSync } from "node:fs"; + +const BASE = "http://localhost:3111"; +const OUT = "runs/manual/shots"; +mkdirSync(OUT, { recursive: true }); + +const browser = await chromium.launch(); +const ctx = await browser.newContext({ viewport: { width: 1440, height: 900 }, deviceScaleFactor: 2 }); +const page = await ctx.newPage(); +const errors = []; +page.on("pageerror", (e) => errors.push(String(e).slice(0, 150))); + +async function shot(name) { + await page.waitForTimeout(500); + await page.screenshot({ path: `${OUT}/${name}.png` }); + console.log(`✓ ${name}`); +} + +async function safeClick(text, fallback) { + try { await page.getByRole("button", { name: text, timeout: 5000 }).click(); } + catch { try { await page.getByText(text, { timeout: 3000 }).first().click(); } + catch { if (fallback) fallback(); else console.log(` (skip: ${text})`); } } +} + +async function safeType(selector, text) { + try { const el = page.locator(selector).first(); await el.click({ timeout: 5000 }); await el.fill(text); } + catch { console.log(` (skip type: ${selector})`); } +} + +// ONBOARDING +await page.goto(`${BASE}/onboard`); await page.waitForTimeout(1500); +// redirect to /onboarding +await page.goto(`${BASE}/onboarding`); await page.waitForTimeout(1500); +await shot("onboard-01-start"); +safeType("input", "Global News Desk"); +await shot("onboard-02-named"); +safeClick("Continue"); +await shot("onboard-03-invite"); +safeType("textarea", "sarah@globalnews.io, marcus@globalnews.io"); +await shot("onboard-04-emails-filled"); +safeClick("Continue"); +await shot("onboard-05-connect"); + +// STREAMS +await page.goto(`${BASE}/streams`); await page.waitForTimeout(1000); +await shot("streams-01-overview"); +await page.keyboard.press("j"); await page.keyboard.press("j"); await page.keyboard.press("j"); +await shot("streams-02-item-selected"); +await page.keyboard.press("r"); +await shot("streams-03-repost-toast"); +await page.keyboard.press("q"); +await shot("streams-04-quote-popover"); +try { const ta = page.locator("textarea").first(); if (await ta.isVisible({ timeout: 3000 })) { await ta.fill("This analysis is essential reading for anyone tracking the Fed's next move."); } } catch {} +await shot("streams-05-quote-typed"); +await page.keyboard.press("Escape"); + +// COMPOSE +await page.goto(`${BASE}/compose`); await page.waitForTimeout(1000); +await shot("compose-01-blank"); +try { const ed = page.locator(".ProseMirror").first(); await ed.click({ timeout: 5000 }); await page.keyboard.type("Breaking: Central bank signals first rate cut of the year as inflation cools to 2.1%. Full analysis at globalnews.io/fed-rates #markets #economy"); } catch { console.log(" (compose editor not found)"); } +await shot("compose-02-typed"); + +// CONNECTIONS +await page.goto(`${BASE}/connections`); await page.waitForTimeout(800); +await shot("connections-01-overview"); + +// TEAM +await page.goto(`${BASE}/team`); await page.waitForTimeout(800); +await shot("team-01-overview"); + +// INBOX +await page.goto(`${BASE}/inbox`); await page.waitForTimeout(800); +await shot("inbox-01-overview"); + +// AUDIT +await page.goto(`${BASE}/audit`); await page.waitForTimeout(800); +await shot("audit-01-overview"); +try { await page.locator("tr").filter({ hasText: "credential" }).first().click({ timeout: 5000 }); } catch {} +await shot("audit-02-diff-open"); + +await browser.close(); +console.log(`\nShots: ${OUT}/`); +console.log(`Errors: ${errors.length}`); +for (const e of errors.slice(0, 5)) console.log(` ${e}`); diff --git a/apps/web/scripts/mobile-phase1-audit.cjs b/apps/web/scripts/mobile-phase1-audit.cjs new file mode 100644 index 0000000..5bf10c8 --- /dev/null +++ b/apps/web/scripts/mobile-phase1-audit.cjs @@ -0,0 +1,58 @@ +const { chromium } = require("playwright"); +const fs = require("node:fs"); +const path = require("node:path"); + +const origin = "http://audit.local"; +const mime = { ".js": "application/javascript", ".css": "text/css", ".svg": "image/svg+xml", ".png": "image/png", ".html": "text/html", ".rsc": "text/x-component", ".json": "application/json" }; + +const routes = ["/streams", "/compose", "/inbox", "/approvals", "/connections", "/team", "/video", "/audit", "/onboarding", "/analytics", "/calendar", "/settings", "/more", "/search"]; + +async function run() { + const browser = await chromium.launch({ headless: true }); + const page = await browser.newPage({ viewport: { width: 390, height: 844 }, deviceScaleFactor: 1, isMobile: true, hasTouch: true }); + await page.route("**/*", async (route) => { + const request = route.request(); + const url = new URL(request.url()); + if (url.hostname !== "audit.local") return route.abort(); + let file; + if (url.pathname.startsWith("/_next/")) file = path.join(process.cwd(), ".next", url.pathname.slice(7)); + else if (url.pathname.startsWith("/brand/")) file = path.join(process.cwd(), "public", url.pathname); + else { + const name = url.pathname === "/" ? "index" : url.pathname.slice(1); + const isRsc = request.headers().rsc === "1" || url.searchParams.has("_rsc"); + file = path.join(process.cwd(), ".next/server/app", `${name}.${isRsc ? "rsc" : "html"}`); + } + if (!fs.existsSync(file)) return route.fulfill({ status: 404, body: "Not found" }); + return route.fulfill({ status: 200, contentType: mime[path.extname(file)] || "application/octet-stream", body: fs.readFileSync(file) }); + }); + const results = []; + for (const route of routes) { + await page.goto(`${origin}${route}`, { waitUntil: "networkidle" }); + await page.waitForTimeout(200); + const result = await page.evaluate(() => { + const width = document.documentElement.clientWidth; + const all = Array.from(document.querySelectorAll("button, a[href], input, select, textarea, [role='tab']")); + const undersized = all.filter((el) => { + const rect = el.getBoundingClientRect(); + const style = getComputedStyle(el); + return rect.width > 0 && rect.height > 0 && rect.left < innerWidth && rect.right > 0 && rect.top < innerHeight && rect.bottom > 0 && style.visibility !== "hidden" && style.display !== "none" && (rect.width < 43.5 || rect.height < 43.5); + }).map((el) => ({ tag: el.tagName, text: (el.getAttribute("aria-label") || el.textContent || "").trim().slice(0, 40), width: Math.round(el.getBoundingClientRect().width), height: Math.round(el.getBoundingClientRect().height) })); + return { title: document.title, width, scrollWidth: document.documentElement.scrollWidth, bodyScrollWidth: document.body.scrollWidth, nav: !!document.querySelector("nav[aria-label='Primary navigation']"), undersized }; + }); + results.push({ route, ...result }); + } + await page.goto(`${origin}/more`, { waitUntil: "networkidle" }); + const scroller = page.locator("[data-mobile-main]"); + await scroller.evaluate((el) => { el.scrollTop = 300; }); + const before = await scroller.evaluate((el) => el.scrollTop); + await page.getByRole("navigation", { name: "Primary navigation" }).getByRole("link", { name: "Home" }).click(); + await page.waitForURL(`${origin}/streams`); + await page.getByRole("navigation", { name: "Primary navigation" }).getByRole("link", { name: "More" }).click(); + await page.waitForURL(`${origin}/more`); + await page.waitForTimeout(100); + const after = await scroller.evaluate((el) => el.scrollTop); + console.log(JSON.stringify({ routes: results, scrollPreservation: { before, after } }, null, 2)); + await browser.close(); + if (results.some((r) => r.scrollWidth > r.width || r.bodyScrollWidth > r.width || r.undersized.length > 0 || !r.nav) || before < 1 || Math.abs(before - after) > 1) process.exitCode = 1; +} +run().catch((error) => { console.error(error); process.exitCode = 1; }); diff --git a/apps/web/scripts/mobile-phase2-audit.cjs b/apps/web/scripts/mobile-phase2-audit.cjs new file mode 100644 index 0000000..2490e9a --- /dev/null +++ b/apps/web/scripts/mobile-phase2-audit.cjs @@ -0,0 +1,121 @@ +// The repo's Node 18 runtime can run this Chromium audit; Playwright 1.63's +// bootstrap refuses it before launch, so bypass only that version gate here. +if (Number(process.versions.node.split(".")[0]) < 20) Object.defineProperty(process.versions, "node", { value: "20.0.0" }); +const { chromium } = require("playwright"); +const fs = require("node:fs"); +const path = require("node:path"); + +const origin = "http://audit.local"; +const routes = ["/streams", "/search", "/inbox", "/inbox/1", "/approvals", "/connections", "/compose", "/audit"]; +const mime = { ".js": "application/javascript", ".css": "text/css", ".svg": "image/svg+xml", ".png": "image/png", ".html": "text/html", ".rsc": "text/x-component", ".json": "application/json" }; + +async function serveBuild(context) { + await context.route(`${origin}/**`, async (route) => { + const request = route.request(); + const url = new URL(request.url()); + let file; + if (url.pathname.startsWith("/_next/")) file = path.join(process.cwd(), ".next", url.pathname.slice(7)); + else if (url.pathname.startsWith("/brand/")) file = path.join(process.cwd(), "public", url.pathname); + else { + const name = url.pathname === "/" ? "index" : url.pathname.slice(1); + const isRsc = request.headers().rsc === "1" || url.searchParams.has("_rsc"); + file = path.join(process.cwd(), ".next/server/app", `${name}.${isRsc ? "rsc" : "html"}`); + } + if (!fs.existsSync(file)) return route.fulfill({ status: 404, body: "Not found" }); + return route.fulfill({ status: 200, contentType: mime[path.extname(file)] || "application/octet-stream", body: fs.readFileSync(file) }); + }); +} + +async function inspect(page) { + return page.evaluate(() => { + const width = document.documentElement.clientWidth; + const undersized = Array.from(document.querySelectorAll("button, a[href], input, select, textarea, [role='tab']")).filter((element) => { + const rect = element.getBoundingClientRect(); + const style = getComputedStyle(element); + return rect.width > 0 && rect.height > 0 && rect.left < innerWidth && rect.right > 0 && rect.top < innerHeight && rect.bottom > 0 && style.visibility !== "hidden" && style.display !== "none" && style.opacity !== "0" && !element.classList.contains("sr-only") && (rect.width < 43.5 || rect.height < 43.5); + }).map((element) => ({ text: (element.getAttribute("aria-label") || element.textContent || "").trim().slice(0, 36), width: Math.round(element.getBoundingClientRect().width), height: Math.round(element.getBoundingClientRect().height) })); + return { width, scrollWidth: document.documentElement.scrollWidth, bodyScrollWidth: document.body.scrollWidth, undersized }; + }); +} + +async function routeAudit(browser) { + const context = await browser.newContext({ viewport: { width: 390, height: 844 }, isMobile: true, hasTouch: true, deviceScaleFactor: 1 }); + await serveBuild(context); + await context.route("http://localhost:3000/**", (route) => route.abort()); + const page = await context.newPage(); + const results = []; + for (const route of routes) { + const parent = route.startsWith("/inbox/") || route === "/approvals" ? "/inbox" : "/more"; + await page.goto(`${origin}${parent}`, { waitUntil: "networkidle" }); + await page.goto(`${origin}${route}`, { waitUntil: "networkidle" }); + await page.waitForTimeout(150); + const sizes = await inspect(page); + const back = page.getByRole("button", { name: "Go back" }); + if (await back.count()) await back.click(); + else await page.goBack({ waitUntil: "domcontentloaded" }); + await page.waitForTimeout(100); + results.push({ route, ...sizes, backResolved: new URL(page.url()).pathname === parent }); + } + await page.goto(`${origin}/streams`, { waitUntil: "networkidle" }); + await page.getByRole("tab", { name: /Mastodon/ }).click(); + await page.getByRole("button", { name: /Amplify @/ }).first().click(); + const mastodonLabels = await page.getByRole("dialog").innerText(); + await page.getByRole("button", { name: "Close", exact: true }).click(); + await page.getByRole("tab", { name: /Telegram/ }).click(); + await page.getByRole("button", { name: /Amplify @/ }).first().click(); + const telegramLabels = await page.getByRole("dialog").innerText(); + await context.close(); + return { routes: results, capabilityLabels: { mastodonBoost: mastodonLabels.includes("Boost"), mastodonQuote: mastodonLabels.includes("Quote"), telegramForward: telegramLabels.includes("Forward"), telegramNoQuote: !telegramLabels.includes("Quote") } }; +} + +async function approvalAudit(browser) { + const context = await browser.newContext({ viewport: { width: 390, height: 844 }, isMobile: true, hasTouch: true }); + await serveBuild(context); + await context.addInitScript(() => localStorage.setItem("tmbh.session", "audit-token")); + const drafts = ["approve", "reject"].map((id) => ({ id, workspaceId: "w", authorUserId: "contributor-1", state: "in_review", createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), post: { text: `${id} this draft`, mediaAssetIds: [], visibility: "public", providerMetadata: { targetConnectionIds: [] } } })); + const audit = []; + const calls = []; + await context.route("http://localhost:3000/**", async (route) => { + const request = route.request(); + const url = new URL(request.url()); + const headers = { "access-control-allow-origin": origin, "access-control-allow-methods": "GET,POST,OPTIONS", "access-control-allow-headers": "authorization,content-type", "content-type": "application/json" }; + if (request.method() === "OPTIONS") return route.fulfill({ status: 204, headers, body: "" }); + let body; + if (url.pathname === "/auth/me") body = { user: { id: "editor-1", email: "editor@example.test", displayName: "Editor" }, workspace: { id: "w", name: "Test", slug: "test" }, role: "Editor", permissions: ["approval.grant", "draft.view"], expiresAt: Date.now() + 60000 }; + else if (url.pathname === "/publishing/drafts" && request.method() === "GET") body = drafts.filter((draft) => draft.state === "in_review"); + else if (url.pathname.match(/^\/publishing\/drafts\/(approve|reject)\/(approve|reject)$/) && request.method() === "POST") { + const [, id, decision] = url.pathname.match(/^\/publishing\/drafts\/(approve|reject)\/(approve|reject)$/); + const draft = drafts.find((item) => item.id === id); + draft.state = decision === "approve" ? "approved" : "rejected"; + calls.push({ id, decision, path: url.pathname }); + audit.unshift({ id: `audit-${audit.length}`, seq: String(audit.length + 1), at: new Date().toISOString(), actorUserId: "editor-1", actorEmail: "editor@example.test", actorRole: "Editor", action: `draft.${decision === "approve" ? "approved" : "rejected"}`, targetType: "draft", targetId: id, diff: null }); + body = draft; + } else if (url.pathname === "/audit") body = { entries: audit, nextCursor: null, facets: { actions: [...new Set(audit.map((item) => item.action))], actors: [{ id: "editor-1", email: "editor@example.test" }] } }; + else if (url.pathname === "/connections") body = []; + else body = []; + return route.fulfill({ status: 200, headers, body: JSON.stringify(body) }); + }); + const page = await context.newPage(); + await page.goto(`${origin}/approvals`, { waitUntil: "networkidle" }); + await page.getByRole("button", { name: "Approve", exact: true }).first().click(); + await page.getByRole("button", { name: "Reject", exact: true }).first().click(); + await page.getByRole("button", { name: "Needs edits" }).click(); + await page.getByRole("button", { name: "Reject with reason" }).click(); + await page.goto(`${origin}/audit`, { waitUntil: "networkidle" }); + const text = await page.locator("[data-mobile-main]").innerText(); + await context.close(); + return { calls, auditActions: audit.map((item) => item.action), visibleInAudit: text.includes("draft.approved") && text.includes("draft.rejected") }; +} + +async function main() { + let browser; + try { + browser = await chromium.launch({ headless: true }); + const routes = await routeAudit(browser); + const approvals = await approvalAudit(browser); + const result = { ...routes, approvals }; + console.log(JSON.stringify(result, null, 2)); + if (routes.routes.some((row) => row.scrollWidth > row.width || row.bodyScrollWidth > row.width || row.undersized.length > 0 || !row.backResolved) || Object.values(routes.capabilityLabels).some((value) => !value) || approvals.calls.length !== 2 || !approvals.visibleInAudit) process.exitCode = 1; + } finally { await browser?.close(); } +} +main().catch((error) => { console.error(error); process.exitCode = 1; }); diff --git a/apps/web/scripts/mobile-swipe.test.cjs b/apps/web/scripts/mobile-swipe.test.cjs new file mode 100644 index 0000000..2c2a2b1 --- /dev/null +++ b/apps/web/scripts/mobile-swipe.test.cjs @@ -0,0 +1,33 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const Module = require("node:module"); +const ts = require("typescript"); +function load(file) { + const source = fs.readFileSync(file, "utf8"); + const compiled = ts.transpileModule(source, { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 } }).outputText; + const loaded = new Module(file, module); + loaded._compile(compiled, file); + return loaded.exports; +} +const { swipeIntent, swipeResult } = load("src/lib/gestures.ts"); +const { nativeAmplifyActions } = load("src/lib/amplify.ts"); +const { PLATFORMS } = load("src/lib/platforms.ts"); + +test("vertical scroll never locks to horizontal reveal", () => { + for (const [dx, dy] of [[0, 12], [9, 20], [-20, 60], [15, 50], [-40, 120]]) assert.equal(swipeIntent(dx, dy), "vertical"); +}); +test("small motion remains pending and deliberate horizontal motion reveals", () => { + assert.equal(swipeIntent(6, 3), "pending"); + assert.equal(swipeIntent(-40, 10), "horizontal"); + assert.equal(swipeResult(-50, 0.2, "left"), "closed"); + assert.equal(swipeResult(-90, 0.2, "left"), "open"); + assert.equal(swipeResult(-180, 0.2, "left"), "full"); +}); +test("native amplify labels follow platform capabilities", () => { + assert.deepEqual(nativeAmplifyActions(PLATFORMS.mastodon).map((item) => item.label), ["Boost", "Quote"]); + assert.deepEqual(nativeAmplifyActions(PLATFORMS.telegram).map((item) => item.label), ["Forward"]); + assert.deepEqual(nativeAmplifyActions(PLATFORMS.facebook).map((item) => item.label), ["Share link"]); + assert.deepEqual(nativeAmplifyActions(PLATFORMS.instagram), []); + assert.deepEqual(nativeAmplifyActions(PLATFORMS.limited), []); +}); diff --git a/apps/web/scripts/visual-qa.mjs b/apps/web/scripts/visual-qa.mjs new file mode 100644 index 0000000..b601e98 --- /dev/null +++ b/apps/web/scripts/visual-qa.mjs @@ -0,0 +1,75 @@ +import { chromium } from "playwright"; +import { mkdirSync } from "node:fs"; + +const BASE = process.env.QA_BASE ?? "http://localhost:3111"; +const OUT = "runs/qa"; +mkdirSync(OUT, { recursive: true }); + +const browser = await chromium.launch(); +const page = await browser.newPage({ viewport: { width: 1440, height: 900 } }); + +async function shot(name, { fullPage = false } = {}) { + await page.waitForTimeout(350); + await page.screenshot({ path: `${OUT}/${name}.png`, fullPage }); + console.log(`✓ ${name}`); +} + +console.log("console errors will be collected at the end"); +const errors = []; +page.on("console", (msg) => { + if (msg.type() === "error") errors.push(msg.text().slice(0, 200)); +}); +page.on("pageerror", (err) => errors.push(`PAGEERROR: ${String(err).slice(0, 200)}`)); + +await page.goto(`${BASE}/streams`); +await shot("01-streams"); + +await page.keyboard.press("j"); +await page.keyboard.press("j"); +await page.keyboard.press("j"); +await shot("02-streams-selected"); + +await page.keyboard.press("r"); +await shot("03-streams-repost-toast"); + +await page.keyboard.press("q"); +await shot("04-streams-quote-popover"); +await page.keyboard.press("Escape"); + +await page.keyboard.press("j"); +await page.keyboard.press("k"); +await page.keyboard.press("k"); +await shot("05-streams-keyboard-moved"); + +await page.goto(`${BASE}/compose`); +await page.locator(".ProseMirror").click(); +await page.keyboard.type("Shipping the visual QA pass — capability-gated amplify, keyboard-first streams."); +await shot("06-compose-typed"); + +await page.goto(`${BASE}/connections`); +await shot("07-connections"); + +await page.goto(`${BASE}/inbox`); +await shot("08-inbox"); + +await page.goto(`${BASE}/team`); +await shot("09-team"); + +await page.goto(`${BASE}/audit`); +await shot("10-audit"); +await page.locator("tr", { hasText: "credential.rotated" }).first().click(); +await shot("11-audit-diff-open"); + +await page.goto(`${BASE}/onboarding`); +await shot("12-onboarding-step1"); +await page.getByPlaceholder("Studio, Newsroom, Acme Social…").fill("Studio"); +await page.getByRole("button", { name: "Continue" }).click(); +await shot("13-onboarding-step2"); + +await page.goto(`${BASE}/streams`); +await page.keyboard.press("Meta+k"); +await shot("14-command-palette"); + +await browser.close(); +console.log(`\nconsole errors: ${errors.length}`); +for (const e of errors.slice(0, 10)) console.log(` - ${e}`); diff --git a/apps/web/src/app/(main)/analytics/page.tsx b/apps/web/src/app/(main)/analytics/page.tsx new file mode 100644 index 0000000..a3e6962 --- /dev/null +++ b/apps/web/src/app/(main)/analytics/page.tsx @@ -0,0 +1,481 @@ +"use client"; + +import { Fragment, useCallback, useEffect, useState } from "react"; +import { Badge, Button, Card } from "../../../components/ui/primitives"; +import { EmptyState } from "../../../components/brand/empty-state"; +import { Loading } from "../../../components/brand/loading"; +import { OfflineBadge } from "../../../components/ui/offline-badge"; +import { api, ApiUnreachableError, errorMessage } from "../../../lib/api-client"; +import { useAuth } from "../../../lib/auth-store"; +import { useApiResource } from "../../../lib/use-api"; +import type { DataSource } from "../../../lib/use-api"; + +/** + * Analytics wiring: GET /analytics/summary (current vs previous month with + * deltas) and GET /analytics/snapshots (month history, keyset-paginated by + * cursor). Response types mirror apps/api/src/analytics/snapshot.service.ts. + * + * Honesty rules (audit-page pattern): no analytics fixtures exist, so the + * offline fallback is an empty screen behind the offline badge — never invented + * numbers — and on a live API error the data is withheld rather than faked. + * The production API may not serve /analytics yet, in which case the page + * degrades to the error state with a retry. + */ + +interface SnapshotRecord { + id: string; + workspaceId: string; + month: string; // YYYY-MM + publicationsTotal: number; + publicationsByProvider: Record; + postsArchivedTotal: number; + activeConnections: number; + draftsCreatedTotal: number; + publishingAttemptsTotal: number; + publishFailuresTotal: number; + snapshotStartedAt: string; + snapshotCompletedAt: string; +} + +type SnapshotMetrics = Pick< + SnapshotRecord, + | "publicationsTotal" + | "postsArchivedTotal" + | "activeConnections" + | "draftsCreatedTotal" + | "publishingAttemptsTotal" + | "publishFailuresTotal" +>; + +/** Deltas from GET /analytics/summary (monthDeltas shape); provider deltas stay per-provider. */ +interface MetricDeltas { + publicationsTotal: number; + publicationsByProvider: Record; + postsArchivedTotal: number; + activeConnections: number; + draftsCreatedTotal: number; + publishingAttemptsTotal: number; + publishFailuresTotal: number; +} + +interface SummaryResponse { + currentMonth: string; + previousMonth: string; + current: SnapshotRecord | null; + previous: SnapshotRecord | null; + deltas: MetricDeltas | null; +} + +interface SnapshotPage { + items: SnapshotRecord[]; + nextCursor: string | null; +} + +const identity = (value: T): T => value; +const PAGE_SIZE = 12; // server default; hard cap is 24 + +interface MetricDef { + key: keyof SnapshotMetrics; + label: string; + /** Fewer failures is good, so invert the up/down coloring for that metric. */ + invert?: boolean; +} + +const METRICS: MetricDef[] = [ + { key: "publicationsTotal", label: "Publications" }, + { key: "postsArchivedTotal", label: "Posts archived" }, + { key: "activeConnections", label: "Active channels" }, + { key: "draftsCreatedTotal", label: "Drafts created" }, + { key: "publishingAttemptsTotal", label: "Publish attempts" }, + { key: "publishFailuresTotal", label: "Publish failures", invert: true }, +]; + +function monthLabel(month: string): string { + return new Date(`${month}-01T00:00:00Z`).toLocaleDateString(undefined, { + year: "numeric", + month: "long", + timeZone: "UTC", + }); +} + +function numberDelta(deltas: MetricDeltas | null, key: keyof SnapshotMetrics): number | null { + return deltas === null ? null : deltas[key]; +} + +function providerDeltas(deltas: MetricDeltas | null): Record | null { + return deltas === null ? null : deltas.publicationsByProvider; +} + +function Delta({ + value, + previousMonth, + invert, +}: { + value: number; + previousMonth: string; + invert?: boolean; +}): React.ReactElement { + if (value === 0) { + return ( + + — + + ); + } + const up = value > 0; + const good = invert ? !up : up; + return ( + + {up ? "▲" : "▼"} {Math.abs(value).toLocaleString()} + + ); +} + +function MetricCard({ + metric, + value, + delta, + previousMonth, +}: { + metric: MetricDef; + value: number; + delta: number | null; + previousMonth: string; +}): React.ReactElement { + return ( +
+
{metric.label}
+
+ {value.toLocaleString()} + {delta !== null && } +
+
+ ); +} + +/** Provider breakdown chips with a relative bar; optional per-provider deltas. */ +function ProviderBreakdown({ + byProvider, + deltas, + previousMonth, +}: { + byProvider: Record; + deltas: Record | null; + previousMonth: string; +}): React.ReactElement { + const entries = Object.entries(byProvider).sort((a, b) => b[1] - a[1]); + if (entries.length === 0) { + return

No publications by any provider.

; + } + const max = Math.max(...entries.map(([, n]) => n), 1); + return ( +
+ {entries.map(([provider, count]) => { + const delta = deltas === null ? undefined : deltas[provider]; + return ( + + + {provider} + {count.toLocaleString()} + + + {delta !== undefined && } + + ); + })} +
+ ); +} + +export default function Page(): React.ReactElement { + const { status, can } = useAuth(); + const live = status === "authenticated"; + // Offline screens render sample data (house pattern); permission checks only + // bite against a real session — can() is false for everyone while offline. + const allowed = !live || can("analytics.view"); + + // Summary panel: current month vs previous month, via the house hook. + const summary = useApiResource("/analytics/summary", identity, null); + + // Snapshot history: audit-page-style manual fetch because keyset "load older" + // appends pages instead of replacing the view. + const [historySource, setHistorySource] = useState("loading"); + const [rows, setRows] = useState([]); + const [cursor, setCursor] = useState(null); + const [historyError, setHistoryError] = useState(null); + const [loadingMore, setLoadingMore] = useState(false); + const [expanded, setExpanded] = useState(null); + const [nonce, setNonce] = useState(0); + const reload = useCallback(() => setNonce((n) => n + 1), []); + + const [generating, setGenerating] = useState(false); + const [message, setMessage] = useState<{ tone: "ok" | "error"; text: string } | null>(null); + + useEffect(() => { + if (status === "loading" || !allowed) return; + if (status !== "authenticated") { + setHistorySource("sample"); + return; + } + const controller = new AbortController(); + setHistorySource("loading"); + api + .get("/analytics/snapshots", { limit: PAGE_SIZE }, controller.signal) + .then((page) => { + setRows(page.items); + setCursor(page.nextCursor); + setHistorySource("live"); + setHistoryError(null); + }) + .catch((err: unknown) => { + if (controller.signal.aborted) return; + if (err instanceof ApiUnreachableError) setHistorySource("sample"); + else { + setHistorySource("live"); + setHistoryError(errorMessage(err)); + } + }); + return () => controller.abort(); + }, [status, allowed, nonce]); + + async function loadOlder(): Promise { + if (cursor === null || historySource !== "live") return; + setLoadingMore(true); + try { + const page = await api.get("/analytics/snapshots", { cursor, limit: PAGE_SIZE }); + setRows((current) => [...current, ...page.items]); + setCursor(page.nextCursor); + } catch (err) { + setHistoryError(errorMessage(err)); + } finally { + setLoadingMore(false); + } + } + + /** Manual trigger for the current month (POST /analytics/snapshot). */ + async function generateSnapshot(): Promise { + setGenerating(true); + setMessage(null); + try { + const record = await api.post("/analytics/snapshot", {}); + setMessage({ tone: "ok", text: `Snapshot generated for ${monthLabel(record.month)}.` }); + reload(); + } catch (err) { + setMessage({ tone: "error", text: errorMessage(err) }); + } finally { + setGenerating(false); + } + } + + if (!allowed) { + return ( +
+ +
+ ); + } + + const sampleMode = summary.source === "sample" || historySource === "sample"; + const view = summary.data; + const canGenerate = live && historySource !== "loading"; + + return ( +
+
+
+

Analytics

+ +
+

+ Monthly rollups of publishing activity · deltas compare the current month with the previous one +

+
+ + {/* Summary: current month + previous-month deltas */} + {summary.source === "loading" ? ( + + Loading summary… + + ) : summary.error !== null ? ( + +

Could not load the monthly summary — {summary.error}

+ +
+ ) : view !== null ? ( + +
+

{monthLabel(view.currentMonth)}

+ vs {monthLabel(view.previousMonth)} +
+ {view.current === null ? ( +
+ No snapshot exists for {monthLabel(view.currentMonth)} yet — it is generated on schedule, or + immediately via POST /analytics/snapshot. +
+ ) : ( + <> +
+ {METRICS.map((metric) => ( + + ))} +
+
+

+ Publications by provider +

+ +
+ {view.previous === null && ( +

+ No snapshot for {monthLabel(view.previousMonth)} — deltas appear once the previous month has one. +

+ )} + + )} +
+ ) : null} + + {/* Snapshot history */} + +
+

Snapshot history

+ {rows.length > 0 && {rows.length} months} +
+ + {message !== null && ( +

{message.text}

+ )} + + {historySource === "loading" ? ( +
+ Loading snapshots… +
+ ) : historyError !== null && rows.length === 0 ? ( +
+

Failed to load analytics snapshots — {historyError}

+ +
+ ) : rows.length === 0 ? ( + <> + + {!sampleMode && historyError === null && ( +
+ +
+ )} + + ) : ( + <> + {historyError !== null && ( +

Could not load older months — {historyError}

+ )} +
+ + + + + + + + + + + + + + {rows.map((record) => ( + + setExpanded(expanded === record.id ? null : record.id)} + > + + + + + + + + + + {expanded === record.id && ( + + + + )} + + ))} + +
MonthPublicationsArchivedDraftsAttemptsFailuresChannels +
{monthLabel(record.month)}{record.publicationsTotal.toLocaleString()}{record.postsArchivedTotal.toLocaleString()}{record.draftsCreatedTotal.toLocaleString()}{record.publishingAttemptsTotal.toLocaleString()} + 0 ? "text-danger" : undefined}> + {record.publishFailuresTotal.toLocaleString()} + + {record.activeConnections.toLocaleString()}{expanded === record.id ? "▾" : "▸"}
+
+ {METRICS.map((metric) => ( +
+
{metric.label}
+
{record[metric.key].toLocaleString()}
+
+ ))} +
+
+ +
+

+ Generated {new Date(record.snapshotCompletedAt).toLocaleString()} +

+
+
+ {historySource === "live" && cursor !== null && ( +
+ +
+ )} + + )} +

+ Snapshots are recomputed per (workspace, month) from source tables — re-running a month overwrites it in place. +

+
+
+ ); +} diff --git a/apps/web/src/app/(main)/approvals/page.tsx b/apps/web/src/app/(main)/approvals/page.tsx new file mode 100644 index 0000000..6f1b928 --- /dev/null +++ b/apps/web/src/app/(main)/approvals/page.tsx @@ -0,0 +1,5 @@ +import InboxPage from "../inbox/page"; + +export default function ApprovalsPage(): React.ReactElement { + return ; +} diff --git a/apps/web/src/app/(main)/archive/page.tsx b/apps/web/src/app/(main)/archive/page.tsx new file mode 100644 index 0000000..2327468 --- /dev/null +++ b/apps/web/src/app/(main)/archive/page.tsx @@ -0,0 +1,139 @@ +"use client"; + +import { useState } from "react"; +import { isBotConnection } from "../../../lib/connections-client"; +import { Badge, Button, Card } from "../../../components/ui/primitives"; +import { OfflineBadge } from "../../../components/ui/offline-badge"; +import { EmptyState } from "../../../components/brand/empty-state"; +import { api, errorMessage } from "../../../lib/api-client"; +import { useAuth } from "../../../lib/auth-store"; +import { useApiResource } from "../../../lib/use-api"; +import type { ConnectionSummary, DraftRecord, DraftState } from "../../../lib/api-types"; + +interface MediaRef { fileId: string; status: string; thumbUrl: string | null; contentType: string } +interface ArchiveItem { + id: string; remoteRef: string; text: string; media: MediaRef[]; + attribution: { channelTitle?: string | null; channelUsername?: string | null; authorSignature?: string | null; forwardOrigin?: Record | null }; + protectedContent: boolean; postedAt: string; revision: number; +} +interface Page { items: ArchiveItem[]; nextCursor: string | null } +const SAMPLE: Page = { items: [{ + id: "sample", remoteRef: "-100123456789:42", text: "A new channel post appears here for review.", + media: [], attribution: { channelTitle: "Example channel", authorSignature: "Editor" }, + protectedContent: false, postedAt: "2026-10-07T12:00:00.000Z", revision: 1, +}], nextCursor: null }; +const identity = (page: Page): Page => page; +const noConnections: ConnectionSummary[] = []; +const same = (value: ConnectionSummary[]): ConnectionSummary[] => value; + +export default function ArchivePage(): React.ReactElement { + const { can } = useAuth(); + const { data, source, error, reload } = useApiResource("/archive/items", identity, SAMPLE); + const { data: connections } = useApiResource("/connections", same, noConnections); + const [more, setMore] = useState([]); + const [cursor, setCursor] = useState(null); + const [drafts, setDrafts] = useState>({}); + const [draftStates, setDraftStates] = useState>({}); + const [connectionId, setConnectionId] = useState(""); + const [attribution, setAttribution] = useState(true); + const [busy, setBusy] = useState(null); + const [actionError, setActionError] = useState(null); + const [notice, setNotice] = useState(null); + const telegramConnections = connections.filter(isBotConnection); + const nextCursor = cursor ?? data.nextCursor; + const items = [...data.items, ...more]; + + async function createDraft(item: ArchiveItem): Promise { + setBusy(item.id); setActionError(null); + try { + const draft = await api.post(`/archive/items/${item.id}/drafts`, {}); + setDrafts((current) => ({ ...current, [item.id]: draft.id })); + setDraftStates((current) => ({ ...current, [item.id]: draft.state })); + setNotice("Draft created. Submit it for review, then approve it before amplifying."); + } catch (err) { setActionError(errorMessage(err)); } + finally { setBusy(null); } + } + async function submitDraft(item: ArchiveItem): Promise { + const draftId = drafts[item.id]; + if (!draftId) return; + setBusy(item.id); setActionError(null); + try { + const draft = await api.post(`/publishing/drafts/${draftId}/submit`); + setDraftStates((current) => ({ ...current, [item.id]: draft.state })); + setNotice("Submitted for review."); + } catch (err) { setActionError(errorMessage(err)); } + finally { setBusy(null); } + } + async function approveDraft(item: ArchiveItem): Promise { + const draftId = drafts[item.id]; + if (!draftId) return; + setBusy(item.id); setActionError(null); + try { + const draft = await api.post(`/publishing/drafts/${draftId}/approve`); + setDraftStates((current) => ({ ...current, [item.id]: draft.state })); + setNotice("Draft approved — it can now be amplified."); + } catch (err) { setActionError(errorMessage(err)); } + finally { setBusy(null); } + } + async function amplify(item: ArchiveItem): Promise { + const draftId = drafts[item.id]; + if (!draftId || !connectionId) return; + setBusy(item.id); setActionError(null); + try { + await api.post(`/archive/items/${item.id}/amplify`, { draftId, connectionId, attribution }); + setNotice("Amplified to Telegram."); + } catch (err) { setActionError(errorMessage(err)); } + finally { setBusy(null); } + } + async function loadMore(): Promise { + if (!nextCursor) return; + setBusy("more"); + try { + const page = await api.get("/archive/items", { cursor: nextCursor }); + setMore((current) => [...current, ...page.items]); setCursor(page.nextCursor); + } catch (err) { setActionError(errorMessage(err)); } + finally { setBusy(null); } + } + + return
+

Archive

+

Posts from channels where your Telegram bot is an admin arrive here. Check that you have the right to reuse someone else's work before sharing it.

+ {error &&

{error}

} + {actionError &&

{actionError}

} + {notice &&

{notice}

} + {items.length === 0 && } +
{items.map((item) => +
+ {item.attribution.channelTitle ?? item.attribution.channelUsername ?? item.remoteRef} + {item.attribution.authorSignature && by {item.attribution.authorSignature}} + + {item.revision > 1 && edited} + {item.protectedContent && amplify unavailable} +
+ {item.text &&

{item.text}

} + {item.attribution.forwardOrigin &&

Forwarded origin retained in attribution.

} + {item.media.length > 0 &&
{item.media.map((media: MediaRef) => typeof media.thumbUrl === "string" && media.contentType.startsWith("image/") + ? Archived Telegram media + : {media.status})}
} + {source === "live" &&
+ {can("draft.create") && } + {drafts[item.id] && <> + Draft {drafts[item.id]?.slice(0, 8)} · state: {draftStates[item.id] ?? "editing"} + {(draftStates[item.id] === undefined || draftStates[item.id] === "editing" || draftStates[item.id] === "rejected") && ( + + )} + {draftStates[item.id] === "in_review" && can("approval.grant") && ( + + )} + + + {can("publication.schedule") && } + } +
} +
)}
+ {nextCursor && source === "live" && } + {source === "live" && } +
; +} diff --git a/apps/web/src/app/(main)/audit/page.tsx b/apps/web/src/app/(main)/audit/page.tsx new file mode 100644 index 0000000..2a7b486 --- /dev/null +++ b/apps/web/src/app/(main)/audit/page.tsx @@ -0,0 +1,267 @@ +"use client"; + +import { Fragment, useEffect, useState } from "react"; +import { Badge, Button } from "../../../components/ui/primitives"; +import { OfflineBadge } from "../../../components/ui/offline-badge"; +import { AUDIT_ENTRIES, ageLabel } from "../../../lib/fixtures"; +import { api, ApiUnreachableError, errorMessage } from "../../../lib/api-client"; +import type { AuditEntryDto, AuditPage as AuditPageDto } from "../../../lib/api-types"; +import { useAuth } from "../../../lib/auth-store"; +import type { DataSource } from "../../../lib/use-api"; +import { Loading } from "../../../components/brand/loading"; + +type DiffValue = string | number | boolean | null | unknown[] | Record; + +function DiffCell({ label, value }: { label: string; value: DiffValue }): React.ReactElement { + return ( +
+ {label} + {JSON.stringify(value)} +
+ ); +} + +function DiffViewer({ diff }: { diff: Record }): React.ReactElement { + return ( +
+ {Object.entries(diff).map(([key, value]) => { + if (Array.isArray(value) && value.length === 2 && typeof value[0] !== "object") { + return ( +
+ {key} + + {JSON.stringify(value[0])} + → + {JSON.stringify(value[1])} + +
+ ); + } + return ; + })} +
+ ); +} + +interface AuditRow { + id: string; + actor: string; + action: string; + targetType: string; + target: string; + minutes: number; + diff: Record | null; +} + +interface FilterOption { + value: string; + label: string; +} + +const PAGE_SIZE = 25; +const ALL = "all"; + +function fromApi(e: AuditEntryDto): AuditRow { + return { + id: e.id, + actor: e.actorEmail ?? (e.actorUserId === null ? "system" : e.actorUserId), + action: e.action, + targetType: e.targetType, + target: e.targetId, + minutes: Math.max(0, Math.floor((Date.now() - Date.parse(e.at)) / 60_000)), + diff: e.diff, + }; +} + +const SAMPLE_ROWS: AuditRow[] = AUDIT_ENTRIES.map((e) => ({ ...e, id: String(e.id) })); + +function csvCell(value: string | number | null): string { + const text = value === null ? "" : String(value); + return /[",\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text; +} + +function toCsv(rows: AuditRow[]): string { + const header = ["id", "actor", "action", "target_type", "target", "minutes_ago", "diff"]; + const lines = rows.map((e) => + [e.id, e.actor, e.action, e.targetType, e.target, e.minutes, e.diff === null ? null : JSON.stringify(e.diff)].map(csvCell).join(",")); + return [header.join(","), ...lines].join("\n"); +} + +function downloadCsv(rows: AuditRow[]): void { + const blob = new Blob([toCsv(rows)], { type: "text/csv;charset=utf-8" }); + const url = URL.createObjectURL(blob); + const anchor = document.createElement("a"); + anchor.href = url; + anchor.download = `audit-${new Date().toISOString().slice(0, 10)}.csv`; + document.body.append(anchor); + anchor.click(); + anchor.remove(); + URL.revokeObjectURL(url); +} + +export default function AuditPage(): React.ReactElement { + const { status } = useAuth(); + const [actor, setActor] = useState(ALL); + const [action, setAction] = useState(ALL); + const [expanded, setExpanded] = useState(null); + const [source, setSource] = useState("loading"); + const [rows, setRows] = useState([]); + const [cursor, setCursor] = useState(null); + const [actorOptions, setActorOptions] = useState([]); + const [actionOptions, setActionOptions] = useState([]); + const [error, setError] = useState(null); + const [loadingMore, setLoadingMore] = useState(false); + + // Live: filters are applied server-side and pages are fetched by seq cursor. + useEffect(() => { + if (status === "loading") return; + if (status !== "authenticated") { + setSource("sample"); + return; + } + const controller = new AbortController(); + api + .get("/audit", { action: action === ALL ? undefined : action, actor: actor === ALL ? undefined : actor, limit: PAGE_SIZE }, controller.signal) + .then((page) => { + setRows(page.entries.map(fromApi)); + setCursor(page.nextCursor); + setActionOptions(page.facets.actions); + setActorOptions(page.facets.actors.map((a) => ({ value: a.id ?? "system", label: a.email ?? (a.id === null ? "system" : a.id) }))); + setSource("live"); + setError(null); + }) + .catch((err: unknown) => { + if (controller.signal.aborted) return; + if (err instanceof ApiUnreachableError) setSource("sample"); + else { + setSource("live"); + setError(errorMessage(err)); + } + }); + return () => controller.abort(); + }, [status, action, actor]); + + async function loadMore(): Promise { + if (cursor === null) return; + setLoadingMore(true); + try { + const page = await api.get("/audit", { + action: action === ALL ? undefined : action, + actor: actor === ALL ? undefined : actor, + before: cursor, + limit: PAGE_SIZE, + }); + setRows((current) => [...current, ...page.entries.map(fromApi)]); + setCursor(page.nextCursor); + } catch (err) { + setError(errorMessage(err)); + } finally { + setLoadingMore(false); + } + } + + const live = source === "live"; + const actors: FilterOption[] = live + ? actorOptions + : [...new Set(SAMPLE_ROWS.map((e) => e.actor))].map((a) => ({ value: a, label: a })); + const actions = live ? actionOptions : [...new Set(SAMPLE_ROWS.map((e) => e.action))]; + const visible = live + ? rows + : SAMPLE_ROWS.filter((e) => (actor === ALL || e.actor === actor) && (action === ALL || e.action === action)); + + return ( +
+
+
+

Audit trail

+ +
+ +
+ +
+ + + hash-chained · tamper-evident +
+ + {error !== null &&

{error}

} + +
+ + + + + + + + + + + {source === "loading" && ( + + )} + {source !== "loading" && visible.length === 0 && ( + error !== null + ? + : + )} + {source !== "loading" && visible.map((e) => ( + + setExpanded(expanded === e.id ? null : e.id)} + > + + + + + + + {expanded === e.id && ( + + + + )} + + ))} + +
ActorActionTargetWhen +
Loading audit trail…
Failed to load audit entries — {error}
No audit entries match.
{e.actor}{e.action}{e.targetType}:{e.target}{ageLabel(e.minutes)} ago{expanded === e.id ? "▾" : "▸"}
+ {e.diff === null ? ( + no payload + ) : ( + + )} +
+
+ {live && cursor !== null && ( +
+ +
+ )} +

+ Secret-shaped keys render as [REDACTED] by construction — they never enter the chain. +

+
+ ); +} diff --git a/apps/web/src/app/(main)/calendar/page.tsx b/apps/web/src/app/(main)/calendar/page.tsx new file mode 100644 index 0000000..f27508b --- /dev/null +++ b/apps/web/src/app/(main)/calendar/page.tsx @@ -0,0 +1,291 @@ +"use client"; + +import { useEffect, useMemo, useState } from "react"; +import { Badge, Button } from "../../../components/ui/primitives"; +import { OfflineBadge } from "../../../components/ui/offline-badge"; +import { EmptyState } from "../../../components/brand/empty-state"; +import { LoadingPanel } from "../../../components/brand/loading"; +import { api, errorMessage } from "../../../lib/api-client"; +import { PLATFORMS } from "../../../lib/platforms"; +import { useAuth } from "../../../lib/auth-store"; +import { useApiResource } from "../../../lib/use-api"; +import type { ConnectionSummary, DraftRecord } from "../../../lib/api-types"; + +/** + * Calendar = the scheduled slice of the publishing pipeline. There is no + * dedicated schedule-list endpoint, so this reads the same draft list the + * approvals queue uses (`GET /publishing/drafts?state=scheduled`) and joins + * each draft to its target connection for a readable label. + */ + +interface ScheduledRow { + id: string; + text: string; + /** ISO — the queue's `post.scheduledAt` when recorded, else the draft's last update. */ + publishAt: string; + fromScheduledAt: boolean; + targetLabel: string; +} + +interface DayGroup { + key: number; + label: string; + rows: ScheduledRow[]; +} + +type Notice = { tone: "ok" | "error"; text: string }; + +const identity = (drafts: DraftRecord[]): DraftRecord[] => drafts; +const same = (connections: ConnectionSummary[]): ConnectionSummary[] => connections; +const noConnections: ConnectionSummary[] = []; + +/** Offline / unauthenticated fallback — one future-dated row so the layout is visible. */ +const SAMPLE: DraftRecord[] = [ + { + id: "sample-scheduled", + workspaceId: "sample", + authorUserId: "sample-author", + post: { + text: "Sample scheduled post — the weekly roundup goes out to every connected channel.", + mediaAssetIds: [], + visibility: "public", + providerMetadata: { connectionId: "sample-connection" }, + scheduledAt: new Date(Date.now() + 3_600_000).toISOString(), + }, + state: "scheduled", + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }, +]; + +function connectionIdOf(draft: DraftRecord): string | null { + const id = draft.post.providerMetadata.connectionId; + return typeof id === "string" && id !== "" ? id : null; +} + +function labelFor(connection: ConnectionSummary | undefined, id: string): string { + if (connection === undefined) return `unknown connection ${id.slice(0, 8)}`; + const handle = typeof connection.fields.handle === "string" ? connection.fields.handle : connection.channelId; + return `${PLATFORMS[connection.provider]?.displayName ?? connection.provider} · ${handle}`; +} + +function toRows(drafts: DraftRecord[], connections: ConnectionSummary[]): ScheduledRow[] { + return drafts + .map((draft): ScheduledRow => { + const connectionId = connectionIdOf(draft); + const connection = connectionId === null ? undefined : connections.find((c) => c.id === connectionId); + return { + id: draft.id, + text: draft.post.text, + publishAt: draft.post.scheduledAt ?? draft.updatedAt, + fromScheduledAt: draft.post.scheduledAt !== undefined, + targetLabel: connectionId === null ? "no target recorded" : labelFor(connection, connectionId), + }; + }) + .sort((a, b) => Date.parse(a.publishAt) - Date.parse(b.publishAt)); +} + +/** Group by the viewer's local calendar day — the list equivalent of a week view. */ +function groupByDay(rows: ScheduledRow[]): DayGroup[] { + const groups = new Map(); + for (const row of rows) { + const date = new Date(row.publishAt); + const key = new Date(date.getFullYear(), date.getMonth(), date.getDate()).getTime(); + const bucket = groups.get(key); + if (bucket === undefined) groups.set(key, [row]); + else bucket.push(row); + } + return [...groups.entries()].map(([key, dayRows]) => ({ + key, + label: new Date(key).toLocaleDateString(undefined, { weekday: "long", month: "short", day: "numeric" }), + rows: dayRows, + })); +} + +const localTime = (iso: string): string => + new Date(iso).toLocaleTimeString(undefined, { hour: "2-digit", minute: "2-digit" }); + +const utcTime = (iso: string): string => `${new Date(iso).toISOString().slice(0, 16).replace("T", " ")} UTC`; + +function relativeLabel(iso: string, now: number): { text: string; overdue: boolean } { + const delta = Date.parse(iso) - now; + const minutes = Math.max(0, Math.round(Math.abs(delta) / 60_000)); + const span = + minutes < 60 + ? `${minutes}m` + : minutes < 1_440 + ? `${Math.floor(minutes / 60)}h ${minutes % 60}m` + : `${Math.floor(minutes / 1_440)}d ${Math.floor((minutes % 1_440) / 60)}h`; + return { text: delta >= 0 ? `in ${span}` : `${span} overdue`, overdue: delta < 0 }; +} + +export default function CalendarPage(): React.ReactElement { + const { can } = useAuth(); + const drafts = useApiResource("/publishing/drafts", identity, SAMPLE, { + state: "scheduled", + limit: 100, + }); + const connections = useApiResource("/connections", same, noConnections); + const live = drafts.source === "live"; + const canSchedule = can("publication.schedule"); + + const [cancelling, setCancelling] = useState(null); + const [busyId, setBusyId] = useState(null); + const [notice, setNotice] = useState(null); + const [now, setNow] = useState(Date.now()); + + // Same cadence as the streams wall: refresh on a 30s timer. + useEffect(() => { + const timer = setInterval(() => { + drafts.reload(); + connections.reload(); + setNow(Date.now()); + }, 30_000); + return () => clearInterval(timer); + }, [drafts.reload, connections.reload]); + + const rows = useMemo(() => toRows(drafts.data, connections.data), [drafts.data, connections.data]); + const groups = useMemo(() => groupByDay(rows), [rows]); + + async function cancel(row: ScheduledRow): Promise { + if (!live) { + setNotice({ tone: "error", text: "Offline — sample mode, nothing was sent." }); + setCancelling(null); + return; + } + setBusyId(row.id); + setNotice(null); + try { + const draft = await api.del(`/publishing/drafts/${row.id}/schedule`); + drafts.setData((current) => current.filter((item) => item.id !== row.id)); + setNotice({ tone: "ok", text: `Schedule cancelled — the draft is back to ${draft.state.replace("_", " ")}.` }); + setCancelling(null); + drafts.reload(); + } catch (err) { + setNotice({ tone: "error", text: errorMessage(err) }); + } finally { + setBusyId(null); + } + } + + return ( +
+
+

Calendar

+ + + {rows.length} scheduled · refreshes every 30s + + +
+

+ Publications queued by the publishing pipeline, grouped by your local day. Each row shows the local time with the + UTC time beside it; execution state lives in the audit log, not this list. +

+ + {notice !== null && ( +

+ {notice.text} +

+ )} + {drafts.error !== null &&

{drafts.error}

} + {!live && drafts.error === null && ( +

Offline sample — connect to the API to see and cancel real schedules.

+ )} + + {drafts.source === "loading" && } + {drafts.source !== "loading" && rows.length === 0 && drafts.error === null && ( + + )} + {drafts.source !== "loading" && + groups.map((group) => ( +
+

+ {group.label} + + {group.rows.length} + +

+
    + {group.rows.map((row) => { + const relative = relativeLabel(row.publishAt, now); + return ( +
  • +
    + {localTime(row.publishAt)} + {utcTime(row.publishAt)} + {relative.text} +
    +
    +

    {row.text}

    +
    + scheduled + {row.targetLabel} + {!row.fromScheduledAt && ( + + time from last draft update + + )} +
    +
    +
    + {live && canSchedule && ( + + )} + {live && !canSchedule && ( + + cancelling needs publication.schedule + + )} +
    +
  • + ); + })} +
+
+ ))} + + {cancelling !== null && ( +
{ if (busyId === null) setCancelling(null); }} + > +
e.stopPropagation()} + > +

Cancel this scheduled publication?

+

+ {utcTime(cancelling.publishAt)} · {cancelling.targetLabel} +

+

+ {cancelling.text} +

+

+ Nothing is published. The draft returns to editing so it can be rescheduled or changed, and the cancellation + is recorded in the audit log. +

+
+ + +
+
+
+ )} +
+ ); +} diff --git a/apps/web/src/app/(main)/compose/page.tsx b/apps/web/src/app/(main)/compose/page.tsx new file mode 100644 index 0000000..6904ebd --- /dev/null +++ b/apps/web/src/app/(main)/compose/page.tsx @@ -0,0 +1,324 @@ +"use client"; + +import { useEffect, useRef, useState } from "react"; +import { useRouter } from "next/navigation"; +import { useEditor, EditorContent } from "@tiptap/react"; +import StarterKit from "@tiptap/starter-kit"; +import Placeholder from "@tiptap/extension-placeholder"; +import CharacterCount from "@tiptap/extension-character-count"; +import { Badge, Button, Input } from "../../../components/ui/primitives"; +import { OfflineBadge } from "../../../components/ui/offline-badge"; +import { MOCK_TARGETS, PLATFORMS } from "../../../lib/platforms"; +import { api, errorMessage } from "../../../lib/api-client"; +import type { ConnectionSummary, DraftPost, DraftRecord, DraftState, ScheduledPublicationRecord, Visibility } from "../../../lib/api-types"; +import { useAuth } from "../../../lib/auth-store"; +import { useApiResource } from "../../../lib/use-api"; +import { Sheet } from "../../../components/mobile/sheet"; + +/** A publish target: a real connection when live, a fixture platform when offline. */ +interface Target { + key: string; + provider: string; + label: string; +} + +function connectionTargets(connections: ConnectionSummary[]): Target[] { + return connections.map((c) => { + const handle = typeof c.fields.handle === "string" ? c.fields.handle : c.channelId; + return { key: c.id, provider: c.provider, label: `${PLATFORMS[c.provider]?.displayName ?? c.provider} · ${handle}` }; + }); +} + +const SAMPLE_TARGETS: Target[] = MOCK_TARGETS.map((t) => ({ key: t, provider: t, label: PLATFORMS[t]?.displayName ?? t })); + +const identity = (value: T): T => value; +const NO_DRAFTS: DraftRecord[] = []; + +const STATE_TONE: Record = { + editing: "neutral", + in_review: "warn", + approved: "accent", + rejected: "danger", + scheduled: "accent", + published: "live", +}; + +type Notice = { tone: "ok" | "error"; text: string }; + +export default function ComposePage(): React.ReactElement { + const router = useRouter(); + const { can } = useAuth(); + const targetsResource = useApiResource("/connections", connectionTargets, SAMPLE_TARGETS); + const drafts = useApiResource("/publishing/drafts", identity, NO_DRAFTS, { limit: 10 }); + const live = targetsResource.source === "live"; + const available = targetsResource.data; + + const [picked, setPicked] = useState(null); + const [scheduledAt, setScheduledAt] = useState(""); + const [visibility, setVisibility] = useState("public"); + const [approval, setApproval] = useState("draft"); + const [busy, setBusy] = useState(false); + const [notice, setNotice] = useState(null); + const [scheduleOpen, setScheduleOpen] = useState(false); + const [previewOpen, setPreviewOpen] = useState(false); + const [keyboardOffset, setKeyboardOffset] = useState(0); + const [media, setMedia] = useState>([]); + const mediaRef = useRef(media); + mediaRef.current = media; + + // Default selection: the first available target, until the user picks. + const targets = (picked ?? available.slice(0, 1).map((t) => t.key)).filter((k) => available.some((t) => t.key === k)); + const selected = available.filter((t) => targets.includes(t.key)); + const limit = selected.length === 0 ? 500 : Math.min(...selected.map((t) => PLATFORMS[t.provider]?.charLimit ?? 500)); + + const editor = useEditor({ + extensions: [ + StarterKit, + Placeholder.configure({ placeholder: "What's shipping? (bold **so** / heading # supported)" }), + CharacterCount, + ], + content: "", + immediatelyRender: false, + // Tiptap v3 no longer re-renders per keystroke by default; the char counter, + // previews and button enablement all read editor state during render. + shouldRerenderOnTransaction: true, + }); + + const chars = editor?.storage.characterCount.characters() ?? 0; + const over = chars > limit; + const mediaBlocked = media.length > 0; + useEffect(() => { + const viewport = window.visualViewport; + if (!viewport) return; + const update = (): void => setKeyboardOffset(Math.max(0, Math.round(window.innerHeight - viewport.height - viewport.offsetTop))); + update(); + viewport.addEventListener("resize", update); + viewport.addEventListener("scroll", update); + return () => { viewport.removeEventListener("resize", update); viewport.removeEventListener("scroll", update); }; + }, []); + useEffect(() => { + if (!editor) return; + const params = new URLSearchParams(window.location.search); + const text = params.get("text"); + if (text) editor.commands.setContent(text); + const requested = params.get("targets")?.split(",").filter(Boolean); + if (requested?.length) setPicked(requested); + }, [editor]); + useEffect(() => () => { mediaRef.current.forEach((item) => URL.revokeObjectURL(item.url)); }, []); + + function postFor(providerMetadata: Record): DraftPost { + return { + text: editor?.getText() ?? "", + mediaAssetIds: [], + visibility, + providerMetadata, + ...(scheduledAt !== "" ? { scheduledAt: new Date(scheduledAt).toISOString() } : {}), + }; + } + + async function run(action: () => Promise): Promise { + if (!live) { + setNotice({ tone: "error", text: "Offline — sample mode, nothing was sent." }); + return; + } + setBusy(true); + setNotice(null); + try { + setNotice({ tone: "ok", text: await action() }); + drafts.reload(); + } catch (err) { + setNotice({ tone: "error", text: errorMessage(err) }); + } finally { + setBusy(false); + } + } + + const saveDraft = (): Promise => + run(async () => { + await api.post("/publishing/drafts", { post: postFor({ targetConnectionIds: targets }) }); + return "Draft saved."; + }); + + /** + * A scheduled publication binds one draft to one connection (UNIQUE draft_id), + * so multi-target posts become one draft per target. Each runs the pipeline as + * far as the caller's permissions allow: create → submit → approve → schedule. + */ + const publish = (): Promise => + run(async () => { + const stopAtReview = approval === "review" || !can("approval.grant"); + const canSchedule = can("publication.schedule"); + const publishAt = scheduledAt !== "" ? new Date(scheduledAt).toISOString() : new Date().toISOString(); + const outcomes: string[] = []; + for (const target of selected) { + const draft = await api.post("/publishing/drafts", { post: postFor({ connectionId: target.key }) }); + await api.post(`/publishing/drafts/${draft.id}/submit`); + if (stopAtReview) { + outcomes.push(`${target.label}: submitted for review`); + continue; + } + await api.post(`/publishing/drafts/${draft.id}/approve`); + if (!canSchedule) { + outcomes.push(`${target.label}: approved (you lack publication.schedule)`); + continue; + } + await api.post(`/publishing/drafts/${draft.id}/schedule`, { connectionId: target.key, publishAt }); + outcomes.push(`${target.label}: ${scheduledAt !== "" ? `scheduled for ${new Date(publishAt).toLocaleString()}` : "queued to publish now"}`); + } + return outcomes.join(" · "); + }); + + return ( +
+
+
+

Compose

+ +
+ +
+ {live && available.length === 0 && ( + No channels connected — add one under Channels to publish. + )} + {available.map((t) => { + const active = targets.includes(t.key); + return ( + + ); + })} +
+ +
+ +
+ +
+
= limit * 0.9 ? "border-warn text-warn" : "border-accent text-accent"}`}>{Math.max(0, limit - chars)}
{selected.map((target) => { const max = PLATFORMS[target.provider]?.charLimit ?? 500; return ; })}
+ {media.length > 0 &&
{media.map((item) =>
{item.alt
setMedia((current) => current.map((value) => value.url === item.url ? { ...value, alt: event.target.value } : value))} placeholder="ALT text" className="mt-1 min-h-11 w-full bg-transparent text-2xs outline-none" />{item.alt ? "ALT ready" : "Add ALT"}
)}
} + {mediaBlocked &&

Media upload is not available in the publishing API yet. Remove previews to save or post text.

} + + + {!can("approval.grant") &&

Your role submits posts for editor approval.

} +
+ +
+ + {chars} / {limit} + + + + + + + + + + + + setScheduledAt(e.target.value)} + className="w-52 py-1 text-xs" + /> + +
+ + +
+
+ {notice !== null && ( +

{notice.text}

+ )} +
+

{approval === "review" || !can("approval.grant") ? "→ Editors approve" : "→ Ready to publish"} · {scheduledAt ? `publishes ${new Date(scheduledAt).toLocaleString()}` : "post now"} · {selected.length} accounts

+
+
+ setScheduleOpen(false)} title="Schedule post" footer={}> setScheduledAt(event.target.value)} className="mt-2 min-h-12" />
{[1, 24, 48].map((hours) => )}
+ setPreviewOpen(false)} title="Target previews">{selected.map((target) =>

{target.label}

{editor?.getText() || "Your post preview appears here…"}

)}
+ {/* Anchor target for the mobile "Resume drafts" shortcut — rendered at every breakpoint. */} + {drafts.source === "live" && drafts.data.length > 0 &&
+

Recent drafts

+ {drafts.data.map((draft) => )} +
} +
+ + +
+ ); +} diff --git a/apps/web/src/app/(main)/connections/page.tsx b/apps/web/src/app/(main)/connections/page.tsx new file mode 100644 index 0000000..d529478 --- /dev/null +++ b/apps/web/src/app/(main)/connections/page.tsx @@ -0,0 +1,236 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { capacitySnapshot, humanizeMs } from "@tactical-media-broadcast-hub/connector-sdk/rate-policy"; +import { Badge, Button, Card } from "../../../components/ui/primitives"; +import { OfflineBadge } from "../../../components/ui/offline-badge"; +import { CONNECTED_ACCOUNTS, CONNECTION_CAPACITIES, DOCUMENTED_RATES, PLATFORMS, type ConnectionCapacity } from "../../../lib/platforms"; +import { CapacityChip } from "../../../components/connections/capacity-chip"; +import { EmptyState } from "../../../components/brand/empty-state"; +import { Loading } from "../../../components/brand/loading"; +import type { ConnectionSummary } from "../../../lib/api-types"; +import { useApiResource } from "../../../lib/use-api"; +import { api, errorMessage } from "../../../lib/api-client"; +import { useAuth } from "../../../lib/auth-store"; +import { reissueSession } from "../../../lib/api-client"; +import { setSessionToken } from "../../../lib/session-token"; +import { linkBotToken, type ConnectProvider } from "../../../lib/connections-client"; + +const NO_PROVIDERS: ConnectProvider[] = []; +const providersFromApi = (providers: ConnectProvider[]): ConnectProvider[] => providers; + +interface AccountView { + key: string; + provider: string; + handle: string; + health: "healthy" | "expiring" | "needs_credentials" | "unavailable"; + capabilities?: string[]; + instanceUrl?: string; + capacity: ConnectionCapacity; +} + +function handleOf(c: ConnectionSummary): string { + const fromFields = c.fields.handle ?? c.fields.displayName; + return typeof fromFields === "string" && fromFields !== "" ? fromFields : c.channelId; +} + +function fromApi(connections: ConnectionSummary[]): AccountView[] { + return connections.map((c) => { + const handle = handleOf(c); + return { + key: c.id, + provider: c.provider, + handle, + health: c.health, + capabilities: c.capabilities, + instanceUrl: typeof c.fields.instanceUrl === "string" ? c.fields.instanceUrl : undefined, + capacity: { + handle, + platform: c.provider, + resolvedCap: DOCUMENTED_RATES[c.provider]?.dailyPostCap, + used: c.recentPublishes.length, + recentPublishes: c.recentPublishes, + }, + }; + }); +} + +const SAMPLE: AccountView[] = CONNECTED_ACCOUNTS.map((a) => ({ + key: a.handle, + provider: a.platform, + handle: a.handle, + health: a.status, + capacity: CONNECTION_CAPACITIES.find((c) => c.handle === a.handle) ?? { handle: a.handle, platform: a.platform, used: 0, recentPublishes: [] }, +})); + +function HealthBadge({ health }: { health: AccountView["health"] }): React.ReactElement { + if (health === "unavailable") return provider unavailable; + if (health === "expiring") return token expiring; + if (health === "needs_credentials") return needs credentials; + return healthy; +} + + +export default function ConnectionsPage(): React.ReactElement { + const { data: accounts, source, error, reload } = useApiResource("/connections", fromApi, SAMPLE); + const live = source === "live"; + const { can } = useAuth(); + const { data: providers } = useApiResource("/connections/providers", providersFromApi, NO_PROVIDERS); + const [selected, setSelected] = useState(null); + const [reconnectId, setReconnectId] = useState(); + const [instanceUrl, setInstanceUrl] = useState(""); + const [token, setToken] = useState(""); + const [target, setTarget] = useState(""); + const [busy, setBusy] = useState(false); + const [notice, setNotice] = useState(null); + const manageable = live && can("connection.manage") && !busy; + + useEffect(() => { + const url = new URL(window.location.href); + const result = url.searchParams.get("connect"); + if (result === null) return; + if (result === "success") void reissueSession().then(() => { reload(); }).catch(() => setNotice("Sign in again to refresh account access.")); + setNotice(result === "success" ? "Account connected successfully." : "Account linking failed or expired. Please try again."); + url.searchParams.delete("connect"); + window.history.replaceState(null, "", url); + }, []); + + // Onboarding (and any deep link) opens the connect dialog via /connections?start=. + useEffect(() => { + if (source !== "live") return; + const url = new URL(window.location.href); + const startId = url.searchParams.get("start"); + if (startId === null) return; + const provider = providers.find((item) => item.id === startId); + url.searchParams.delete("start"); + window.history.replaceState(null, "", url); + if (provider !== undefined) { setSelected(provider); setNotice(null); } + }, [source, providers]); + + function openConnect(provider: ConnectProvider, account?: AccountView): void { + setSelected(provider); setReconnectId(account?.key); setInstanceUrl(account?.instanceUrl ?? ""); setToken(""); setTarget(""); setNotice(null); + } + async function submitConnect(event: React.FormEvent): Promise { + event.preventDefault(); + if (!selected || !manageable) return; + setBusy(true); + try { + if (selected.kind === "token") { + await linkBotToken(token, target, reconnectId); + setToken(""); setSelected(null); reload(); setNotice("Bot connected successfully."); + } else { + const result = reconnectId + ? await api.post<{ redirectUrl: string }>(`/connections/${reconnectId}/reconnect`) + : await api.post<{ redirectUrl: string }>("/connections/authorize", { provider: selected.id, instanceUrl }); + window.location.assign(result.redirectUrl); + } + } catch (err) { setNotice(errorMessage(err)); } + finally { setBusy(false); } + } + function reconnect(account: AccountView): void { + const provider = providers.find((item) => item.id === account.provider); + if (provider) openConnect(provider, account); + } + async function remove(account: AccountView): Promise { + if (!window.confirm(`Remove ${account.handle}? This removes local schedules and publication records for this account. Published posts stay on the provider.`)) return; + setBusy(true); + try { const result = await api.del<{ removed: true; token: string }>(`/connections/${account.key}`); setSessionToken(result.token); reload(); setNotice("Account removed."); } + catch (err) { setNotice(errorMessage(err)); } + finally { setBusy(false); } + } + const [filter, setFilter] = useState<"all" | "attention" | "capacity">("all"); + const atCapacity = (account: AccountView): boolean => account.capacity.resolvedCap !== undefined && account.capacity.used >= account.capacity.resolvedCap; + const urgent = [...accounts].sort((a, b) => Number(b.health !== "healthy") - Number(a.health !== "healthy") || Number(atCapacity(b)) - Number(atCapacity(a))); + const filtered = urgent.filter((account) => filter === "all" || (filter === "attention" && account.health !== "healthy") || (filter === "capacity" && atCapacity(account))); + + return ( +
+
+
+

Channels

+ +
+ +
+ + {notice &&
{notice}
} + {selected &&
void submitConnect(event)} className="mb-6 space-y-3 rounded-lg border border-accent bg-surface p-4"> +

{reconnectId ? "Reconnect" : "Connect"} {selected.displayName}

+ {!reconnectId && } + {selected.instanceRequired && } + {selected.kind === "token" && <>} + {selected.note &&

{selected.note}

} +
+
} +
{([ + { id: "all", label: "Connected", count: accounts.length }, + { id: "attention", label: "Need attention", count: accounts.filter((account) => account.health !== "healthy").length }, + { id: "capacity", label: "At capacity", count: accounts.filter(atCapacity).length }, + ] as const).map((tile) => )}
+
{filtered.map((account) => { + const platform = PLATFORMS[account.provider]; + const cap = account.capacity.resolvedCap; + const snapshot = capacitySnapshot({ used: account.capacity.used, cap, recentPublishes: account.capacity.recentPublishes }, { requestsPerSecond: DOCUMENTED_RATES[account.provider]?.requestsPerSecond ?? 1, dailyPostCap: cap }); + return
{(platform?.displayName ?? account.provider).slice(0, 1)}

{platform?.displayName ?? account.provider}

{account.handle}

{account.health !== "healthy" && }
+
{cap === undefined ? `${account.capacity.used} posts · no documented cap` : `${account.capacity.used}/${cap} posts in 24h`}{snapshot.remaining === 0 && snapshot.nextSlotInMs !== undefined && frees in {humanizeMs(snapshot.nextSlotInMs)}}
+
+
{cap !== undefined &&
}
+
; + })}{filtered.length === 0 &&

No accounts in this filter.

}
+ + {error !== null &&

{error}

} + {source === "loading" && ( +
+ Loading channels… +
+ )} +
+ {live && accounts.length === 0 && ( +
+ +
+ )} + {source !== "loading" && accounts.map((account) => { + const platform = PLATFORMS[account.provider]; + const caps = account.capabilities ?? []; + return ( + +
+
+
+ {platform?.displayName ?? account.provider} + +
+
{account.handle}
+
+
+ + + +
+
+
+ {caps.map((cap) => ( + {cap} + ))} +
+ +
+ ); + })} + +
+
+ {providers.map((provider) =>
{provider.displayName}
{provider.note &&

{provider.note}

}
)} +
+ +

+ Connections store OAuth tokens or bot tokens — passwords never transit this system. Tokens live in the + workspace vault (envelope-encrypted); rotate or revoke any time. +

+
+ ); +} diff --git a/apps/web/src/app/(main)/control-room/control-room.css b/apps/web/src/app/(main)/control-room/control-room.css new file mode 100644 index 0000000..265c367 --- /dev/null +++ b/apps/web/src/app/(main)/control-room/control-room.css @@ -0,0 +1,16 @@ +.control-room .cr-field { + display: block; + width: 100%; + max-width: 100%; + min-width: 44px; + min-height: 44px; + border: 1px solid var(--color-border, #444); + border-radius: 6px; + background: var(--color-surface-2, #222); + padding: 8px 10px; + color: inherit; + font-size: 14px; +} +.control-room label { max-width: 100%; } +.control-room button { min-width: 44px; } +.control-room :focus-visible { outline: 2px solid #ff1610; outline-offset: 3px; } diff --git a/apps/web/src/app/(main)/control-room/page.tsx b/apps/web/src/app/(main)/control-room/page.tsx new file mode 100644 index 0000000..ae50a63 --- /dev/null +++ b/apps/web/src/app/(main)/control-room/page.tsx @@ -0,0 +1,111 @@ +"use client"; +import { useEffect, useMemo, useState } from "react"; +import { useAuth } from "../../../lib/auth-store"; +import { API_URL, ApiError, ApiUnreachableError } from "../../../lib/api-client"; +import { getSessionToken } from "../../../lib/session-token"; +import { Badge, Button } from "../../../components/ui/primitives"; +import { AudioMonitor } from "../../../components/control-room/audio-monitor"; +import { MultiviewGrid, liveTapIds, type GridTier } from "../../../components/control-room/multiview-grid"; +import { OverlayEditor, type RenderInput } from "../../../components/control-room/overlay-editor"; +import { PstPreview } from "../../../components/control-room/pst-preview"; +import { SwitcherBar, type DirectorAction } from "../../../components/control-room/switcher-bar"; +import { controlRequest } from "../../../components/control-room/client"; +import { fixtureChannel } from "../../../components/control-room/fixtures"; +import { RealtimeClient } from "../../../components/control-room/state-store"; +import { initialTally, refetchTally, tallyReducer, type TallyState } from "../../../components/control-room/tally"; +import type { Receipt, Status } from "../../../components/control-room/types"; +import "./control-room.css"; + +export default function ControlRoom(): React.ReactElement { + const { session, can, status: authStatus } = useAuth(); + const [channels, setChannels] = useState([]); + const [selected, setSelected] = useState(""); + const [state, setState] = useState(null); + const [fixture, setFixture] = useState(false); + const [connection, setConnection] = useState("loading"); + const [message, setMessage] = useState(""); + const [busy, setBusy] = useState(false); + const [mobile, setMobile] = useState(true); + const [tier, setTier] = useState("2x2"); + const [focused, setFocused] = useState(null); + const [upgrades, setUpgrades] = useState([]); + const [reload, setReload] = useState(0); + useEffect(() => { + const query = matchMedia("(max-width: 1023px)"); const update = (): void => setMobile(query.matches); + update(); query.addEventListener("change", update); return () => query.removeEventListener("change", update); + }, []); + useEffect(() => { + if (authStatus === "loading") return; + let cancelled = false; + const abort = new AbortController(); + setConnection("loading"); setState(null); setFixture(false); setMessage(""); + void controlRequest("/media-control/status", undefined, abort.signal).then(result => { + if (cancelled) return; + setChannels(result.channels); setSelected(result.channels[0]?.channelId ?? ""); setConnection(result.channels.length ? "connecting" : "empty"); + }).catch(error => { + if (cancelled) return; + if (error instanceof ApiUnreachableError || error instanceof ApiError && error.status >= 500) { + const sample = fixtureChannel(); setFixture(true); setChannels([sample]); setSelected(sample.channelId); setConnection("fixture"); + } else { setChannels([]); setSelected(""); setConnection("error"); setMessage(error instanceof Error ? error.message : "Status unavailable"); } + }); + return () => { cancelled = true; abort.abort(); }; + }, [authStatus, session?.workspace.id, reload]); + useEffect(() => { + const channel = channels.find(item => item.channelId === selected); + setFocused(null); setUpgrades([]); + if (!channel) { setState(null); return; } + setState(initialTally(channel)); + if (fixture || !session) return; + const url = new URL(`${API_URL}/realtime`); url.protocol = url.protocol === "https:" ? "wss:" : "ws:"; + const client = new RealtimeClient({ url: url.href, token: getSessionToken, workspaceId: session.workspace.id, channelId: selected, + socket: address => new WebSocket(address), status: () => controlRequest("/media-control/status"), + onFrame: frame => setState(previous => previous ? tallyReducer(previous, frame) : previous), + onStatus: result => { const current = result.channels.find(item => item.channelId === selected); setState(previous => current ? refetchTally(current, previous) : null); }, + onConnection: setConnection }); + client.start(); return () => client.stop(); + }, [channels, selected, fixture, session]); + const production = state?.channel.desired?.production; + const liveIds = useMemo(() => liveTapIds(production?.sources ?? [], tier, focused, upgrades, mobile), [production, tier, focused, upgrades, mobile]); + const action = async (operation: DirectorAction, body: object): Promise => { + if (!state?.channel.desired) return; + setBusy(true); setMessage(""); + try { + if (fixture) { + const desired = structuredClone(state.channel.desired); + if (desired.production) { + if (operation === "pst-select" && "sceneId" in body) desired.production.buses.preview = String(body.sceneId); + if (operation === "cut" || operation === "take") desired.production.buses.program = desired.production.buses.preview; + if (operation === "lock" && "enabled" in body && desired.director) desired.director.locked = Boolean(body.enabled); + if (operation === "override" && "mode" in body && desired.director) desired.director.override = body.mode === null ? null : { mode: body.mode as NonNullable["mode"], expiresAt: null }; + } + desired.desiredVersion += 1; + setState(previous => previous ? tallyReducer(previous, { type: "state", workspaceId: "fixture", channelId: selected, kind: "desired", seq: previous.lastSeq + 1, data: desired }) : previous); + setMessage("Fixture request recorded. Tally awaits a simulated observed frame."); + } else { + const receipt = await controlRequest(`/media-control/channels/${selected}/${operation}`, body); + setMessage(`Request accepted · version ${receipt.desiredVersion}. Waiting for observed state.`); + } + } catch (error) { setMessage(error instanceof Error ? error.message : "Command failed"); } + finally { setBusy(false); } + }; + const render = async (id: string, input: RenderInput): Promise => { + if (fixture) throw new Error("Fixture mode has no render service. Reconnect to render an overlay."); + await controlRequest(`/production/overlays/${id}`, input); + }; + const simulate = (): void => setState(previous => previous ? tallyReducer(previous, { type: "state", workspaceId: "fixture", channelId: selected, kind: "observed", seq: previous.lastSeq + 1, data: { workspaceId: "fixture", channelId: selected, workerId: previous.channel.workerId, assignmentEpoch: previous.channel.assignmentEpoch, realizedVersion: previous.channel.desired?.desiredVersion ?? null, state: "RUNNING", reason: "Fixture simulation", updatedAt: new Date().toISOString() } }) : previous); + const offline = connection !== "live" && connection !== "polling" && !fixture; + return
+

Control Room

Preview taps · server composition · observed tally

{connection === "fixture" ? "FIXTURE · sample data" : connection}
+ {fixture &&

API unavailable. This is a fixture rehearsal with sample scenes, no broadcast, and no actual program state. Actions stay local.

} + {message &&

{message}

} + {channels.length > 0 && } + {!production || production.scenes.length === 0 ?

{connection === "loading" ? "Loading production status…" : connection === "error" ? "Production status unavailable" : "No production scenes configured"}

{connection === "error" ? "Reconnect or ask a workspace administrator to check your media.view permission. Production state cannot be determined from this response." : "Configure a virtual channel, assign a media worker, add ingest sources and production scenes, then select the preview bus. Program tally appears only after the worker confirms a realized version."}

: <> +
Desired v{state?.channel.desired?.desiredVersion}Observed {state?.channel.observed?.realizedVersion === null || !state?.channel.observed ? "unconfirmed" : `v${state.channel.observed.realizedVersion} · ${state.channel.observed.state}`}{state?.observedProduction ? "Tally confirmed" : "Tally unknown · awaiting matching observed version"}
+ { if (!fixture && !can(operation === "lock" ? "director.lock_program" : operation === "override" ? "director.override_auto" : "director.take")) { setMessage("Your role cannot perform this action."); return; } void action(operation, body); }} /> +
+ { setTier(value); setUpgrades([]); }} upgrades={upgrades} onUpgrade={id => setUpgrades(previous => previous.includes(id) ? previous.filter(item => item !== id) : liveIds.length < 4 ? [...previous, id] : previous)} /> +
+
+ } +
; +} diff --git a/apps/web/src/app/(main)/inbox/[conversationId]/conversation-view.tsx b/apps/web/src/app/(main)/inbox/[conversationId]/conversation-view.tsx new file mode 100644 index 0000000..abc7741 --- /dev/null +++ b/apps/web/src/app/(main)/inbox/[conversationId]/conversation-view.tsx @@ -0,0 +1,18 @@ +"use client"; + +import { useState } from "react"; +import Link from "next/link"; +import { CONVERSATIONS, THREADS, ageLabel } from "../../../../lib/fixtures"; +import { Avatar } from "../../../../components/ui/primitives"; + +export function ConversationView({ conversationId }: { conversationId: string }): React.ReactElement { + const [reply, setReply] = useState(""); + const id = Number(conversationId); + const conversation = CONVERSATIONS.find((item) => item.id === id); + const thread = THREADS[id] ?? []; + return
+
←{conversation && }

{conversation?.participant ?? "Conversation unavailable"}

{conversation?.handle}

+
{thread.map((message) =>
{message.from} · {ageLabel(message.minutes)} ago{message.text}
)}
+
setReply(event.target.value)} placeholder={`Reply to ${conversation?.handle ?? "conversation"}`} className="min-h-11 min-w-0 flex-1 rounded-md border border-border bg-surface-2 px-3 text-sm" />

Sample thread · messaging API is not connected yet.

+
; +} diff --git a/apps/web/src/app/(main)/inbox/[conversationId]/page.tsx b/apps/web/src/app/(main)/inbox/[conversationId]/page.tsx new file mode 100644 index 0000000..497b30d --- /dev/null +++ b/apps/web/src/app/(main)/inbox/[conversationId]/page.tsx @@ -0,0 +1,5 @@ +import { ConversationView } from "./conversation-view"; + +export default function ConversationPage({ params }: { params: { conversationId: string } }): React.ReactElement { + return ; +} diff --git a/apps/web/src/app/(main)/inbox/page.tsx b/apps/web/src/app/(main)/inbox/page.tsx new file mode 100644 index 0000000..7b6331d --- /dev/null +++ b/apps/web/src/app/(main)/inbox/page.tsx @@ -0,0 +1,61 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { useRouter } from "next/navigation"; +import { Avatar } from "../../../components/ui/primitives"; +import { CONVERSATIONS, THREADS, ageLabel, type Conversation, type FeedItem } from "../../../lib/fixtures"; +import { getPlatform, MOCK_COLUMN_DEFS, PLATFORMS } from "../../../lib/platforms"; +import { Segmented } from "../../../components/mobile/segmented"; +import { SwipeItem } from "../../../components/mobile/swipe-item"; +import { ApprovalsSegment } from "../../../components/mobile/approvals-segment"; +import { AmplifySheet } from "../../../components/mobile/amplify-sheet"; +import { useApiResource } from "../../../lib/use-api"; +import type { DraftRecord } from "../../../lib/api-types"; + +type Segment = "all" | "mentions" | "approvals" | "dms"; +type Kind = "spike" | "mention" | "approval" | "reply" | "dm" | "health"; +interface NoticeRow { id: string; kind: Kind; text: string; conversation?: Conversation; unread: boolean; } +const extra: NoticeRow[] = [ + { id: "spike", kind: "spike", text: "Sample spike alert · a post is getting attention", unread: true }, + { id: "approval-sample", kind: "approval", text: "Sample draft waiting for review", unread: true }, + { id: "health", kind: "health", text: "Facebook connection needs attention", unread: true }, +]; +const initial: NoticeRow[] = [...extra, ...CONVERSATIONS.map((conversation, i) => ({ id: `conversation-${conversation.id}`, kind: (i === 0 ? "mention" : i === 1 ? "dm" : "reply") as Kind, text: conversation.preview, conversation, unread: conversation.unread > 0 }))]; +const identity = (value: DraftRecord[]): DraftRecord[] => value; +const empty: DraftRecord[] = []; +const spikePost: FeedItem = { id: -1, platform: MOCK_COLUMN_DEFS[0]?.platformId ?? "unknown", author: "Sample author", handle: "@sample", text: "Sample spike alert post", ageMinutes: 4, likes: 120, reposts: 30, replies: 12, hasMedia: false }; + +export default function InboxPage({ defaultSegment = "all" }: { defaultSegment?: "all" | "approvals" }): React.ReactElement { + const router = useRouter(); + const [segment, setSegment] = useState(defaultSegment); + const [rows, setRows] = useState(initial); + const [backup, setBackup] = useState(null); + const [activeId, setActiveId] = useState(CONVERSATIONS[0]?.id ?? 1); + const [amplify, setAmplify] = useState(false); + const drafts = useApiResource("/publishing/drafts", identity, empty, { state: "in_review", limit: 100 }); + const approvals = drafts.data.length; + // Fixture rows are only shown while the API is unreachable (sample mode). + const sample = drafts.source === "sample"; + useEffect(() => { const requested = new URLSearchParams(window.location.search).get("segment"); if (requested === "approvals" || requested === "mentions" || requested === "dms") setSegment(requested); }, []); + function change(value: Segment): void { setSegment(value); window.history.replaceState(window.history.state, "", value === "all" ? "/inbox" : `/inbox?segment=${value}`); } + function changeRows(update: (current: NoticeRow[]) => NoticeRow[]): void { setBackup(rows); setRows(update); } + const visible = sample ? rows.filter((row) => segment === "all" || (segment === "mentions" && row.kind === "mention") || (segment === "dms" && row.kind === "dm")) : []; + const selected = CONVERSATIONS.find((conversation) => conversation.id === activeId) ?? CONVERSATIONS[0]; + return
+
+

Inbox

{sample && }
+ {backup &&
Inbox updated
} +
row.unread).length + approvals }, { value: "mentions", label: "Mentions", count: visible.filter((row) => row.unread && row.kind === "mention").length }, { value: "approvals", label: "Approvals", count: approvals }, { value: "dms", label: "DMs", count: visible.filter((row) => row.unread && row.kind === "dm").length }]} value={segment} onChange={change} />
+ {segment === "approvals" ? :
{visible.map((row) => changeRows((current) => current.filter((item) => item.id !== row.id)) }]} rightActions={row.kind === "mention" || row.kind === "dm" ? [{ label: "Reply", onClick: () => { if (row.conversation) router.push(`/inbox/${row.conversation.id}`); } }] : []}> +
+ {row.conversation ? : {row.kind === "spike" ? "↗" : "!"}} +
{row.kind}sample{row.conversation && {row.conversation.participant}}{row.conversation && {ageLabel(row.conversation.lastMinutes)}}

{row.text}

{row.conversation && }{row.kind === "spike" && }{row.kind === "approval" && }{row.kind === "health" && }
+
)} + {!sample && segment === "all" &&

No messages yet — direct messages arrive when connector messaging support lands.

} +
} +
+ {segment === "approvals" ?
: sample ? <> +
{selected?.participant} · {PLATFORMS[selected?.platform ?? ""]?.displayName}
{(THREADS[selected?.id ?? 1] ?? []).map((message, index) =>

{message.text}

)}

Sample thread · replies await connector messaging support.

:
Inbox
No messages yet — direct messages arrive when connector messaging support lands.
} + setAmplify(false)} /> +
; +} diff --git a/apps/web/src/app/(main)/layout.tsx b/apps/web/src/app/(main)/layout.tsx new file mode 100644 index 0000000..c01602d --- /dev/null +++ b/apps/web/src/app/(main)/layout.tsx @@ -0,0 +1,73 @@ +"use client"; + +import { useEffect, useRef, type ReactNode } from "react"; +import { usePathname } from "next/navigation"; +import { Sidebar } from "../../components/shell/sidebar"; +import { CommandPalette } from "../../components/shell/command-palette"; +import { BottomNav } from "../../components/mobile/bottom-nav"; +import { MobileTopBar } from "../../components/mobile/top-bar"; +import { AuthProvider, useAuth } from "../../lib/auth-store"; +import { AuthGate, SessionStatus } from "../../components/auth/auth-gate"; +import { useApiResource } from "../../lib/use-api"; +import type { DraftRecord } from "../../lib/api-types"; +import { useIsMobile } from "../../hooks/use-is-mobile"; + +const titles: Record = { + streams: "Home", search: "Search", inbox: "Inbox", approvals: "Approvals", more: "More", compose: "New post", + calendar: "Calendar", video: "Video", analytics: "Analytics", connections: "Connections", team: "Team", + audit: "Audit log", onboarding: "Welcome", settings: "Settings", archive: "Archive", +}; +const rootTabs = new Set(["/streams", "/search", "/inbox", "/more"]); +const scrollPositions = new Map(); +const identity = (drafts: DraftRecord[]): DraftRecord[] => drafts; +const emptyDrafts: DraftRecord[] = []; + +function scrollableChildren(main: HTMLElement): HTMLElement[] { + return Array.from(main.querySelectorAll("*")).filter((element) => + element.scrollHeight > element.clientHeight && /auto|scroll/.test(getComputedStyle(element).overflowY)); +} + +function Shell({ children }: { children: ReactNode }): React.ReactElement { + const pathname = usePathname(); + const mobile = useIsMobile(); + const { can } = useAuth(); + const drafts = useApiResource("/publishing/drafts", identity, emptyDrafts, { state: "in_review", limit: 100 }); + // Badge comes from live approvals-in-review only — 0 while loading/sample/forbidden, never fixture counts. + const approvals = can("approval.grant") && drafts.source === "live" ? drafts.data.length : 0; + const mainRef = useRef(null); + useEffect(() => { + const main = mainRef.current; + if (main === null || !mobile) return; + const saved = scrollPositions.get(pathname); + const restore = window.setTimeout(() => { + main.scrollTop = saved?.main ?? 0; + scrollableChildren(main).forEach((element, index) => { element.scrollTop = saved?.nested[index] ?? 0; }); + }, 0); + return () => { + window.clearTimeout(restore); + scrollPositions.set(pathname, { main: main.scrollTop, nested: scrollableChildren(main).map((element) => element.scrollTop) }); + }; + }, [pathname, mobile]); + const title = titles[pathname.split("/")[1] ?? ""] ?? "Tactical Media Broadcast Hub"; + return ( +
+ +
+
+
Press ⌘K for commands
+
+
+ {mobile && } +
+ {children} +
+ {mobile && } +
+ {!mobile && } +
+ ); +} + +export default function MainLayout({ children }: { children: ReactNode }): React.ReactNode { + return {children}; +} diff --git a/apps/web/src/app/(main)/more/page.tsx b/apps/web/src/app/(main)/more/page.tsx new file mode 100644 index 0000000..c7c25ea --- /dev/null +++ b/apps/web/src/app/(main)/more/page.tsx @@ -0,0 +1,70 @@ +"use client"; + +import Link from "next/link"; +import { useAuth } from "../../../lib/auth-store"; +import { useApiResource } from "../../../lib/use-api"; +import { Avatar } from "../../../components/ui/primitives"; +import type { DraftRecord } from "../../../lib/api-types"; + +interface GroupSummary { id: string; name: string; } +const identity = (groups: GroupSummary[]): GroupSummary[] => groups; +const identityDrafts = (drafts: DraftRecord[]): DraftRecord[] => drafts; +const emptyDrafts: DraftRecord[] = []; + +function Tile({ href, title, detail }: { href: string; title: string; detail: string }): React.ReactElement { + return + ✦{title}{detail} + ; +} + +function Row({ href, title, detail }: { href: string; title: string; detail?: string }): React.ReactElement { + return + {title}{detail && {detail}} + + ; +} + +export default function MorePage(): React.ReactElement { + const { session, can, logout } = useAuth(); + const groups = useApiResource("/groups/mine", identity, []); + const drafts = useApiResource("/publishing/drafts", identityDrafts, emptyDrafts, { limit: 100 }); + const canApprove = can("approval.grant") || session?.role === "Editor"; + const canManage = can("workspace.members.manage"); + const canAudit = can("audit.view"); + return
+
+
+
+

{session?.user.displayName || session?.user.username || "Tactical Media Broadcast Hub"}

+

@{session?.user.username || "offline"}{session?.user.email ? ` · ${session.user.email}` : ""}

+
{session?.role || "Preview"}{session?.workspace.name || "Studio"}
+
+
+

Your groups

+
{groups.data.length ? groups.data.map((group) => {group.name}) : No groups yet}
+ {(canApprove || canManage || canAudit) && <> +

Team

+
+ {canApprove && draft.state === "in_review").length} waiting`} />} + {canManage && } + {canManage && } + {canAudit && } +
+ } +

Help

+
+ +
+

Settings

+
+ + + + + + + + +
+
; +} diff --git a/apps/web/src/app/(main)/onboarding/page.tsx b/apps/web/src/app/(main)/onboarding/page.tsx new file mode 100644 index 0000000..a5cc719 --- /dev/null +++ b/apps/web/src/app/(main)/onboarding/page.tsx @@ -0,0 +1,363 @@ +"use client"; + +import { useEffect, useState, type FormEvent } from "react"; +import Link from "next/link"; +import { Badge, Button, Card, Input } from "../../../components/ui/primitives"; +import { OfflineBadge } from "../../../components/ui/offline-badge"; +import { ArrowGlyph } from "../../../components/brand/logo"; +import { PLATFORMS } from "../../../lib/platforms"; +import { useApiResource } from "../../../lib/use-api"; +import { api, errorMessage } from "../../../lib/api-client"; +import type { MemberMutationResult } from "../../../lib/api-types"; +import type { ConnectProvider } from "../../../lib/connections-client"; +import { useAuth } from "../../../lib/auth-store"; + +const STEPS = ["Create workspace", "Invite team", "Connect accounts"]; +const MIN_PASSWORD = 10; +const INVITE_ROLE_CHOICES = ["Owner", "Editor", "Viewer"] as const; +const DEFAULT_INVITE_ROLE = "Editor"; +const NO_PROVIDERS: ConnectProvider[] = []; + +function platformName(id: string): string { + return PLATFORMS[id]?.displayName ?? id; +} + +function SignInForm({ onSwitch }: { onSwitch: () => void }): React.ReactElement { + const { login } = useAuth(); + const [username, setUsername] = useState(""); + const [password, setPassword] = useState(""); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + + async function submit(e: FormEvent): Promise { + e.preventDefault(); + setBusy(true); + setError(null); + try { + await login(username.trim(), password); + window.location.assign("/streams"); + } catch (err) { + setError(errorMessage(err)); + setBusy(false); + } + } + + return ( +
void submit(e)}> +

Sign in

+

Use the username and password you registered with.

+
+ setUsername(e.target.value)} placeholder="username" autoComplete="username" autoFocus /> + setPassword(e.target.value)} placeholder="Password" autoComplete="current-password" /> +
+ {error !== null &&

{error}

} +
+ + +
+
+ ); +} + +export default function OnboardingPage(): React.ReactElement { + const { status, session, register, adoptToken, logout } = useAuth(); + const offline = status === "offline"; + const [mode, setMode] = useState<"register" | "signin">("register"); + const [step, setStep] = useState(0); + const [name, setName] = useState(""); + const [displayName, setDisplayName] = useState(""); + const [username, setUsername] = useState(""); + const [email, setEmail] = useState(""); + const [password, setPassword] = useState(""); + const [emails, setEmails] = useState(""); + const [inviteRole, setInviteRole] = useState(DEFAULT_INVITE_ROLE); + const { data: providers, source: providerSource } = useApiResource("/connections/providers", (p) => p, NO_PROVIDERS); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const [sessionExpired, setSessionExpired] = useState(false); + + // The API client redirects here with ?reason=session on a 401: default to sign-in. + useEffect(() => { + if (new URLSearchParams(window.location.search).get("reason") === "session") { + setSessionExpired(true); + setMode("signin"); + } + }, []); + + async function createWorkspace(): Promise { + if (offline) { + setStep(1); + return; + } + setBusy(true); + setError(null); + try { + await register({ + workspaceName: name.trim(), + username: username.trim(), + password, + ...(displayName.trim() === "" ? {} : { displayName: displayName.trim() }), + ...(email.trim() === "" ? {} : { email: email.trim() }), + }); + setStep(1); + } catch (err) { + setError(errorMessage(err)); + } finally { + setBusy(false); + } + } + + async function sendInvites(): Promise { + const list = emails.split(/[\s,;]+/).map((e) => e.trim()).filter((e) => e.includes("@")); + if (offline || list.length === 0) { + setStep(2); + return; + } + setBusy(true); + setError(null); + const failures: string[] = []; + // Sequential: each invite bumps the authz epoch and returns the token the next call must use. + for (const invitee of list) { + try { + const result = await api.post("/team/members/invite", { email: invitee, role: inviteRole }); + adoptToken(result.token); + } catch (err) { + failures.push(`${invitee}: ${errorMessage(err)}`); + } + } + setBusy(false); + if (failures.length > 0) setError(failures.join(" · ")); + else setStep(2); + } + + const usernameValid = /^[a-z0-9][a-z0-9_.-]{1,31}$/.test(username.trim()); + const registerReady = + name.trim().length >= 2 && (offline || (usernameValid && password.length >= MIN_PASSWORD)); + + // Signed-in visitors who land here (e.g. from the sidebar) get a way out rather than a second registration form. + if (status === "authenticated" && session !== null && step === 0) { + return ( +
+ +

You're signed in

+

+ @{session.user.username} · {session.role} of {session.workspace.name} +

+
+ + + Continue to Streams + +
+
+
+ ); + } + + return ( +
+ +
+ +
+ {offline && ( +
+ +
+ )} + {sessionExpired && step === 0 && ( +

+ Your session ended (expired, or your role changed). Sign in again. +

+ )} + + {mode === "signin" && step === 0 && !offline ? ( + setMode("register")} /> + ) : ( + <> +
+ {STEPS.map((s, i) => ( +
+ + {i < step ? "✓" : i + 1} + + {s} + {i < STEPS.length - 1 && } +
+ ))} +
+ + {step === 0 && ( +
{ + e.preventDefault(); + void createWorkspace(); + }} + > +
+
+ Tactical Media Broadcast Hub — arrows from a circle, warm red on black +
+
+

Tactical Media Broadcast Hub

+

Your platform. Their feeds. No algorithm in between.

+
+
+

Name your workspace

+

The tenant boundary for accounts, drafts, and audit.

+ setName(e.target.value)} placeholder="Your collective, your studio…" autoFocus /> + {!offline && ( +
+

Your owner account

+ setDisplayName(e.target.value)} placeholder="Your name (optional)" autoComplete="name" /> + setUsername(e.target.value.toLowerCase())} + placeholder="Username — this is how you sign in" + autoComplete="username" + pattern="[a-z0-9][a-z0-9_.-]{1,31}" + title="2-32 characters: lowercase letters, digits, dot, underscore or hyphen" + /> + setPassword(e.target.value)} + placeholder={`Password (min ${MIN_PASSWORD} characters)`} + autoComplete="new-password" + /> + setEmail(e.target.value)} + placeholder="Email (optional)" + autoComplete="email" + /> +
+ )} + {error !== null &&

{error}

} +
+ {offline ? : ( + + )} + +
+
+ )} + + {step === 1 && ( +
+

Invite your team

+

+ Comma-separated emails. Invitees join at the chosen role; adjust roles on the Team page. +

+ ", + "", + "", + // Encoding / comment / malformed evasions + "<script>alert(1)</script>", + "", + "-->", + "alert(1)]]>", + "", + "alert(1)", + "quotes", + "", + "

", + "nested", + "", + // Spoofing + "https://bank.example/login", + "paypal.com", + "

file‮gnp.exe and zero​width⁦isolate⁩

", +]; + +export const MALICIOUS_PLAIN: readonly string[] = [ + "", + "", + "javascript:alert(1)", + "click javascript://%0aalert(1) now", + "data:text/html,", + "https://evil.example/\">", + "see https://user:pass@evil.example/ and //evil.example", + "@alice@evil.example #tag bold", + "‮txt.exe ​‍", + "<script>", +]; + +export const MALICIOUS_LINK_PREVIEWS: readonly unknown[] = [ + { url: "javascript:alert(1)", title: "x" }, + { url: "data:text/html,", title: "x" }, + { url: "//evil.example/", title: "protocol-relative" }, + { url: "https://ok.example/", title: "", description: "", image: "https://evil.example/og.png" }, + { url: "https://ok.example/", image: "javascript:alert(1)" }, + { url: "https://ok.example/", image: "data:image/svg+xml," }, + { url: "https://ok.example/", image: "//evil.example/og.png" }, + { url: "https://ok.example/", siteName: "‮lapyap", title: "x".repeat(100_000) }, + { type: "linkPreview", href: "javascript:alert(1)", host: "ok.example" }, + { type: "linkPreview", href: "https://ok.example/", host: "ok.example", image: "https://evil.example/direct.png" }, + { url: { toString: () => "javascript:alert(1)" } }, + null, + "https://ok.example/", +]; + +/** Hand-forged ASTs, as if from a tampered DB row or websocket frame. */ +export const FORGED_DOCUMENTS: readonly unknown[] = [ + { + type: "doc", + version: 1, + truncated: false, + children: [ + { + type: "paragraph", + children: [ + { type: "link", href: "javascript:alert(1)", host: "ok.example", children: [{ type: "text", text: "x" }] }, + { type: "link", href: "https://ok.example/", host: "ok.example", children: [{ type: "html", html: "" }] }, + { type: "emoji", shortcode: "x", src: "https://evil.example/e.png" }, + { type: "emoji", shortcode: "x", src: "/api/media-proxy?url=javascript%3Aalert(1)" }, + { type: "emoji", shortcode: "x\"onerror=alert(1)", src: "/api/media-proxy?url=https%3A%2F%2Fok.example%2Fe.png" }, + { type: "html", html: "" }, + { type: "text", text: "" }, + { type: "mention", handle: "x\">", + ], + }, + { type: "doc", children: "nope" }, + "", + null, + [], +]; diff --git a/packages/safe-content/src/entities.ts b/packages/safe-content/src/entities.ts new file mode 100644 index 0000000..04d5e8a --- /dev/null +++ b/packages/safe-content/src/entities.ts @@ -0,0 +1,18 @@ +const NAMED: Record = { + amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: "\u00A0", + hellip: "…", mdash: "—", ndash: "–", lsquo: "‘", rsquo: "’", ldquo: "“", rdquo: "”", + copy: "©", reg: "®", trade: "™", laquo: "«", raquo: "»", middot: "·", bull: "•", +}; + +/** Decodes HTML character references to plain text. Output is still only ever a React text node. */ +export function decodeEntities(input: string): string { + if (!input.includes("&")) return input; + return input.replace(/&(?:#[xX]([0-9a-fA-F]{1,8})|#([0-9]{1,10})|([A-Za-z]{2,8}));?/g, (whole, hex, dec, name) => { + if (hex !== undefined || dec !== undefined) { + const code = hex !== undefined ? parseInt(hex, 16) : parseInt(dec, 10); + if (code === 0 || code > 0x10ffff || (code >= 0xd800 && code <= 0xdfff)) return "�"; + return String.fromCodePoint(code); + } + return Object.hasOwn(NAMED, name) && whole.endsWith(";") ? (NAMED[name] as string) : whole; + }); +} diff --git a/packages/safe-content/src/eslint.ts b/packages/safe-content/src/eslint.ts new file mode 100644 index 0000000..7f4c985 --- /dev/null +++ b/packages/safe-content/src/eslint.ts @@ -0,0 +1,31 @@ +/** + * Flat-config block banning raw-HTML sinks (proposal §4.5 (1)). Spread it into + * the root `eslint.config.mjs`; `packages/safe-content/**` is the only + * sanctioned renderer and is excluded via `ignores`. + * + * import { noRawHtmlSinks } from "@tactical-media-broadcast-hub/safe-content/eslint"; + * export default tseslint.config(..., noRawHtmlSinks); + */ +const MESSAGE = "Raw HTML sink — render provider content only through @tactical-media-broadcast-hub/safe-content (proposal §4.5)."; + +export const RAW_HTML_SINK_SELECTORS: readonly string[] = [ + "JSXAttribute[name.name='dangerouslySetInnerHTML']", + "Property[key.name='dangerouslySetInnerHTML']", + "AssignmentExpression > MemberExpression.left[property.name=/^(innerHTML|outerHTML)$/]", + "CallExpression[callee.property.name=/^(insertAdjacentHTML|createContextualFragment|parseFromString|setHTMLUnsafe)$/]", + "CallExpression[callee.object.name='document'][callee.property.name=/^(write|writeln)$/]", + "NewExpression[callee.name='Function']", + "CallExpression[callee.name='eval']", +]; + +export const noRawHtmlSinks: { + files: string[]; + ignores: string[]; + rules: { "no-restricted-syntax": ["error", ...{ selector: string; message: string }[]] }; +} = { + files: ["**/*.ts", "**/*.tsx", "**/*.js", "**/*.jsx"], + ignores: ["packages/safe-content/**"], + rules: { + "no-restricted-syntax": ["error", ...RAW_HTML_SINK_SELECTORS.map((selector) => ({ selector, message: MESSAGE }))], + }, +}; diff --git a/packages/safe-content/src/html.ts b/packages/safe-content/src/html.ts new file mode 100644 index 0000000..a84a5e8 --- /dev/null +++ b/packages/safe-content/src/html.ts @@ -0,0 +1,269 @@ +import type { EmojiNode, RichTextDocument, TextNode } from "./ast.js"; +import { DocBuilder, type ParseOptions } from "./builder.js"; +import { decodeEntities } from "./entities.js"; + +/* + * Converts the Mastodon/PeerTube HTML subset into the rich-text AST. This is + * a converter, not an HTML renderer: no tag or attribute survives as markup. + * Recognised structure (p, br, a, Mastodon mention/hashtag/invisible/ellipsis + * spans) maps to AST nodes; dangerous containers are dropped with their + * content; every other tag is unwrapped to its text. The tokenizer is a single + * forward scan (no regex backtracking), so huge or malformed input is linear. + */ + +/** Element content parsed as raw text by browsers; skipped up to the matching end tag. */ +const RAW_TEXT = new Set(["script", "style", "textarea", "title", "xmp", "iframe", "noembed", "noframes", "noscript"]); +/** Elements whose entire content is discarded. */ +const DROP_CONTENT = new Set([ + "svg", "math", "template", "object", "embed", "applet", "frame", "frameset", "head", "select", "option", + "form", "button", "audio", "video", "canvas", "map", "picture", "source", "track", "dialog", "portal", +]); +const VOID = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"]); +const PARAGRAPH = new Set([ + "p", "div", "blockquote", "pre", "ul", "ol", "li", "h1", "h2", "h3", "h4", "h5", "h6", + "section", "article", "header", "footer", "aside", "table", "tr", "dl", "dt", "dd", "figure", "figcaption", "hr", +]); + +type Token = + | { kind: "text"; text: string } + | { kind: "start"; name: string; attrs: Map; selfClosing: boolean } + | { kind: "end"; name: string }; + +const MAX_ATTRS = 16; + +function isAlpha(c: number): boolean { + return (c >= 65 && c <= 90) || (c >= 97 && c <= 122); +} +function isSpace(c: number): boolean { + return c === 32 || c === 9 || c === 10 || c === 12 || c === 13; +} + +function* tokenize(src: string): Generator { + const lower = src.toLowerCase(); + const n = src.length; + let i = 0; + while (i < n) { + const lt = src.indexOf("<", i); + if (lt === -1) { + yield { kind: "text", text: src.slice(i) }; + return; + } + if (lt > i) yield { kind: "text", text: src.slice(i, lt) }; + i = lt; + const next = src.charCodeAt(i + 1); + if (src.startsWith("", i + 4); + i = close === -1 ? n : close + 3; + continue; + } + if (next === 33 /* ! */ || next === 63 /* ? */ || src.startsWith("", i)) { + // Bogus comment / doctype / processing instruction / "". + const close = src.indexOf(">", i + 1); + i = close === -1 ? n : close + 1; + continue; + } + const isEnd = next === 47; + const nameStart = isEnd ? i + 2 : i + 1; + if (!isAlpha(src.charCodeAt(nameStart))) { + yield { kind: "text", text: "<" }; + i++; + continue; + } + let j = nameStart; + while (j < n && !isSpace(src.charCodeAt(j)) && src[j] !== "/" && src[j] !== ">") j++; + const name = lower.slice(nameStart, j); + // Attributes: name[=value], value quoted or unquoted. + const attrs = new Map(); + let selfClosing = false; + while (j < n && src[j] !== ">") { + const c = src.charCodeAt(j); + if (isSpace(c)) { j++; continue; } + if (src[j] === "/") { selfClosing = true; j++; continue; } + selfClosing = false; + const an = j; + while (j < n && !isSpace(src.charCodeAt(j)) && src[j] !== "/" && src[j] !== ">" && src[j] !== "=") j++; + if (j === an) j++; // stray "=": consume to guarantee progress + const attrName = lower.slice(an, j); + let value = ""; + while (j < n && isSpace(src.charCodeAt(j))) j++; + if (src[j] === "=") { + j++; + while (j < n && isSpace(src.charCodeAt(j))) j++; + const q = src[j]; + if (q === '"' || q === "'") { + const close = src.indexOf(q, j + 1); + const end = close === -1 ? n : close; + value = src.slice(j + 1, end); + j = end + 1; + } else { + const vs = j; + while (j < n && !isSpace(src.charCodeAt(j)) && src[j] !== ">") j++; + value = src.slice(vs, j); + } + } + if (attrName && !attrs.has(attrName) && attrs.size < MAX_ATTRS) attrs.set(attrName, decodeEntities(value)); + } + i = j + 1; + if (isEnd) { + yield { kind: "end", name }; + continue; + } + if (name === "plaintext") return; + if (RAW_TEXT.has(name)) { + const close = lower.indexOf(" { + const gt = src.indexOf(">", close); + return gt === -1 ? n : gt + 1; + })(); + continue; + } + yield { kind: "start", name, attrs, selfClosing }; + } +} + +type Frame = { name: string; role: "drop" | "link" | "invisible" | "ellipsis" | "other" }; + +type OpenLink = { href: string; classes: Set; rel: Set; label: (TextNode | EmojiNode)[] }; + +function words(value: string | undefined): Set { + return new Set((value ?? "").toLowerCase().split(/\s+/).filter(Boolean)); +} + +/** Mastodon/PeerTube-style HTML → rich-text AST. */ +export function parseProviderHtml(input: unknown, options: ParseOptions = {}): RichTextDocument { + const builder = new DocBuilder(options); + if (typeof input !== "string") return builder.finish(); + const src = builder.clip(input); + const stack: Frame[] = []; + let dropDepth = 0; + let invisibleDepth = 0; + let link: OpenLink | null = null; + + const flushLink = () => { + if (!link) return; + const current = link; + link = null; + const labelText = current.label.map((n) => (n.type === "text" ? n.text : `:${n.shortcode}:`)).join("").trim(); + if (current.classes.has("mention") && (current.classes.has("hashtag") || current.rel.has("tag"))) { + builder.hashtag(labelText, current.href); + } else if (current.classes.has("mention")) { + builder.mention(handleFor(labelText, current.href), current.href); + } else { + builder.link(current.href, current.label); + } + }; + + const emitText = (raw: string) => { + if (dropDepth > 0) return; + if (link && invisibleDepth > 0) return; + const text = decodeEntities(raw).replace(/[\t\n\f\r ]+/g, " "); + if (link) link.label.push(...builder.inlineText(text)); + else builder.text(text); + }; + + const close = (index: number) => { + while (stack.length > index) { + const frame = stack.pop() as Frame; + if (frame.role === "drop") dropDepth--; + else if (frame.role === "invisible") invisibleDepth--; + else if (frame.role === "ellipsis" && link && dropDepth === 0) link.label.push({ type: "text", text: "…" }); + else if (frame.role === "link") flushLink(); + if (PARAGRAPH.has(frame.name) && dropDepth === 0) { + flushLink(); + builder.endParagraph(); + } + } + }; + + for (const token of tokenize(src)) { + if (token.kind === "text") { + emitText(token.text); + continue; + } + if (token.kind === "end") { + if (token.name === "br" && dropDepth === 0) builder.lineBreak(); // "
" acts as
in browsers + for (let k = stack.length - 1; k >= 0; k--) { + const frame = stack[k] as Frame; + // Inside dropped content only the dropped elements' own end tags count. + if (frame.name === token.name && (dropDepth === 0 || frame.role === "drop")) { + close(k); + break; + } + } + continue; + } + const { name, attrs } = token; + if (dropDepth > 0) { + if (DROP_CONTENT.has(name) && !VOID.has(name) && !token.selfClosing && stack.length < builder.limits.maxDepth) { + stack.push({ name, role: "drop" }); + dropDepth++; + } + continue; + } + if (name === "br") { + if (link) link.label.push({ type: "text", text: " " }); + else builder.lineBreak(); + continue; + } + if (name === "img") { + const alt = attrs.get("alt") ?? ""; + if (alt) emitText(alt); + continue; + } + if (PARAGRAPH.has(name)) { + flushLink(); + builder.endParagraph(); + } + if (VOID.has(name) || token.selfClosing && !DROP_CONTENT.has(name) && name !== "a" && name !== "span") continue; + if (stack.length >= builder.limits.maxDepth) { + // Too deep: stop tracking structure, keep text (inert) and flag it. + builder.truncated = true; + if (DROP_CONTENT.has(name)) { + // Can't track the matching end tag any more; drop the rest of the input. + break; + } + continue; + } + if (DROP_CONTENT.has(name)) { + if (token.selfClosing) continue; + stack.push({ name, role: "drop" }); + dropDepth++; + continue; + } + if (name === "a") { + flushLink(); + link = { href: attrs.get("href") ?? "", classes: words(attrs.get("class")), rel: words(attrs.get("rel")), label: [] }; + stack.push({ name, role: "link" }); + continue; + } + if (name === "span" && link) { + const classes = words(attrs.get("class")); + if (classes.has("invisible")) { + invisibleDepth++; + stack.push({ name, role: "invisible" }); + continue; + } + if (classes.has("ellipsis")) { + stack.push({ name, role: "ellipsis" }); + continue; + } + } + stack.push({ name, role: "other" }); + } + close(0); + flushLink(); + return builder.finish(); +} + +/** Mastodon renders only the local username in mention text; qualify it with the href host. */ +function handleFor(label: string, href: string): string { + const handle = label.replace(/^@+/, ""); + if (handle.includes("@")) return handle; + try { + const url = new URL(href); + if (url.protocol === "http:" || url.protocol === "https:") return `${handle}@${url.hostname}`; + } catch { + // fall through: unqualified handle + } + return handle; +} diff --git a/packages/safe-content/src/index.ts b/packages/safe-content/src/index.ts new file mode 100644 index 0000000..1045cdb --- /dev/null +++ b/packages/safe-content/src/index.ts @@ -0,0 +1,15 @@ +export * from "./ast.js"; +export { DEFAULT_LIMITS, LINK_PREVIEW_LIMITS, type SafeContentLimits } from "./limits.js"; +export { hasMixedScriptConfusables, neutralizeDisplayName, neutralizeText } from "./text.js"; +export { + DEFAULT_MEDIA_PROXY, + isProxiedMediaUrl, + parseSafeHttpUrl, + toProxiedMediaUrl, + type MediaProxyPolicy, + type SafeUrl, +} from "./url.js"; +export type { CustomEmoji, ParseOptions } from "./builder.js"; +export { parsePlainText } from "./plain.js"; +export { parseProviderHtml } from "./html.js"; +export { coerceRichTextDocument, parseLinkPreview, type RawLinkPreview } from "./validate.js"; diff --git a/packages/safe-content/src/limits.ts b/packages/safe-content/src/limits.ts new file mode 100644 index 0000000..986cb23 --- /dev/null +++ b/packages/safe-content/src/limits.ts @@ -0,0 +1,23 @@ +export type SafeContentLimits = { + /** Max input characters considered; the rest is dropped and `truncated` set. */ + readonly maxInputLength: number; + /** Max element nesting tracked by the HTML parser; deeper structure is flattened to text. */ + readonly maxDepth: number; + /** Max AST nodes emitted per document. */ + readonly maxNodes: number; + /** Max characters in any single URL. */ + readonly maxUrlLength: number; +}; + +export const DEFAULT_LIMITS: SafeContentLimits = { + maxInputLength: 20_000, + maxDepth: 32, + maxNodes: 2_000, + maxUrlLength: 2_048, +}; + +export const LINK_PREVIEW_LIMITS = { + title: 300, + description: 1_000, + siteName: 100, +} as const; diff --git a/packages/safe-content/src/parse.test.tsx b/packages/safe-content/src/parse.test.tsx new file mode 100644 index 0000000..ea96503 --- /dev/null +++ b/packages/safe-content/src/parse.test.tsx @@ -0,0 +1,179 @@ +import { Linter } from "eslint"; +import { renderToStaticMarkup } from "react-dom/server"; +import { describe, expect, it } from "vitest"; +import { noRawHtmlSinks } from "./eslint.js"; +import { parseProviderHtml } from "./html.js"; +import { parsePlainText } from "./plain.js"; +import { RichText } from "./react.js"; +import { isProxiedMediaUrl, parseSafeHttpUrl, toProxiedMediaUrl } from "./url.js"; +import { coerceRichTextDocument, parseLinkPreview } from "./validate.js"; + +describe("Mastodon HTML", () => { + const status = + '

@alice ' + + 'look :blobcat:

' + + '

read ' + + 'news.example/articles/2026/v
bye

'; + + it("maps mentions, hashtags, links, emoji and breaks to the AST", () => { + const doc = parseProviderHtml(status, { emojis: [{ shortcode: "blobcat", url: "https://cdn.example/blobcat.png" }] }); + expect(doc).toEqual({ + type: "doc", + version: 1, + truncated: false, + children: [ + { + type: "paragraph", + children: [ + { type: "mention", handle: "alice@social.example", href: "https://social.example/@alice" }, + { type: "text", text: " look " }, + { type: "emoji", shortcode: "blobcat", src: "/api/media-proxy?url=https%3A%2F%2Fcdn.example%2Fblobcat.png" }, + { type: "text", text: " " }, + { type: "hashtag", tag: "fediverse", href: "https://social.example/tags/fediverse" }, + ], + }, + { + type: "paragraph", + children: [ + { type: "text", text: "read " }, + { + type: "link", + href: "https://news.example/articles/2026/very-long-path", + host: "news.example", + children: [{ type: "text", text: "news.example/articles/2026/v…" }], + }, + { type: "linebreak" }, + { type: "text", text: "bye" }, + ], + }, + ], + }); + }); + + it("renders the expected markup", () => { + const html = renderToStaticMarkup(); + expect(html).toContain('@alice@social.example'); + expect(html).toContain(":blobcat:"); + expect(html).not.toContain("sc-link-host"); + }); +}); + +describe("plain text", () => { + it("linkifies http(s) URLs, mentions, hashtags and newlines", () => { + const doc = parsePlainText("hi @bob@x.example see https://a.example/p?q=1. #tag\nline2\n\npara2"); + expect(doc.children).toHaveLength(2); + expect(doc.children[0]!.children).toEqual([ + { type: "text", text: "hi " }, + { type: "mention", handle: "bob@x.example" }, + { type: "text", text: " see " }, + { type: "link", href: "https://a.example/p?q=1", host: "a.example", children: [{ type: "text", text: "a.example/p?q=1" }] }, + { type: "text", text: ". " }, + { type: "hashtag", tag: "tag" }, + { type: "linebreak" }, + { type: "text", text: "line2" }, + ]); + }); + + it("ignores non-string input", () => { + expect(parsePlainText(42).children).toEqual([]); + }); +}); + +describe("URL policy", () => { + it.each([ + "javascript:alert(1)", " javascript:alert(1)", "JAVASCRIPT:alert(1)", "java\tscript:alert(1)", "data:text/html,x", + "vbscript:x", "file:///etc/passwd", "//evil.example", "/relative", "https:evil.example", "https:/evil.example", + "https://user@evil.example/", "https://:pw@evil.example/", "http://evil.example/\u0000", "https://evil.example/‮", + "ftp://evil.example/", "https://" + "a".repeat(3_000) + ".example/", "", + ])("rejects %j", (raw) => { + expect(parseSafeHttpUrl(raw)).toBeNull(); + }); + + it("accepts http(s) and normalises", () => { + expect(parseSafeHttpUrl("HTTPS://Ok.Example/a b".replace(" ", "%20"))).toEqual({ href: "https://ok.example/a%20b", host: "ok.example" }); + }); + + it("media proxy URLs are canonical and same-origin", () => { + const src = toProxiedMediaUrl("https://cdn.example/a.png"); + expect(src).toBe("/api/media-proxy?url=https%3A%2F%2Fcdn.example%2Fa.png"); + expect(isProxiedMediaUrl(src)).toBe(true); + expect(isProxiedMediaUrl("https://cdn.example/a.png")).toBe(false); + expect(isProxiedMediaUrl("/api/media-proxy?url=https://cdn.example/a.png")).toBe(false); + expect(isProxiedMediaUrl("/api/media-proxy?url=javascript%3Aalert(1)")).toBe(false); + expect(toProxiedMediaUrl("data:image/png;base64,AAAA")).toBeNull(); + expect(() => toProxiedMediaUrl("https://cdn.example/a.png", { proxyPath: "//evil.example/p" })).toThrow(); + expect(() => toProxiedMediaUrl("https://cdn.example/a.png", { proxyPath: "https://evil.example/p" })).toThrow(); + }); + + it("supports a custom proxy path end to end", () => { + const mediaProxy = { proxyPath: "/media/p" }; + const doc = parseProviderHtml("hi :e:", { mediaProxy, emojis: [{ shortcode: "e", url: "https://cdn.example/e.png" }] }); + const html = renderToStaticMarkup(); + expect(html).toContain('src="/media/p?url=https%3A%2F%2Fcdn.example%2Fe.png"'); + // Same doc rendered against a different proxy: emoji degrades to text. + expect(renderToStaticMarkup()).not.toContain(" { + it("round-trips a valid document unchanged", () => { + const doc = parseProviderHtml('

a ok.example @x #y

'); + expect(coerceRichTextDocument(JSON.parse(JSON.stringify(doc)))).toEqual(doc); + }); + + it("re-applies node limits", () => { + const forged = { type: "doc", children: [{ type: "paragraph", children: Array.from({ length: 10_000 }, (_, i) => (i % 2 ? { type: "linebreak" } : { type: "mention", handle: "a" })) }] }; + const doc = coerceRichTextDocument(forged); + expect(doc.truncated).toBe(true); + expect(doc.children[0]!.children.length).toBeLessThanOrEqual(2_000); + }); + + it("bounds total text from forged nodes", () => { + const forged = { type: "doc", children: [{ type: "paragraph", children: [{ type: "text", text: "x".repeat(5_000_000) }] }] }; + const doc = coerceRichTextDocument(forged); + expect(doc.truncated).toBe(true); + expect(JSON.stringify(doc).length).toBeLessThan(25_000); + }); +}); + +describe("parseLinkPreview", () => { + it("builds a preview with proxied image and plain-text fields", () => { + expect(parseLinkPreview({ url: "https://ok.example/a", title: "A & B", siteName: "Ok", image: "https://cdn.example/i.png" })).toEqual({ + type: "linkPreview", + href: "https://ok.example/a", + host: "ok.example", + title: "A & B", + siteName: "Ok", + image: "/api/media-proxy?url=https%3A%2F%2Fcdn.example%2Fi.png", + }); + }); +}); + +describe("eslint raw-HTML sink rule", () => { + const linter = new Linter({ configType: "flat" }); + const lint = (code: string) => + linter.verify(code, [ + { + languageOptions: { ecmaVersion: "latest", sourceType: "module", parserOptions: { ecmaFeatures: { jsx: true } } }, + rules: noRawHtmlSinks.rules, + }, + ]); + + it.each([ + "const x =
;", + "React.createElement('div', { dangerouslySetInnerHTML: { __html: s } });", + "el.innerHTML = s;", + "el.outerHTML = s;", + "el.insertAdjacentHTML('beforeend', s);", + "document.write(s);", + "new Function(s)();", + "eval(s);", + "new DOMParser().parseFromString(s, 'text/html');", + ])("flags %s", (code) => { + expect(lint(code).map((m) => m.ruleId)).toContain("no-restricted-syntax"); + }); + + it("allows text rendering", () => { + expect(lint("const x =
{post.text}
; el.textContent = s;")).toEqual([]); + }); +}); diff --git a/packages/safe-content/src/plain.ts b/packages/safe-content/src/plain.ts new file mode 100644 index 0000000..548bf85 --- /dev/null +++ b/packages/safe-content/src/plain.ts @@ -0,0 +1,44 @@ +import type { RichTextDocument } from "./ast.js"; +import { DocBuilder, type ParseOptions } from "./builder.js"; + +// Single-pass, backtracking-safe token scan: URL | @mention | #hashtag. +const TOKENS = + /(https?:\/\/[^\s<>"'`\u0000-\u001F]{1,2048})|(? last) builder.text(line.slice(last, match.index)); + if (url !== undefined) { + const trimmed = url.replace(TRAILING_PUNCT, ""); + consumed = trimmed; + builder.link(trimmed, builder.inlineText(trimmed.replace(/^https?:\/\//i, ""))); + } else if (handle !== undefined) { + const clean = handle.replace(/[.-]+$/, ""); + consumed = "@" + clean; + builder.mention(clean); + } else if (tag !== undefined) { + builder.hashtag(tag); + } + last = match.index + consumed.length; + } + if (last < line.length) builder.text(line.slice(last)); +} + +/** Plain provider text (Bluesky, Telegram, bios, …): newlines, URLs, mentions, hashtags. */ +export function parsePlainText(input: unknown, options: ParseOptions = {}): RichTextDocument { + const builder = new DocBuilder(options); + if (typeof input !== "string") return builder.finish(); + const text = builder.clip(input).replace(/\r\n?/g, "\n"); + for (const paragraph of text.split(/\n{2,}/)) { + paragraph.split("\n").forEach((line, i) => { + if (i > 0) builder.lineBreak(); + lineToNodes(builder, line); + }); + builder.endParagraph(); + } + return builder.finish(); +} diff --git a/packages/safe-content/src/react.tsx b/packages/safe-content/src/react.tsx new file mode 100644 index 0000000..88a2948 --- /dev/null +++ b/packages/safe-content/src/react.tsx @@ -0,0 +1,135 @@ +import type { ReactNode } from "react"; +import type { EmojiNode, InlineNode, LinkPreview, RichTextDocument, TextNode } from "./ast.js"; +import { neutralizeDisplayName, hasMixedScriptConfusables } from "./text.js"; +import { DEFAULT_MEDIA_PROXY, isProxiedMediaUrl, parseSafeHttpUrl, type MediaProxyPolicy } from "./url.js"; +import { coerceRichTextDocument, parseLinkPreview } from "./validate.js"; + +/* + * The single reviewed renderer for provider content (proposal §4.5). Every + * provider string reaches the DOM as a React text node; the only attributes + * set from provider data are re-validated http(s) `href`s and media-proxy + * `src`s. No dangerouslySetInnerHTML, no style, no event handlers. + */ + +export const EXTERNAL_LINK_REL = "noopener noreferrer nofollow ugc"; + +type CommonProps = { readonly className?: string; readonly mediaProxy?: MediaProxyPolicy }; + +export function SafeLink(props: { readonly href: string; readonly children?: ReactNode; readonly className?: string }): ReactNode { + const safe = parseSafeHttpUrl(props.href); + if (!safe) return {props.children}; + return ( + + {props.children ?? safe.host} + + ); +} + +function Emoji({ node, mediaProxy }: { node: EmojiNode; mediaProxy: MediaProxyPolicy }): ReactNode { + const label = `:${node.shortcode}:`; + if (!isProxiedMediaUrl(node.src, mediaProxy)) return label; + return ( + {label} + ); +} + +function leaf(node: TextNode | EmojiNode, key: number, mediaProxy: MediaProxyPolicy): ReactNode { + return node.type === "text" ? node.text : ; +} + +function labelStartsWithHost(children: readonly (TextNode | EmojiNode)[], host: string): boolean { + const label = children + .map((c) => (c.type === "text" ? c.text : "")) + .join("") + .trim() + .toLowerCase() + .replace(/^https?:\/\//, "") + .replace(/^www\./, ""); + const h = host.toLowerCase().replace(/^www\./, ""); + return label === h || label.startsWith(h + "/") || label.startsWith(h + "?") || label.startsWith(h + "#") || label.startsWith(h + ":"); +} + +function inline(node: InlineNode, key: number, mediaProxy: MediaProxyPolicy): ReactNode { + switch (node.type) { + case "text": + return node.text; + case "linebreak": + return
; + case "emoji": + return ; + case "link": + return ( + + {node.children.map((c, i) => leaf(c, i, mediaProxy))} + {/* The real destination host is always visible when the label could mislead. */} + {labelStartsWithHost(node.children, node.host) ? null : ({node.host})} + + ); + case "mention": + return node.href ? ( + {`@${node.handle}`} + ) : ( + {`@${node.handle}`} + ); + case "hashtag": + return node.href ? ( + {`#${node.tag}`} + ) : ( + {`#${node.tag}`} + ); + default: + return null; + } +} + +/** Renders a rich-text document. `doc` may be untrusted JSON; it is re-validated first. */ +export function RichText(props: CommonProps & { readonly doc: RichTextDocument | unknown }): ReactNode { + const mediaProxy = props.mediaProxy ?? DEFAULT_MEDIA_PROXY; + const doc = coerceRichTextDocument(props.doc, { mediaProxy }); + return ( +
+ {doc.children.map((p, i) => ( +

+ {p.children.map((n, j) => inline(n, j, mediaProxy))} +

+ ))} +
+ ); +} + +/** Link-preview card. `preview` may be raw provider card data or a built `LinkPreview`. */ +export function LinkPreviewCard(props: CommonProps & { readonly preview: LinkPreview | unknown }): ReactNode { + const mediaProxy = props.mediaProxy ?? DEFAULT_MEDIA_PROXY; + const preview = parseLinkPreview(props.preview, mediaProxy); + if (!preview) return null; + return ( + + {preview.image && isProxiedMediaUrl(preview.image, mediaProxy) ? ( + + ) : null} + {preview.siteName ? {preview.siteName} : null} + {preview.title ? {preview.title} : null} + {preview.description ? {preview.description} : null} + {preview.host} + + ); +} + +/** Display names / usernames: bidi + zero-width neutralised, isolated, confusables flagged. */ +export function DisplayName(props: { readonly name: unknown; readonly maxLength?: number; readonly className?: string }): ReactNode { + const name = typeof props.name === "string" ? neutralizeDisplayName(props.name, props.maxLength) : ""; + return ( + + {name} + + ); +} diff --git a/packages/safe-content/src/text.ts b/packages/safe-content/src/text.ts new file mode 100644 index 0000000..d921a48 --- /dev/null +++ b/packages/safe-content/src/text.ts @@ -0,0 +1,57 @@ +/** + * Bidi embedding/override/isolate controls (U+202A–U+202E, U+2066–U+2069) can + * visually reorder text ("evil" + U+202E + "txt.exe"); they are always removed. + */ +const BIDI_CONTROLS = /[\u202A-\u202E\u2066-\u2069]/g; +/** Marks and zero-width characters used for spoofing look-alike names. */ +const INVISIBLES = /[\u200B-\u200F\u061C\u180E\u2060-\u2064\uFEFF\u034F\u115F\u1160\u3164\uFFA0]/g; +/** C0/C1 controls except tab and newline; lone surrogates. */ +const CONTROLS = /[\u0000-\u0008\u000B-\u001F\u007F-\u009F]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(? 1) return true; + } + return false; +} diff --git a/packages/safe-content/src/url.ts b/packages/safe-content/src/url.ts new file mode 100644 index 0000000..23adebf --- /dev/null +++ b/packages/safe-content/src/url.ts @@ -0,0 +1,69 @@ +import { DEFAULT_LIMITS } from "./limits.js"; + +export type SafeUrl = { readonly href: string; readonly host: string }; + +const ABSOLUTE_HTTP = /^https?:\/\/[^/\\]/i; + +/** + * Accepts only absolute, credential-free http(s) URLs. Everything else — + * `javascript:`, `data:`, `vbscript:`, `file:`, protocol-relative, relative, + * backslash tricks, embedded controls/whitespace — returns null so the caller + * renders the text inertly. + */ +export function parseSafeHttpUrl(raw: unknown, maxLength = DEFAULT_LIMITS.maxUrlLength): SafeUrl | null { + if (typeof raw !== "string") return null; + const input = raw.trim(); + if (input.length === 0 || input.length > maxLength) return null; + // Browsers strip tab/newline inside URLs ("java\tscript:"); refuse instead of guessing. + if (/[\u0000-\u0020\u007F-\u00A0\u1680\u2000-\u200F\u2028-\u202F\u205F-\u206F\u3000\uFEFF]/.test(input)) return null; + if (!ABSOLUTE_HTTP.test(input)) return null; + let url: URL; + try { + url = new URL(input); + } catch { + return null; + } + if (url.protocol !== "http:" && url.protocol !== "https:") return null; + if (url.username !== "" || url.password !== "") return null; + if (url.hostname === "") return null; + if (url.href.length > maxLength) return null; + // `URL` serialises IDN hosts as punycode (xn--…), which exposes homoglyph domains. + return { href: url.href, host: url.host }; +} + +export type MediaProxyPolicy = { + /** Same-origin path of the app's media proxy, e.g. `/api/media-proxy`. */ + readonly proxyPath: string; +}; + +export const DEFAULT_MEDIA_PROXY: MediaProxyPolicy = { proxyPath: "/api/media-proxy" }; + +function assertProxyPath(path: string): void { + if (!/^\/[A-Za-z0-9._~\-/]*$/.test(path) || path.startsWith("//") || path.includes("/../") || path.endsWith("/..")) { + throw new Error(`safe-content: media proxy path must be a same-origin absolute path, got ${JSON.stringify(path)}`); + } +} + +/** Rewrites a remote media URL into a media-proxy URL; null if the origin URL is unsafe. */ +export function toProxiedMediaUrl(raw: unknown, policy: MediaProxyPolicy = DEFAULT_MEDIA_PROXY): string | null { + assertProxyPath(policy.proxyPath); + const safe = parseSafeHttpUrl(raw); + if (!safe) return null; + return `${policy.proxyPath}?url=${encodeURIComponent(safe.href)}`; +} + +/** True when `src` is exactly a media-proxy URL produced by `toProxiedMediaUrl`. */ +export function isProxiedMediaUrl(src: unknown, policy: MediaProxyPolicy = DEFAULT_MEDIA_PROXY): src is string { + if (typeof src !== "string") return false; + assertProxyPath(policy.proxyPath); + const prefix = `${policy.proxyPath}?url=`; + if (!src.startsWith(prefix)) return false; + let decoded: string; + try { + decoded = decodeURIComponent(src.slice(prefix.length)); + } catch { + return false; + } + // Canonical form only: re-deriving the proxy URL must reproduce `src` exactly. + return toProxiedMediaUrl(decoded, policy) === src; +} diff --git a/packages/safe-content/src/validate.ts b/packages/safe-content/src/validate.ts new file mode 100644 index 0000000..1e94e2e --- /dev/null +++ b/packages/safe-content/src/validate.ts @@ -0,0 +1,107 @@ +import type { EmojiNode, LinkPreview, RichTextDocument, TextNode } from "./ast.js"; +import { DocBuilder, normalizeHandle, normalizeTag, type ParseOptions } from "./builder.js"; +import { decodeEntities } from "./entities.js"; +import { LINK_PREVIEW_LIMITS } from "./limits.js"; +import { neutralizeText, truncateCodePoints } from "./text.js"; +import { DEFAULT_MEDIA_PROXY, isProxiedMediaUrl, parseSafeHttpUrl, toProxiedMediaUrl, type MediaProxyPolicy } from "./url.js"; + +type Unknown = Record; +const isObj = (v: unknown): v is Unknown => typeof v === "object" && v !== null && !Array.isArray(v); +const arr = (v: unknown): readonly unknown[] => (Array.isArray(v) ? v : []); + +/** + * Rebuilds a document from untrusted JSON (DB row, API payload, websocket + * frame). Unknown node types, bad URLs and non-proxied media are dropped; + * text is re-neutralised; limits are re-applied. The renderer always calls + * this, so a forged AST is as inert as a parsed one. + */ +export function coerceRichTextDocument(value: unknown, options: ParseOptions = {}): RichTextDocument { + const builder = new DocBuilder({ ...options, emojis: [] }); + const policy = options.mediaProxy ?? DEFAULT_MEDIA_PROXY; + if (!isObj(value) || value.type !== "doc") return builder.finish(); + if (value.truncated === true) builder.truncated = true; + const leaf = (n: unknown): (TextNode | EmojiNode)[] => { + if (!isObj(n)) return []; + if (n.type === "text" && typeof n.text === "string") return builder.inlineText(n.text); + if (n.type === "emoji" && typeof n.shortcode === "string" && /^[A-Za-z0-9_+-]{1,64}$/.test(n.shortcode)) { + return isProxiedMediaUrl(n.src, policy) + ? [{ type: "emoji", shortcode: n.shortcode, src: n.src }] + : [{ type: "text", text: `:${n.shortcode}:` }]; + } + return []; + }; + const capped = (items: unknown): readonly unknown[] => { + const all = arr(items); + if (all.length <= builder.limits.maxNodes) return all; + builder.truncated = true; + return all.slice(0, builder.limits.maxNodes); + }; + for (const p of capped(value.children)) { + if (!isObj(p) || p.type !== "paragraph") continue; + for (const n of capped(p.children)) { + if (!isObj(n)) continue; + switch (n.type) { + case "text": + if (typeof n.text === "string") builder.text(n.text); + break; + case "emoji": + for (const e of leaf(n)) builder.pushLeaf(e); + break; + case "linebreak": + builder.lineBreak(); + break; + case "link": + builder.link(typeof n.href === "string" ? n.href : "", arr(n.children).slice(0, 64).flatMap(leaf)); + break; + case "mention": + if (typeof n.handle === "string" && normalizeHandle(n.handle)) builder.mention(n.handle, typeof n.href === "string" ? n.href : undefined); + break; + case "hashtag": + if (typeof n.tag === "string" && normalizeTag(n.tag)) builder.hashtag(n.tag, typeof n.href === "string" ? n.href : undefined); + break; + } + } + builder.endParagraph(); + } + return builder.finish(); +} + +export type RawLinkPreview = { + readonly url?: unknown; + readonly title?: unknown; + readonly description?: unknown; + readonly siteName?: unknown; + readonly image?: unknown; +}; + +function previewText(value: unknown, max: number): string | undefined { + if (typeof value !== "string") return undefined; + // OpenGraph fields are text: entities are decoded, tags are NOT interpreted. + const text = neutralizeText(decodeEntities(value.slice(0, max * 4))).replace(/\s+/g, " ").trim(); + return text === "" ? undefined : truncateCodePoints(text, max); +} + +/** + * Link-preview card data (OpenGraph/oEmbed/Mastodon `card`). Returns null when + * the target is not a safe http(s) URL. The image is rewritten to the proxy + * or dropped. Accepts either raw provider data or a previously built preview. + */ +export function parseLinkPreview(raw: unknown, mediaProxy: MediaProxyPolicy = DEFAULT_MEDIA_PROXY): LinkPreview | null { + if (!isObj(raw)) return null; + const safe = parseSafeHttpUrl(raw.type === "linkPreview" ? raw.href : raw.url); + if (!safe) return null; + const image = isProxiedMediaUrl(raw.image, mediaProxy) ? raw.image : toProxiedMediaUrl(raw.image, mediaProxy); + const title = previewText(raw.title, LINK_PREVIEW_LIMITS.title); + const description = previewText(raw.description, LINK_PREVIEW_LIMITS.description); + const siteName = previewText(raw.siteName, LINK_PREVIEW_LIMITS.siteName); + return { + type: "linkPreview", + href: safe.href, + host: safe.host, + ...(title !== undefined && { title }), + ...(description !== undefined && { description }), + ...(siteName !== undefined && { siteName }), + ...(image !== null && { image }), + }; +} + diff --git a/packages/safe-content/tsconfig.json b/packages/safe-content/tsconfig.json new file mode 100644 index 0000000..5ec248a --- /dev/null +++ b/packages/safe-content/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "lib": ["ES2022", "DOM"], + "jsx": "react-jsx" + }, + "include": ["src"] +} diff --git a/packages/scheduling/README.md b/packages/scheduling/README.md new file mode 100644 index 0000000..adbc7de --- /dev/null +++ b/packages/scheduling/README.md @@ -0,0 +1,56 @@ +# Scheduling + +Shared schedule time semantics for WP3 (publishing) and WP13 (multi-output +schedules). Pure functions and types only — no app, database or Temporal +imports, and no runtime dependency beyond Node's `Intl` tzdata. + +A schedule is a five-field cron expression plus an explicit IANA timezone +(never server-local; `UTC` allowed). Occurrences are evaluated in that zone as +nominal local date-times, then materialized to UTC fire instants. The +occurrence key is the materialized UTC instant, so `UNIQUE (workspace_id, +schedule_id, fire_at_utc)` holds by construction. + +## Policies + +- **DST gap** (nonexistent local time): fires at the first valid instant after + the gap (shift forward). Works for 1-hour gaps and Pacific/Apia's whole-day + skip alike. +- **DST fold** (ambiguous local time): fires ONCE, at its first (earlier-offset) + instant. +- **Collapse**: several nominal local times materializing onto one post-gap + instant produce exactly one occurrence, labelled with the latest collapsed + slot — the conservative value for the `last_fired_nominal_local` gate. +- **Re-materialization** on tzdata or timezone change: deletes all unclaimed + future occurrences, then regenerates only occurrences strictly after + max(change instant, last claimed fire instant) whose nominal slot is strictly + later than `last_fired_nominal_local`. A slot already fired under the old + zone (daily 09:00 fired in New York, zone switched to Los Angeles) never + fires again. The tzdata version is reported for recording. + +## API + +```ts +import { + parseCron, // five-field cron -> sets (rejects 6-field/`?`/`L`/`W`/`#`/macros) + nextOccurrences, // (spec, from, count, {exclusive}) -> Occurrence[] + materializeOccurrence, // ("2024-03-10T02:30", zone) -> epoch ms + rematerialize, // (oldSpec, newSpec, storedRows, {changeInstant, ...}) + canonicalTimezone, // validates + canonicalizes; rejects "local"/offsets + currentTzdataVersion, // process.versions.tz +} from "@tactical-media-broadcast-hub/scheduling"; +``` + +Errors: `SchedulingError` base, with `InvalidCronError`, `ImpossibleScheduleError`, +`InvalidTimezoneError`, `InvalidNominalLocalError`, `InvalidInstantError`. Note +`toInstantMs` rejects offset-less ISO datetimes that would otherwise parse in +the server's local zone. + +## Verification + +``` +pnpm exec tsc -p tsconfig.json --noEmit +pnpm exec vitest run +``` + +`pnpm install` is required once so the workspace links this package's +`vitest`/`typescript` devDependencies into `packages/scheduling/node_modules`. diff --git a/packages/scheduling/package.json b/packages/scheduling/package.json new file mode 100644 index 0000000..22fa5d0 --- /dev/null +++ b/packages/scheduling/package.json @@ -0,0 +1,20 @@ +{ + "name": "@tactical-media-broadcast-hub/scheduling", + "private": true, + "version": "0.1.0", + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { ".": "./dist/index.js" }, + "scripts": { + "build": "tsc -p tsconfig.json", + "lint": "eslint src", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "vitest run" + }, + "devDependencies": { + "@types/node": "^22.20.5", + "typescript": "^5.5.4", + "vitest": "^2.0.5" + } +} diff --git a/packages/scheduling/src/cron.test.ts b/packages/scheduling/src/cron.test.ts new file mode 100644 index 0000000..d14e012 --- /dev/null +++ b/packages/scheduling/src/cron.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from "vitest"; +import { parseCron, cronMatchesTime } from "./cron.js"; +import { ImpossibleScheduleError, InvalidCronError } from "./errors.js"; + +describe("parseCron — valid syntax", () => { + it("parses the every-minute expression", () => { + const c = parseCron("* * * * *"); + expect(c.minute.values.size).toBe(60); + expect(c.minute.restricted).toBe(false); + expect(c.dayOfMonth.restricted).toBe(false); + expect(c.dayOfWeek.restricted).toBe(false); + }); + it("parses steps, ranges and lists", () => { + const c = parseCron("*/15 8-18/2 * * 1,3,5"); + expect([...c.minute.values]).toEqual([0, 15, 30, 45]); + expect([...c.hour.values]).toEqual([8, 10, 12, 14, 16, 18]); + expect([...c.dayOfWeek.values]).toEqual([1, 3, 5]); + expect(c.minute.restricted).toBe(true); + expect(c.hour.restricted).toBe(true); + expect(c.dayOfWeek.restricted).toBe(true); + }); + it("parses month and dow names case-insensitively", () => { + const c = parseCron("0 0 1 jan-mar mon-fri"); + expect([...c.month.values]).toEqual([1, 2, 3]); + expect([...c.dayOfWeek.values]).toEqual([1, 2, 3, 4, 5]); + }); + it("accepts a list element of one value", () => { + expect([...parseCron("5,10 * * * *").minute.values]).toEqual([5, 10]); + }); + it("treats n/step as n..max/step", () => { + expect([...parseCron("20/15 * * * *").minute.values]).toEqual([20, 35, 50]); + }); +}); + +describe("parseCron — rejection", () => { + const bad = [ + "", + " ", + "* * * *", + "* * * * * *", + "60 * * * *", + "* 24 * * *", + "* * 0 * *", + "* * 32 * *", + "* * * 0 *", + "* * * 13 *", + "* * * * 7", + "* * * * MON-BR", + "a * * * *", + "*/0 * * * *", + "*/-1 * * * *", + "5-1 * * * *", + "1--2 * * * *", + "@reboot", + "@daily", + "? * * * *", + "* * ? * *", + "* * * * * 2025", + "0 0 * * MON#1", + "0 0 L * *", + "0 0 1 W *", + "1/2/3 * * * *", + "*/ * * * *", + "1,,2 * * * *", + "-5 * * * *", + ]; + it.each(bad.map((e) => [e]))("rejects %s", (expr) => { + expect(() => parseCron(expr)).toThrow(InvalidCronError); + }); + it("rejects non-string input", () => { + expect(() => parseCron(undefined as unknown as string)).toThrow(InvalidCronError); + expect(() => parseCron(5 as unknown as string)).toThrow(InvalidCronError); + }); + it("rejects impossible schedules", () => { + expect(() => parseCron("0 0 31 2 *")).toThrow(ImpossibleScheduleError); + expect(() => parseCron("0 0 30 2 *")).toThrow(ImpossibleScheduleError); + expect(() => parseCron("0 0 31 4 *")).toThrow(ImpossibleScheduleError); + }); + it("accepts Feb 29 (leap years exist)", () => { + expect(() => parseCron("0 0 29 2 *")).not.toThrow(); + }); + it("accepts the 31st in long months", () => { + expect(() => parseCron("0 0 31 1 *")).not.toThrow(); + expect(() => parseCron("0 0 31 * *")).not.toThrow(); + }); +}); + +describe("cronMatchesTime", () => { + const cron = parseCron("*/15 9 * * *"); + it("matches on the quarter hour in the 9 hour", () => { + expect(cronMatchesTime(cron, 9, 0, 0)).toBe(true); + expect(cronMatchesTime(cron, 9, 45, 0)).toBe(true); + }); + it("rejects other minutes/hours and any nonzero second", () => { + expect(cronMatchesTime(cron, 9, 20, 0)).toBe(false); + expect(cronMatchesTime(cron, 10, 0, 0)).toBe(false); + expect(cronMatchesTime(cron, 9, 0, 1)).toBe(false); + }); +}); diff --git a/packages/scheduling/src/cron.ts b/packages/scheduling/src/cron.ts new file mode 100644 index 0000000..94e70ae --- /dev/null +++ b/packages/scheduling/src/cron.ts @@ -0,0 +1,249 @@ +import { ImpossibleScheduleError, InvalidCronError } from "./errors.js"; + +/** + * A single field of a five-field cron expression after alias resolution. + * + * `values` is the expanded set of allowed integers; `restricted` is false only + * for the literal `*`, which matters for the cron dom/dow OR rule: when both + * fields are restricted the day matches if EITHER matches, and when only one is + * restricted it alone constrains the day. + */ +export interface CronField { + readonly values: ReadonlySet; + readonly restricted: boolean; +} + +/** + * Parsed five-field cron expression: minute hour day-of-month month + * day-of-week. minute/hour/dow are zero-based; month is 1-12, dom 1-31. + */ +export interface ParsedCron { + readonly minute: CronField; + readonly hour: CronField; + readonly dayOfMonth: CronField; + readonly month: CronField; + readonly dayOfWeek: CronField; +} + +interface FieldBounds { + readonly min: number; + readonly max: number; + /** alias -> value, applied before range parsing (MON, JAN, ...) */ + readonly aliases?: Readonly>; +} + +const MONTH_ALIASES: Readonly> = { + JAN: 1, + FEB: 2, + MAR: 3, + APR: 4, + MAY: 5, + JUN: 6, + JUL: 7, + AUG: 8, + SEP: 9, + OCT: 10, + NOV: 11, + DEC: 12, +}; + +const DOW_ALIASES: Readonly> = { + SUN: 0, + MON: 1, + TUE: 2, + WED: 3, + THU: 4, + FRI: 5, + SAT: 6, +}; + +const MINUTE: FieldBounds = { min: 0, max: 59 }; +const HOUR: FieldBounds = { min: 0, max: 23 }; +const DOM: FieldBounds = { min: 1, max: 31 }; +const MONTH: FieldBounds = { min: 1, max: 12, aliases: MONTH_ALIASES }; +const DOW: FieldBounds = { min: 0, max: 6, aliases: DOW_ALIASES }; + +export const FIELD_NAMES = ["minute", "hour", "day-of-month", "month", "day-of-week"] as const; + +/** Month lengths ignoring leap years (February counted as 29 to stay permissive). */ +const MONTH_LENGTHS: readonly number[] = [0, 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31]; + +function applyAliases(token: string, bounds: FieldBounds): string { + const aliases = bounds.aliases; + if (!aliases) return token; + const mapped = aliases[token.toUpperCase()]; + return mapped === undefined ? token : String(mapped); +} + +function parseValue(token: string, bounds: FieldBounds, fieldName: string): number { + const mapped = applyAliases(token, bounds); + if (!/^\d+$/.test(mapped)) { + throw new InvalidCronError(`invalid value "${token}" in ${fieldName} field`); + } + const value = Number(mapped); + if (value < bounds.min || value > bounds.max) { + throw new InvalidCronError( + `value ${value} out of range ${bounds.min}-${bounds.max} in ${fieldName} field`, + ); + } + return value; +} + +function parseStep(token: string, fieldName: string): number { + const trimmed = token.trim(); + if (!/^\d+$/.test(trimmed)) { + throw new InvalidCronError(`invalid step "${token}" in ${fieldName} field`); + } + const step = Number(trimmed); + if (!Number.isInteger(step) || step <= 0) { + throw new InvalidCronError(`step must be a positive integer, got "${token}" in ${fieldName} field`); + } + return step; +} + +function parseField(field: string, bounds: FieldBounds, fieldName: string): CronField { + const raw = field.trim(); + if (raw === "") { + throw new InvalidCronError(`empty ${fieldName} field`); + } + const values = new Set(); + for (const part of raw.split(",")) { + const token = part.trim(); + if (token === "") { + throw new InvalidCronError(`empty list element in ${fieldName} field "${field}"`); + } + // Split off an optional /step suffix, being careful not to break on a bare value. + const slash = token.indexOf("/"); + const range = (slash === -1 ? token : token.slice(0, slash)).trim(); + const stepPart = slash === -1 ? null : token.slice(slash + 1).trim(); + if (range === "") { + throw new InvalidCronError(`malformed element "${token}" in ${fieldName} field`); + } + if ((token.match(/\//g) ?? []).length > 1) { + throw new InvalidCronError(`malformed element "${token}" in ${fieldName} field`); + } + const step = stepPart === null ? 1 : parseStep(stepPart, fieldName); + let lo: number; + let hi: number; + if (range === "*") { + lo = bounds.min; + hi = bounds.max; + } else if (range.includes("-")) { + const dash = range.indexOf("-"); + const loToken = range.slice(0, dash).trim(); + const hiToken = range.slice(dash + 1).trim(); + if (loToken === "" || hiToken === "") { + throw new InvalidCronError(`malformed range "${range}" in ${fieldName} field`); + } + lo = parseValue(loToken, bounds, fieldName); + hi = parseValue(hiToken, bounds, fieldName); + if (lo > hi) { + throw new InvalidCronError(`descending range "${range}" in ${fieldName} field (use ascending ranges)`); + } + } else { + const value = parseValue(range, bounds, fieldName); + if (stepPart === null) { + values.add(value); + continue; + } + // `n/step` means "starting at n, every step" — the vixie shorthand for n-max/step. + lo = value; + hi = bounds.max; + } + for (let v = lo; v <= hi; v += step) values.add(v); + } + if (values.size === 0) { + throw new InvalidCronError(`${fieldName} field "${field}" selects no values`); + } + // Only the bare `*` leaves a field unrestricted; `*/15`, `1-5`, lists and + // even `*,5` all constrain the field. + return { values, restricted: raw !== "*" }; +} + +/** + * Parse a five-field cron expression. Supported syntax: `*`, integers, ranges + * `a-b`, steps `n/step`, `a-b/step`, comma lists, and month/day-of-week names + * (`JAN-DEC`, `MON-SUN`, case-insensitive). Anything else — more or fewer than + * five fields, seconds/year fields, `@reboot` macros, `?`, `L`, `W`, `#` — is + * rejected with `InvalidCronError` rather than silently misinterpreted. + * + * Throws `ImpossibleScheduleError` when the expression can never fire (for + * example `0 0 31 2 *`). + */ +export function parseCron(expression: string): ParsedCron { + if (typeof expression !== "string") { + throw new InvalidCronError(`cron expression must be a string, got ${typeof expression}`); + } + const trimmed = expression.trim(); + if (trimmed === "") { + throw new InvalidCronError("cron expression must not be empty"); + } + if (trimmed.startsWith("@")) { + throw new InvalidCronError( + `cron macros like "${trimmed}" are not supported; use a five-field expression`, + ); + } + for (const ch of ["?", "L", "W", "#"] as const) { + if (trimmed.includes(ch)) { + throw new InvalidCronError( + `unsupported cron syntax "${ch}" in "${expression}"; this library implements the five-field syntax only`, + ); + } + } + const fields = trimmed.split(/\s+/); + if (fields.length !== 5) { + throw new InvalidCronError( + `cron expression must have exactly 5 fields (minute hour day-of-month month day-of-week), got ${fields.length} in "${expression}"`, + ); + } + const minute = parseField(fields[0]!, MINUTE, FIELD_NAMES[0]); + const hour = parseField(fields[1]!, HOUR, FIELD_NAMES[1]); + const dayOfMonth = parseField(fields[2]!, DOM, FIELD_NAMES[2]); + const month = parseField(fields[3]!, MONTH, FIELD_NAMES[3]); + const dayOfWeek = parseField(fields[4]!, DOW, FIELD_NAMES[4]); + assertPossible(month, dayOfMonth); + return { minute, hour, dayOfMonth, month, dayOfWeek }; +} + +function assertPossible(month: CronField, dayOfMonth: CronField): void { + for (const m of month.values) { + const maxDay = MONTH_LENGTHS[m] ?? 0; + for (const d of dayOfMonth.values) { + if (d <= maxDay) return; + } + } + throw new ImpossibleScheduleError( + `expression can never fire: no allowed month is long enough for day-of-month ${[ + ...dayOfMonth.values, + ].join(", ")}`, + ); +} + +/** + * Does this cron match the given local wall-clock day? Implements the vixie + * dom/dow OR rule: if both fields are restricted the day matches when either + * matches, otherwise the restricted field alone decides. + */ +export function cronMatchesDay(cron: ParsedCron, year: number, month: number, day: number): boolean { + if (!cron.month.values.has(month)) return false; + const domMatch = cron.dayOfMonth.values.has(day); + const dowMatch = cron.dayOfWeek.values.has(dayOfWeekOf(year, month, day)); + const domRestricted = cron.dayOfMonth.restricted; + const dowRestricted = cron.dayOfWeek.restricted; + // Vixie rule: neither restricted -> every day; exactly one restricted -> + // that field alone; both restricted -> match if EITHER matches. + if (domRestricted && dowRestricted) return domMatch || dowMatch; + if (domRestricted) return domMatch; + if (dowRestricted) return dowMatch; + return true; +} + +function dayOfWeekOf(year: number, month: number, day: number): number { + return new Date(Date.UTC(year, month - 1, day)).getUTCDay(); +} + +/** Does this cron match the given local time-of-day (hour/minute)? */ +export function cronMatchesTime(cron: ParsedCron, hour: number, minute: number, second: number): boolean { + if (second !== 0) return false; + return cron.minute.values.has(minute) && cron.hour.values.has(hour); +} diff --git a/packages/scheduling/src/errors.ts b/packages/scheduling/src/errors.ts new file mode 100644 index 0000000..8d0afdb --- /dev/null +++ b/packages/scheduling/src/errors.ts @@ -0,0 +1,31 @@ +/** + * Error hierarchy for `@tactical-media-broadcast-hub/scheduling`. + * + * Every failure mode of the library is one of these types; callers can branch + * on the class without string matching. + */ +export class SchedulingError extends Error { + constructor(message: string) { + super(message); + this.name = new.target.name; + } +} + +/** The cron expression is syntactically invalid or uses unsupported syntax. */ +export class InvalidCronError extends SchedulingError {} + +/** The timezone is not a usable IANA zone id (never silently server-local). */ +export class InvalidTimezoneError extends SchedulingError {} + +/** A nominal local date-time string is malformed or not a real calendar date. */ +export class InvalidNominalLocalError extends SchedulingError {} + +/** + * An instant input (Date | number | ISO string) is not usable. Notably, an ISO + * string that carries a time but no explicit UTC offset is rejected instead of + * being parsed in the server's local zone. + */ +export class InvalidInstantError extends SchedulingError {} + +/** The schedule can never fire (e.g. `0 0 31 2 *` — February 31st). */ +export class ImpossibleScheduleError extends SchedulingError {} diff --git a/packages/scheduling/src/index.ts b/packages/scheduling/src/index.ts new file mode 100644 index 0000000..877bdff --- /dev/null +++ b/packages/scheduling/src/index.ts @@ -0,0 +1,58 @@ +/** + * @tactical-media-broadcast-hub/scheduling — shared schedule time semantics (WP3, reused by WP13). + * + * A schedule is a five-field cron expression plus an explicit IANA timezone + * (never server-local; UTC allowed). Occurrences are evaluated in that zone as + * nominal local times, then materialized to UTC fire instants. The occurrence + * key is the materialized UTC instant, so `UNIQUE (workspace_id, schedule_id, + * fire_at_utc)` holds by construction. + * + * - DST gap: a nonexistent local time fires at the first valid instant after + * the gap (shift forward). + * - DST fold: an ambiguous local time fires once, at its earlier instant. + * - Collapse: several nominal local times mapping onto one post-gap instant + * yield exactly one occurrence. + * - Re-materialization on tzdata/timezone change deletes unclaimed future + * occurrences and regenerates only strictly-later slots, never refiring a + * slot already fired under the old zone. + * + * Pure functions only: no app, DB or Temporal imports. + */ +export { + SchedulingError, + InvalidCronError, + InvalidTimezoneError, + InvalidNominalLocalError, + InvalidInstantError, + ImpossibleScheduleError, +} from "./errors.js"; +export { + canonicalTimezone, + isValidTimezone, + assertValidTimezone, + wallOfInstant, + offsetMs, + toInstantMs, + currentTzdataVersion, + epochOfWall, +} from "./timezone.js"; +export type { WallClock, NominalLocal } from "./timezone.js"; +export { parseCron, cronMatchesDay, cronMatchesTime, FIELD_NAMES } from "./cron.js"; +export type { ParsedCron, CronField } from "./cron.js"; +export { formatNominal, parseNominal, compareWall, wallEquals, wallAsEpochMs } from "./nominal.js"; +export { resolveWallToInstant } from "./materialize.js"; +export { materializeOccurrence } from "./materialize-api.js"; +export { + prepareSchedule, + sortedHours, + sortedMinutes, + slotNominal, + matchingLocalDays, + stepOneLocalDay, +} from "./schedule.js"; +export type { ScheduleSpec, Occurrence, PreparedSchedule } from "./schedule.js"; +export { nextOccurrencesPrepared } from "./occurrences.js"; +export type { NextOccurrencesOptions } from "./occurrences.js"; +export { nextOccurrences, materializeOccurrences } from "./occurrences_extra.js"; +export { rematerialize } from "./rematerialize.js"; +export type { StoredOccurrence, RematerializeOptions, RematerializeResult } from "./rematerialize.js"; diff --git a/packages/scheduling/src/materialize-api.ts b/packages/scheduling/src/materialize-api.ts new file mode 100644 index 0000000..9c6d9ba --- /dev/null +++ b/packages/scheduling/src/materialize-api.ts @@ -0,0 +1,16 @@ +import { resolveWallToInstant } from "./materialize.js"; +import { canonicalTimezone } from "./timezone.js"; +import { parseNominal } from "./nominal.js"; +import type { NominalLocal } from "./timezone.js"; + +/** + * Materialize one nominal local date-time (e.g. `2024-03-10T02:30`) in an + * explicit IANA zone to its UTC fire instant in epoch milliseconds. + * + * Gap -> first valid instant after the gap; fold -> single firing at the + * earlier instant. Throws on a malformed nominal string or an invalid zone. + */ +export function materializeOccurrence(nominalLocal: NominalLocal, timezone: string): number { + const zone = canonicalTimezone(timezone); + return resolveWallToInstant(parseNominal(nominalLocal), zone); +} diff --git a/packages/scheduling/src/materialize.test.ts b/packages/scheduling/src/materialize.test.ts new file mode 100644 index 0000000..fc173c9 --- /dev/null +++ b/packages/scheduling/src/materialize.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from "vitest"; +import { materializeOccurrence } from "./materialize-api.js"; +import { compareWall } from "./nominal.js"; +import { InvalidNominalLocalError } from "./errors.js"; +import { wallOfInstant } from "./timezone.js"; + +const ISO = (ms: number) => new Date(ms).toISOString().replace(".000Z", "Z"); + +describe("materializeOccurrence — normal (unambiguous) times", () => { + it("resolves a plain New York evening slot", () => { + expect(ISO(materializeOccurrence("2024-03-09T21:00", "America/New_York"))) + .toBe("2024-03-10T02:00:00Z"); + }); + it("passes UTC straight through", () => { + expect(ISO(materializeOccurrence("2024-03-10T02:30", "UTC"))) + .toBe("2024-03-10T02:30:00Z"); + }); + it("resolves Kolkata (no DST, half-hour offset)", () => { + expect(ISO(materializeOccurrence("2024-06-01T09:00", "Asia/Kolkata"))) + .toBe("2024-06-01T03:30:00Z"); + }); +}); + +describe("DST gap — nonexistent local time fires at the first valid instant after", () => { + it("2:30 on New York spring-forward (2024-03-10) shifts to 3:00 local / 07:00Z", () => { + const fire = materializeOccurrence("2024-03-10T02:30", "America/New_York"); + expect(ISO(fire)).toBe("2024-03-10T07:00:00Z"); + const wall = wallOfInstant(fire, "America/New_York"); + expect(`${wall.hour}:${String(wall.minute).padStart(2, "0")}`).toBe("3:00"); + }); + it("2:45 shifts to 3:00 as well", () => { + expect(ISO(materializeOccurrence("2024-03-10T02:45", "America/New_York"))) + .toBe("2024-03-10T07:00:00Z"); + }); + it("2:00 (whole gap hour) shifts to 3:00", () => { + expect(ISO(materializeOccurrence("2024-03-10T02:00", "America/New_York"))) + .toBe("2024-03-10T07:00:00Z"); + }); + it("1:59 — last pre-gap minute — is untouched", () => { + expect(ISO(materializeOccurrence("2024-03-10T01:59", "America/New_York"))) + .toBe("2024-03-10T06:59:00Z"); + }); + it("Berlin 02:30 spring-forward shifts to 03:00 local", () => { + // EU transitions at 01:00 UTC: 2024-03-31 02:00->03:00 local + expect(ISO(materializeOccurrence("2024-03-31T02:30", "Europe/Berlin"))) + .toBe("2024-03-31T01:00:00Z"); + }); + it("whole-day gap: Pacific/Apia skipped 2011-12-30; a slot on that day fires at the first valid instant after the gap", () => { + // The zone jumped 2011-12-29T23:59:59-10:00 -> 2011-12-31T00:00:00+14:00, + // so 2011-12-30 never existed. First valid instant after the gap is + // 2011-12-31T00:00 local == 2011-12-30T10:00Z. + const fire = materializeOccurrence("2011-12-30T09:00", "Pacific/Apia"); + expect(ISO(fire)).toBe("2011-12-30T10:00:00Z"); + const wall = wallOfInstant(fire, "Pacific/Apia"); + expect(`${wall.year}-${wall.month}-${String(wall.day).padStart(2, "0")}T${wall.hour}:${wall.minute}`).toBe("2011-12-31T0:0"); + }); + it("a slot just before the Apia skip (23:00 on 12-29) is untouched", () => { + expect(ISO(materializeOccurrence("2011-12-29T23:00", "Pacific/Apia"))) + .toBe("2011-12-30T09:00:00Z"); + }); +}); + +describe("DST fold — ambiguous local time fires once, at the earlier instant", () => { + it("1:30 on New York fall-back (2024-11-03) fires at EDT (earlier offset)", () => { + // 01:30 EDT == 05:30Z; the later reading 01:30 EST == 06:30Z is NOT used. + const fire = materializeOccurrence("2024-11-03T01:30", "America/New_York"); + expect(ISO(fire)).toBe("2024-11-03T05:30:00Z"); + expect(fire).not.toBe(Date.parse("2024-11-03T06:30:00Z")); + }); + it("1:30 fires exactly once at 05:30Z (not 06:30Z)", () => { + expect(ISO(materializeOccurrence("2024-11-03T01:30", "America/New_York"))) + .toBe("2024-11-03T05:30:00Z"); + }); + it("1:00 fold hour fires at 05:00Z (earlier)", () => { + expect(ISO(materializeOccurrence("2024-11-03T01:00", "America/New_York"))) + .toBe("2024-11-03T05:00:00Z"); + }); + it("2:00 after the fold is unambiguous again", () => { + expect(ISO(materializeOccurrence("2024-11-03T02:00", "America/New_York"))) + .toBe("2024-11-03T07:00:00Z"); + }); +}); + +describe("validation", () => { + it("rejects malformed nominal strings", () => { + for (const bad of ["2024-3-10T02:30", "2024-03-10", "2024-03-10T2:30", "2024-13-01T00:00", "2024-03-32T00:00", "2024-03-10T25:00", "nonsense"]) { + expect(() => materializeOccurrence(bad, "UTC")).toThrow(InvalidNominalLocalError); + } + }); + it("rejects invalid zones", () => { + expect(() => materializeOccurrence("2024-03-10T02:30", "America/Nowhere")).toThrow(); + expect(() => materializeOccurrence("2024-03-10T02:30", "local")).toThrow(); + }); + it("is pure — repeated calls agree", () => { + const a = materializeOccurrence("2024-03-10T02:30", "America/New_York"); + const b = materializeOccurrence("2024-03-10T02:30", "America/New_York"); + const c = materializeOccurrence("2024-03-10T02:30", "America/New_York"); + expect(a).toBe(b); + expect(b).toBe(c); + }); + it("compareWall orders wall clocks chronologically", () => { + const w = (s: string) => ({ + year: +s.slice(0, 4), month: +s.slice(5, 7), day: +s.slice(8, 10), + hour: +s.slice(11, 13), minute: +s.slice(14, 16), second: 0, + }); + expect(compareWall(w("2024-03-10T01:59"), w("2024-03-10T03:00"))).toBeLessThan(0); + expect(compareWall(w("2024-11-03T01:30"), w("2024-11-03T01:30"))).toBe(0); + }); +}); + +export {}; diff --git a/packages/scheduling/src/materialize.ts b/packages/scheduling/src/materialize.ts new file mode 100644 index 0000000..75f5341 --- /dev/null +++ b/packages/scheduling/src/materialize.ts @@ -0,0 +1,96 @@ +import { compareWall, wallAsEpochMs } from "./nominal.js"; +import { offsetMs, wallOfInstant } from "./timezone.js"; +import type { WallClock } from "./timezone.js"; + +export { materializeOccurrence } from "./materialize-api.js"; + +const SIX_HOURS = 6 * 3600_000; + +/** + * Resolve one nominal local wall time in a zone to its UTC fire instant + * (epoch ms). + * + * Normal: the unique instant reading that wall time. Fold (ambiguous): fires + * once, at the earlier instant. Gap (nonexistent): the first valid instant + * after the gap — the earliest instant whose local reading is strictly later + * than the target. + * + * The candidate instant is found by fixed-point iteration (guess = local-as-UTC + * minus the offset at the guess), which converges for normal and fold times. + * Neighbouring offsets are probed to catch the second reading of a fold; the + * earliest matching instant wins. A non-matching iteration means the wall time + * does not exist (gap), resolved by a bracketed binary search for the exact + * first valid instant after the transition — correct for 1-hour gaps and + * Pacific/Apia's whole-day skip alike. + */ +export function resolveWallToInstant(wall: WallClock, zone: string): number { + if (zone === "UTC") return wallAsEpochMs(wall); + const asUtc = wallAsEpochMs(wall); + + // Fixed-point iteration from the naive reading. + let guess = asUtc; + let resolved: number | null = null; + for (let i = 0; i < 6; i++) { + guess = asUtc - offsetMs(guess, zone); + if (compareWall(wallOfInstant(guess, zone), wall) === 0) { + resolved = guess; + break; + } + } + + // Collect every instant reading this wall time: the resolved one plus + // candidates derived from the offsets in effect on either side of it. + const matches: number[] = []; + const consider = (instant: number) => { + if (compareWall(wallOfInstant(instant, zone), wall) === 0) matches.push(instant); + }; + if (resolved !== null) { + consider(resolved); + consider(asUtc - offsetMs(resolved - SIX_HOURS, zone)); + consider(asUtc - offsetMs(resolved + SIX_HOURS, zone)); + } + if (matches.length > 0) { + // Fold: exactly one firing, at the earliest (earlier-offset) instant. + return Math.min(...matches); + } + + // Gap: the wall time does not exist. First valid instant after the gap. + return firstInstantAfterGap(asUtc, wall, zone); +} + +/** + * Earliest instant whose wall clock in `zone` is strictly after `wall`. + * A coarse 15-minute scan brackets the transition, then a binary search + * narrows to the exact instant (works for 1-hour and whole-day gaps). + */ +function firstInstantAfterGap(asUtc: number, wall: WallClock, zone: string): number { + const stepMs = 15 * 60_000; + // Establish the search invariant wall(lo) <= wall first: the local-as-UTC + // reading can already lie past the gap when the transition is large (e.g. a + // whole-day skip), so step back until the reading is at or before the target. + let lo = asUtc; + for (let i = 0; i < 8 * 96 && compareWall(wallOfInstant(lo, zone), wall) > 0; i++) { + lo -= stepMs; + } + for (let i = 0; i < 16 * 96; i++) { + const next = lo + stepMs; + if (compareWall(wallOfInstant(next, zone), wall) > 0) { + return binarySearchFirstAfter(lo, next, wall, zone); + } + lo = next; + } + throw new Error(`no instant found after gap at ${JSON.stringify(wall)} in ${zone}`); +} + +/** Invariant: wall(lo) <= wall < wall(hi); returns the smallest such hi. */ +function binarySearchFirstAfter(lo: number, hi: number, wall: WallClock, zone: string): number { + while (hi - lo > 1) { + const mid = lo + Math.floor((hi - lo) / 2); + if (compareWall(wallOfInstant(mid, zone), wall) > 0) { + hi = mid; + } else { + lo = mid; + } + } + return hi; +} diff --git a/packages/scheduling/src/nominal.ts b/packages/scheduling/src/nominal.ts new file mode 100644 index 0000000..592e094 --- /dev/null +++ b/packages/scheduling/src/nominal.ts @@ -0,0 +1,65 @@ +import { InvalidNominalLocalError } from "./errors.js"; +import type { NominalLocal, WallClock } from "./timezone.js"; + +/** Format a wall clock as a nominal local date-time string, `YYYY-MM-DDTHH:mm`. */ +export function formatNominal(wall: WallClock): NominalLocal { + const y = String(wall.year).padStart(4, "0"); + const mo = String(wall.month).padStart(2, "0"); + const d = String(wall.day).padStart(2, "0"); + const h = String(wall.hour).padStart(2, "0"); + const mi = String(wall.minute).padStart(2, "0"); + return `${y}-${mo}-${d}T${h}:${mi}`; +} + +const NOMINAL_RE = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})(?::(\d{2}))?$/; + +/** + * Parse a nominal local date-time string back into a wall clock. Rejects + * malformed strings and impossible calendar values with + * `InvalidNominalLocalError`. Note the string may denote a wall time that does + * not exist in a zone (gap) or exists twice (fold); existence is a + * materialization concern, not a parse concern. + */ +export function parseNominal(nominal: NominalLocal): WallClock { + const match = NOMINAL_RE.exec(nominal); + if (!match) { + throw new InvalidNominalLocalError(`malformed nominal local date-time: "${nominal}"`); + } + const wall: WallClock = { + year: Number(match[1]), + month: Number(match[2]), + day: Number(match[3]), + hour: Number(match[4]), + minute: Number(match[5]), + second: match[6] === undefined ? 0 : Number(match[6]), + }; + if (wall.month < 1 || wall.month > 12) { + throw new InvalidNominalLocalError(`month out of range in "${nominal}"`); + } + if (wall.day < 1 || wall.day > 31) { + throw new InvalidNominalLocalError(`day out of range in "${nominal}"`); + } + if (wall.hour > 23 || wall.minute > 59 || wall.second > 59) { + throw new InvalidNominalLocalError(`time of day out of range in "${nominal}"`); + } + return wall; +} + +/** Total ordering over wall clocks; safe because fields are validated. */ +export function compareWall(a: WallClock, b: WallClock): number { + if (a.year !== b.year) return a.year < b.year ? -1 : 1; + if (a.month !== b.month) return a.month < b.month ? -1 : 1; + if (a.day !== b.day) return a.day < b.day ? -1 : 1; + if (a.hour !== b.hour) return a.hour < b.hour ? -1 : 1; + if (a.minute !== b.minute) return a.minute < b.minute ? -1 : 1; + return a.second < b.second ? -1 : a.second > b.second ? 1 : 0; +} + +export function wallEquals(a: WallClock, b: WallClock): boolean { + return compareWall(a, b) === 0; +} + +/** Milliseconds between two wall readings when interpreted as UTC. */ +export function wallAsEpochMs(wall: WallClock): number { + return Date.UTC(wall.year, wall.month - 1, wall.day, wall.hour, wall.minute, wall.second); +} diff --git a/packages/scheduling/src/occurrences.test.ts b/packages/scheduling/src/occurrences.test.ts new file mode 100644 index 0000000..f344c48 --- /dev/null +++ b/packages/scheduling/src/occurrences.test.ts @@ -0,0 +1,249 @@ +import { describe, expect, it } from "vitest"; +import { nextOccurrences, materializeOccurrences } from "./occurrences_extra.js"; +import { prepareSchedule } from "./schedule.js"; +import { nextOccurrencesPrepared } from "./occurrences.js"; +import { InvalidCronError } from "./errors.js"; + +const ISO = (occ: { fireAtUtc: number }) => new Date(occ.fireAtUtc).toISOString().replace(".000Z", "Z"); + +describe("UTC passthrough", () => { + it("emits exactly the cron slots with no offset math", () => { + const out = nextOccurrences({ cron: "0 9 * * *", timezone: "UTC" }, "2024-03-09T00:00:00Z", 3); + expect(out.map(ISO)).toEqual([ + "2024-03-09T09:00:00Z", + "2024-03-10T09:00:00Z", + "2024-03-11T09:00:00Z", + ]); + expect(out.map((o) => o.nominalLocal)).toEqual([ + "2024-03-09T09:00", + "2024-03-10T09:00", + "2024-03-11T09:00", + ]); + }); + it("daily at 02:30 UTC is unaffected by any DST", () => { + const out = nextOccurrences({ cron: "30 2 * * *", timezone: "UTC" }, "2024-03-09T00:00:00Z", 3); + expect(out.map(ISO)).toEqual([ + "2024-03-09T02:30:00Z", + "2024-03-10T02:30:00", + "2024-03-11T02:30:00Z", + ].map((x) => (x.endsWith("Z") ? x : `${x}Z`))); + }); + it("respects hour and minute fields", () => { + const out = nextOccurrences({ cron: "15,45 6 * * *", timezone: "UTC" }, "2024-03-09T00:00:00Z", 4); + expect(out.map(ISO)).toEqual([ + "2024-03-09T06:15:00Z", + "2024-03-09T06:45:00Z", + "2024-03-10T06:15:00Z", + "2024-03-10T06:45:00Z", + ]); + }); +}); + +describe("Europe/New_York spring-forward gap (2024-03-10 02:00->03:00)", () => { + it("daily 02:30: the gap day fires at the shifted-forward instant 03:00 local (07:00Z)", () => { + const out = nextOccurrences({ cron: "30 2 * * *", timezone: "America/New_York" }, "2024-03-08T00:00:00Z", 3); + expect(out.map(ISO)).toEqual([ + "2024-03-08T07:30:00Z", // 02:30 EST + "2024-03-09T07:30:00Z", // 02:30 EST + "2024-03-10T07:00:00Z", // 02:30 does not exist; fires 03:00 EDT + ]); + // The nominal slot is preserved on the gap day even though the instant + // shifted — that is what last_fired_nominal_local records. + expect(out[2]!.nominalLocal).toBe("2024-03-10T02:30"); + }); + it("a 02:00-only schedule still fires on the gap day, shifted to 03:00 local", () => { + const out = nextOccurrences({ cron: "0 2 * * *", timezone: "America/New_York" }, "2024-03-09T00:00:00Z", 3); + expect(out.map(ISO)).toEqual([ + "2024-03-09T07:00:00Z", // 02:00 EST + "2024-03-10T07:00:00Z", // 02:00 nonexistent -> 03:00 EDT + "2024-03-11T06:00:00Z", // 02:00 EDT + ]); + }); +}); + +describe("collapse-dedup: */15 across the spring-forward hour fires once per instant", () => { + it("02:00..02:45 nominal slots collapse; the gap instant appears exactly once", () => { + const out = nextOccurrences( + { cron: "*/15 * * * *", timezone: "America/New_York" }, + "2024-03-10T05:00:00Z", + 8, + ); + // 03:00 EDT == 07:00Z is the first valid instant after the gap. The slots + // 02:00, 02:15, 02:30, 02:45 all materialize there -> ONE occurrence. + // 05:00Z == 00:15 local. The next slots are 00:30(05:30Z), 00:45(05:45Z), + // 01:00(06:00Z), 01:15(06:15Z), then the four nominal slots 01:30(06:30Z), + // 01:45(06:45Z), 02:00, 02:15, 02:30, 02:45 — of which everything in the + // gap collapses onto 03:00 local == 07:00Z — ONE occurrence — followed by + // the normal 03:15 cadence. + expect(out.map(ISO)).toEqual([ + "2024-03-10T05:15:00Z", + "2024-03-10T05:30:00Z", + "2024-03-10T05:45:00Z", + "2024-03-10T06:00:00Z", + "2024-03-10T06:15:00Z", + "2024-03-10T06:30:00Z", + "2024-03-10T06:45:00Z", + "2024-03-10T07:00:00Z", + ]); + // The single surviving occurrence at the gap edge carries the LATEST + // nominal slot that collapsed onto it — the conservative never-refire label. + expect(out[7]!.nominalLocal).toBe("2024-03-10T03:00"); + }); + it("the collapse instant appears exactly once even when asking for many", () => { + const out = nextOccurrences({ cron: "*/15 * * * *", timezone: "America/New_York" }, "2024-03-10T05:00:00Z", 40); + const atEdge = out.filter((o) => o.fireAtUtc === Date.parse("2024-03-10T07:00:00Z")); + expect(atEdge.length).toBe(1); + }); + it("produces globally unique fire instants (occurrence key uniqueness)", () => { + const out = nextOccurrences({ cron: "*/15 * * * *", timezone: "America/New_York" }, "2024-03-09T00:00:00Z", 200); + const instants = out.map((o) => o.fireAtUtc); + expect(new Set(instants).size).toBe(instants.length); + }); + it("fold hour: */15 emits each ambiguous quarter-hour once, at the earlier instant", () => { + const out = nextOccurrences( + { cron: "*/15 * * * *", timezone: "America/New_York" }, + "2024-11-03T04:00:00Z", + 8, + { exclusive: false }, + ); + expect(out.map(ISO)).toEqual([ + "2024-11-03T04:00:00Z", // 00:00 EDT + "2024-11-03T04:15:00Z", + "2024-11-03T04:30:00Z", + "2024-11-03T04:45:00Z", + "2024-11-03T05:00:00Z", // 01:00 EDT (earlier reading) + "2024-11-03T05:15:00Z", + "2024-11-03T05:30:00Z", + "2024-11-03T05:45:00Z", + ]); + }); +}); + +describe("fall-back fold — daily 01:30 fires once at the earlier instant", () => { + it("fires 05:30Z (EDT), never 06:30Z, and continues next day", () => { + const out = nextOccurrences({ cron: "30 1 * * *", timezone: "America/New_York" }, "2024-11-01T00:00:00Z", 3); + expect(out.map(ISO)).toEqual([ + "2024-11-01T05:30:00Z", // 01:30 EDT + "2024-11-02T05:30:00Z", // 01:30 EDT + "2024-11-03T05:30:00Z", // 01:30 exists twice; fires ONCE at EDT (05:30Z) + ]); + // and never the later EST reading + expect(out.map(ISO)).not.toContain("2024-11-03T06:30:00Z"); + }); +}); + +describe("cron day-of-month/month handling", () => { + it("monthly on the 1st at 09:00", () => { + const out = nextOccurrences({ cron: "0 9 1 * *", timezone: "UTC" }, "2024-01-15T00:00:00Z", 3); + expect(out.map(ISO)).toEqual([ + "2024-02-01T09:00:00Z", + "2024-03-01T09:00:00Z", + "2024-04-01T09:00:00Z", + ]); + }); + it("monthly last-day-ish: the 31st only in long months", () => { + const out = nextOccurrences({ cron: "0 9 31 * *", timezone: "UTC" }, "2024-01-01T00:00:00Z", 4); + expect(out.map(ISO)).toEqual([ + "2024-01-31T09:00:00Z", + "2024-03-31T09:00:00Z", + "2024-05-31T09:00:00Z", + "2024-07-31T09:00:00Z", + ]); + }); + it("day-of-week: weekly on Monday 09:00", () => { + const out = nextOccurrences({ cron: "0 9 * * 1", timezone: "UTC" }, "2024-03-08T00:00:00Z", 3); + // 2024-03-08 is a Friday and does not match; the Mondays follow. + expect(out.map(ISO)).toEqual([ + "2024-03-11T09:00:00Z", + "2024-03-18T09:00:00Z", + "2024-03-25T09:00:00Z", + ]); + expect(out.every((o) => new Date(o.fireAtUtc).getUTCDay() === 1)).toBe(true); + }); + it("dom/dow OR rule: both restricted means either matches", () => { + // 13th OR every Friday (2024-03-01 is itself a Friday but sits exactly at + // `from`, so the exclusive default drops it; 03-02 would keep it). + const out = nextOccurrences({ cron: "0 0 13 * 5", timezone: "UTC" }, "2024-03-01T00:00:00Z", 6); + expect(out.map((o) => o.nominalLocal)).toEqual([ + "2024-03-08T00:00", + "2024-03-13T00:00", + "2024-03-15T00:00", + "2024-03-22T00:00", + "2024-03-29T00:00", + "2024-04-05T00:00", + ]); + }); + it("dow-only restriction drives the day when dom is *", () => { + // Mondays and Fridays only + const out = nextOccurrences({ cron: "0 12 * * 1,5", timezone: "UTC" }, "2024-03-04T00:00:00Z", 4); + expect(out.map((o) => o.nominalLocal)).toEqual([ + "2024-03-04T12:00", // Monday + "2024-03-08T12:00", // Friday + "2024-03-11T12:00", // Monday + "2024-03-15T12:00", // Friday + ]); + }); + it("hour range with step: 8-18/2 hours", () => { + const out = nextOccurrences({ cron: "0 8-18/2 * * *", timezone: "UTC" }, "2024-03-09T00:00:00Z", 7); + expect(out.map((o) => o.nominalLocal.slice(11))).toEqual(["08:00", "10:00", "12:00", "14:00", "16:00", "18:00", "08:00"]); + }); + it("month names and dow names", () => { + const out = nextOccurrences({ cron: "0 9 1 JAN-MAR *", timezone: "UTC" }, "2024-01-02T00:00:00Z", 2); + expect(out.map(ISO)).toEqual(["2024-02-01T09:00:00Z", "2024-03-01T09:00:00Z"]); + }); +}); + +describe("Pacific/Apia whole-day skip in generation", () => { + it("daily 09:00 skips 2011-12-30 entirely and resumes 12-31", () => { + const out = nextOccurrences({ cron: "0 9 * * *", timezone: "Pacific/Apia" }, "2011-12-28T00:00:00Z", 3); + expect(out.map((o) => o.nominalLocal)).toEqual([ + "2011-12-28T09:00", + "2011-12-29T09:00", + // The skipped day's slot (12-30 09:00) still fires — shifted forward to + // the first valid instant — and keeps its own nominal label so it can + // never be refired as a "new" slot later. + "2011-12-30T09:00", + ]); + // Its fire instant is the zone's first valid instant after the skip. + expect(new Date(out[2]!.fireAtUtc).toISOString()).toBe("2011-12-30T10:00:00.000Z"); + }); +}); + +describe("generation hygiene", () => { + it("prepared and unprepared agree", () => { + const spec = { cron: "*/20 * * * *", timezone: "America/New_York" }; + const a = nextOccurrences(spec, "2024-03-09T00:00:00Z", 50); + const b = nextOccurrencesPrepared(prepareSchedule(spec), "2024-03-09T00:00:00Z", 50); + expect(a).toEqual(b); + }); + it("exclusive default: regenerating from the last emitted instant does not re-emit it", () => { + const spec = { cron: "0 * * * *", timezone: "UTC" }; + const first = nextOccurrences(spec, "2024-03-09T00:00:00Z", 5); + const last = first[first.length - 1]!; + const second = nextOccurrences(spec, last.fireAtUtc, 5); + expect(second[0]!.fireAtUtc).toBeGreaterThan(last.fireAtUtc); + expect(second.map(ISO)).toEqual([ + "2024-03-09T06:00:00Z", + "2024-03-09T07:00:00Z", + "2024-03-09T08:00:00Z", + "2024-03-09T09:00:00Z", + "2024-03-09T10:00:00Z", + ]); + }); + it("inclusive option includes a slot exactly at from", () => { + const out = nextOccurrences({ cron: "0 * * * *", timezone: "UTC" }, "2024-03-09T05:00:00Z", 2, { exclusive: false }); + expect(out.map(ISO)).toEqual(["2024-03-09T05:00:00Z", "2024-03-09T06:00:00Z"]); + }); + it("rejects a bad count", () => { + expect(() => nextOccurrences({ cron: "* * * * *", timezone: "UTC" }, "2024-01-01T00:00:00Z", 0)).toThrow(RangeError); + expect(() => nextOccurrences({ cron: "* * * * *", timezone: "UTC" }, "2024-01-01T00:00:00Z", 1.5)).toThrow(RangeError); + }); + it("materializeOccurrences is the generation batch", () => { + const out = materializeOccurrences({ cron: "0 9 * * *", timezone: "UTC" }, "2024-03-09T00:00:00Z", 2); + expect(out.map(ISO)).toEqual(["2024-03-09T09:00:00Z", "2024-03-10T09:00:00Z"]); + }); + it("rejects invalid cron/zone at generation time", () => { + expect(() => nextOccurrences({ cron: "not cron", timezone: "UTC" }, "2024-01-01T00:00:00Z", 1)).toThrow(InvalidCronError); + expect(() => nextOccurrences({ cron: "* * * * *", timezone: "local" }, "2024-01-01T00:00:00Z", 1)).toThrow(); + }); +}); diff --git a/packages/scheduling/src/occurrences.ts b/packages/scheduling/src/occurrences.ts new file mode 100644 index 0000000..550ece3 --- /dev/null +++ b/packages/scheduling/src/occurrences.ts @@ -0,0 +1,120 @@ +import { cronMatchesDay } from "./cron.js"; +import { resolveWallToInstant } from "./materialize.js"; +import { slotNominal, sortedHours, sortedMinutes, type PreparedSchedule } from "./schedule.js"; +import { toInstantMs, wallOfInstant } from "./timezone.js"; +import type { WallClock } from "./timezone.js"; +import type { Occurrence } from "./schedule.js"; + +export interface NextOccurrencesOptions { + /** + * When true (default) a slot whose fire instant is exactly `from` is + * skipped; when false it is included. The default keeps re-generation + * idempotent: re-running generation from the last emitted instant never + * re-emits it. + */ + readonly exclusive?: boolean; +} + + +/** Nominal calendar day (local midnight reading) containing `epochMs`. */ +function startNominalDay(epochMs: number, zone: string): WallClock { + const wall = wallOfInstant(epochMs, zone); + return { year: wall.year, month: wall.month, day: wall.day, hour: 0, minute: 0, second: 0 }; +} + +/** Advance a nominal calendar day by one, purely calendar arithmetic. */ +function nextNominalDay(day: WallClock): WallClock { + const next = new Date(Date.UTC(day.year, day.month - 1, day.day + 1)); + return { + year: next.getUTCFullYear(), + month: next.getUTCMonth() + 1, + day: next.getUTCDate(), + hour: 0, + minute: 0, + second: 0, + }; +} + +/** + * The next `count` occurrences at or after `from`, evaluated in the schedule + * zone and materialized to UTC instants. + * + * Each local day's matching slots are emitted in ascending local order; a + * slot's materialization (gap shift-forward, fold earlier-instant) may collide + * with another slot's instant — such collisions are deduplicated so each + * materialized instant appears exactly once (occurrence key stays unique). + * Generation is capped at `maxYears` of local days to bound work for + * sparse schedules. + */ +export function nextOccurrencesPrepared( + prepared: PreparedSchedule, + from: Date | number | string, + count: number, + options?: NextOccurrencesOptions, + maxYears = 8, +): Occurrence[] { + if (!Number.isInteger(count) || count <= 0) { + throw new RangeError(`count must be a positive integer, got ${count}`); + } + const zone = prepared.timezone; + const cron = prepared.cron; + const fromMs = toInstantMs(from); + const exclusive = options?.exclusive ?? true; + const hours = sortedHours(cron); + const minutes = sortedMinutes(cron); + const out: Occurrence[] = []; + const seen = new Set(); + // Index of the emitted occurrence per fire instant, so a later nominal slot + // collapsing onto an already-emitted instant can take over the label. + const indexAt = new Map(); + const dayLimit = Math.ceil(maxYears * 366); + // Iterate NOMINAL calendar days (not materialized midnights) so a skipped + // local day (Pacific/Apia 2011-12-30) still contributes its slots: every + // slot of that day materializes to the first valid instant after the gap and + // collapses onto a single occurrence there. + let day = startNominalDay(fromMs, zone); + // Once `count` occurrences exist, generation may continue only far enough to + // finish the collapse set of the LAST emitted instant — later slots of that + // same instant must still be able to take over its nominal label — and then + // stops. Occurrences are emitted in ascending instant order, so the first + // slot materializing strictly after the boundary ends generation. + let boundary: number | null = null; + for (let dayIndex = 0; dayIndex < dayLimit; dayIndex++) { + if (cronMatchesDay(cron, day.year, day.month, day.day)) { + for (const hour of hours) { + for (const minute of minutes) { + const slotWall: WallClock = { + year: day.year, month: day.month, day: day.day, + hour, minute, second: 0, + }; + const fireAt = resolveWallToInstant(slotWall, zone); + if (exclusive ? fireAt <= fromMs : fireAt < fromMs) continue; + if (boundary !== null && fireAt > boundary) return out; + if (seen.has(fireAt)) { + // Collapse: several nominal slots materialized onto one instant + // (post-gap). Exactly one occurrence survives per instant, carrying + // the LATEST collapsed nominal slot — the conservative value for + // the last_fired_nominal_local never-refire gate. + const index = indexAt.get(fireAt); + if (index !== undefined) { + out[index] = { + nominalLocal: slotNominal(day.year, day.month, day.day, hour, minute), + fireAtUtc: fireAt, + }; + } + continue; + } + seen.add(fireAt); + indexAt.set(fireAt, out.length); + out.push({ + nominalLocal: slotNominal(day.year, day.month, day.day, hour, minute), + fireAtUtc: fireAt, + }); + if (out.length >= count) boundary = fireAt; + } + } + } + day = nextNominalDay(day); + } + return out; +} diff --git a/packages/scheduling/src/occurrences_extra.ts b/packages/scheduling/src/occurrences_extra.ts new file mode 100644 index 0000000..dbc8bd5 --- /dev/null +++ b/packages/scheduling/src/occurrences_extra.ts @@ -0,0 +1,26 @@ +// Convenience wrappers kept in a separate module so occurrences.ts stays a +// single-purpose iterator. Re-exported through index.ts. +import { nextOccurrencesPrepared } from "./occurrences.js"; +import { prepareSchedule, type Occurrence, type ScheduleSpec } from "./schedule.js"; + +/** The next `count` occurrences of `spec` at or after `from` (strictly after). */ +export function nextOccurrences( + spec: ScheduleSpec, + from: Date | number | string, + count: number, + options?: { exclusive?: boolean }, +): Occurrence[] { + return nextOccurrencesPrepared(prepareSchedule(spec), from, count, options); +} + +/** + * Occurrence rows a worker should insert for `spec` from an instant — the + * materialization batch keyed by (schedule_id, fire_at_utc). + */ +export function materializeOccurrences( + spec: ScheduleSpec, + from: Date | number | string, + count: number, +): Occurrence[] { + return nextOccurrences(spec, from, count); +} diff --git a/packages/scheduling/src/property.test.ts b/packages/scheduling/src/property.test.ts new file mode 100644 index 0000000..f33d9d0 --- /dev/null +++ b/packages/scheduling/src/property.test.ts @@ -0,0 +1,131 @@ +import { describe, expect, it } from "vitest"; +import { nextOccurrences } from "./occurrences_extra.js"; +import { materializeOccurrence } from "./materialize-api.js"; +import { wallOfInstant } from "./timezone.js"; + +/** + * Brute-force oracle: scan every minute with Intl and apply the spec rules + * independently of the engine's search strategy. + * + * - a wall reading that matches a cron slot fires at that instant; + * - a fold (reading repeats one already seen) fires only at its first instant; + * - a gap (the wall clock jumps forward over slot(s)) fires at the instant of + * the jump, labelled with the LATEST skipped slot; + * - several slots landing on one instant collapse to a single occurrence. + */ +function oracle(cronMinutes: number[], cronHours: number[], zone: string, fromMs: number, toMs: number) { + const wanted = new Set(); + for (const h of cronHours) for (const m of cronMinutes) wanted.add(`${h}:${m}`); + const wallKey = (y: number, mo: number, d: number, h: number, mi: number) => + Date.UTC(y, mo - 1, d, h, mi); + const byInstant = new Map(); + const seenWalls = new Set(); + let prevKey: number | null = null; + for (let t = fromMs; t <= toMs - 60_000; t += 60_000) { + const w = wallOfInstant(t, zone); + const key = wallKey(w.year, w.month, w.day, w.hour, w.minute); + const label = `${w.year}-${pad(w.month)}-${pad(w.day)}T${pad(w.hour)}:${pad(w.minute)}`; + // Fold: this wall reading already fired at its earlier instant. + if (seenWalls.has(key)) { + prevKey = key; + continue; + } + seenWalls.add(key); + // Gap: the wall clock jumped. Any wanted slot strictly between the previous + // reading and this one did not exist; it fires HERE (shift forward), and + // several skipped slots collapse onto this single instant. + if (prevKey !== null && key - prevKey > 60_000) { + for (let k = prevKey + 60_000; k < key; k += 60_000) { + const kk = new Date(k); + if (wanted.has(`${kk.getUTCHours()}:${kk.getUTCMinutes()}`)) { + const d = kk.getUTCDate(); + const mo = kk.getUTCMonth() + 1; + const y = kk.getUTCFullYear(); + byInstant.set(t, `${y}-${pad(mo)}-${pad(d)}T${pad(kk.getUTCHours())}:${pad(kk.getUTCMinutes())}`); + } + } + } + if (wanted.has(`${w.hour}:${w.minute}`)) { + byInstant.set(t, label); + } + prevKey = key; + } + return [...byInstant.entries()] + .sort((a, b) => a[0] - b[0]) + .map(([fireAtUtc, nominalLocal]) => ({ nominalLocal, fireAtUtc })); +} + +const pad = (n: number) => String(n).padStart(2, "0"); + +describe("cross-zone property: generator matches a brute-force minute oracle", () => { + const cases: Array<{ name: string; zone: string; from: string; to: string; cron: string }> = [ + { name: "NY hourly across both 2024 transitions", zone: "America/New_York", from: "2024-03-09T00:00:00Z", to: "2024-03-12T00:00:00Z", cron: "0 * * * *" }, + { name: "NY quarter-hours across spring-forward", zone: "America/New_York", from: "2024-03-10T00:00:00Z", to: "2024-03-11T00:00:00Z", cron: "*/15 * * * *" }, + { name: "NY quarter-hours across fall-back", zone: "America/New_York", from: "2024-11-03T00:00:00Z", to: "2024-11-04T00:00:00Z", cron: "*/15 * * * *" }, + { name: "Berlin hourly across spring-forward", zone: "Europe/Berlin", from: "2024-03-30T00:00:00Z", to: "2024-04-02T00:00:00Z", cron: "0 * * * *" }, + { name: "Apia daily across the 2011 whole-day skip", zone: "Pacific/Apia", from: "2011-12-27T00:00:00Z", to: "2012-01-02T00:00:00Z", cron: "0 9 * * *" }, + { name: "Apia hourly across the skip", zone: "Pacific/Apia", from: "2011-12-29T00:00:00Z", to: "2011-12-31T00:00:00Z", cron: "30 * * * *" }, + { name: "Lord Howe 30-minute transition (spring)", zone: "Australia/Lord_Howe", from: "2024-10-04T00:00:00Z", to: "2024-10-08T00:00:00Z", cron: "0 * * * *" }, + { name: "Lord Howe 30-minute transition (fall)", zone: "Australia/Lord_Howe", from: "2024-04-04T00:00:00Z", to: "2024-04-08T00:00:00Z", cron: "0 * * * *" }, + { name: "Kathmandu 45-minute offset (no DST)", zone: "Asia/Kathmandu", from: "2024-03-01T00:00:00Z", to: "2024-03-04T00:00:00Z", cron: "*/20 * * * *" }, + { name: "Chatham Islands 45-min offset with DST", zone: "Pacific/Chatham", from: "2024-04-05T00:00:00Z", to: "2024-04-09T00:00:00Z", cron: "0 * * * *" }, + { name: "UTC passthrough", zone: "UTC", from: "2024-03-09T00:00:00Z", to: "2024-03-11T00:00:00Z", cron: "*/30 * * * *" }, + { name: "Santiago delayed transition (midnight)", zone: "America/Santiago", from: "2024-09-07T00:00:00Z", to: "2024-09-10T00:00:00Z", cron: "0 * * * *" }, + ]; + + it.each(cases.map((c) => [c.name, c] as const))("%s", (_name, c) => { + const fromMs = Date.parse(c.from); + const toMs = Date.parse(c.to); + // Derive the oracle's slot sets from the same cron fields the generator + // parses, so the comparison exercises materialization and ordering — not + // cron parsing (covered separately in cron.test.ts). + const fields = c.cron.split(/\s+/); + const minuteField = fields[0]!; + const hourField = fields[1]!; + const minutes = minuteField.startsWith("*/") + ? Array.from({ length: Math.ceil(60 / Number(minuteField.slice(2))) }, (_, i) => i * Number(minuteField.slice(2))).filter((m) => m < 60) + : minuteField.split(",").map(Number); + const hours = hourField === "*" ? Array.from({ length: 24 }, (_, i) => i) : hourField.split(",").map(Number); + const wanted = oracle(minutes, hours, c.zone, fromMs, toMs); + // Inclusive so the generator's window matches the oracle's inclusive scan. + const got = nextOccurrences({ cron: c.cron, timezone: c.zone }, c.from, wanted.length || 1, { exclusive: false }); + expect(got.map((o) => [o.nominalLocal, o.fireAtUtc])).toEqual(wanted.map((o) => [o.nominalLocal, o.fireAtUtc])); + }); +}); + +describe("materializeOccurrence inverts wallOfInstant", () => { + const zones = ["America/New_York", "Europe/Berlin", "Pacific/Apia", "Australia/Lord_Howe", "Pacific/Chatham", "Asia/Kathmandu", "UTC"]; + // Hourly around each hemisphere's 2024 transitions (where inversions can + // happen) plus a coarse 12-hourly annual sweep. + const windows = [ + ["2024-03-08T00:00:00Z", "2024-03-13T00:00:00Z"], + ["2024-11-01T00:00:00Z", "2024-11-06T00:00:00Z"], + ["2024-09-28T00:00:00Z", "2024-10-10T00:00:00Z"], + ["2024-04-01T00:00:00Z", "2024-04-09T00:00:00Z"], + ["2011-12-27T00:00:00Z", "2012-01-03T00:00:00Z"], + ] as const; + const steps: Array<[number, number]> = [ + ...windows.map((w) => [Date.parse(w[0]), Date.parse(w[1])] as [number, number]), + [Date.parse("2024-01-01T00:00:00Z"), Date.parse("2025-01-01T00:00:00Z")], + ]; + it.each(zones.map((z) => [z]))("round-trips wall readings in %s", (zone) => { + let folds = 0; + let checked = 0; + for (const [start, end] of steps) { + const step = end - start > 20 * 86400_000 ? 12 * 3600_000 : 3600_000; + for (let t = start; t < end; t += step) { + const w = wallOfInstant(t, zone); + const nominal = `${w.year}-${String(w.month).padStart(2, "0")}-${String(w.day).padStart(2, "0")}T${String(w.hour).padStart(2, "0")}:${String(w.minute).padStart(2, "0")}`; + const back = materializeOccurrence(nominal, zone); + checked++; + if (back === t) continue; + // Only fold readings may differ, and only by the offset delta (back < t). + expect(back).toBeLessThan(t); + expect(t - back).toBeLessThanOrEqual(3600_000); + folds++; + } + } + expect(checked).toBeGreaterThan(100); + expect(folds).toBeLessThanOrEqual(6); + }); +}); diff --git a/packages/scheduling/src/rematerialize.test.ts b/packages/scheduling/src/rematerialize.test.ts new file mode 100644 index 0000000..5970ce6 --- /dev/null +++ b/packages/scheduling/src/rematerialize.test.ts @@ -0,0 +1,198 @@ +import { describe, expect, it } from "vitest"; +import { rematerialize } from "./rematerialize.js"; +import { nextOccurrences } from "./occurrences_extra.js"; +import type { StoredOccurrence } from "./rematerialize.js"; + +/** Build stored rows as a worker would have persisted them. */ +function storedFrom(spec: { cron: string; timezone: string }, from: string, count: number): StoredOccurrence[] { + return nextOccurrences(spec, from, count).map((o) => ({ + nominalLocal: o.nominalLocal, + fireAtUtc: o.fireAtUtc, + claimed: false, + })); +} + +describe("rematerialize — invalidation and regeneration", () => { + it("deletes unclaimed future occurrences and regenerates from the change instant", () => { + const oldSpec = { cron: "0 9 * * *", timezone: "America/New_York" }; + const stored = storedFrom(oldSpec, "2024-03-01T00:00:00Z", 5); + // One past occurrence was claimed. + stored[0] = { ...stored[0]!, claimed: true }; + const changeInstant = "2024-03-02T00:00:00Z"; + + const result = rematerialize(oldSpec, { cron: "0 10 * * *", timezone: "America/New_York" }, stored, { + changeInstant, + lastFiredNominalLocal: stored[0]!.nominalLocal, + count: 3, + }); + + // Unclaimed future rows are invalidated. + expect(result.toDeleteFireAtUtc).toEqual( + stored.slice(1).map((r) => r.fireAtUtc), + ); + // New occurrences start strictly after the change instant and the last claim. + expect(result.toInsert.length).toBe(3); + expect(result.toInsert[0]!.fireAtUtc).toBeGreaterThan(Date.parse(changeInstant)); + expect(result.toInsert[0]!.nominalLocal).toBe("2024-03-02T10:00"); + // The new tzdata version is reported for recording. + expect(result.tzdataVersion).toMatch(/.+/); + }); + + it("never refires a nominal slot already fired under the old zone (NY -> LA same day)", () => { + // Daily 09:00. Under New York the 2024-03-01 09:00 slot fired at 14:00Z. + const oldSpec = { cron: "0 9 * * *", timezone: "America/New_York" }; + const stored: StoredOccurrence[] = [ + { nominalLocal: "2024-03-01T09:00", fireAtUtc: Date.parse("2024-03-01T14:00:00Z"), claimed: true }, + ...storedFrom(oldSpec, "2024-03-02T00:00:00Z", 3), + ]; + const result = rematerialize(oldSpec, { cron: "0 9 * * *", timezone: "America/Los_Angeles" }, stored, { + changeInstant: "2024-03-01T15:00:00Z", // later the same day, after the NY firing + lastFiredNominalLocal: "2024-03-01T09:00", + count: 3, + }); + // 09:00 in Los Angeles on 03-01 is 17:00Z — but that nominal slot was + // already fired under New York, so it must NOT appear. + expect(result.toInsert.map((o) => o.nominalLocal)).not.toContain("2024-03-01T09:00"); + expect(result.toInsert.map((o) => o.nominalLocal)).toEqual([ + "2024-03-02T09:00", + "2024-03-03T09:00", + "2024-03-04T09:00", + ].slice(0, 3)); + expect(result.toInsert[0]!.fireAtUtc).toBe(Date.parse("2024-03-02T17:00:00Z")); + }); + + it("claimed rows are never deleted", () => invariants()); + + function invariants(): boolean { + const oldSpec = { cron: "0 9 * * *", timezone: "America/New_York" }; + const stored = storedFrom(oldSpec, "2024-03-01T00:00:00Z", 5); + stored[0] = { ...stored[0]!, claimed: true }; + const result = rematerialize(oldSpec, oldSpec, stored, { + changeInstant: "2024-03-02T00:00:00Z", + lastFiredNominalLocal: stored[0]!.nominalLocal, + count: 3, + }); + expect(result.toDeleteFireAtUtc).not.toContain(stored[0]!.fireAtUtc); + return true; + } + + it("inserts never collide with SURVIVING rows (deletes are applied first)", () => { + const oldSpec = { cron: "0 9 * * *", timezone: "America/New_York" }; + const stored = storedFrom(oldSpec, "2024-03-01T00:00:00Z", 5); + stored[0] = { ...stored[0]!, claimed: true }; + const result = rematerialize(oldSpec, oldSpec, stored, { + changeInstant: "2024-03-01T00:00:00Z", + lastFiredNominalLocal: stored[0]!.nominalLocal, + count: 10, + }); + const deleting = new Set(result.toDeleteFireAtUtc); + const surviving = stored.filter((r) => !deleting.has(r.fireAtUtc)).map((r) => r.fireAtUtc); + expect(surviving).toEqual([stored[0]!.fireAtUtc]); // only the claimed row survives + for (const o of result.toInsert) { + expect(surviving.includes(o.fireAtUtc)).toBe(false); + } + }); + + it("unchanged spec regenerates the same instants it deleted (idempotent)", () => { + const spec = { cron: "0 9 * * *", timezone: "America/New_York" }; + const stored = storedFrom(spec, "2024-03-01T00:00:00Z", 5); + stored[0] = { ...stored[0]!, claimed: true }; + const result = rematerialize(spec, spec, stored, { + changeInstant: "2024-03-01T00:00:00Z", + lastFiredNominalLocal: stored[0]!.nominalLocal, + count: stored.length - 1, // regenerate exactly the deleted horizon + }); + const deletedSorted = [...result.toDeleteFireAtUtc].sort((a, b) => a - b); + expect(result.toInsert.map((o) => o.fireAtUtc)).toEqual(deletedSorted); + }); + + it("toInsert fire instants are unique among themselves and cross the gap once", () => { + const spec = { cron: "*/15 * * * *", timezone: "America/New_York" }; + const stored = storedFrom(spec, "2024-03-08T00:00:00Z", 20); + // 150 quarter-hours from 03-09 00:15Z reaches past the 03-10 07:00Z edge. + const result = rematerialize(spec, spec, stored, { changeInstant: "2024-03-09T00:00:00Z", count: 150 }); + const instants = result.toInsert.map((o) => o.fireAtUtc); + expect(new Set(instants).size).toBe(instants.length); + const edge = instants.filter((i) => i === Date.parse("2024-03-10T07:00:00Z")); + expect(edge.length).toBe(1); + // The nominal slots 02:00..02:45 collapse onto that one instant, and the + // surviving occurrence carries the latest collapsed nominal slot. + const atEdge = result.toInsert.find((o) => o.fireAtUtc === Date.parse("2024-03-10T07:00:00Z")); + expect(atEdge?.nominalLocal).toBe("2024-03-10T03:00"); + }); + + it("tzdata-only change (same zone) still regenerates correctly", () => { + const spec = { cron: "0 9 * * *", timezone: "America/New_York" }; + const stored = storedFrom(spec, "2024-03-01T00:00:00Z", 3); + const result = rematerialize(spec, spec, stored, { + changeInstant: "2024-03-01T00:00:00Z", + tzdataVersion: "2025a", + count: 2, + }); + expect(result.tzdataVersion).toBe("2025a"); + expect(result.toInsert.length).toBe(2); + }); + + it("reports the current tzdata version when not overridden", () => { + const spec = { cron: "0 9 * * *", timezone: "UTC" }; + const result = rematerialize(spec, spec, [], { changeInstant: "2024-01-01T00:00:00Z" }); + expect(typeof result.tzdataVersion).toBe("string"); + expect(result.tzdataVersion.length).toBeGreaterThan(0); + }); + + it("a schedule with no history regenerates from the change instant alone", () => { + const spec = { cron: "0 9 * * *", timezone: "UTC" }; + const result = rematerialize(spec, { ...spec, timezone: "Europe/Berlin" }, [], { + changeInstant: "2024-03-05T00:00:00Z", + count: 2, + }); + expect(result.toInsert.map((o) => o.nominalLocal)).toEqual([ + "2024-03-05T09:00", + "2024-03-06T09:00", + ]); + expect(result.toDeleteFireAtUtc).toEqual([]); + }); + + it("unclaimed PAST occurrences are not deleted (only future ones)", () => { + const spec = { cron: "0 9 * * *", timezone: "UTC" }; + const stored: StoredOccurrence[] = [ + { nominalLocal: "2024-02-01T09:00", fireAtUtc: Date.parse("2024-02-01T09:00:00Z"), claimed: false }, + { nominalLocal: "2024-04-01T09:00", fireAtUtc: Date.parse("2024-04-01T09:00:00Z"), claimed: false }, + ]; + const result = rematerialize(spec, spec, stored, { changeInstant: "2024-03-01T00:00:00Z" }); + expect(result.toDeleteFireAtUtc).toEqual([Date.parse("2024-04-01T09:00:00Z")]); + }); + + it("survives a whole-day skip: Apia daily 09:00 across the 2011 change", () => { + const spec = { cron: "0 9 * * *", timezone: "Pacific/Apia" }; + const stored = storedFrom(spec, "2011-12-28T00:00:00Z", 3); + const result = rematerialize(spec, spec, stored, { + changeInstant: "2011-12-29T12:00:00Z", + lastFiredNominalLocal: "2011-12-28T09:00", + count: 2, + }); + expect(result.toInsert.map((o) => o.nominalLocal)).toEqual([ + "2011-12-29T09:00", + "2011-12-30T09:00", // skipped day, shifted forward + ]); + }); +}); + +describe("rematerialize — input validation", () => { + it("rejects an invalid new zone", () => { + const spec = { cron: "0 9 * * *", timezone: "UTC" }; + expect(() => + rematerialize(spec, { cron: "0 9 * * *", timezone: "local" }, [], { changeInstant: "2024-01-01T00:00:00Z" }), + ).toThrow(); + }); + it("rejects an invalid new cron", () => { + const spec = { cron: "0 9 * * *", timezone: "UTC" }; + expect(() => + rematerialize(spec, { cron: "99 * * * *", timezone: "UTC" }, [], { changeInstant: "2024-01-01T00:00:00Z" }), + ).toThrow(); + }); + it("rejects an offset-less change instant", () => { + const spec = { cron: "0 9 * * *", timezone: "UTC" }; + expect(() => rematerialize(spec, spec, [], { changeInstant: "2024-01-01T00:00:00" })).toThrow(); + }); +}); diff --git a/packages/scheduling/src/rematerialize.ts b/packages/scheduling/src/rematerialize.ts new file mode 100644 index 0000000..ef19e27 --- /dev/null +++ b/packages/scheduling/src/rematerialize.ts @@ -0,0 +1,121 @@ +import { nextOccurrencesPrepared } from "./occurrences.js"; +import { prepareSchedule, type Occurrence, type ScheduleSpec } from "./schedule.js"; +import { currentTzdataVersion, toInstantMs } from "./timezone.js"; +import type { NominalLocal } from "./timezone.js"; + +/** A stored occurrence row, as read from `schedule_occurrences`. */ +export interface StoredOccurrence { + /** Nominal local slot of this row (e.g. `2024-03-10T09:00`). */ + readonly nominalLocal: NominalLocal; + /** Materialized UTC fire instant, epoch ms. */ + readonly fireAtUtc: number; + /** Whether a claim has been taken on this row. */ + readonly claimed: boolean; +} + +export interface RematerializeOptions { + /** + * Instant the zone/tzdata change was applied (epoch ms). New occurrences + * must fire strictly after max(this, last claimed fire instant). + */ + readonly changeInstant: Date | number | string; + /** + * `last_fired_nominal_local` recorded on the schedule — the local slot of + * the last claimed occurrence. New occurrences must be in a strictly later + * nominal slot, so a slot already fired under the old zone never refires. + */ + readonly lastFiredNominalLocal?: NominalLocal; + /** How many future occurrences to generate after the change. */ + readonly count?: number; + /** Override the recorded tzdata version in tests. */ + readonly tzdataVersion?: string; +} + +export interface RematerializeResult { + /** Occurrences to insert, ascending by fire instant. */ + readonly toInsert: Occurrence[]; + /** Fire instants (epoch ms) of unclaimed future rows to delete. */ + readonly toDeleteFireAtUtc: number[]; + /** The tzdata version backing this materialization, to record on the schedule. */ + readonly tzdataVersion: string; +} + +/** + * Re-materialize a schedule after a tzdata or timezone change. + * + * Order of operations mirrors the spec: first, all *unclaimed* future + * occurrences are invalidated (their fire instants are returned for deletion); + * then new occurrences are generated — only those whose fire instant is + * strictly after max(change instant, last claimed fire instant) AND whose + * nominal local slot is strictly later than `last_fired_nominal_local`, so a + * slot already fired under the old zone (daily 09:00 fired in New York, then + * zone switched to Los Angeles) is never fired again. + * + * Claimed rows are never deleted and never regenerate: their fire instants + * already happened or are owned by a claim. + */ +export function rematerialize( + oldSpec: ScheduleSpec, + newSpec: ScheduleSpec, + stored: readonly StoredOccurrence[], + options: RematerializeOptions, +): RematerializeResult { + const changeMs = toInstantMs(options.changeInstant); + const lastFiredNominal = options.lastFiredNominalLocal; + const tzVersion = options.tzdataVersion ?? currentTzdataVersion(); + + // 1) Invalidate every unclaimed future occurrence: its fire instant is + // returned for deletion. Claimed rows survive untouched. + const toDeleteFireAtUtc: number[] = []; + let lastClaimedFireMs: number | null = null; + let maxClaimedNominal: NominalLocal | null = null; + for (const row of stored) { + if (row.claimed) { + if (lastClaimedFireMs === null || row.fireAtUtc > lastClaimedFireMs) { + lastClaimedFireMs = row.fireAtUtc; + maxClaimedNominal = row.nominalLocal; + } + continue; + } + if (row.fireAtUtc > changeMs) { + toDeleteFireAtUtc.push(row.fireAtUtc); + } + } + + // Effective refire gate: max(change instant, last claimed fire instant). + const gateMs = Math.max(changeMs, lastClaimedFireMs ?? Number.NEGATIVE_INFINITY); + // Effective nominal gate: the later of the recorded last-fired slot and the + // latest claimed row's slot (rows are the runtime truth). + let nominalGate: NominalLocal | null = lastFiredNominal ?? null; + if (maxClaimedNominal !== null && (nominalGate === null || maxClaimedNominal > nominalGate)) { + nominalGate = maxClaimedNominal; + } + + // 2) Generate forward from the gate, skipping any slot not strictly later + // than the nominal gate. + const count = options.count ?? 32; + const prepared = prepareSchedule(newSpec); + const candidates = nextOccurrencesPrepared( + prepared, + gateMs, + count * 4, + { exclusive: true }, + ); + // Only rows that SURVIVE the change may block regeneration: claimed rows and + // unclaimed rows already in the past. Rows being deleted are being replaced, + // so their instants must not suppress the regenerated occurrences. + const deleting = new Set(toDeleteFireAtUtc); + const existing = new Set( + stored.filter((row) => !deleting.has(row.fireAtUtc)).map((row) => row.fireAtUtc), + ); + const toInsert: Occurrence[] = []; + for (const occurrence of candidates) { + if (occurrence.fireAtUtc <= gateMs) continue; + if (nominalGate !== null && occurrence.nominalLocal <= nominalGate) continue; + if (existing.has(occurrence.fireAtUtc)) continue; + toInsert.push(occurrence); + if (toInsert.length >= count) break; + } + toInsert.sort((a, b) => a.fireAtUtc - b.fireAtUtc); + return { toInsert, toDeleteFireAtUtc, tzdataVersion: tzVersion }; +} diff --git a/packages/scheduling/src/schedule.ts b/packages/scheduling/src/schedule.ts new file mode 100644 index 0000000..6ff35e6 --- /dev/null +++ b/packages/scheduling/src/schedule.ts @@ -0,0 +1,96 @@ +import { cronMatchesDay, type ParsedCron } from "./cron.js"; +import { formatNominal } from "./nominal.js"; +import { resolveWallToInstant } from "./materialize.js"; +import { canonicalTimezone, wallOfInstant } from "./timezone.js"; +import { parseCron } from "./cron.js"; +import type { NominalLocal } from "./timezone.js"; + +/** A schedule's time specification: five-field cron in an explicit IANA zone. */ +export interface ScheduleSpec { + /** Five-field cron expression, e.g. `*\/15 * * * *`. */ + readonly cron: string; + /** Explicit IANA zone id — never server-local. `UTC` is allowed. */ + readonly timezone: string; +} + +/** A materialized occurrence: nominal local slot plus its UTC fire instant. */ +export interface Occurrence { + /** Local date-time of this slot in the schedule zone, e.g. `2024-03-10T02:30`. */ + nominalLocal: NominalLocal; + /** Materialized UTC fire instant in epoch milliseconds. */ + fireAtUtc: number; +} + +/** A schedule validated once, reusable across calls. */ +export interface PreparedSchedule { + readonly cron: ParsedCron; + readonly timezone: string; + readonly spec: ScheduleSpec; +} + +/** + * Validate a spec (cron + zone) once. Throws `InvalidCronError`, + * `ImpossibleScheduleError` or `InvalidTimezoneError` on bad input. + */ +export function prepareSchedule(spec: ScheduleSpec): PreparedSchedule { + return { + cron: parseCron(spec.cron), + timezone: canonicalTimezone(spec.timezone), + spec: { cron: spec.cron, timezone: spec.timezone }, + }; +} + +/** Hour values sorted ascending for deterministic day-slot iteration. */ +export function sortedHours(cron: ParsedCron): number[] { + return [...cron.hour.values].sort((a, b) => a - b); +} + + + +/** Builds the nominal-local string for one slot on a matching local day. */ +export function slotNominal(year: number, month: number, day: number, hour: number, minute: number): NominalLocal { + const y = String(year).padStart(4, "0"); + const mo = String(month).padStart(2, "0"); + const d = String(day).padStart(2, "0"); + const h = String(hour).padStart(2, "0"); + const mi = String(minute).padStart(2, "0"); + return `${y}-${mo}-${d}T${h}:${mi}`; +} + +/** + * Iterate matching local days at or after a starting local day. Returns each + * matching day's wall clock at local midnight (materialized to an instant so + * the caller can step by absolute time across DST). + */ +export function* matchingLocalDays( + cron: ParsedCron, + startDayEpoch: number, + zone: string, + maxDays: number, +): Generator<{ wall: ReturnType; midnightInstant: number }> { + let cursor = startDayEpoch; + for (let i = 0; i < maxDays; i++) { + const wall = wallOfInstant(cursor, zone); + if (cronMatchesDay(cron, wall.year, wall.month, wall.day)) { + yield { wall, midnightInstant: cursor }; + } + cursor = stepOneLocalDay(cursor, zone); + } +} + +/** Advance to the next local midnight, robust across 23/24/25-hour days. */ +export function stepOneLocalDay(dayEpoch: number, zone: string): number { + const nextApprox = dayEpoch + 24 * 3600_000; + const nextWall = wallOfInstant(nextApprox, zone); + return resolveWallToInstant( + { year: nextWall.year, month: nextWall.month, day: nextWall.day, hour: 0, minute: 0, second: 0 }, + zone, + ); +} + +export { formatNominal }; + +/** Minute values sorted ascending for deterministic day-slot iteration. */ +export function sortedMinutes(cron: ParsedCron): number[] { + return [...cron.minute.values].sort((a, b) => a - b); +} diff --git a/packages/scheduling/src/timezone.test.ts b/packages/scheduling/src/timezone.test.ts new file mode 100644 index 0000000..454327e --- /dev/null +++ b/packages/scheduling/src/timezone.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, it } from "vitest"; +import { + canonicalTimezone, + currentTzdataVersion, + isValidTimezone, + offsetMs, + toInstantMs, + wallOfInstant, +} from "./timezone.js"; +import { InvalidInstantError, InvalidTimezoneError } from "./errors.js"; + +describe("canonicalTimezone", () => { + it("accepts and canonicalizes IANA ids", () => { + expect(canonicalTimezone("America/New_York")).toBe("America/New_York"); + expect(canonicalTimezone("US/Eastern")).toBe("America/New_York"); + expect(canonicalTimezone(" Europe/Paris ")).toBe("Europe/Paris"); + expect(canonicalTimezone("Etc/GMT+5")).toBe("Etc/GMT+5"); + }); + it("accepts UTC in several spellings", () => { + expect(canonicalTimezone("UTC")).toBe("UTC"); + expect(canonicalTimezone("gmt")).toBe("UTC"); + expect(canonicalTimezone("Zulu")).toBe("UTC"); + }); + it("rejects unknown zones", () => { + expect(() => canonicalTimezone("America/Nowhere")).toThrow(InvalidTimezoneError); + expect(() => canonicalTimezone("Mars/Olympus")).toThrow(InvalidTimezoneError); + }); + it("rejects offset spellings and empty input", () => { + for (const bad of ["UTC+3", "+05:30", "-08:00", "5", ""]) { + expect(() => canonicalTimezone(bad)).toThrow(InvalidTimezoneError); + } + }); + it("rejects server-local idioms — a schedule zone must be explicit", () => { + for (const bad of ["local", "system", "server", "SERVER", "server-local", "machine"]) { + expect(() => canonicalTimezone(bad)).toThrow(InvalidTimezoneError); + } + }); + it("isValidTimezone mirrors canonicalTimezone", () => { + expect(isValidTimezone("Europe/Berlin")).toBe(true); + expect(isValidTimezone("local")).toBe(false); + expect(isValidTimezone("America/Nowhere")).toBe(false); + }); +}); + +describe("wallOfInstant / offsetMs", () => { + it("reads New York offsets across the 2024 transitions", () => { + expect(offsetMs(Date.parse("2024-03-10T06:59:59Z"), "America/New_York")).toBe(-5 * 3600_000); + expect(offsetMs(Date.parse("2024-03-10T07:00:00Z"), "America/New_York")).toBe(-4 * 3600_000); + expect(offsetMs(Date.parse("2024-11-03T05:59:59Z"), "America/New_York")).toBe(-4 * 3600_000); + expect(offsetMs(Date.parse("2024-11-03T06:00:00Z"), "America/New_York")).toBe(-5 * 3600_000); + }); + it("reads half-hour and 45-minute zones", () => { + expect(offsetMs(Date.parse("2024-06-01T12:00:00Z"), "Asia/Kolkata")).toBe(5.5 * 3600_000); + expect(offsetMs(Date.parse("2024-06-01T12:00:00Z"), "Asia/Kathmandu")).toBe(5.75 * 3600_000); + }); + it("sees Pacific/Apia's 2011 whole-day skip", () => { + const before = wallOfInstant(Date.parse("2011-12-29T09:59:59Z"), "Pacific/Apia"); + const after = wallOfInstant(Date.parse("2011-12-30T10:00:00Z"), "Pacific/Apia"); + expect(`${before.year}-${before.month}-${before.day}`).toBe("2011-12-28"); + expect(`${after.year}-${after.month}-${after.day}`).toBe("2011-12-31"); + }); + it("returns UTC fields without a formatter", () => { + const wall = wallOfInstant(Date.parse("2024-03-10T02:30:00Z"), "UTC"); + expect([wall.year, wall.month, wall.day, wall.hour, wall.minute]).toEqual([2024, 3, 10, 2, 30]); + }); +}); + +describe("toInstantMs", () => { + it("accepts dates, numbers and offset-carrying ISO strings", () => { + expect(toInstantMs(new Date("2024-03-10T02:30:00Z"))).toBe(Date.parse("2024-03-10T02:30:00Z")); + expect(toInstantMs(1710038400000)).toBe(1710038400000); + expect(toInstantMs("2024-03-10T02:30:00Z")).toBe(Date.parse("2024-03-10T02:30:00Z")); + expect(toInstantMs("2024-03-10T04:30:00+02:00")).toBe(Date.parse("2024-03-10T02:30:00Z")); + }); + it("rejects offset-less datetimes that would parse server-local", () => { + for (const bad of ["2024-03-10T02:30:00", "2024-03-10 02:30", "2024-03-10T02:30:00.500"]) { + expect(() => toInstantMs(bad)).toThrow(InvalidInstantError); + } + }); + it("rejects NaN/Infinity/garbage", () => { + expect(() => toInstantMs(new Date(NaN))).toThrow(InvalidInstantError); + expect(() => toInstantMs(Number.NaN)).toThrow(InvalidInstantError); + expect(() => toInstantMs(Number.POSITIVE_INFINITY)).toThrow(InvalidInstantError); + expect(() => toInstantMs("not a date")).toThrow(InvalidInstantError); + }); +}); + +describe("currentTzdataVersion", () => { + it("returns a non-empty version string", () => { + expect(currentTzdataVersion()).toMatch(/.+/); + }); +}); diff --git a/packages/scheduling/src/timezone.ts b/packages/scheduling/src/timezone.ts new file mode 100644 index 0000000..6040218 --- /dev/null +++ b/packages/scheduling/src/timezone.ts @@ -0,0 +1,212 @@ +import { InvalidInstantError, InvalidTimezoneError } from "./errors.js"; + +/** + * Wall-clock reading of an instant in some zone. Fields are the raw numeric + * calendar values (month 1-12, day 1-31, hour 0-23). + */ +export interface WallClock { + readonly year: number; + readonly month: number; + readonly day: number; + readonly hour: number; + readonly minute: number; + readonly second: number; +} + +/** + * Nominal local date-time, e.g. `2024-03-10T02:30`. Lexicographic order of + * these strings is chronological order, which is what the + * `last_fired_nominal_local` refire gate relies on. + */ +export type NominalLocal = string; + +const WALL_FORMAT_OPTIONS: Intl.DateTimeFormatOptions = { + hourCycle: "h23", + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", +}; + +const wallFormatters = new Map(); + +function wallFormatter(zone: string): Intl.DateTimeFormat { + const cached = wallFormatters.get(zone); + if (cached) return cached; + const formatter = new Intl.DateTimeFormat("en-US", { + ...WALL_FORMAT_OPTIONS, + timeZone: zone, + }); + wallFormatters.set(zone, formatter); + return formatter; +} + +const IDIOMS_REJECTED = new Set(["local", "system", "server", "server-local", "machine"]); + +/** + * Validate a timezone and return its canonical IANA id. + * + * Accepts real IANA ids (including `Etc/GMT+5` style fixed zones and `UTC`), + * canonicalizing whatever Intl resolves (e.g. `US/Eastern` -> `America/New_York`). + * Rejects unknown ids, offset spellings like `UTC+3`/`+05:30`, empty input and + * the strings `local`/`system` — the schedule zone is never allowed to fall + * back to the process zone. + */ +export function canonicalTimezone(timezone: string): string { + if (typeof timezone !== "string") { + throw new InvalidTimezoneError(`timezone must be a string, got ${typeof timezone}`); + } + const trimmed = timezone.trim(); + if (trimmed === "") { + throw new InvalidTimezoneError("timezone must be an explicit IANA zone id, not an empty string"); + } + if (IDIOMS_REJECTED.has(trimmed.toLowerCase())) { + throw new InvalidTimezoneError( + `timezone "${timezone}" is not an IANA zone id; a schedule must carry an explicit zone (never server-local)`, + ); + } + if (/^[+-]?\d{1,2}(:?\d{2})?$/.test(trimmed) || /^UTC[+-]\d/.test(trimmed)) { + throw new InvalidTimezoneError( + `timezone "${timezone}" is a UTC offset, not an IANA zone id; store an IANA zone (UTC is allowed)`, + ); + } + let resolved: string; + try { + resolved = new Intl.DateTimeFormat("en-US", { timeZone: trimmed }).resolvedOptions().timeZone; + } catch { + throw new InvalidTimezoneError(`unknown IANA time zone: "${timezone}"`); + } + if (!resolved) { + throw new InvalidTimezoneError(`unknown IANA time zone: "${timezone}"`); + } + return resolved; +} + +/** True when `canonicalTimezone` would accept the value. */ +export function isValidTimezone(timezone: string): boolean { + try { + canonicalTimezone(timezone); + return true; + } catch { + return false; + } +} + +/** Validate and canonicalize, throwing `InvalidTimezoneError` on rejection. */ +export function assertValidTimezone(timezone: string): string { + return canonicalTimezone(timezone); +} + +/** Local wall clock struck by an instant, in the given zone. */ +export function wallOfInstant(epochMs: number, zone: string): WallClock { + if (zone === "UTC") { + return decodeUtc(epochMs); + } + const parts = wallFormatter(zone).formatToParts(new Date(epochMs)); + let year = 0; + let month = 0; + let day = 0; + let hour = 0; + let minute = 0; + let second = 0; + for (const part of parts) { + switch (part.type) { + case "year": + year = Number(part.value); + break; + case "month": + month = Number(part.value); + break; + case "day": + day = Number(part.value); + break; + case "hour": + hour = Number(part.value); + break; + case "minute": + minute = Number(part.value); + break; + case "second": + second = Number(part.value); + break; + default: + break; + } + } + return { year, month, day, hour, minute, second }; +} + +/** The UTC instant a wall-clock reading would denote if the offset were zero. */ +export function epochOfWall(wall: WallClock): number { + return Date.UTC(wall.year, wall.month - 1, wall.day, wall.hour, wall.minute, wall.second); +} + +function decodeUtc(epochMs: number): WallClock { + const d = new Date(epochMs); + return { + year: d.getUTCFullYear(), + month: d.getUTCMonth() + 1, + day: d.getUTCDate(), + hour: d.getUTCHours(), + minute: d.getUTCMinutes(), + second: d.getUTCSeconds(), + }; +} + +/** + * Zone offset (ms east of UTC) in effect at an instant. Minute-precision + * arithmetic against `Intl`; UTC short-circuits without touching a formatter. + */ +export function offsetMs(epochMs: number, zone: string): number { + if (zone === "UTC") return 0; + return epochOfWall(wallOfInstant(epochMs, zone)) - epochMs; +} + +const OFFSET_ISO_TAIL = /(?:Z|[+-]\d{2}:\d{2})$/; + +/** + * Coerce an instant input to epoch milliseconds. + * + * `Date` and finite numbers pass through. Strings must be ISO-8601; a string + * that carries a time component must also carry an explicit UTC offset (`Z` or + * `±hh:mm`) — otherwise `Date.parse` would silently apply the server's local + * zone, which this library exists to prevent. + */ +export function toInstantMs(value: Date | number | string): number { + if (value instanceof Date) { + const ms = value.getTime(); + if (Number.isNaN(ms)) throw new InvalidInstantError("invalid Date (NaN)"); + return ms; + } + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new InvalidInstantError(`invalid instant number: ${value}`); + return value; + } + if (typeof value === "string") { + const trimmed = value.trim(); + const hasTime = /[T ]\d/.test(trimmed); + if (hasTime && !OFFSET_ISO_TAIL.test(trimmed)) { + throw new InvalidInstantError( + `instant "${value}" has a time but no UTC offset; supply ...Z or ...±hh:mm (never server-local)`, + ); + } + const ms = Date.parse(trimmed); + if (Number.isNaN(ms)) throw new InvalidInstantError(`unparseable instant: "${value}"`); + return ms; + } + throw new InvalidInstantError(`unsupported instant input: ${String(value)}`); +} + +type ProcessVersionsLike = { tz?: string }; + +/** + * tzdata version backing this process (`process.versions.tz`, e.g. `2026a`), + * or `"unknown"` outside Node. Recorded per schedule so tzdata-driven + * re-materialization can detect drift. + */ +export function currentTzdataVersion(): string { + const proc = (globalThis as { process?: { versions?: ProcessVersionsLike } }).process; + return proc?.versions?.tz ?? "unknown"; +} diff --git a/packages/scheduling/src/verify.test.ts b/packages/scheduling/src/verify.test.ts new file mode 100644 index 0000000..eb3dcda --- /dev/null +++ b/packages/scheduling/src/verify.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import { nextOccurrences } from "./occurrences_extra.js"; + +const ISO = (o: { fireAtUtc: number }) => new Date(o.fireAtUtc).toISOString().replace(".000Z", "Z"); + +describe("count boundary semantics", () => { + it("never returns more than count", () => { + for (const count of [1, 2, 5, 17]) { + const out = nextOccurrences({ cron: "*/15 * * * *", timezone: "America/New_York" }, "2024-03-09T00:00:00Z", count); + expect(out.length).toBe(count); + } + }); + it("stops at the collapse edge with the latest collapsed label", () => { + // Exclusive from 05:00Z (== 00:00 local): the first slot is 00:15. The + // eighth occurrence is the collapse edge — nominal slots 02:00..03:00 all + // materialize onto 07:00Z — and it carries the LATEST label (03:00), with + // nothing beyond the edge emitted. + const out = nextOccurrences({ cron: "*/15 * * * *", timezone: "America/New_York" }, "2024-03-10T05:00:00Z", 8); + expect(out.length).toBe(8); + expect(out.map(ISO)[7]).toBe("2024-03-10T07:00:00Z"); + expect(out[7]!.nominalLocal).toBe("2024-03-10T03:00"); + // The same result whether the boundary is reached with 8 or 80 requested, + // because generation stops at the first slot after the boundary instant. + const wide = nextOccurrences({ cron: "*/15 * * * *", timezone: "America/New_York" }, "2024-03-10T05:00:00Z", 80); + expect(wide[7]!.nominalLocal).toBe("2024-03-10T03:00"); + }); +}); diff --git a/packages/scheduling/tsconfig.json b/packages/scheduling/tsconfig.json new file mode 100644 index 0000000..5a24989 --- /dev/null +++ b/packages/scheduling/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src"] +} diff --git a/packages/testing/package.json b/packages/testing/package.json new file mode 100644 index 0000000..eaac368 --- /dev/null +++ b/packages/testing/package.json @@ -0,0 +1,22 @@ +{ + "name": "@tactical-media-broadcast-hub/testing", + "private": true, + "version": "0.1.0", + "type": "module", + "bin": { + "live-source-simulator": "src/simulator.ts" + }, + "scripts": { + "build": "tsc -p tsconfig.json", + "lint": "eslint src", + "typecheck": "tsc -p tsconfig.json --noEmit", + "simulate": "tsx src/simulator.ts" + }, + "dependencies": { + "tsx": "^4.17.0" + }, + "devDependencies": { + "@types/node": "^22.5.0", + "typescript": "^5.5.4" + } +} diff --git a/packages/testing/src/scenarios.ts b/packages/testing/src/scenarios.ts new file mode 100644 index 0000000..0f256e5 --- /dev/null +++ b/packages/testing/src/scenarios.ts @@ -0,0 +1,32 @@ +export interface ScenarioStep { + /** Source that changes state in this step. */ + source: "A" | "B" | "C"; + action: "go-live" | "fail" | "recover" | "go-offline"; + /** Seconds to hold after this step before the next one. */ + holdSeconds: number; +} + +export interface Scenario { + name: string; + description: string; + steps: ScenarioStep[]; +} + +export const SCENARIOS: Record = { + "dod-flap": { + name: "dod-flap", + description: "Deterministic Auto Director exercise: A live → B live → A fails → C live → A recovers", + steps: [ + { source: "A", action: "go-live", holdSeconds: 20 }, + { source: "B", action: "go-live", holdSeconds: 20 }, + { source: "A", action: "fail", holdSeconds: 10 }, + { source: "C", action: "go-live", holdSeconds: 20 }, + { source: "A", action: "recover", holdSeconds: 20 }, + ], + }, + "single-live": { + name: "single-live", + description: "One source goes live and holds — baseline relay check", + steps: [{ source: "A", action: "go-live", holdSeconds: 60 }], + }, +}; diff --git a/packages/testing/src/simulator.ts b/packages/testing/src/simulator.ts new file mode 100644 index 0000000..9369d81 --- /dev/null +++ b/packages/testing/src/simulator.ts @@ -0,0 +1,69 @@ +#!/usr/bin/env node +/** + * Live-source simulator: drives ffmpeg testsrc-based streams through a + * scripted live/offline/fail scenario so the Auto Director can be exercised + * deterministically without real PeerTube instances. + */ +import { spawnSync } from "node:child_process"; +import process from "node:process"; +import { SCENARIOS, type Scenario } from "./scenarios.js"; + +function ffmpegAvailable(): boolean { + const probe = spawnSync("ffmpeg", ["-version"], { encoding: "utf8" }); + return probe.status === 0; +} + +function runStep(step: Scenario["steps"][number], sinkUrl: string): void { + const label = `${step.source}-${step.action}`; + const duration = step.holdSeconds; + const args = [ + "-re", + "-f", "lavfi", + "-i", `testsrc2=size=640x360:rate=30:duration=${duration}`, + "-f", "lavfi", + "-i", `sine=frequency=${step.source === "A" ? 440 : step.source === "B" ? 550 : 660}:duration=${duration}`, + "-c:v", "libx264", "-preset", "veryfast", "-tune", "zerolatency", + "-c:a", "aac", "-b:a", "128k", + "-f", "flv", + `${sinkUrl}/${label}`, + ]; + process.stdout.write(`[${new Date().toISOString()}] ${label}: pushing ${duration}s to ${sinkUrl}\n`); + const result = spawnSync("ffmpeg", args, { stdio: "inherit" }); + if (result.status !== 0) { + process.stderr.write(`[${label}] ffmpeg exited ${result.status}\n`); + } +} + +function usage(): never { + process.stderr.write( + "usage: live-source-simulator --scenario [--sink rtmp://localhost:1935/mock]\n" + + "scenarios: " + Object.keys(SCENARIOS).join(", ") + "\n", + ); + process.exit(2); +} + +const args = process.argv.slice(2); +if (args.includes("--help") || args.includes("-h")) { + process.stdout.write( + "usage: live-source-simulator --scenario [--sink rtmp://localhost:1935/mock]\n" + + "scenarios: " + Object.keys(SCENARIOS).join(", ") + "\n", + ); + process.exit(0); +} +const scenarioIdx = args.indexOf("--scenario"); +const sinkIdx = args.indexOf("--sink"); +if (scenarioIdx === -1 || !args[scenarioIdx + 1]) usage(); +const scenario: Scenario | undefined = SCENARIOS[args[scenarioIdx + 1] ?? ""]; +if (!scenario) usage(); +const sink = (sinkIdx !== -1 && args[sinkIdx + 1]) || "rtmp://localhost:1935/mock"; + +if (!ffmpegAvailable()) { + process.stderr.write("ffmpeg not found on PATH — install ffmpeg (the simulator is the deterministic test source)\n"); + process.exit(1); +} + +process.stdout.write(`scenario ${scenario.name}: ${scenario.description}\n`); +for (const step of scenario.steps) { + runStep(step, sink); +} +process.stdout.write("scenario complete\n"); diff --git a/packages/testing/tsconfig.json b/packages/testing/tsconfig.json new file mode 100644 index 0000000..7962fd2 --- /dev/null +++ b/packages/testing/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "module": "ESNext", + "moduleResolution": "bundler" + }, + "include": ["src"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml new file mode 100644 index 0000000..e5ff257 --- /dev/null +++ b/pnpm-lock.yaml @@ -0,0 +1,8685 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + devDependencies: + eslint: + specifier: ^9.9.0 + version: 9.39.5(jiti@2.7.0) + turbo: + specifier: ^2.1.3 + version: 2.11.7 + typescript: + specifier: ^5.5.4 + version: 5.9.3 + typescript-eslint: + specifier: ^8.5.0 + version: 8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@5.9.3) + + apps/api: + dependencies: + '@aws-sdk/client-s3': + specifier: ^3.1146.0 + version: 3.1146.0 + '@aws-sdk/s3-request-presigner': + specifier: ^3.1146.0 + version: 3.1146.0 + '@nestjs/common': + specifier: ^10.4.1 + version: 10.4.22(reflect-metadata@0.2.2)(rxjs@7.8.2) + '@nestjs/core': + specifier: ^10.4.1 + version: 10.4.22(@nestjs/common@10.4.22(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/platform-express@10.4.22)(reflect-metadata@0.2.2)(rxjs@7.8.2) + '@nestjs/platform-express': + specifier: ^10.4.1 + version: 10.4.22(@nestjs/common@10.4.22(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@10.4.22) + '@opentelemetry/api': + specifier: ^1.9.1 + version: 1.9.1 + '@opentelemetry/auto-instrumentations-node': + specifier: ^0.80.0 + version: 0.80.0(@opentelemetry/api@1.9.1)(@opentelemetry/core@2.11.0(@opentelemetry/api@1.9.1)) + '@opentelemetry/exporter-trace-otlp-http': + specifier: ^0.222.0 + version: 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-node': + specifier: ^0.222.0 + version: 0.222.0(@opentelemetry/api@1.9.1) + '@tactical-media-broadcast-hub/connector-mastodon': + specifier: workspace:* + version: link:../../packages/connector-mastodon + '@tactical-media-broadcast-hub/connector-runtime': + specifier: workspace:* + version: link:../../packages/connector-runtime + '@tactical-media-broadcast-hub/connector-sdk': + specifier: workspace:* + version: link:../../packages/connector-sdk + '@tactical-media-broadcast-hub/connector-telegram': + specifier: workspace:* + version: link:../../packages/connector-telegram + '@tactical-media-broadcast-hub/core': + specifier: workspace:* + version: link:../../packages/core + '@tactical-media-broadcast-hub/publishing-execution': + specifier: workspace:* + version: link:../../packages/publishing-execution + '@temporalio/client': + specifier: 1.24.0 + version: 1.24.0 + jose: + specifier: ^6.2.12 + version: 6.2.12 + pg: + specifier: ^8.23.1 + version: 8.23.1 + pino: + specifier: ^9.3.2 + version: 9.14.0 + reflect-metadata: + specifier: ^0.2.2 + version: 0.2.2 + rxjs: + specifier: ^7.8.1 + version: 7.8.2 + devDependencies: + '@tactical-media-broadcast-hub/connector-mock-livestream': + specifier: workspace:^ + version: link:../../packages/connector-mock-livestream + '@types/express': + specifier: ^4.17.21 + version: 4.17.25 + '@types/node': + specifier: ^22.5.0 + version: 22.20.5 + '@types/pg': + specifier: ^8.23.1 + version: 8.23.1 + tsx: + specifier: ^4.17.0 + version: 4.23.15 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + apps/media-worker: + dependencies: + pino: + specifier: ^9.3.2 + version: 9.14.0 + devDependencies: + '@types/node': + specifier: ^22.5.0 + version: 22.20.5 + tsx: + specifier: ^4.17.0 + version: 4.23.15 + + apps/scheduler: + dependencies: + '@tactical-media-broadcast-hub/connector-mastodon': + specifier: workspace:* + version: link:../../packages/connector-mastodon + '@tactical-media-broadcast-hub/connector-runtime': + specifier: workspace:* + version: link:../../packages/connector-runtime + '@tactical-media-broadcast-hub/connector-sdk': + specifier: workspace:* + version: link:../../packages/connector-sdk + '@tactical-media-broadcast-hub/connector-telegram': + specifier: workspace:* + version: link:../../packages/connector-telegram + '@tactical-media-broadcast-hub/core': + specifier: workspace:* + version: link:../../packages/core + '@tactical-media-broadcast-hub/publishing-execution': + specifier: workspace:* + version: link:../../packages/publishing-execution + '@temporalio/activity': + specifier: 1.24.0 + version: 1.24.0 + '@temporalio/client': + specifier: 1.24.0 + version: 1.24.0 + '@temporalio/common': + specifier: 1.24.0 + version: 1.24.0 + '@temporalio/worker': + specifier: 1.24.0 + version: 1.24.0(lightningcss@1.32.0) + '@temporalio/workflow': + specifier: 1.24.0 + version: 1.24.0 + pg: + specifier: ^8.23.1 + version: 8.23.1 + pino: + specifier: ^9.3.2 + version: 9.14.0 + devDependencies: + '@types/node': + specifier: ^22.5.0 + version: 22.20.5 + '@types/pg': + specifier: ^8.23.1 + version: 8.23.1 + tsx: + specifier: ^4.17.0 + version: 4.23.15 + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + apps/web: + dependencies: + '@tactical-media-broadcast-hub/connector-sdk': + specifier: workspace:^ + version: link:../../packages/connector-sdk + '@tailwindcss/postcss': + specifier: ^4.3.3 + version: 4.3.3 + '@tanstack/react-virtual': + specifier: ^3.14.13 + version: 3.14.13(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@tiptap/extension-character-count': + specifier: ^3.31.4 + version: 3.31.4(@tiptap/extensions@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4)) + '@tiptap/extension-placeholder': + specifier: ^3.31.4 + version: 3.31.4(@tiptap/extensions@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4)) + '@tiptap/react': + specifier: ^3.31.4 + version: 3.31.4(@floating-ui/dom@1.8.0)(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4)(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@tiptap/starter-kit': + specifier: ^3.31.4 + version: 3.31.4 + cmdk: + specifier: ^1.1.1 + version: 1.1.1(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + next: + specifier: ^14.2.5 + version: 14.2.35(@opentelemetry/api@1.9.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + postcss: + specifier: ^8.5.29 + version: 8.5.29 + react: + specifier: ^18.3.1 + version: 18.3.1 + react-dom: + specifier: ^18.3.1 + version: 18.3.1(react@18.3.1) + tailwindcss: + specifier: ^4.3.3 + version: 4.3.3 + devDependencies: + '@types/node': + specifier: ^22.5.0 + version: 22.20.5 + '@types/react': + specifier: ^18.3.3 + version: 18.3.31 + '@types/react-dom': + specifier: ^18.3.0 + version: 18.3.7(@types/react@18.3.31) + playwright: + specifier: ^1.63.0 + version: 1.63.0 + typescript: + specifier: ^5.5.4 + version: 5.9.3 + + packages/connector-bluesky: + dependencies: + '@tactical-media-broadcast-hub/connector-sdk': + specifier: workspace:* + version: link:../connector-sdk + '@tactical-media-broadcast-hub/core': + specifier: workspace:* + version: link:../core + devDependencies: + '@types/node': + specifier: ^22.20.5 + version: 22.20.5 + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/connector-discord: + dependencies: + '@tactical-media-broadcast-hub/connector-sdk': + specifier: workspace:* + version: link:../connector-sdk + '@tactical-media-broadcast-hub/core': + specifier: workspace:* + version: link:../core + devDependencies: + '@types/node': + specifier: ^22.20.5 + version: 22.20.5 + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/connector-example: + dependencies: + '@tactical-media-broadcast-hub/connector-sdk': + specifier: workspace:* + version: link:../connector-sdk + '@tactical-media-broadcast-hub/core': + specifier: workspace:* + version: link:../core + devDependencies: + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/connector-facebook: + dependencies: + '@tactical-media-broadcast-hub/connector-sdk': + specifier: workspace:* + version: link:../connector-sdk + '@tactical-media-broadcast-hub/core': + specifier: workspace:* + version: link:../core + devDependencies: + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/connector-instagram: + dependencies: + '@tactical-media-broadcast-hub/connector-sdk': + specifier: workspace:* + version: link:../connector-sdk + '@tactical-media-broadcast-hub/core': + specifier: workspace:* + version: link:../core + devDependencies: + '@types/node': + specifier: ^22.20.5 + version: 22.20.5 + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/connector-linkedin: + dependencies: + '@tactical-media-broadcast-hub/connector-sdk': + specifier: workspace:* + version: link:../connector-sdk + '@tactical-media-broadcast-hub/core': + specifier: workspace:* + version: link:../core + devDependencies: + '@types/node': + specifier: ^22.20.5 + version: 22.20.5 + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/connector-mastodon: + dependencies: + '@tactical-media-broadcast-hub/connector-sdk': + specifier: workspace:* + version: link:../connector-sdk + '@tactical-media-broadcast-hub/core': + specifier: workspace:* + version: link:../core + devDependencies: + '@types/node': + specifier: ^22.20.5 + version: 22.20.5 + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/connector-mock-livestream: + dependencies: + '@tactical-media-broadcast-hub/connector-sdk': + specifier: workspace:* + version: link:../connector-sdk + '@tactical-media-broadcast-hub/core': + specifier: workspace:* + version: link:../core + devDependencies: + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/connector-mock-social: + dependencies: + '@tactical-media-broadcast-hub/connector-sdk': + specifier: workspace:* + version: link:../connector-sdk + '@tactical-media-broadcast-hub/core': + specifier: workspace:* + version: link:../core + devDependencies: + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/connector-mock-video: + dependencies: + '@tactical-media-broadcast-hub/connector-sdk': + specifier: workspace:* + version: link:../connector-sdk + '@tactical-media-broadcast-hub/core': + specifier: workspace:* + version: link:../core + devDependencies: + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/connector-peertube: + dependencies: + '@tactical-media-broadcast-hub/connector-sdk': + specifier: workspace:* + version: link:../connector-sdk + '@tactical-media-broadcast-hub/core': + specifier: workspace:* + version: link:../core + devDependencies: + '@types/node': + specifier: ^22.20.5 + version: 22.20.5 + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/connector-reddit: + dependencies: + '@tactical-media-broadcast-hub/connector-sdk': + specifier: workspace:* + version: link:../connector-sdk + '@tactical-media-broadcast-hub/core': + specifier: workspace:* + version: link:../core + devDependencies: + '@types/node': + specifier: ^22.20.5 + version: 22.20.5 + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/connector-runtime: + dependencies: + '@tactical-media-broadcast-hub/connector-mastodon': + specifier: workspace:* + version: link:../connector-mastodon + '@tactical-media-broadcast-hub/connector-sdk': + specifier: workspace:* + version: link:../connector-sdk + '@tactical-media-broadcast-hub/connector-telegram': + specifier: workspace:* + version: link:../connector-telegram + '@tactical-media-broadcast-hub/core': + specifier: workspace:* + version: link:../core + pg: + specifier: ^8.23.1 + version: 8.23.1 + devDependencies: + '@types/node': + specifier: ^22.20.5 + version: 22.20.5 + '@types/pg': + specifier: ^8.23.1 + version: 8.23.1 + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/connector-sdk: + dependencies: + '@tactical-media-broadcast-hub/core': + specifier: workspace:* + version: link:../core + devDependencies: + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/connector-slack: + dependencies: + '@tactical-media-broadcast-hub/connector-sdk': + specifier: workspace:* + version: link:../connector-sdk + '@tactical-media-broadcast-hub/core': + specifier: workspace:* + version: link:../core + devDependencies: + '@types/node': + specifier: ^22.20.5 + version: 22.20.5 + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/connector-telegram: + dependencies: + '@tactical-media-broadcast-hub/connector-sdk': + specifier: workspace:* + version: link:../connector-sdk + '@tactical-media-broadcast-hub/core': + specifier: workspace:* + version: link:../core + devDependencies: + '@types/node': + specifier: ^22.20.5 + version: 22.20.5 + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/connector-threads: + dependencies: + '@tactical-media-broadcast-hub/connector-sdk': + specifier: workspace:* + version: link:../connector-sdk + '@tactical-media-broadcast-hub/core': + specifier: workspace:* + version: link:../core + devDependencies: + '@types/node': + specifier: ^22.20.5 + version: 22.20.5 + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/connector-twitch: + dependencies: + '@tactical-media-broadcast-hub/connector-sdk': + specifier: workspace:* + version: link:../connector-sdk + '@tactical-media-broadcast-hub/core': + specifier: workspace:* + version: link:../core + devDependencies: + '@types/node': + specifier: ^22.20.5 + version: 22.20.5 + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/core: + devDependencies: + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/egress: + dependencies: + pg: + specifier: ^8.23.1 + version: 8.23.1 + devDependencies: + '@types/node': + specifier: ^22.20.5 + version: 22.20.5 + '@types/pg': + specifier: ^8.23.1 + version: 8.23.1 + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/media-sandbox: + devDependencies: + '@types/node': + specifier: ^22.20.5 + version: 22.20.5 + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/publishing-execution: + dependencies: + '@tactical-media-broadcast-hub/core': + specifier: workspace:* + version: link:../core + pg: + specifier: ^8.23.1 + version: 8.23.1 + devDependencies: + '@types/node': + specifier: ^22.20.5 + version: 22.20.5 + '@types/pg': + specifier: ^8.23.1 + version: 8.23.1 + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/safe-content: + devDependencies: + '@types/node': + specifier: ^22.20.5 + version: 22.20.5 + '@types/react': + specifier: ^18.3.3 + version: 18.3.31 + '@types/react-dom': + specifier: ^18.3.0 + version: 18.3.7(@types/react@18.3.31) + eslint: + specifier: ^9.9.0 + version: 9.39.5(jiti@2.7.0) + react: + specifier: ^18.3.1 + version: 18.3.1 + react-dom: + specifier: ^18.3.1 + version: 18.3.1(react@18.3.1) + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/scheduling: + devDependencies: + '@types/node': + specifier: ^22.20.5 + version: 22.20.5 + typescript: + specifier: ^5.5.4 + version: 5.9.3 + vitest: + specifier: ^2.0.5 + version: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + packages/testing: + dependencies: + tsx: + specifier: ^4.17.0 + version: 4.23.15 + devDependencies: + '@types/node': + specifier: ^22.5.0 + version: 22.20.5 + typescript: + specifier: ^5.5.4 + version: 5.9.3 + +packages: + + '@alloc/quick-lru@5.3.0': + resolution: {integrity: sha512-U4+70Pc5ZS9osnCBCE5Jha/ciHM+Yp+CNMNC/7HvYbNRk1Ldd+f7qO65W5qfhu/TCv+/ozljlXXe9Nj8419DMA==} + engines: {node: '>=10'} + + '@aws-sdk/checksums@3.1001.1': + resolution: {integrity: sha512-x12Q17KYlJAd3nKf8LV5LV0vt8sh8/6YfQLGPtrGnQf/tW4jqxPGq5GPpuVitpQYM3eUR4XB7CbxZf751NMbLw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/client-s3@3.1146.0': + resolution: {integrity: sha512-WY0YCBzxc4muFfY6UbGpA+oib0nGT/Px2aEPGYX7pp1la2OmDLChnK81mu8H7LdYvwGU1pLAsLIa/qgad0Wd7g==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/core@3.978.1': + resolution: {integrity: sha512-LbY9aGsEiznDWmUc30Nwv3aIX/+dbwTx8KfS0yOC3NPYMO+O91e6jkT1azf34FwjOndq8/Q+RcVVZz5xnerwdg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-env@3.972.72': + resolution: {integrity: sha512-xTKO/FWJPozTIXbozVnVGoNBhaGba8TBcx+KyUjRVeOlXE+dUc7GTR1cLvu0uTdIdmemzaFbqqCshXeZA1fZew==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-http@3.972.74': + resolution: {integrity: sha512-u91E/hT8f4d1xy0Jl7VG4nVKJ3lxbrZkoBTeSVoJdWBiSEUMwMS/9+e0H/aJVQV//Lt5wuzP+E69v4aRSsNTmw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-ini@3.973.17': + resolution: {integrity: sha512-ged4KXdBkvIC81bLvNHHuQKdKak/VXhQTR1NWYTTqW0474nlmsxy9O/vlgTIohDDWH3xpBdtVMZRyjb+DnocDA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-login@3.972.79': + resolution: {integrity: sha512-L+Z85anONJd8MaiuraO4wRxATCdEejBZ3K3eymzWI5JPXa9sOS9CkIm72PBKqXKX+Z9p9NGMX5AIMXm0LEflgw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-node@3.972.84': + resolution: {integrity: sha512-oHt854odINVwzwsh+c5x69j0ajm4DbqqqVJ+O1ECsCIZeMDAbzFpXItaqP7UZstJj/ATdTk/KFSH0LaNAgV+kA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-process@3.972.72': + resolution: {integrity: sha512-rLIp2xbMjX/k9/od7APpqq1ZgXXnV0pOL1Th3ZsL8Wu0TRtBsDTVS8iPqcfRFcHakFxPvR04OSTv2ka2qOb/2A==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-sso@3.973.16': + resolution: {integrity: sha512-IGihaJfFZYacJJr/odqILCoK7W/mvrZ7cuK7ECn3sAu4vLC6u0V8bS7mCGbdugJ8Aum2tnvqmx0F2MRFp2rn9g==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-web-identity@3.972.78': + resolution: {integrity: sha512-/y9WvNtlcPBGLR0qc1a+9J/xtYZfVczvLUOuXaVWylzttH7ewsxwHtjmiJSolNrVSDorIxHGHMU61CbonRkmwA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-sdk-s3@3.972.77': + resolution: {integrity: sha512-E7W2UOeUoc+lg3uIfR/dM7ZwusHwhBQrKMnlkRv4EXRR+C0YtV1pg25xC7GdZIhXH+NAMgZPCbE7o5to2cjFiw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/nested-clients@3.997.46': + resolution: {integrity: sha512-oRxtBcka/JGHGs9l9p9IVajGoTP8vTPmoAzdHGy4Qcy9P5vPnDf6nhIeM/COQNY9k/OahImTRaLkHftoXvfcmQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/s3-request-presigner@3.1146.0': + resolution: {integrity: sha512-5BEfVr2OYX2bwB45NSWAXPoTN5M2NPLOHohgKV+XfMNcL5TF7UQN4hAOVF4S6DwrA4BrI3MsAry7vqdZ2aJEjg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/signature-v4-multi-region@3.996.47': + resolution: {integrity: sha512-Zk08macMvQTHzQJCLJVkOlviVoqwYMrpXv4lmLN7b7sAbiMoOK7Go0NYdR5UeF+MW8LIbRmwrNy9u/5VvX1U5g==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/token-providers@3.1138.0': + resolution: {integrity: sha512-GpyAr0DD63YOEmYFM6Df+gJuIgC92MMTiBK4FTKfxii5MJ9ge20epR7LyroulscYlG89J+ZB2ivFDPjvfQhzdw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/types@3.974.6': + resolution: {integrity: sha512-v/clNZzZnDxGyvpHMOGpJKVXFAExJzUNAAjaWGdcx8QAcXLGwTaOkw33p5SHAi0YAioK32xB3hWwOekRVfmfKg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/xml-builder@3.972.41': + resolution: {integrity: sha512-ctjVSyCMegrWfXlx6VqzSBFI6UqmQ5ZlnfMhdLIiWmhoH8UAQxSCP5N3OpG7X3k4LnS7ou74C4mt20+bfTW2aQ==} + engines: {node: '>=20.0.0'} + + '@aws/lambda-invoke-store@0.3.0': + resolution: {integrity: sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==} + engines: {node: '>=18.0.0'} + + '@borewit/text-codec@0.2.2': + resolution: {integrity: sha512-DDaRehssg1aNrH4+2hnj1B7vnUGEjU6OIlyRdkMd0aUdIUvKXrJfXsy8LVtXAy7DRvYVluWbMspsRhz2lcW0mQ==} + + '@esbuild/aix-ppc64@0.21.5': + resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [aix] + + '@esbuild/aix-ppc64@0.28.2': + resolution: {integrity: sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.21.5': + resolution: {integrity: sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==} + engines: {node: '>=12'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm64@0.28.2': + resolution: {integrity: sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.21.5': + resolution: {integrity: sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==} + engines: {node: '>=12'} + cpu: [arm] + os: [android] + + '@esbuild/android-arm@0.28.2': + resolution: {integrity: sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.21.5': + resolution: {integrity: sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==} + engines: {node: '>=12'} + cpu: [x64] + os: [android] + + '@esbuild/android-x64@0.28.2': + resolution: {integrity: sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.21.5': + resolution: {integrity: sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==} + engines: {node: '>=12'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-arm64@0.28.2': + resolution: {integrity: sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.21.5': + resolution: {integrity: sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==} + engines: {node: '>=12'} + cpu: [x64] + os: [darwin] + + '@esbuild/darwin-x64@0.28.2': + resolution: {integrity: sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.21.5': + resolution: {integrity: sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==} + engines: {node: '>=12'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-arm64@0.28.2': + resolution: {integrity: sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.21.5': + resolution: {integrity: sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.28.2': + resolution: {integrity: sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.21.5': + resolution: {integrity: sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==} + engines: {node: '>=12'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm64@0.28.2': + resolution: {integrity: sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.21.5': + resolution: {integrity: sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==} + engines: {node: '>=12'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-arm@0.28.2': + resolution: {integrity: sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.21.5': + resolution: {integrity: sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==} + engines: {node: '>=12'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-ia32@0.28.2': + resolution: {integrity: sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.21.5': + resolution: {integrity: sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==} + engines: {node: '>=12'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-loong64@0.28.2': + resolution: {integrity: sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.21.5': + resolution: {integrity: sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==} + engines: {node: '>=12'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-mips64el@0.28.2': + resolution: {integrity: sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.21.5': + resolution: {integrity: sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-ppc64@0.28.2': + resolution: {integrity: sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.21.5': + resolution: {integrity: sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==} + engines: {node: '>=12'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-riscv64@0.28.2': + resolution: {integrity: sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.21.5': + resolution: {integrity: sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==} + engines: {node: '>=12'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-s390x@0.28.2': + resolution: {integrity: sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.21.5': + resolution: {integrity: sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [linux] + + '@esbuild/linux-x64@0.28.2': + resolution: {integrity: sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.28.2': + resolution: {integrity: sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.21.5': + resolution: {integrity: sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==} + engines: {node: '>=12'} + cpu: [x64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.28.2': + resolution: {integrity: sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.28.2': + resolution: {integrity: sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.21.5': + resolution: {integrity: sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==} + engines: {node: '>=12'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.28.2': + resolution: {integrity: sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.28.2': + resolution: {integrity: sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.21.5': + resolution: {integrity: sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==} + engines: {node: '>=12'} + cpu: [x64] + os: [sunos] + + '@esbuild/sunos-x64@0.28.2': + resolution: {integrity: sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.21.5': + resolution: {integrity: sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==} + engines: {node: '>=12'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-arm64@0.28.2': + resolution: {integrity: sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.21.5': + resolution: {integrity: sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==} + engines: {node: '>=12'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-ia32@0.28.2': + resolution: {integrity: sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.21.5': + resolution: {integrity: sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==} + engines: {node: '>=12'} + cpu: [x64] + os: [win32] + + '@esbuild/win32-x64@0.28.2': + resolution: {integrity: sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@eslint-community/eslint-utils@4.10.1': + resolution: {integrity: sha512-cuadcxVFE8sDK6iWJbs8Sn0av2Nrh2QSGQhVlBW9AaAHqHwjWsZHT8LJ4hFGPh7ASBV2deFdM7H/DPjulmh8rg==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + peerDependencies: + eslint: ^6.0.0 || ^7.0.0 || >=8.0.0 + + '@eslint-community/regexpp@4.12.2': + resolution: {integrity: sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==} + engines: {node: ^12.0.0 || ^14.0.0 || >=16.0.0} + + '@eslint/config-array@0.21.2': + resolution: {integrity: sha512-nJl2KGTlrf9GjLimgIru+V/mzgSK0ABCDQRvxw5BjURL7WfH5uoWmizbH7QB6MmnMBd8cIC9uceWnezL1VZWWw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/config-helpers@0.4.2': + resolution: {integrity: sha512-gBrxN88gOIf3R7ja5K9slwNayVcZgK6SOUORm2uBzTeIEfeVaIhOpCtTox3P6R7o2jLFwLFTLnC7kU/RGcYEgw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/core@0.17.0': + resolution: {integrity: sha512-yL/sLrpmtDaFEiUj1osRP4TI2MDz1AddJL+jZ7KSqvBuliN4xqYY54IfdN8qD8Toa6g1iloph1fxQNkjOxrrpQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/eslintrc@3.3.7': + resolution: {integrity: sha512-F42g89Qd5oAWtp0k0nnSrjziAKza7w8SVT4mStc18LZMaRb4J1HQAHLCalEtDCxrTuksx7NU9qsmeLwpOfPqWw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/js@9.39.5': + resolution: {integrity: sha512-QywQuszQh77pIXCsq998c8hbhSTI/azTty1Z6N53dmAudKHhy573j3yvRLsX2BSp8YpLtoCEG8E9DJe+8zUh4A==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/object-schema@2.1.7': + resolution: {integrity: sha512-VtAOaymWVfZcmZbp6E2mympDIHvyjXs/12LqWYjVw6qjrfF+VK+fyG33kChz3nnK+SU5/NeHOqrTEHS8sXO3OA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@eslint/plugin-kit@0.4.1': + resolution: {integrity: sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@floating-ui/core@1.8.0': + resolution: {integrity: sha512-0CIZ5itps/8x7BG8dEIhs53BvCUH2PCoogtakwRTut+Arm58sJooJ0AuZhLw2HJYIR5cMLNPBSS728sPho2khQ==} + + '@floating-ui/dom@1.8.0': + resolution: {integrity: sha512-yXSrzeHZBTZadLOlfyhCkJHNeLJnHRnRInwdZ40L7ZiaAtrBwoYlsDrX3v5zB1Utk7CLfzcOVnVVWoXEky7Ceg==} + + '@floating-ui/utils@0.2.12': + resolution: {integrity: sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww==} + + '@grpc/grpc-js@1.14.5': + resolution: {integrity: sha512-7VZM+SVdEcUUqSQeNI3zM8Qs/BhQKZndPo2h5VkYkAM8Iz0wJIa8mKV5ekQGqG8UUsnkQ0NMxIxwkIHYvj0qOw==} + engines: {node: '>=12.10.0'} + + '@grpc/proto-loader@0.8.1': + resolution: {integrity: sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==} + engines: {node: '>=6'} + hasBin: true + + '@humanfs/core@0.19.2': + resolution: {integrity: sha512-UhXNm+CFMWcbChXywFwkmhqjs3PRCmcSa/hfBgLIb7oQ5HNb1wS0icWsGtSAUNgefHeI+eBrA8I1fxmbHsGdvA==} + engines: {node: '>=18.18.0'} + + '@humanfs/node@0.16.8': + resolution: {integrity: sha512-gE1eQNZ3R++kTzFUpdGlpmy8kDZD/MLyHqDwqjkVQI0JMdI1D51sy1H958PNXYkM2rAac7e5/CnIKZrHtPh3BQ==} + engines: {node: '>=18.18.0'} + + '@humanfs/types@0.15.0': + resolution: {integrity: sha512-ZZ1w0aoQkwuUuC7Yf+7sdeaNfqQiiLcSRbfI08oAxqLtpXQr9AIVX7Ay7HLDuiLYAaFPu8oBYNq/QIi9URHJ3Q==} + engines: {node: '>=18.18.0'} + + '@humanwhocodes/module-importer@1.0.1': + resolution: {integrity: sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==} + engines: {node: '>=12.22'} + + '@humanwhocodes/retry@0.4.3': + resolution: {integrity: sha512-bV0Tgo9K4hfPCek+aMAn81RppFKv2ySDQeMoSZuvTASywNTnVJCArCZE2FWqpvIatKu7VMRLWlR1EazvVhDyhQ==} + engines: {node: '>=18.18'} + + '@isaacs/cliui@8.0.2': + resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} + engines: {node: '>=12'} + + '@jridgewell/gen-mapping@0.3.13': + resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} + + '@jridgewell/remapping@2.3.5': + resolution: {integrity: sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==} + + '@jridgewell/resolve-uri@3.1.2': + resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} + engines: {node: '>=6.0.0'} + + '@jridgewell/source-map@0.3.11': + resolution: {integrity: sha512-ZMp1V8ZFcPG5dIWnQLr3NSI1MiCU7UETdS/A0G8V/XWHvJv3ZsFqutJn1Y5RPmAPX6F3BiE397OqveU/9NCuIA==} + + '@jridgewell/sourcemap-codec@1.6.0': + resolution: {integrity: sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==} + + '@jridgewell/trace-mapping@0.3.31': + resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + + '@js-sdsl/ordered-map@4.4.2': + resolution: {integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==} + + '@jsonjoy.com/base64@1.1.2': + resolution: {integrity: sha512-q6XAnWQDIMA3+FTiOYajoYqySkO+JSat0ytXGSuRdq9uXE7o92gzuQwQM14xaCRlBLGq3v5miDGC4vkVTn54xA==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/base64@17.67.0': + resolution: {integrity: sha512-5SEsJGsm15aP8TQGkDfJvz9axgPwAEm98S5DxOuYe8e1EbfajcDmgeXXzccEjh+mLnjqEKrkBdjHWS5vFNwDdw==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/buffers@1.2.1': + resolution: {integrity: sha512-12cdlDwX4RUM3QxmUbVJWqZ/mrK6dFQH4Zxq6+r1YXKXYBNgZXndx2qbCJwh3+WWkCSn67IjnlG3XYTvmvYtgA==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/buffers@17.67.0': + resolution: {integrity: sha512-tfExRpYxBvi32vPs9ZHaTjSP4fHAfzSmcahOfNxtvGHcyJel+aibkPlGeBB+7AoC6hL7lXIE++8okecBxx7lcw==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/codegen@1.0.0': + resolution: {integrity: sha512-E8Oy+08cmCf0EK/NMxpaJZmOxPqM+6iSe2S4nlSBrPZOORoDJILxtbSUEDKQyTamm/BVAhIGllOBNU79/dwf0g==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/codegen@17.67.0': + resolution: {integrity: sha512-idnkUplROpdBOV0HMcwhsCUS5TRUi9poagdGs70A6S4ux9+/aPuKbh8+UYRTLYQHtXvAdNfQWXDqZEx5k4Dj2Q==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/fs-core@4.80.0': + resolution: {integrity: sha512-qMKWshnyjbyhm+NbzqGE3w5y1mckFFEMj0aDPU2/JBNh1BzIjVy9esq9lBmRnNVmhMEkxwDzLZ8Gqh5G1P5Dag==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/fs-fsa@4.80.0': + resolution: {integrity: sha512-ZBEKl7J6dbkRCfmrFV48Re211A/FHqniTUhsWCsaRteNSOqMgkX3qF1UreVgwB9oSPm81JLAwhIi6Lb/NH/PoA==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/fs-node-builtins@4.80.0': + resolution: {integrity: sha512-OIOIhqaWiwUySFMny4epIAoviXsyMaQQ63PTc3zH+5HEOvarl5hpUmZIhp2m5+OfyKAh5ylxXnQIgew1OgJDVw==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/fs-node-to-fsa@4.80.0': + resolution: {integrity: sha512-ljeaR5xwtX1EyRmB0wvXANUMQZ3mjKcm7z8s2N8D8SLQ04CcAGtUhHFRZQXFb9mhgFyEVMydQGQtSOFS7pcrFQ==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/fs-node-utils@4.80.0': + resolution: {integrity: sha512-xwHeVVi3f+Khg/Fmb8i5qoYVmQ08QznE5TWhQokUmIZjz4KBPzU+zt/+bahBI6G1ghtqajt27lWuAciIxApuzA==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/fs-node@4.80.0': + resolution: {integrity: sha512-yiBlUfkMMRFFGy8CD/h2zd5rPnrM6bjlF+nx2LRAxNV6hOSiuObQIUEE7m7y2VTMisjxCRtzINH94bFGySWL7g==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/fs-print@4.80.0': + resolution: {integrity: sha512-1ocZLJw/m/xlq0moaIHO7HV9nHRVBUvdc2zMzhsJGkFOrqfwWvexfCv5cYCPPLRhP+9p3yunoi4Ep5YhRORC9Q==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/fs-snapshot@4.80.0': + resolution: {integrity: sha512-3AV23Q1H228KTtJKTUE0nNq8FCtxSvtENOLrEat80wkAJCKHuqQEv8yjhpS8Bei2OEFJAnluGXp9Ri2SWBlROw==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/json-pack@1.21.0': + resolution: {integrity: sha512-+AKG+R2cfZMShzrF2uQw34v3zbeDYUqnQ+jg7ORic3BGtfw9p/+N6RJbq/kkV8JmYZaINknaEQ2m0/f693ZPpg==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/json-pack@17.67.0': + resolution: {integrity: sha512-t0ejURcGaZsn1ClbJ/3kFqSOjlryd92eQY465IYrezsXmPcfHPE/av4twRSxf6WE+TkZgLY+71vCZbiIiFKA/w==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/json-pointer@1.0.2': + resolution: {integrity: sha512-Fsn6wM2zlDzY1U+v4Nc8bo3bVqgfNTGcn6dMgs6FjrEnt4ZCe60o6ByKRjOGlI2gow0aE/Q41QOigdTqkyK5fg==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/json-pointer@17.67.0': + resolution: {integrity: sha512-+iqOFInH+QZGmSuaybBUNdh7yvNrXvqR+h3wjXm0N/3JK1EyyFAeGJvqnmQL61d1ARLlk/wJdFKSL+LHJ1eaUA==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/util@1.9.0': + resolution: {integrity: sha512-pLuQo+VPRnN8hfPqUTLTHk126wuYdXVxE6aDmjSeV4NCAgyxWbiOIeNJVtID3h1Vzpoi9m4jXezf73I6LgabgQ==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@jsonjoy.com/util@17.67.0': + resolution: {integrity: sha512-6+8xBaz1rLSohlGh68D1pdw3AwDi9xydm8QNlAFkvnavCJYSze+pxoW2VKP8p308jtlMRLs5NTHfPlZLd4w7ew==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + '@lukeed/csprng@1.1.0': + resolution: {integrity: sha512-Z7C/xXCiGWsg0KuKsHTKJxbWhpI3Vs5GwLfOean7MGyVFGqdRgBbAjOCh6u4bbjPc/8MJ2pZmK/0DLdCbivLDA==} + engines: {node: '>=8'} + + '@napi-rs/lzma-linux-x64-gnu@1.5.1': + resolution: {integrity: sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==} + engines: {node: ^22.20 || ^24.12 || >=25} + cpu: [x64] + os: [linux] + + '@nestjs/common@10.4.22': + resolution: {integrity: sha512-fxJ4v85nDHaqT1PmfNCQ37b/jcv2OojtXTaK1P2uAXhzLf9qq6WNUOFvxBrV4fhQek1EQoT1o9oj5xAZmv3NRw==} + peerDependencies: + class-transformer: '*' + class-validator: '*' + reflect-metadata: ^0.1.12 || ^0.2.0 + rxjs: ^7.1.0 + peerDependenciesMeta: + class-transformer: + optional: true + class-validator: + optional: true + + '@nestjs/core@10.4.22': + resolution: {integrity: sha512-6IX9+VwjiKtCjx+mXVPncpkQ5ZjKfmssOZPFexmT+6T9H9wZ3svpYACAo7+9e7Nr9DZSoRZw3pffkJP7Z0UjaA==} + peerDependencies: + '@nestjs/common': ^10.0.0 + '@nestjs/microservices': ^10.0.0 + '@nestjs/platform-express': ^10.0.0 + '@nestjs/websockets': ^10.0.0 + reflect-metadata: ^0.1.12 || ^0.2.0 + rxjs: ^7.1.0 + peerDependenciesMeta: + '@nestjs/microservices': + optional: true + '@nestjs/platform-express': + optional: true + '@nestjs/websockets': + optional: true + + '@nestjs/platform-express@10.4.22': + resolution: {integrity: sha512-ySSq7Py/DFozzZdNDH67m/vHoeVdphDniWBnl6q5QVoXldDdrZIHLXLRMPayTDh5A95nt7jjJzmD4qpTbNQ6tA==} + peerDependencies: + '@nestjs/common': ^10.0.0 + '@nestjs/core': ^10.0.0 + + '@next/env@14.2.35': + resolution: {integrity: sha512-DuhvCtj4t9Gwrx80dmz2F4t/zKQ4ktN8WrMwOuVzkJfBilwAwGr6v16M5eI8yCuZ63H9TTuEU09Iu2HqkzFPVQ==} + + '@next/swc-darwin-arm64@14.2.33': + resolution: {integrity: sha512-HqYnb6pxlsshoSTubdXKu15g3iivcbsMXg4bYpjL2iS/V6aQot+iyF4BUc2qA/J/n55YtvE4PHMKWBKGCF/+wA==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@next/swc-darwin-x64@14.2.33': + resolution: {integrity: sha512-8HGBeAE5rX3jzKvF593XTTFg3gxeU4f+UWnswa6JPhzaR6+zblO5+fjltJWIZc4aUalqTclvN2QtTC37LxvZAA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@next/swc-linux-arm64-gnu@14.2.33': + resolution: {integrity: sha512-JXMBka6lNNmqbkvcTtaX8Gu5by9547bukHQvPoLe9VRBx1gHwzf5tdt4AaezW85HAB3pikcvyqBToRTDA4DeLw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@next/swc-linux-arm64-musl@14.2.33': + resolution: {integrity: sha512-Bm+QulsAItD/x6Ih8wGIMfRJy4G73tu1HJsrccPW6AfqdZd0Sfm5Imhgkgq2+kly065rYMnCOxTBvmvFY1BKfg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@next/swc-linux-x64-gnu@14.2.33': + resolution: {integrity: sha512-FnFn+ZBgsVMbGDsTqo8zsnRzydvsGV8vfiWwUo1LD8FTmPTdV+otGSWKc4LJec0oSexFnCYVO4hX8P8qQKaSlg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@next/swc-linux-x64-musl@14.2.33': + resolution: {integrity: sha512-345tsIWMzoXaQndUTDv1qypDRiebFxGYx9pYkhwY4hBRaOLt8UGfiWKr9FSSHs25dFIf8ZqIFaPdy5MljdoawA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@next/swc-win32-arm64-msvc@14.2.33': + resolution: {integrity: sha512-nscpt0G6UCTkrT2ppnJnFsYbPDQwmum4GNXYTeoTIdsmMydSKFz9Iny2jpaRupTb+Wl298+Rh82WKzt9LCcqSQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@next/swc-win32-ia32-msvc@14.2.33': + resolution: {integrity: sha512-pc9LpGNKhJ0dXQhZ5QMmYxtARwwmWLpeocFmVG5Z0DzWq5Uf0izcI8tLc+qOpqxO1PWqZ5A7J1blrUIKrIFc7Q==} + engines: {node: '>= 10'} + cpu: [ia32] + os: [win32] + + '@next/swc-win32-x64-msvc@14.2.33': + resolution: {integrity: sha512-nOjfZMy8B94MdisuzZo9/57xuFVLHJaDj5e/xrduJp9CV2/HrfxTRH2fbyLe+K9QT41WBLUd4iXX3R7jBp0EUg==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@nuxtjs/opencollective@0.3.2': + resolution: {integrity: sha512-um0xL3fO7Mf4fDxcqx9KryrB7zgRM5JSlvGN5AGkP6JLM5XEKyjeAiPbNxdXVXQ16isuAhYpvP88NgL2BGd6aA==} + engines: {node: '>=8.0.0', npm: '>=5.0.0'} + hasBin: true + + '@opentelemetry/api-logs@0.222.0': + resolution: {integrity: sha512-9mb1If+IF6u0ZVXkHQ6ogEae5HwA6ajIVUgpSDQyRASxft6BSXHvBvPooRle3yFN/fKnCdSOnuu0OC3PLcF6+g==} + engines: {node: '>=8.0.0'} + + '@opentelemetry/api@1.9.1': + resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==} + engines: {node: '>=8.0.0'} + + '@opentelemetry/auto-instrumentations-node@0.80.0': + resolution: {integrity: sha512-McPSxdNZqudZnC8WsYti63QyDoj+QOB4EKVp3Ndrq+gq5UOoi2GcDvXjphIxgXUFoJ3jzWoiczhReKW23OKsOw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.4.1 + '@opentelemetry/core': ^2.0.0 + + '@opentelemetry/configuration@0.222.0': + resolution: {integrity: sha512-zNWGmEvyX7CUbS6F6PKi7hf3CF/SffzeBYyWVQAfadZ8hODrvLmqV3IUGVt1SiqyYanRPHc8VuhoO0Qh9OEuug==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.9.0 + + '@opentelemetry/context-async-hooks@2.11.0': + resolution: {integrity: sha512-Tr79DyWI8itsBdg+jH+opjfrwLzX+erk1/ExkIwhWoAVjVrJIn2y5+cGjTC0Vy8fyNIA/y8wuJPZwr1T3xCZeQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/core@2.11.0': + resolution: {integrity: sha512-7YP44XH0tV6+Mb54x2YGf84i7yi+31MBZlE8JwvozkxyTvXbSp10X7cI7YE49ChJ3shMJoBmCJF3+1QFBJctGA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/exporter-logs-otlp-grpc@0.222.0': + resolution: {integrity: sha512-HH4Oi/E8HXBKtcjgcVPJGsrg+lSstdw1m93brRdpVj5CjMpyTkv/O0ZYMXP6QXVEaLjjFivtzUu4Y8+i1UJpjw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-logs-otlp-http@0.222.0': + resolution: {integrity: sha512-GqwLZohJTf/sttYlOpT1oFTJ9ScYaKnk0sOx4Qe/+KmXYVOa+v94cfnkUkFKDuvTODWhCL50Hz3gvsZD6ngQpg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-logs-otlp-proto@0.222.0': + resolution: {integrity: sha512-LlCDJuaznfFPzjHoKXjN/r9T5pgCZpmVheZ/EmwMUhMvpxI7RXDmsdLXoYpl5suYQNkwyXAUZhVEXKe3lev8Hg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-metrics-otlp-grpc@0.222.0': + resolution: {integrity: sha512-nPU2gpQWJfQttf+A71aLkKEOQoO6n6QGs5QTtPagi/rb3Lc1I55bqRXoLQEZsWqWimc8eas2Vo168NkTaByxQw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-metrics-otlp-http@0.222.0': + resolution: {integrity: sha512-6Ko+0bgNP6V4G/RsmNBSk5lO1B/lo9z6tvLpFY3ykPPevXNzFmOW3C9hEmTt0zKCvHovyrol3TYlg4HVb5gwjg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-metrics-otlp-proto@0.222.0': + resolution: {integrity: sha512-s1ZPwS8Ig7cpRmL3kCobPaFbemM81dJ03LNPrsuN32h3mCDvNF2NQ1pRHruOju0P0o1TZ24LkZA685rtQal1KA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-prometheus@0.222.0': + resolution: {integrity: sha512-I7nalnnBZrhlTgjlqsVeEjJGcQCFK5kKAYVk7XTz/cbubvj5fvjLOal/Many0taHF2GIkOWmegsbQ8vbQJjhFg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-trace-otlp-grpc@0.222.0': + resolution: {integrity: sha512-uisilePEdOMa3hIfXW82XXU6FofF8gw3iv+PoboIMWEUBC7XbJB3XnIHSdrSDsqCw3njR/If08qD0yzOY1U9Sg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-trace-otlp-http@0.222.0': + resolution: {integrity: sha512-RCnPWcHppwiquQ+cV3nWvNwdf0MG1w26e5jewW2T83nTZOlXgcg88sY9ulCVgagGHcq1mj0L0GP6YHgzk2v8oA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-trace-otlp-proto@0.222.0': + resolution: {integrity: sha512-imLrLzNhnABJt3tUyT63h6MqfZrG3Gdv3TE7+OUQTvwD088RfCUx3rF01lcgfu2KXBFk13vwi0yKZLvFEalkIA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-zipkin@2.11.0': + resolution: {integrity: sha512-mFibA45yXLEETIj+VcTVycFFe4R91iD41d+r3CU0f3DDWJCrICiHxUIjmYBTCZIuywXEeNo++2lncQeWnH7lWQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.0.0 + + '@opentelemetry/instrumentation-amqplib@0.69.0': + resolution: {integrity: sha512-hoSfNGxqospEFw0S2RH7cY8MF5U1BRKsAfX7PhiiZgGDWxPcmfAKYdXilmn0DqPBGRFYzq+HbtSp+R+2sy4mXA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-aws-lambda@0.74.0': + resolution: {integrity: sha512-N7L+NIyndEUWC0EjtN7OmNBF0J5beXQ81+46qt1NCA2YDC95os9TIh2iWfKL+U3wINkJeStMWBEQgFZhEvj1qw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-aws-sdk@0.77.0': + resolution: {integrity: sha512-igri8DF7ZWlYtQjT1u2A3v6Grb59VSsYiheGrLpnI7rlEAxmFeWHPwiwzzko39lM0LO7Ntf3VJPRBA1ZNGwF1w==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-bunyan@0.67.0': + resolution: {integrity: sha512-a/ucku1yx4qW2zpYpepTM/rZlv92DG/C+k/Kxn/nqhh5HOoe3pHFbXUXunvy2EtHW/7l81PI++AUm+rIxXqlTg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-cassandra-driver@0.67.0': + resolution: {integrity: sha512-PIHbjmBWfYeYqAH8XDCyoJoXaR6TkEakNw3BX8uDt8b9zjYk3uetjCoP8nubRDb8m30vkDbs09Qg09pFOXKj2w==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-connect@0.65.0': + resolution: {integrity: sha512-i7xkOkzzeaYbaK2Gjpxa6g9FJd1nqoH0+KFyRXV0iweH/F2+xZPintRFD5gcSN5/H4mpfslsLbDqRT897CvzEQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-cucumber@0.38.0': + resolution: {integrity: sha512-sorgrR3FOBy+YEwVutxcm84JTI0luCNIGp0kiL+jSoUrH9W/pYVSpP4isLB3VtQMgNnD4cd0e9qNvvr7RwKLpA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.0.0 + + '@opentelemetry/instrumentation-dataloader@0.39.0': + resolution: {integrity: sha512-01+jNP1hESK8GYTCuRMoiQAEf9ZI5b3XwfSk7bWDiXLB2/zml5Md/U/xaWyFHQ2kYyU553Qvr8dcyup5Beu7qA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-dns@0.65.0': + resolution: {integrity: sha512-7QXICs4eV/GeihRqPnsR7TnK6ugAK8/B7tWr8UFxRzGTfNG97LEfByevhNfMEE8pFFLHUhiCE5metED2lO8nhw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-express@0.70.0': + resolution: {integrity: sha512-OHdj/2apRZNhMKkd6w4OV4SnX/aS2LWMLyHwGGNYhWratgh88o+cVJS9FvqC+HCYWSDWTEWEfl/8d7BdEI7c+g==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-fs@0.41.0': + resolution: {integrity: sha512-rk1RcxvOeb5j7Gi6P/PB6M9MRRYQzp4t9ZYN/i6jVM+m0j2zCUYGH4Xo6omNh0FhTDhhytDPG6o95p82eXFL2w==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-generic-pool@0.65.0': + resolution: {integrity: sha512-iKkn/8m0oerf00qCxz9/xeua+EieX6tlSmLhGiVLgLsOspmV1sb33ee8RPiWnJiandGVJgKA0GTGUZXwVKgPnQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-graphql@0.70.0': + resolution: {integrity: sha512-zyBTEG0Dxm6rYf0IaoIqt1ZRAuiq4AUPDhAi0vl/hBbYweU56CleS5zH8n+GQ3+EsTBW6KYKULHvXk21+UJo9w==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-grpc@0.222.0': + resolution: {integrity: sha512-5gUFhJF5X8ss60apt2p7czvIeIzTHZQSq1QowiKVS4gBfP5nHZNhBYoOtq1j14A8IXs/qL8Hn/sfni4kaPVLRA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-hapi@0.68.0': + resolution: {integrity: sha512-wJShEI4aKajQT5Sg/sJ6jV3xF6SqJKfXHTLDdrmWMv0h1sDXp4mpSTD1XUJfO5H0lZFUtEJapxtWl126zAmf9w==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-host-metrics@0.5.0': + resolution: {integrity: sha512-rAw5lmY2u3bwUFohRtR4ZYNDnhMTiSeQqq8BB4Bo+aJQ+7vIOOwW/3d4awLPMDQ69VeA5jsh+ICL5oJinrWIwA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-http@0.222.0': + resolution: {integrity: sha512-HhMHW32LyGzIJEmUElxaML4oUe6JiCq0Hvjt/Awxb60SlMsSFetFLFjed45g/0wrPr1n7z+xW+x1B/584WBAkg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-ioredis@0.70.0': + resolution: {integrity: sha512-AUyn8K5vlSGkIF2AXPlUsa5d5ZXRidHMXlAHNAykvgW3u8juDZqGTgsOKJMYnLfllawygawLrqK6Fr2lnPcBUQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-kafkajs@0.31.0': + resolution: {integrity: sha512-9/EBOb1aW5ai4+U7mu2ER5fhJ7OYZNK/ghRSETfCf92xCeoQkZcS8LEF8e05w6OlrgZCwq7x+vLuUxXtkRPv3w==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-knex@0.66.0': + resolution: {integrity: sha512-tPI1T0ur+P2qvsOWcadv9ViWtJ8g0XsKOAMPiR5rasZBZAC6nEI4u4BAnAhaZtdZkuMJrGa4uZur5xTn1bv7MQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-koa@0.70.0': + resolution: {integrity: sha512-+CFyL7V0auN0FzKvGUwOIOGT/8VTC3BM8cd8X0UoHRmPKEdYfwNWuHIkA6zknZPDMXN9WLfZCxdZ0Ln/YqLx4w==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.9.0 + + '@opentelemetry/instrumentation-lru-memoizer@0.66.0': + resolution: {integrity: sha512-kYJQFqkR2eEwGzqTRw2Nc/NiQyrQRU3rLHmsEvdnlf40YRfCBadbDu3I8C5GWoVn1XyEKFTvmn85UoslRpbUjA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-memcached@0.65.0': + resolution: {integrity: sha512-QLGrx33THENogCdkBi1DJxPwvWe4QInoKUbx7u15zb37bxrKww68ZrqbMF4luq9YZbGQUFqEsnSprHV8kU93QA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-mongodb@0.75.0': + resolution: {integrity: sha512-Bf9Az6zdOsnRFUew1CToTKpfhcFkAJ4SgmnGpxID/vuzVHI31oZhAwuKBTHny/Eq7ydXfncDbET8jV6d3L8REw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-mongoose@0.68.0': + resolution: {integrity: sha512-GrDPpy9XHwWJUpKb4NjH8bdRU9EvtS37D102LhMOnjNOp6ouvtVBmFcPLMsJKsZcFOG4YykSERnau4l5a+sy2w==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-mysql2@0.68.0': + resolution: {integrity: sha512-xNvUL0nwk/t0gGPb1GSQJWKiR3gj4Cd3vk4palrttLy/FYbnRBOOBRCobik3grAMzjsynEgyYdzEziXx/CvpyQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-mysql@0.68.0': + resolution: {integrity: sha512-N6TCMiWmisdG9DzSuMHxZ+N5qD3meoCy70mVE6fneZj6M24EXj86e56BzLFQ8Pkd6BgaPBJcdMTT/GBfPuW15w==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-nestjs-core@0.68.0': + resolution: {integrity: sha512-BCsqKPLWVDgrTDyVScRwENWg8uNH7LA0QBy+ojuPQx9C9F85yOhNkale5MdmQG0su7L6BLLTNPByZOZeKp3c5A==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-net@0.66.0': + resolution: {integrity: sha512-OxnzC0696+Ne7zHisHo95RZkYT4e2C4OoB+q0dqQyZ0/9klLQ9CyC/Yu+OzYrQzPjdzRQVVPNDhoW+5+cXk3XQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-openai@0.20.0': + resolution: {integrity: sha512-vl8ReH9dq9goRWbajF0xQt/mnCZDl2KCcmOvgC1sSKCRA8DnqSMq7Z9ZVKPA9PnaV4tGInb39qtGdGIORUkEqQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-oracledb@0.47.0': + resolution: {integrity: sha512-3DDgcsl2CFX4829TxZiNYD29pFN9xyT71HUBN475CuFQ94eq+1jyMZPshO+K7qJltrJY2Df3MWV2ANrlw8H8fQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-pg@0.74.0': + resolution: {integrity: sha512-8i59Z7M0FjEwp2hF1eRqEr+NZtiwjvPYGl2xsGWEfL/hF6ffmPERWbySyK1EXa7gGyHCArwNkHLShsMfFywqpQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-pino@0.68.0': + resolution: {integrity: sha512-U0ggAM9eqDe7jsUnZeFMv6QIZwahNfyPZQAlUV9ZQ3aAYpaq+Ag9Okec75Fgzwh72+e4gewfT99HObkDg6Y/bA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-redis@0.70.0': + resolution: {integrity: sha512-jXHduXt6GQWluGtqBPibAvpw27seRjA6SMyLp0Oc41AzSQz1JFZq1CNw6gvdOrRasH4AJPpRykPxw02IjHYr0A==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-restify@0.67.0': + resolution: {integrity: sha512-T4sjjxX0CaeYNEMUNEDbCL9nGPIRkEmAzj7b1HwI0YJgZV459aStVsgJRTKmuaeIoJSihTqH2yUmcB+gdR5Qxg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-router@0.66.0': + resolution: {integrity: sha512-18GWiTNuIMDFxepxYV0PVikAAMmcqLZsMziJXi2PyuP3r9msPjMJnU7RZ+DLS1N8amwWit3EZYmFI4dNE4d9uQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-runtime-node@0.35.0': + resolution: {integrity: sha512-/cOW4Qlv4nnr4QXDCkv0TEtOBa9R9lXHu//JuD5+nRPYR74g9iLmf5tadcYvn32oz2R8vQnV8FTnMLzjfAGMCQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-socket.io@0.69.0': + resolution: {integrity: sha512-VlS6GFeWd32d09WziTZR9cfLPNJ1nV747suBCgUivRVzIwrg1UG+DBCmMX/kvrMmGJcFRAbf6I6s6nLsR6+Jag==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-tedious@0.41.0': + resolution: {integrity: sha512-dtrHwEs40nzD/fR9jkSNbSiHqCIHf2sXcMM8nY29Ekc25pw0EADRBqN4Ya2prrCSNGVtIx6N9o3fTS7fsACysA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-undici@0.32.0': + resolution: {integrity: sha512-Ms8oPkrCHa6hJOYodcFC8QOt1AFlns3KD1CqXBEihXyTFeLXDO84tIXYyKlXNHr8jMrl1Gm2velDvsSDXAuh9w==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.7.0 + + '@opentelemetry/instrumentation-winston@0.66.0': + resolution: {integrity: sha512-wwBZnlq9/2qiBpIoLB5yC88Ws0zoOLFde6DuIbEYL+SmC+bgYBITYV59AIlaDGpvos/n7AM+k8EFw+y/ZQVudw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation@0.222.0': + resolution: {integrity: sha512-fhbRDuAgzPKpq1k5+hX3uS+3QAYNca0on0Ngot/R8dDsQtuFaADeZnG2uSgxABuvcI7S2phAaRhbct9jieeKZw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-exporter-base@0.222.0': + resolution: {integrity: sha512-YbywG3veEm2Fb6TbdxRkuquWob6eVWXuA8/Ba1tXz9jHfUqpdE3keilOHEtPboC4CvS1bjeeVfNkWGOOrLj+lw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-grpc-exporter-base@0.222.0': + resolution: {integrity: sha512-Kw5WqDBYYpCgY+edHWZS/Jp3bdguqNkCyEWOsqFhf8RmtcEuWYzPFA9i+L6UsNoJXTs8ngeQB6dwELH1czNYjA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-transformer@0.222.0': + resolution: {integrity: sha512-/F3BZ89+CJQnZkMh2tCrtcdB+XT2Dxhj4FFE+WPQ//413hmFL0/RfEX6vgOIWGhiSzrkHWTK3+6SiT7K5/g/jQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/propagator-aws-xray@2.2.0': + resolution: {integrity: sha512-Yjvt2EjL+tfpkVOdKbhTPgpM4SIAez9nG6Q/QjQ3yfcJcjIWp59ph70SLfvmkSL6++3DCnuBG3iWcB18PwWavQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/propagator-b3@2.11.0': + resolution: {integrity: sha512-ldyRZXx6qsqu9Spqd7rVxo6SNAUJqP8Ir3vwIi4N8xyZLXNl/2XeQmIHvFneiVOqNwWkU+JGFg2O9/T1fefU0w==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/propagator-jaeger@2.11.0': + resolution: {integrity: sha512-RUOqgoIqXOPQTOhpBrs07Y8sBlxVoLY5Fka6ueZ0prW+k6fWQsba/nHDY4gm+gmfuwtRqbX5Y9YKhDvktjbIqQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/redis-common@0.38.3': + resolution: {integrity: sha512-VCghU1JYs/4gP6Gqf/xro9MEsZ7LrMv2uONVsaESKL38ZOB9BqnI98FfS23wjMnHlpuE+TTaWSoAVNpTwYXzjw==} + engines: {node: ^18.19.0 || >=20.6.0} + + '@opentelemetry/resource-detector-alibaba-cloud@0.37.0': + resolution: {integrity: sha512-tEbi6ZPrPZG79oZdQlzw5xR7sirtYy+LBbt/S/JxDUZnmSWEYmo7l2jp985XU4UhsfMDpcfdOY+Gnt2bPMtvyQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.0.0 + + '@opentelemetry/resource-detector-aws@2.22.0': + resolution: {integrity: sha512-3kM+xfsSHs0hShUAx9IFhX5jniaXxJL3CTjbCMNCFS+EY7bswWbukV3au0jfqEgzeNxGxHEy8RLA1TOhAVLpTQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.0.0 + + '@opentelemetry/resource-detector-azure@0.30.0': + resolution: {integrity: sha512-fwGDPVGJ6PyRMc5CPdHBfASX5prg3mLhZA0rs+n8ZJK4DjpJw9CvB1FujWl8l0QAsTmvrYBm9Deal3fvfqZBkQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.0.0 + + '@opentelemetry/resource-detector-container@0.8.13': + resolution: {integrity: sha512-44AKD/FOpC4amfsZkKvtymUnThWLBPTrZavv5tpRPVE3REEtFCbQV0rMT5lTRXmDkg8rk8M/2RX49NZdTSnpjw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.0.0 + + '@opentelemetry/resource-detector-gcp@0.57.0': + resolution: {integrity: sha512-wp+hk7jD7mxA6swAqa5wQ3jws2egh1DCPZ9FfEgntmFw/mc8MaGZjwuttOoMeRjIlcBCCaNUHuN8B+MZaujAXQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.0.0 + + '@opentelemetry/resources@2.11.0': + resolution: {integrity: sha512-Ie7+8q8MDF4FAEQCKVMTx3ReUvxiIAgIiiW3c9JdmP8+HMcDy20puT+AHjexnExgnbvBxjQ9fjkFDWrikJ2jQA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/sdk-logs@0.222.0': + resolution: {integrity: sha512-+19YHODIjaUCArxleaJtuufFZVpz/xvvK+VllQqE+W8hHolxdoRwHfK/s667zezwh1hkx6FFF+oYzetYgqK+Bg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.4.0 <1.10.0' + + '@opentelemetry/sdk-metrics@2.11.0': + resolution: {integrity: sha512-7GXXcObyHyDUUSG+L+kJoquty01bzm7ivE7+SSgXXJcHuPzGviptxwARmI2c+bnnxjexGQbJnyNlN8HxBP/Y7A==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.9.0 <1.10.0' + + '@opentelemetry/sdk-node@0.222.0': + resolution: {integrity: sha512-t8tlFN/ezYV7iI4k6u3t2jFbIReX99Z0iVMVSYLJscE4McxIW9YGHhdOldeY1elHeoq8mjEO4B9s4Fk7+zJbsg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/sdk-trace-base@2.11.0': + resolution: {integrity: sha512-H19x/TX/LZdqiYOjM7fqtSxwlplC5pgelavqbQdHbhdq0q/AI/TGkM2dfGuuynTXmJPeF2HoZVoPDu+TGoW78A==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/sdk-trace-node@2.11.0': + resolution: {integrity: sha512-CuvCMJmZxswhNLlM2LfuLOW3h3fZujA4hsG4B+Sz4dX2zvaXO8Ng74cnDHWD64gLszTlhiG3c0iNUjj4g+0/sA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/sdk-trace@2.11.0': + resolution: {integrity: sha512-fFnTqGm8/G73GQVnxYi7LXa1ZVYEUvgL6XI1LpvV0bPC7WQ/ZGgKxCSl8FnlZBKto9JHHEFTO6s6CUpvvtwFrA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/semantic-conventions@1.43.0': + resolution: {integrity: sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==} + engines: {node: '>=14'} + + '@opentelemetry/sql-common@0.42.0': + resolution: {integrity: sha512-nwUwUU+8O8a4bnLqk6CodWeegGMEANgC94KTAhXcpGWLrW/2/hek/0ajNbjXnSOoNuCX+nteUPs46HFHhou9Xw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.1.0 + + '@pinojs/redact@0.4.0': + resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} + + '@pkgjs/parseargs@0.11.0': + resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} + engines: {node: '>=14'} + + '@protobufjs/aspromise@1.1.2': + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + + '@protobufjs/base64@1.1.2': + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + + '@protobufjs/codegen@2.0.5': + resolution: {integrity: sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==} + + '@protobufjs/eventemitter@1.1.1': + resolution: {integrity: sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==} + + '@protobufjs/fetch@1.1.1': + resolution: {integrity: sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==} + + '@protobufjs/float@1.0.2': + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + + '@protobufjs/path@1.1.2': + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + + '@protobufjs/pool@1.1.0': + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + + '@protobufjs/utf8@1.1.2': + resolution: {integrity: sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==} + + '@radix-ui/primitive@1.1.7': + resolution: {integrity: sha512-rqWnm76nYT8HoNNqEjpgJ7Pw/DrBj5iBTrmEPo6HTX5+VJyBNOqTdv4g89G63HuR5g0AaENoAcH7Is5fF2kZ8Q==} + + '@radix-ui/react-compose-refs@1.1.5': + resolution: {integrity: sha512-+48PbAAbq3didjJxa+OaWY2ZwgAKsNiRGyeHKszblZMQ+kcpd9pAaT11cMkGEie0vsOi3QdeTE6d5Fe3Gn61kA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-context@1.2.2': + resolution: {integrity: sha512-RHCUGwKHDr0hDGg4X7ma4JG4/+12qxw8rkh5QKdDldlCvtja6nUx1Ef/8HVrJze81lEsgLQlqjzjGNHantgnQA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-dialog@1.2.0': + resolution: {integrity: sha512-l5CX8HdNL+9cc4s7HkNyplumSA3FIx68sg9dttBHKr286fRBwq1MBniDVTeI/tkof92/APntMQvC1YG1Khb0Cg==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-dismissable-layer@1.1.20': + resolution: {integrity: sha512-35WHRrycfnj2m4LvPiKMKHGLM3Y2WvUA744CkRdxjs1BGDX1c8PVMfmiSGTUwnHRbqdtIoxveYTXvcJSt7VfgQ==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-focus-guards@1.1.6': + resolution: {integrity: sha512-RNOJjfZMTyBM6xYmV3IVGXkPjIhcBAuv48POevAXwrGJhkWZ9p1rFoIS1JFooPuT193AZmRsCPhpoVJxx6OPoQ==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-focus-scope@1.2.0': + resolution: {integrity: sha512-I5S7vsAOozx2lWSBAyuXjWIJ/A8qx4fjbv6115WoaRYQSb1yZDaeNHonUAQzXyE8oD+0whJHbQRl4HIE2gQ0Xw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-id@1.1.4': + resolution: {integrity: sha512-TMQp2llA+RYn7JcjnrMnz7wN4pcVttPZnRZo52PLQsoLVKzNlVwUeHmfePgTgRluXFvlD3GD5g5MOVVTJCO0qA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-portal@1.1.18': + resolution: {integrity: sha512-ygFIJ+QXNKf72AAq8x3hd22+ws7pYpmc1S0E4kW0Je+qD4qEb3FvxeKVQRGrVDMOkQaYfsB+Qs+KDh81a1gAOw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-presence@1.1.11': + resolution: {integrity: sha512-Z7/atlkab36Z8hm+V8zYc79r0W+tiOB1o0tun1By94JyadkekLQmtEAnBSP9XKXBNQsStoStS4+gX10JpS5akA==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-primitive@2.1.11': + resolution: {integrity: sha512-RzcaSw5aKl0OPhBJ1wbolKSF6nVvN+0o/QLeIVq+09npiPdnCkPggy2k6gI6yetxl4ZyrlBPYFgCB8BQJZwXcQ==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-slot@1.4.0': + resolution: {integrity: sha512-IuMVHZ+Ah5fkctu3X3qeFrV/ZrBcuWuDTFUUPfS/L5hvoTjzhp/HdWGE14CsQombPdEidsa33HVlYZXi49PZ1Q==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-callback-ref@1.1.4': + resolution: {integrity: sha512-R6OUY2e2fA6Yn6s+VSx5KBV6Nx8LQEhu+cz7LCej18rQ1HLyg9PSC9jP/ZNx0o6FAIK9c0F1kHylzSxKsdlkrQ==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-controllable-state@1.2.6': + resolution: {integrity: sha512-uEQJGT97ZA/TgP/Hydw47lHu+/vQj6z/0jA+WeTbK1o9Rx45GImjpD0tc3W5ad3D6XTSR6e1yEO0FvGq6WQfVQ==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-effect-event@0.0.5': + resolution: {integrity: sha512-7cshFL8HGS/7HEiHH+9kL9HBwp2sa9yX18Knwek6KYWmXwM7pegMgta2AXMQKI+rq3JnfSj9x8wYqFMTdG1Jgg==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-layout-effect@1.1.4': + resolution: {integrity: sha512-K20DkRkUwDnxEYMBPcg3Y6voLkEy5p5QQmszZgLngKKiC7dzBR/aEuK3w1qlx2JWDUNH6FluahYdgR3BP+QbYw==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@rollup/rollup-android-arm-eabi@4.64.0': + resolution: {integrity: sha512-FLLpX941CD/7aBvX3GqLGBM9OvY5PGXTryPJkG5r5/OuBpZpV5k45Uo3RxAvqd35tsyc3hH1bhtz921IKzoIzg==} + cpu: [arm] + os: [android] + + '@rollup/rollup-android-arm64@4.64.0': + resolution: {integrity: sha512-iI32XM/jnDYJsvx5ai/za/7gWO9XoSeFFtU06/ZCaRgyilfJOjaopD3nOpk2WrTCGOyeDt4hleUiGOAnd5tHNg==} + cpu: [arm64] + os: [android] + + '@rollup/rollup-darwin-arm64@4.64.0': + resolution: {integrity: sha512-dRBZlbt2sEtF8Mn4MODThze9fIPdDoOhVSlXMAKy5dQbOYK9pOOoPY4Fat3OTtr0+8zbJo5TTZwlsfxY/kDNJg==} + cpu: [arm64] + os: [darwin] + + '@rollup/rollup-darwin-x64@4.64.0': + resolution: {integrity: sha512-Y3whrJOMucfkhgNkvcM93M6j8C53iEAMmPMYD99C6G1kUy4ELigLWk9ar2/4aCBSRgsql0zdlOemLgW2m5GZgQ==} + cpu: [x64] + os: [darwin] + + '@rollup/rollup-freebsd-arm64@4.64.0': + resolution: {integrity: sha512-S6S1GEzPjFF6IONYJX3AKt/799+8RgBgCcwRnUo+Ep1IvBxUbw5M51iU0TDdK2NWWVnSvCQ8yi6Em0fGASMOng==} + cpu: [arm64] + os: [freebsd] + + '@rollup/rollup-freebsd-x64@4.64.0': + resolution: {integrity: sha512-Ed1DPoFENkXj4IGzSB12SesV40iP6LxCCrbTIkSGIQBpOOxpBM+0kj62lmaeIHfMWTDoddqTOtqc7s8LYfn+Pw==} + cpu: [x64] + os: [freebsd] + + '@rollup/rollup-linux-arm-gnueabihf@4.64.0': + resolution: {integrity: sha512-J/6xhutJPtFyIkmHdf4olOdHHKH5kOJNumojdWn+Elv5ICEYTVOa362Vw27TxsK0BLzb1le3EDlO3UbvFK1lnQ==} + cpu: [arm] + os: [linux] + + '@rollup/rollup-linux-arm-musleabihf@4.64.0': + resolution: {integrity: sha512-G4L/Nnx59d2LvDoV/P4S5lZ7oVLT6hodOHMacqjiq2LL2gts86KbkEBTYL53+fBt0odhf7XPIbIATGoYiJctxg==} + cpu: [arm] + os: [linux] + + '@rollup/rollup-linux-arm64-gnu@4.64.0': + resolution: {integrity: sha512-zF16pvJDEUksGYRbfnXne7pdklAyj2XDmJeYKgP93oq0cUyX9Y8O7lUo1R+IY1GagP/YBvCEDCe7eEcAKpw//g==} + cpu: [arm64] + os: [linux] + + '@rollup/rollup-linux-arm64-musl@4.64.0': + resolution: {integrity: sha512-nIggWn8n+w12+R5lsV+PYYoUY8vuObxPAQqJI7WES+9v0qspYuAHoZMfYM0yLxlipasT5BY9ht5W8t5c/rSCkg==} + cpu: [arm64] + os: [linux] + + '@rollup/rollup-linux-loong64-gnu@4.64.0': + resolution: {integrity: sha512-WLOTiHPzszP0dxIMsHpFeDSOSB5I49Mj8K4+yKE3H1X/D0JRG5XZ5jrctJLMjnUc/LUzaKiwq6EHWNJ8pup8PA==} + cpu: [loong64] + os: [linux] + + '@rollup/rollup-linux-loong64-musl@4.64.0': + resolution: {integrity: sha512-35RK9aRhh7g7Pmeu5gQ01aCvrhXzj9LoBE6xtUcEH1ns+nyTaSzxSpADmBQINu4QiTDGvmQYwo1j5hlg1rRXFw==} + cpu: [loong64] + os: [linux] + + '@rollup/rollup-linux-ppc64-gnu@4.64.0': + resolution: {integrity: sha512-RqTfnpoTjuWfPAmqK5p2edt2AMCDJWm6oTn7sKfmyx+6tRga0ip05SJ977o6udTcIiDeF/CJMUfkaN4gtWt6Aw==} + cpu: [ppc64] + os: [linux] + + '@rollup/rollup-linux-ppc64-musl@4.64.0': + resolution: {integrity: sha512-RQjZPigjxFvFZDijG6rjlSXxhRaTf9zLYVpGK+lVcga64oMqP1GbgVqCb46eHTTceSIgp89Ja0y5aPe3GFBYHg==} + cpu: [ppc64] + os: [linux] + + '@rollup/rollup-linux-riscv64-gnu@4.64.0': + resolution: {integrity: sha512-tjL58mgQcTRnDRpeky+Sth1MKQeYi2Ct6r3Luq9E5aTSjnM/i/JbozqZkKqp9thM5+oDSZxo+05f9kdQedfTGA==} + cpu: [riscv64] + os: [linux] + + '@rollup/rollup-linux-riscv64-musl@4.64.0': + resolution: {integrity: sha512-ictoRyVaoZ8C67myHef+8hhCa6IplXJJ/xUCkVCxziV4zxMZh+jvhkhGsPBLqj1S72fnqD0QH5zL4lcaa4X7DQ==} + cpu: [riscv64] + os: [linux] + + '@rollup/rollup-linux-s390x-gnu@4.64.0': + resolution: {integrity: sha512-8aVvZ2hfr9RLFzBQkQNmYeHLDja8fLeQAmLt757zdKN/B+8L809KlLJKctQjS22wZotptV8G6o7fVPuAs8S1tQ==} + cpu: [s390x] + os: [linux] + + '@rollup/rollup-linux-x64-gnu@4.64.0': + resolution: {integrity: sha512-2dEF8GAcDKwshUfydD+GhosppNyBzhVUHkdFF3CXd3FpOkzj/RZzF5aoEtzklujklT5qFVRU2GK/cFeLaTNKVg==} + cpu: [x64] + os: [linux] + + '@rollup/rollup-linux-x64-musl@4.64.0': + resolution: {integrity: sha512-3XHTvoo2hKkh0MiOfS7SkhHjMaaWF7P4F3drocZZaho/Z8w7h/jpnuBHm91BOQLfvrKHs1yR+zir4E8RI3XzTQ==} + cpu: [x64] + os: [linux] + + '@rollup/rollup-openbsd-x64@4.64.0': + resolution: {integrity: sha512-HKtkzodL4a3KFWDXDoO61cRynAWTR3LnAE0oGtfG9lcNFpNuVYUFOByxDgNqEvic9rvaQbAj+SFwTEWDptRaIw==} + cpu: [x64] + os: [openbsd] + + '@rollup/rollup-openharmony-arm64@4.64.0': + resolution: {integrity: sha512-JBsxqiY16uyyCujtoC3s7zL+SrYyNElptD7K4V+KUoJ2V8/XG3hPWpheK+SuPwNh1cikJvIZmIdsrYXvD6PLJw==} + cpu: [arm64] + os: [openharmony] + + '@rollup/rollup-win32-arm64-msvc@4.64.0': + resolution: {integrity: sha512-S5Qrijh37qhnUYQ0ay8WN3LuWeKjow7wTUyCOPMZtnXiBXYXdsZETI6ZLfYnQAKovltckPLBFdNeD61P2rJc7g==} + cpu: [arm64] + os: [win32] + + '@rollup/rollup-win32-ia32-msvc@4.64.0': + resolution: {integrity: sha512-zbZ6iMmLFEsCtTdc2/y1F/cDFVNNCPF7Ae8PCT/wdPE//hGwYp6r+MyCLMFX+kOeIe09Pz6ASfnnfpSTN1FJQQ==} + cpu: [ia32] + os: [win32] + + '@rollup/rollup-win32-x64-gnu@4.64.0': + resolution: {integrity: sha512-L6Hw69oNpuYaOr1zCG30v0/DqKYCSRwodPGwIcVZ/6f8SOzq9rxtL/xNVI5MO4wjli+Ko0ErK9uHJL3mNPt8wQ==} + cpu: [x64] + os: [win32] + + '@rollup/rollup-win32-x64-msvc@4.64.0': + resolution: {integrity: sha512-nl+CTWIgOGszQmUz/kanfgrfFZ4qiwxKIfT/ryl3N3Ewk5tqizmYtNXiqb7Bu/fDtNgu1kyxdFiAUJNur3iysg==} + cpu: [x64] + os: [win32] + + '@smithy/core@3.35.1': + resolution: {integrity: sha512-i4YPS4B6ts7bjn7UwLnGjiZdprOvHvgGobFZsYK3GIY3E5hIqtj0rReU69BcTpGp+fvtraSNXeG1l+jtJvF55w==} + engines: {node: '>=18.0.0'} + + '@smithy/credential-provider-imds@4.5.2': + resolution: {integrity: sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==} + engines: {node: '>=18.0.0'} + + '@smithy/fetch-http-handler@5.8.0': + resolution: {integrity: sha512-ycSJu3tFAQ4v04CBB0agqFMVsSQ1iG3yw+SpgxRqKfaURpQD4CZ8Wn0zPMmSnOuTpTh65Vz+EA0rMrw089wvkA==} + engines: {node: '>=18.0.0'} + + '@smithy/node-http-handler@4.12.1': + resolution: {integrity: sha512-ThMkboGeONWXAelq9FvGsuJC4rOi+qyC4/zhUF58xYpxUg5sQKx2VXZYJmtNjr4dSuBJ1HeJXETQILCz3wOHvw==} + engines: {node: '>=18.0.0'} + + '@smithy/signature-v4@5.7.4': + resolution: {integrity: sha512-tHy0K0VtqNd5Y7Y41h0a0Lhh0L1GzC08dTWg0F7vRJWFtTENg7IZikf3wQkanYIRdb7ngoIPMTmqgUi401fEeQ==} + engines: {node: '>=18.0.0'} + + '@smithy/types@4.19.0': + resolution: {integrity: sha512-r7jh49VJxGerfAcTQA6gXcKc+98zOp/tqRwzYjgOE+iSQsP6cEU1hq2QzbuipmP68QtYdY9wKEhiCQZIzHgZ4Q==} + engines: {node: '>=18.0.0'} + + '@swc/core-darwin-arm64@1.16.13': + resolution: {integrity: sha512-JrMH2h7ohd4mOIneNRx/diqTmCDSdWfp9uOdQ8VJ2V2CA7jOJ1jWb+qp8RoZLqgiVmwDDC7eySKB1kbvZ+jVXQ==} + engines: {node: '>=10'} + cpu: [arm64] + os: [darwin] + + '@swc/core-darwin-x64@1.16.13': + resolution: {integrity: sha512-HauZMRxpOImRxgY6GXjw5gbD8ts2NcPZuLMM8htY8Mic7j1Izt/lxkXrKYUwqyU56YLwQZ4dHRT+6jFHkztcag==} + engines: {node: '>=10'} + cpu: [x64] + os: [darwin] + + '@swc/core-linux-arm-gnueabihf@1.16.13': + resolution: {integrity: sha512-tUSojGIythnHJKrElDM9OvuhBnK+FF39eM9iIBgj9kdfbAm6PhAM7dKbaSDcnX2gOLG2slkasjD0Z8m/YUlzOA==} + engines: {node: '>=10'} + cpu: [arm] + os: [linux] + + '@swc/core-linux-arm64-gnu@1.16.13': + resolution: {integrity: sha512-qS3bk/ezRrLXGn1b764S/EG3/oKj6uRhnTNRAALpnuu8d9I0YwSiUkQghnX7lOH9X/yRr+qrFI3R0wAruO47Tg==} + engines: {node: '>=10'} + cpu: [arm64] + os: [linux] + + '@swc/core-linux-arm64-musl@1.16.13': + resolution: {integrity: sha512-aeoYoG6fQsOS3mFOuTKvnJo1aaaxYNI0aRRlhdHaGP4q1YQx4XOc5Hkd35IFduWCBG3eZl3fPTjRuYvxpl7eTA==} + engines: {node: '>=10'} + cpu: [arm64] + os: [linux] + + '@swc/core-linux-ppc64-gnu@1.16.13': + resolution: {integrity: sha512-eEFoueZXWZ2S1lsqL8Ve2C8RjhzH9+Ktj8U4IGTUQL6CNG04+VhQiYQWJOf5xfVBU6I+PbH80yvsYegqZOOSZQ==} + engines: {node: '>=10'} + cpu: [ppc64] + os: [linux] + + '@swc/core-linux-s390x-gnu@1.16.13': + resolution: {integrity: sha512-K0q1ReVXL3d5bzBgZxS+yJq5q2xRjg4k2XXGj9yiCEpE5ZqJGQ4Gtk1sRrk8W2lo3KY0DCJD3Myt6x2SMvGArQ==} + engines: {node: '>=10'} + cpu: [s390x] + os: [linux] + + '@swc/core-linux-x64-gnu@1.16.13': + resolution: {integrity: sha512-Qvpihw30irhIjy2f91BR1pamcJBCgf8hmr9XWenztAX1YLkneRkkTB/PFGWQubhaBqQhY2uZKViMj/xV61uMBQ==} + engines: {node: '>=10'} + cpu: [x64] + os: [linux] + + '@swc/core-linux-x64-musl@1.16.13': + resolution: {integrity: sha512-iCGxHbh3Cq5vqRvCPlw3XkkDz4RlkPSyQwfFKbOKTMj5tiEuojY/rLmf4BMfI9Z8yPJTf9VB07t3NbKInoC97g==} + engines: {node: '>=10'} + cpu: [x64] + os: [linux] + + '@swc/core-win32-arm64-msvc@1.16.13': + resolution: {integrity: sha512-iOabn6cd+hg3cnoKlAC8j3Yufv8nQHBVU5OIDX8dCPA95PjAZ0B0TlDn9sWivk8ezo/cLJux7p9n0B9vRej+1Q==} + engines: {node: '>=10'} + cpu: [arm64] + os: [win32] + + '@swc/core-win32-ia32-msvc@1.16.13': + resolution: {integrity: sha512-+1b/3xBNuVbpnJIZUCAaSdscOPYRWq85eohKx44y/3qbPp56UMiCJWBiWtyr7K5F1nf1JENXm2IBSEJaLgWq6Q==} + engines: {node: '>=10'} + cpu: [ia32] + os: [win32] + + '@swc/core-win32-x64-msvc@1.16.13': + resolution: {integrity: sha512-6kEJhYshk7OVoVIWryiHqjCNuwZeOlPTS6et+bKV/WhtHs8fLA01wyTLrQGi+4ultgRe/o0M4b25K/KKdvl0dQ==} + engines: {node: '>=10'} + cpu: [x64] + os: [win32] + + '@swc/core@1.16.13': + resolution: {integrity: sha512-k4uUsIMS0IwOKVsrQs48IZK75O4roV3FnuJEdL7ES+DFESftdgvz73QBqN3VUAbUmUr78We4tzG0kgYj36M22Q==} + engines: {node: '>=10'} + peerDependencies: + '@swc/helpers': '>=0.5.17' + peerDependenciesMeta: + '@swc/helpers': + optional: true + + '@swc/counter@0.1.3': + resolution: {integrity: sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ==} + + '@swc/helpers@0.5.5': + resolution: {integrity: sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A==} + + '@swc/types@0.1.28': + resolution: {integrity: sha512-V6Mnml8v09QALx6K0elJ7o9K/MkVDtW3t6L+7Ou/JcWtb3xwId2AH4FeOceySd2JaO87IMw4+6vSZxLm34LPbw==} + + '@tailwindcss/node@4.3.3': + resolution: {integrity: sha512-/T8IKEsf9VTU6tLjgC7+sv2mOPtQxzE2jMw7u4Tt40Tx+QSZxpzh95/H6cMKoja9XuW7iMdLJYBB0o9G1CaAgg==} + + '@tailwindcss/oxide-android-arm64@4.3.3': + resolution: {integrity: sha512-Y85A2gmPSkl5Ve5qR86GL4HT509cFqQh1aes9p3sSkyTPwt0Pppf3GkwGe4JPACcRYjgJIEhQgM6dBClnr0NYw==} + engines: {node: '>= 20'} + cpu: [arm64] + os: [android] + + '@tailwindcss/oxide-darwin-arm64@4.3.3': + resolution: {integrity: sha512-BiaWatpBcERQFDlOjRDpIVXuFK5PJez5SA4JMg6VYZdBYU+qKfV/vqjcIs+IYmtitf1xYQZTwXvU/8y4lfZUGw==} + engines: {node: '>= 20'} + cpu: [arm64] + os: [darwin] + + '@tailwindcss/oxide-darwin-x64@4.3.3': + resolution: {integrity: sha512-fAeUqfV5ndhxRwai8cXGzdLvul9utWOmeTkv69unv4ZXixjn61Z+p9lCWdwOwA3TYboG3BwdVuN/RDjhBRl0mw==} + engines: {node: '>= 20'} + cpu: [x64] + os: [darwin] + + '@tailwindcss/oxide-freebsd-x64@4.3.3': + resolution: {integrity: sha512-iyf5bV6+wnAlflVeEy7R25dupxTNECZN5QMI0qNT6eT+EgaGdZcKhGkr5SdoaWiLJ3spLqIY9VCeSGrwmtg4kw==} + engines: {node: '>= 20'} + cpu: [x64] + os: [freebsd] + + '@tailwindcss/oxide-linux-arm-gnueabihf@4.3.3': + resolution: {integrity: sha512-aAYUprJAJQWWbRrPvtjdroZ56Md+JM8pMiopS6xGEwDfLhqj+2ver2p4nU4Mb3CRqcMmNBjo8KkUgcxhkzVQGQ==} + engines: {node: '>= 20'} + cpu: [arm] + os: [linux] + + '@tailwindcss/oxide-linux-arm64-gnu@4.3.3': + resolution: {integrity: sha512-nDxldcEENOxZRzC2uu9jrutZdAAQtb+8WWDCSnWL1zvBk1+FN+x6MtDViPB5AJMfttVCUhehGWus3XBPgatM/w==} + engines: {node: '>= 20'} + cpu: [arm64] + os: [linux] + + '@tailwindcss/oxide-linux-arm64-musl@4.3.3': + resolution: {integrity: sha512-Md44bD6veX/PC5iyF8cDVnw4HBIANZepRZZ7a8DQOvkfo5WUBwcp6iAuCUz23u+4SUkhJlD3eL7hNdW8ezd/kA==} + engines: {node: '>= 20'} + cpu: [arm64] + os: [linux] + + '@tailwindcss/oxide-linux-x64-gnu@4.3.3': + resolution: {integrity: sha512-tx7us1muwOKAKWao2v/GaafFeQboE6aj88vC6ziN2NCGcRm8gWUhwjzg+YdVB1e4boAtdtma4L43onunI6NS4w==} + engines: {node: '>= 20'} + cpu: [x64] + os: [linux] + + '@tailwindcss/oxide-linux-x64-musl@4.3.3': + resolution: {integrity: sha512-SJxX60smvHgasZoBy11dX6YRjXJFovwWBoedhbQPOBzgFWBHGB+TVPWB9BxzR7TTxU8FQZAI2AyiNCMzFm8Img==} + engines: {node: '>= 20'} + cpu: [x64] + os: [linux] + + '@tailwindcss/oxide-wasm32-wasi@4.3.3': + resolution: {integrity: sha512-jx1+rPhY/5Ympkktd656HBWEBLxP7dH06losBLjjf5vgCODXvi9KhtftWcMIwTFIDqBr7cRnQkdLnAG+IOlGvQ==} + engines: {node: '>=14.0.0'} + cpu: [wasm32] + bundledDependencies: + - '@napi-rs/wasm-runtime' + - '@emnapi/core' + - '@emnapi/runtime' + - '@tybys/wasm-util' + - '@emnapi/wasi-threads' + - tslib + + '@tailwindcss/oxide-win32-arm64-msvc@4.3.3': + resolution: {integrity: sha512-3rc292Ca2ceK6Ulcc/bAVnTs/3nDtoPhyEKlgPv+yQJQi/JS/AMJlqzxvlDacL1nekbrcf6bTqp/jV4qgnPxNQ==} + engines: {node: '>= 20'} + cpu: [arm64] + os: [win32] + + '@tailwindcss/oxide-win32-x64-msvc@4.3.3': + resolution: {integrity: sha512-yJ0pwIVc/nYeGoV02WtsN8KYyLQv7kyI2wDnkezyJlGGjkd4QLwDGAwl47YpPJeuI0M0ObaXGSPjvWDPeTPggw==} + engines: {node: '>= 20'} + cpu: [x64] + os: [win32] + + '@tailwindcss/oxide@4.3.3': + resolution: {integrity: sha512-krXjAikiaFSPaK/FkAQT5UTx3VormQaiZ5hBFlJZ9UFQGB/rwg1MZIhHAG9smMQRTdyJxP6Qt5MwMtdyU5FWrA==} + engines: {node: '>= 20'} + + '@tailwindcss/postcss@4.3.3': + resolution: {integrity: sha512-JTSZZGQi1AyKirbLN3azmjVzef92tcX7h+iSqPdaeStyFpGpDlKvvpxeOE8njhbUanbRwr3z8DyzhICWnMtQeg==} + + '@tanstack/react-virtual@3.14.13': + resolution: {integrity: sha512-JbDTAwtzZ99aOeCrAfW5EsE5KSq5RWh6Af2dtFwyLIIk48Ja7vm6n4axu/43T3vfjFnEGJalxJ1wyYjSQD6bSg==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + + '@tanstack/virtual-core@3.17.11': + resolution: {integrity: sha512-+ILjvtHup6Y2hzQ6YzwMgX1Q+oQpxEGOXCEsCNaPoIP0VxMbizIBTmYTDtkerkIQS8/CbP1BRuyt8V/8BCsy1g==} + + '@temporalio/activity@1.24.0': + resolution: {integrity: sha512-eBBho/r9XHe2KczYGzmaSp6e9+6c9ta/GGOBldIBr00D/cV34kJ76iAY3Ue1pQTaP7YaX3phMuOz07uUg/IKyA==} + engines: {node: '>= 20.3.0'} + + '@temporalio/client@1.24.0': + resolution: {integrity: sha512-/GGg6ZYl+pN/bQEvq414CyjDvyM+MM1WsMzi/7SCYb3UlivVXx9nOmH/6BGx8OfYFayp11fkmfSh6rt7vW/ePQ==} + engines: {node: '>= 20.3.0'} + + '@temporalio/common@1.24.0': + resolution: {integrity: sha512-jXp77fXwynBcVNc/CaGMT33TVYnhcN26S959HDv7Q71oc0rN6d8RHWR5z0nRCDAu5CQwZQRRhVK0TRdyAiEg7Q==} + engines: {node: '>= 20.3.0'} + + '@temporalio/core-bridge@1.24.0': + resolution: {integrity: sha512-6kzPB8RsHcPU6HDNUSDBFC0JG2UFrV3ThmPmJkeEafKh2DFrKARM8IzxgKeE5MzvdIXYmONFM0UmHg+TZ0/qrQ==} + engines: {node: '>= 20.3.0'} + + '@temporalio/nexus@1.24.0': + resolution: {integrity: sha512-SQOQia98KFowgpgH6Cn+7rcpWVEjmJbV5FlqQVQoDH82Q7yl5gT81GO9vN7YRYj5RE2P+DVzEsCc8ohT7Hy57g==} + engines: {node: '>= 20.3.0'} + + '@temporalio/proto@1.24.0': + resolution: {integrity: sha512-vhlhB9GpoyJx+78r3lDahHywpdri6LKx7kt7djLIw6XzHyb8FQTIi4lX8DSJBqKp3+HVfL1CLwiqO26KFRzg/g==} + engines: {node: '>= 20.3.0'} + + '@temporalio/worker@1.24.0': + resolution: {integrity: sha512-Lq79WAHOQFuCFJe1ZJcwcs/u8ZPhHyoTpnWZq1gr6mnB9IRMImAYrJFXY/MY/VdXVltTpy+7T6u/V+P327J+gQ==} + engines: {node: '>= 20.3.0'} + + '@temporalio/workflow@1.24.0': + resolution: {integrity: sha512-pIA9nxkDCCbBnNuMSffQ44HWHYe3zXFfrDpDLwzh/FUSx8uLbsBM8jwf5oUMf+5eV/qVW9PknZhOu9IFO4XnPA==} + engines: {node: '>= 20.3.0'} + + '@tiptap/core@3.31.4': + resolution: {integrity: sha512-cIVUQYBycighzsk6GbmCef8ZiEDVxoT5EuAe4xg2q9dNkDqCXVqI4YGr9aavCQEiAS4uHiRwqvcsPqJHipFDfg==} + peerDependencies: + '@tiptap/pm': 3.31.4 + + '@tiptap/extension-blockquote@3.31.4': + resolution: {integrity: sha512-FHnbvCIW0eDKgUUFjmvEB0ax9YUBW7/Unm+38qpuQROvU872yRqhnDtycKT9IKk46jNdjM4A4fAGUbge10CrAA==} + peerDependencies: + '@tiptap/core': 3.31.4 + '@tiptap/pm': 3.31.4 + + '@tiptap/extension-bold@3.31.4': + resolution: {integrity: sha512-QX1xoXvO0gz6uVsoLegumqQ4MJXL7lcUDQZ4J7la7LBqiIh4efrCmp2EXbhpr37M/YQL7w8q3g7H/VovSh7xIg==} + peerDependencies: + '@tiptap/core': 3.31.4 + + '@tiptap/extension-bubble-menu@3.31.4': + resolution: {integrity: sha512-Ml/yswuys+3QkOMajfXyAGlUKCxEbT3dMyF8vHwQ5mDk9MDm+dqp10It5CrDBkUT/KkVTa6AJI3kb0TXZJZiAA==} + peerDependencies: + '@tiptap/core': 3.31.4 + '@tiptap/pm': 3.31.4 + + '@tiptap/extension-bullet-list@3.31.4': + resolution: {integrity: sha512-gHn1RDR4olCnmKpH+G9x+f8R+iKPNVdLlvbFkIpR/stjabZi7fB4Rr5GiHWsulbtxAjdQrNjnVeNdGM4rBxApw==} + peerDependencies: + '@tiptap/extension-list': 3.31.4 + + '@tiptap/extension-character-count@3.31.4': + resolution: {integrity: sha512-JWftIQ7wBwgQ5sMGxkGTY+310uJv9yc6irNFC1t+FRyYxClqUNBGWoGG5z+H8x5AyoCQqFyBCkCVG9IKp2kkSA==} + peerDependencies: + '@tiptap/extensions': 3.31.4 + + '@tiptap/extension-code-block@3.31.4': + resolution: {integrity: sha512-su/B/cfQ1HpiHl6JfJ6wa+RVEptiYsV4dCntE8XQNj2dUcII4y6w6PQbbswOoDrU79uXHsAvpy44215rghG4LA==} + peerDependencies: + '@tiptap/core': 3.31.4 + '@tiptap/pm': 3.31.4 + + '@tiptap/extension-code@3.31.4': + resolution: {integrity: sha512-pzi8GNDs5J1DEoO1HNYiJ5WZpgOZzlc/2VIa77mMpEuD4hZxuK0BQzlF/O6Ps/O5P1lGJT4xS0Yowpzb9mD9RA==} + peerDependencies: + '@tiptap/core': 3.31.4 + + '@tiptap/extension-document@3.31.4': + resolution: {integrity: sha512-vT20u62RWR+BLIyNJ54e24MPUY4zm8VF7tcgJnxQXi7BgVJWcNMtMWWDIkPJ4JNXwfoIznKh3jzYsaYmbCJgQQ==} + peerDependencies: + '@tiptap/core': 3.31.4 + + '@tiptap/extension-dropcursor@3.31.4': + resolution: {integrity: sha512-5TbCPCC4RL+QT1NYHjXCYDBHBtVqYOFvFEnk3v98lvXCM2PNL13ofXUfJPuykAVy1OdTAx4DlZOQAD9WKy/IdQ==} + peerDependencies: + '@tiptap/extensions': 3.31.4 + + '@tiptap/extension-floating-menu@3.31.4': + resolution: {integrity: sha512-pKzulkDmolpVt7IP9FSPsC5pKbXQ5pyeqkaAYr8cyOVW+KUObpPRIzAuXvEso8PCjWgwJ1f2qyishOEMw8MbfQ==} + peerDependencies: + '@floating-ui/dom': ^1.0.0 + '@tiptap/core': 3.31.4 + '@tiptap/pm': 3.31.4 + + '@tiptap/extension-gapcursor@3.31.4': + resolution: {integrity: sha512-hBogUAepQH6IQUhaePhPWzQ1Oeu+yqlJVMKO06Eue5m36uRy1PZZAQ780pCGT0SC4R5MH2td1N8EUQ7bupQdDA==} + peerDependencies: + '@tiptap/extensions': 3.31.4 + + '@tiptap/extension-hard-break@3.31.4': + resolution: {integrity: sha512-oLNajLyUvRMg1qtzFgKGduygTlpaHFB9PU+M+pUWdA4sa1bRnxS2Y++Ao4lHYopJ7nxatLndrYoCIHkotqkPDQ==} + peerDependencies: + '@tiptap/core': 3.31.4 + + '@tiptap/extension-heading@3.31.4': + resolution: {integrity: sha512-KqlSeLathpdxCc3kVHBCA/RkHvb+V7u5vgiSEu1Cv3L70r7F19COMGODav/k1AGYBgEXKL6lM9cy8cloE5/28g==} + peerDependencies: + '@tiptap/core': 3.31.4 + + '@tiptap/extension-horizontal-rule@3.31.4': + resolution: {integrity: sha512-ws0Kh8kfHL7ESILSXj0HJAXs8G10ewlW1sedXKm8dm0op9L8il+xl8bEJVoqG/jMf77900Fc2CaQldcCGRpF1g==} + peerDependencies: + '@tiptap/core': 3.31.4 + '@tiptap/pm': 3.31.4 + + '@tiptap/extension-italic@3.31.4': + resolution: {integrity: sha512-pftTQBPYZocNDAj/u6OyDIp7YxmN0+N895cQPogJQ6qMHdLiRhzgawiEewJkniyN1w8k3Wx/Etp8pTKrxaOOBQ==} + peerDependencies: + '@tiptap/core': 3.31.4 + + '@tiptap/extension-link@3.31.4': + resolution: {integrity: sha512-LLxW3qfeL9gvDdMPVdxYFYZ0zZSjsoIDhOqjJ5U2UfaK3U2iwkKTBgV2qT+FWf3uUHmBRjBZ08q3dRPmWBAD5g==} + peerDependencies: + '@tiptap/core': 3.31.4 + '@tiptap/pm': 3.31.4 + + '@tiptap/extension-list-item@3.31.4': + resolution: {integrity: sha512-ao8XAPvhOrtGFILs+fKtOM91omDSpUC8X19m44gbaq5xJDVpIMZ0rP+NAVUS72G7LskhcHa0bu/Pm022bUVjVw==} + peerDependencies: + '@tiptap/extension-list': 3.31.4 + + '@tiptap/extension-list-keymap@3.31.4': + resolution: {integrity: sha512-etdkl/YnYfwtALDxecJ+4WYgJAez1fn1vzzZqJklZBpgqiZtny3NxpvWS7/JhsltNG84Wo/oovpfJgVmzhddrw==} + peerDependencies: + '@tiptap/extension-list': 3.31.4 + + '@tiptap/extension-list@3.31.4': + resolution: {integrity: sha512-Wd4YRYxOdf+TtF33/QsRo1RK1CecVxjYGjUBIKNGHAywfh8tghgV4YOUCiV1iJqYt2IBs+RWlIqTmdIPzBrd4A==} + peerDependencies: + '@tiptap/core': 3.31.4 + '@tiptap/pm': 3.31.4 + + '@tiptap/extension-ordered-list@3.31.4': + resolution: {integrity: sha512-UNWyF+Dhm08znszIMnT8Y2kJ1lvtiCvUGs1aG7m9cPHxDa7VycWnOqdtXWkc+6xRLNiC0zgWCFWgLLHduV8pRQ==} + peerDependencies: + '@tiptap/extension-list': 3.31.4 + + '@tiptap/extension-paragraph@3.31.4': + resolution: {integrity: sha512-WZlBgw+XVKXerRlmEsoQ7x1PwSU+dh3X+b70+ViZsLebdfsmmYBIaNDrvIGwG9UQr5eLnddJnZplMSTI2YKs+g==} + peerDependencies: + '@tiptap/core': 3.31.4 + + '@tiptap/extension-placeholder@3.31.4': + resolution: {integrity: sha512-3SoJopaOzaA9IZHSk9ZXHiO6ib8ErPh7ssRh3bIaxfD+BqRyXcm4hV4yfFXTqrDCnl5FE+DbBRGmeVAVkDko3g==} + peerDependencies: + '@tiptap/extensions': 3.31.4 + + '@tiptap/extension-strike@3.31.4': + resolution: {integrity: sha512-4MBr63TJIMqxCjE+gAPdlY1IdELgvbAP516K3Ntn/BkcpEbEoPMEYVz+XTxULyHQEuARDO6jsXPPxRXpA74jaw==} + peerDependencies: + '@tiptap/core': 3.31.4 + + '@tiptap/extension-text@3.31.4': + resolution: {integrity: sha512-WL2ZRpFksgQcXJN05/UQH09YCH9LlhdLrSEcviASvEH/aAfsKiJlDrouN8PET+YAWK/F5xFc5/Q1z0xj+I23jw==} + peerDependencies: + '@tiptap/core': 3.31.4 + + '@tiptap/extension-underline@3.31.4': + resolution: {integrity: sha512-0iFGPIqmqGkCrOFh1OPo7Dzv5s86dVN3bZaugsz0jK04otkb8bn04ZDKP2/6Wq+lcK0M9DfJ0QqfW5hC2nBZxQ==} + peerDependencies: + '@tiptap/core': 3.31.4 + + '@tiptap/extensions@3.31.4': + resolution: {integrity: sha512-nIupIne/sHpV2Kw1M4MOmr4+G454KI8BXM5+LWKOU2EpzfALZWVSy7koF9lVRna4Zp5Gv8hfg5Z8kvmrYQvhkw==} + peerDependencies: + '@tiptap/core': 3.31.4 + '@tiptap/pm': 3.31.4 + + '@tiptap/pm@3.31.4': + resolution: {integrity: sha512-x7dwth79cQp4QGae66/RJqMDJQv0+n7b7dD7cyFQBWqzPFLJxN/4paHY4qbWedZjNLhRyIOAD9a5q4AYM1zmPA==} + + '@tiptap/react@3.31.4': + resolution: {integrity: sha512-mBvvXgdPhMinT1NPfwHvFK1dFdMXHVWMSeNn0IPj7/zV0XBmOnASkEOwoIvl8NluKYEtpZBE1yA+6pN2enLa5g==} + peerDependencies: + '@tiptap/core': 3.31.4 + '@tiptap/pm': 3.31.4 + '@types/react': ^17.0.0 || ^18.0.0 || ^19.0.0 + '@types/react-dom': ^17.0.0 || ^18.0.0 || ^19.0.0 + react: ^17.0.0 || ^18.0.0 || ^19.0.0 + react-dom: ^17.0.0 || ^18.0.0 || ^19.0.0 + + '@tiptap/starter-kit@3.31.4': + resolution: {integrity: sha512-u7HIJ9vK6kFe4S4J2CIcDLrhRZzoTMSyU0XxjQ6lUNF7pWW3rMi//pvi8RcsZmxGjyb8CpXAqYgNl7f9WyaUeA==} + + '@tokenizer/inflate@0.2.7': + resolution: {integrity: sha512-MADQgmZT1eKjp06jpI2yozxaU9uVs4GzzgSL+uEq7bVcJ9V1ZXQkeGNql1fsSI0gMy1vhvNTNbUqrx+pZfJVmg==} + engines: {node: '>=18'} + + '@tokenizer/token@0.3.0': + resolution: {integrity: sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==} + + '@turbo/darwin-64@2.11.7': + resolution: {integrity: sha512-Z2LFOxwbiJueD1miByXJS/BrW21vv6YaQXZustty5jEX4vQYw6zyh101Sz1QO7WG0mqBzKagZ/DoAeeujI0/Sw==} + cpu: [x64] + os: [darwin] + + '@turbo/darwin-arm64@2.11.7': + resolution: {integrity: sha512-Vhv8ybgIflosNDSSRD9fNXHlCFA60dsGyTqQmc7l28SEX5jOaweCCD4BBYNd8/p0WGkw3e7tZbfA9w6FsujGvw==} + cpu: [arm64] + os: [darwin] + + '@turbo/linux-64@2.11.7': + resolution: {integrity: sha512-rAqRp3blQKk1+ylMGlZHAW0eJjVNuuGUm4ZrC6RU1Im8tLryvaO4a2zLQ/FQi1TRlmKqKV9YXBki2YeytlLbXA==} + cpu: [x64] + os: [android, linux] + + '@turbo/linux-arm64@2.11.7': + resolution: {integrity: sha512-thVYGov7S+llXAxrl3kBb/efalav0xhrjl5NHCR6AfKx14JkbqaetvbVwyss4FvZ9rt+Pv3xsTb+8SIYix8DTg==} + cpu: [arm64] + os: [android, linux] + + '@turbo/windows-64@2.11.7': + resolution: {integrity: sha512-ZIoKT7J7KutgYl4bGxmmgeUQjG+7E0nY8BF9dxUOiMH2vdTsM4JQ2uSwTYFkTR2Llbg6ghGjSmuZ8LQtkLEYjg==} + cpu: [x64] + os: [win32] + + '@turbo/windows-arm64@2.11.7': + resolution: {integrity: sha512-88Xv2f7SLGVJ68lRRx8o375dKmnZYNdDtvlKcPA0kZpcF93eAQd8l96BKQ0/AMHG0bMJA0Z5QqoqtetxzNsC+g==} + cpu: [arm64] + os: [win32] + + '@types/aws-lambda@8.10.164': + resolution: {integrity: sha512-XOnrazWcOd6yWPnR7DxqCBPPYciDjHq+NN8LPwwqBCJNXLyPJgKo47fZpJ113+oyAn9Zti0Cf6H4k6yPcE3WFg==} + + '@types/body-parser@1.19.6': + resolution: {integrity: sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==} + + '@types/connect@3.4.38': + resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==} + + '@types/estree@1.0.9': + resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + + '@types/express-serve-static-core@4.19.9': + resolution: {integrity: sha512-QP2ESEe/ImWY0HDwNAnK9PvEffUyhLTnWkk7KXzHfyeWAnlrDe1fN77bXl6ia8KT3wPlmA7t9/VPRpnf4Ex9sg==} + + '@types/express@4.17.25': + resolution: {integrity: sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==} + + '@types/http-errors@2.0.5': + resolution: {integrity: sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==} + + '@types/json-schema@7.0.15': + resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} + + '@types/memcached@2.2.10': + resolution: {integrity: sha512-AM9smvZN55Gzs2wRrqeMHVP7KE8KWgCJO/XL5yCly2xF6EKa4YlbpK+cLSAH4NG/Ah64HrlegmGqW8kYws7Vxg==} + + '@types/mime@1.3.5': + resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==} + + '@types/node@22.20.5': + resolution: {integrity: sha512-U2+DNr+wSjpsTS/wZGYHq7GcwfuSmKiKvoPvK22zwTlRhU91yOniN4qRR5KhIjvif7ysw/dz/hKmfDH0Ris4aA==} + + '@types/pg-pool@2.0.7': + resolution: {integrity: sha512-U4CwmGVQcbEuqpyju8/ptOKg6gEC+Tqsvj2xS9o1g71bUh8twxnC6ZL5rZKCsGN0iyH0CwgUyc9VR5owNQF9Ng==} + + '@types/pg@8.15.6': + resolution: {integrity: sha512-NoaMtzhxOrubeL/7UZuNTrejB4MPAJ0RpxZqXQf2qXuVlTPuG6Y8p4u9dKRaue4yjmC7ZhzVO2/Yyyn25znrPQ==} + + '@types/pg@8.23.1': + resolution: {integrity: sha512-fKVHpikPdg4GKks3JuLEhvwSyvwzF23hnabPy6DD8ljVbC7+6J5dQzdv4arV6jqq57djnMgs1HKBxX4P8aBI3A==} + + '@types/prop-types@15.7.15': + resolution: {integrity: sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==} + + '@types/qs@6.15.1': + resolution: {integrity: sha512-GZHUBZR9hckSUhrxmp1nG6NwdpM9fCunJwyThLW1X3AyHgd9IlHb6VANpQQqDr2o/qQp6McZ3y/IA2rVzKzSbw==} + + '@types/range-parser@1.2.7': + resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==} + + '@types/react-dom@18.3.7': + resolution: {integrity: sha512-MEe3UeoENYVFXzoXEWsvcpg6ZvlrFNlOQ7EOsvhI3CfAXwzPfO8Qwuxd40nepsYKqyyVQnTdEfv68q91yLcKrQ==} + peerDependencies: + '@types/react': ^18.0.0 + + '@types/react@18.3.31': + resolution: {integrity: sha512-vfEqpXTvwT91yhmwdfouStN2hSKwTvyRs8qpLfADyrq/kxDw0hZM7Wk9Ug1FELj8hIby+S/+kQCSRFF32nv2Qw==} + + '@types/send@0.17.6': + resolution: {integrity: sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og==} + + '@types/send@1.2.1': + resolution: {integrity: sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==} + + '@types/serve-static@1.15.10': + resolution: {integrity: sha512-tRs1dB+g8Itk72rlSI2ZrW6vZg0YrLI81iQSTkMmOqnqCaNr/8Ek4VwWcN5vZgCYWbg/JJSGBlUaYGAOP73qBw==} + + '@types/use-sync-external-store@0.0.6': + resolution: {integrity: sha512-zFDAD+tlpf2r4asuHEj0XH6pY6i0g5NeAHPn+15wk3BV6JA69eERFXC1gyGThDkVa1zCyKr5jox1+2LbV/AMLg==} + + '@typescript-eslint/eslint-plugin@8.71.0': + resolution: {integrity: sha512-pqcS9c1HxZTHt7End4nXqd0s5lJrrFzrgCkKFJrsbUnaL6M3+6oBFZaslg6Gjsl3argl2DDRFROnXARaZ2e4Nw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + '@typescript-eslint/parser': ^8.71.0 + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/parser@8.71.0': + resolution: {integrity: sha512-CG4nPk1f2zc8yw4pALqHsFYH2hdo+h1T9daSp21+Hnxi9LOE3GT9hAfTKJCBXVNM2GmYs1eMEP615wPoeOgk3A==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/project-service@8.71.0': + resolution: {integrity: sha512-aABjw5rjBacYONVPaPiWOCjJu0vEF4a25iQuodlmQYL1trtLZ0X/y+2Vzl3BKI1odM4LnwLE1oUDXYp1wzx1TQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/scope-manager@8.71.0': + resolution: {integrity: sha512-gWF0BhUcnjZxSpLE8ngS/59n2SB0J3YqRxvX1+2aoRJk9hNtHSLOV+TcarFiOr5ipXm3yc1QrI4c9YZc8zyCxw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@typescript-eslint/tsconfig-utils@8.71.0': + resolution: {integrity: sha512-Z1UlWHADEK2Mlb9NpWfDeSjqoZ5EyrOv4R3eQpbkzqn/EwaIdOpXXupEA1+0ZIOSJSZZDBHG0BrQyN8zUG6Pwg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/type-utils@8.71.0': + resolution: {integrity: sha512-i8uO1qbdxeKgRnS5sCRt6On3/nfo2d2DwQe3Yvjx543zLy7r8ySqRuPPiIIXAhS03U0v5NfAFx+rUgxFzKKwNw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/types@8.71.0': + resolution: {integrity: sha512-cJ4OoxPGWvFnBTnSZyaU+qJzGTqPTGJY+gDchj6cRyLRdmIdt4rcsE4twj+zPfrNiWuVi38wijHzShL++Z9atQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@typescript-eslint/typescript-estree@8.71.0': + resolution: {integrity: sha512-PEEF4G5sLLWAS5BpPrUvms4ySZkiBQQZM4z+3ReI46axK5Vqr/vXBQatJQIZZOYdGyPUAKTtsrWzpqKuU+3DEw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/utils@8.71.0': + resolution: {integrity: sha512-pKR/tEMVrXZG23UFKUn5BQf3zfmfk7KQceI2cGzywZ5nxM5Eu3hEJU1utjWzydtzBbcJAQhHN8iPCxobHpPcZQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + '@typescript-eslint/visitor-keys@8.71.0': + resolution: {integrity: sha512-8eQ9R218XORK+KLosnf4bu/QsUXvUyVwTbArg7/0NMB1Pu87OJKvj4nhFblkYE8gQV73mW1dx1ptlPCkwRGa7A==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + '@vitest/expect@2.1.9': + resolution: {integrity: sha512-UJCIkTBenHeKT1TTlKMJWy1laZewsRIzYighyYiJKZreqtdxSos/S1t+ktRMQWu2CKqaarrkeszJx1cgC5tGZw==} + + '@vitest/mocker@2.1.9': + resolution: {integrity: sha512-tVL6uJgoUdi6icpxmdrn5YNo3g3Dxv+IHJBr0GXHaEdTcw3F+cPKnsXFhli6nO+f/6SDKPHEK1UN+k+TQv0Ehg==} + peerDependencies: + msw: ^2.4.9 + vite: ^5.0.0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + + '@vitest/pretty-format@2.1.9': + resolution: {integrity: sha512-KhRIdGV2U9HOUzxfiHmY8IFHTdqtOhIzCpd8WRdJiE7D/HUcZVD0EgQCVjm+Q9gkUXWgBvMmTtZgIG48wq7sOQ==} + + '@vitest/runner@2.1.9': + resolution: {integrity: sha512-ZXSSqTFIrzduD63btIfEyOmNcBmQvgOVsPNPe0jYtESiXkhd8u2erDLnMxmGrDCwHCCHE7hxwRDCT3pt0esT4g==} + + '@vitest/snapshot@2.1.9': + resolution: {integrity: sha512-oBO82rEjsxLNJincVhLhaxxZdEtV0EFHMK5Kmx5sJ6H9L183dHECjiefOAdnqpIgT5eZwT04PoggUnW88vOBNQ==} + + '@vitest/spy@2.1.9': + resolution: {integrity: sha512-E1B35FwzXXTs9FHNK6bDszs7mtydNi5MIfUWpceJ8Xbfb1gBMscAnwLbEu+B44ed6W3XjL9/ehLPHR1fkf1KLQ==} + + '@vitest/utils@2.1.9': + resolution: {integrity: sha512-v0psaMSkNJ3A2NMrUEHFRzJtDPFn+/VWZ5WxImB21T9fjucJRmS7xCS3ppEnARb9y11OAzaD+P2Ps+b+BGX5iQ==} + + '@webassemblyjs/ast@1.14.1': + resolution: {integrity: sha512-nuBEDgQfm1ccRp/8bCQrx1frohyufl4JlbMMZ4P1wpeOfDhF6FQkxZJ1b/e+PLwr6X1Nhw6OLme5usuBWYBvuQ==} + + '@webassemblyjs/floating-point-hex-parser@1.13.2': + resolution: {integrity: sha512-6oXyTOzbKxGH4steLbLNOu71Oj+C8Lg34n6CqRvqfS2O71BxY6ByfMDRhBytzknj9yGUPVJ1qIKhRlAwO1AovA==} + + '@webassemblyjs/helper-api-error@1.13.2': + resolution: {integrity: sha512-U56GMYxy4ZQCbDZd6JuvvNV/WFildOjsaWD3Tzzvmw/mas3cXzRJPMjP83JqEsgSbyrmaGjBfDtV7KDXV9UzFQ==} + + '@webassemblyjs/helper-buffer@1.14.1': + resolution: {integrity: sha512-jyH7wtcHiKssDtFPRB+iQdxlDf96m0E39yb0k5uJVhFGleZFoNw1c4aeIcVUPPbXUVJ94wwnMOAqUHyzoEPVMA==} + + '@webassemblyjs/helper-numbers@1.13.2': + resolution: {integrity: sha512-FE8aCmS5Q6eQYcV3gI35O4J789wlQA+7JrqTTpJqn5emA4U2hvwJmvFRC0HODS+3Ye6WioDklgd6scJ3+PLnEA==} + + '@webassemblyjs/helper-wasm-bytecode@1.13.2': + resolution: {integrity: sha512-3QbLKy93F0EAIXLh0ogEVR6rOubA9AoZ+WRYhNbFyuB70j3dRdwH9g+qXhLAO0kiYGlg3TxDV+I4rQTr/YNXkA==} + + '@webassemblyjs/helper-wasm-section@1.14.1': + resolution: {integrity: sha512-ds5mXEqTJ6oxRoqjhWDU83OgzAYjwsCV8Lo/N+oRsNDmx/ZDpqalmrtgOMkHwxsG0iI//3BwWAErYRHtgn0dZw==} + + '@webassemblyjs/ieee754@1.13.2': + resolution: {integrity: sha512-4LtOzh58S/5lX4ITKxnAK2USuNEvpdVV9AlgGQb8rJDHaLeHciwG4zlGr0j/SNWlr7x3vO1lDEsuePvtcDNCkw==} + + '@webassemblyjs/leb128@1.13.2': + resolution: {integrity: sha512-Lde1oNoIdzVzdkNEAWZ1dZ5orIbff80YPdHx20mrHwHrVNNTjNr8E3xz9BdpcGqRQbAEa+fkrCb+fRFTl/6sQw==} + + '@webassemblyjs/utf8@1.13.2': + resolution: {integrity: sha512-3NQWGjKTASY1xV5m7Hr0iPeXD9+RDobLll3T9d2AO+g3my8xy5peVyjSag4I50mR1bBSN/Ct12lo+R9tJk0NZQ==} + + '@webassemblyjs/wasm-edit@1.14.1': + resolution: {integrity: sha512-RNJUIQH/J8iA/1NzlE4N7KtyZNHi3w7at7hDjvRNm5rcUXa00z1vRz3glZoULfJ5mpvYhLybmVcwcjGrC1pRrQ==} + + '@webassemblyjs/wasm-gen@1.14.1': + resolution: {integrity: sha512-AmomSIjP8ZbfGQhumkNvgC33AY7qtMCXnN6bL2u2Js4gVCg8fp735aEiMSBbDR7UQIj90n4wKAFUSEd0QN2Ukg==} + + '@webassemblyjs/wasm-opt@1.14.1': + resolution: {integrity: sha512-PTcKLUNvBqnY2U6E5bdOQcSM+oVP/PmrDY9NzowJjislEjwP/C4an2303MCVS2Mg9d3AJpIGdUFIQQWbPds0Sw==} + + '@webassemblyjs/wasm-parser@1.14.1': + resolution: {integrity: sha512-JLBl+KZ0R5qB7mCnud/yyX08jWFw5MsoalJ1pQ4EdFlgj9VdXKGuENGsiCIjegI1W7p91rUlcB/LB5yRJKNTcQ==} + + '@webassemblyjs/wast-printer@1.14.1': + resolution: {integrity: sha512-kPSSXE6De1XOR820C90RIo2ogvZG+c3KiHzqUoO/F34Y2shGzesfqv7o57xrxovZJH/MetF5UjroJ/R/3isoiw==} + + '@xtuc/ieee754@1.2.0': + resolution: {integrity: sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==} + + '@xtuc/long@4.2.2': + resolution: {integrity: sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==} + + abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + + accepts@1.3.8: + resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} + engines: {node: '>= 0.6'} + + acorn-jsx@5.3.2: + resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} + peerDependencies: + acorn: ^6.0.0 || ^7.0.0 || ^8.0.0 + + acorn@8.19.0: + resolution: {integrity: sha512-oJlA3XiRm7Cyk6qFD2Jn8ak9B7jSy0qy00ADO3+8dpT0LSjFihQYv4C02LFCSYJV3Q37xYwwRy5m+IpIiUzqWw==} + engines: {node: '>=0.4.0'} + hasBin: true + + agent-base@7.1.4: + resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} + engines: {node: '>= 14'} + + ajv-formats@3.0.1: + resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==} + peerDependencies: + ajv: ^8.0.0 + peerDependenciesMeta: + ajv: + optional: true + + ajv-keywords@5.1.0: + resolution: {integrity: sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==} + peerDependencies: + ajv: ^8.8.2 + + ajv@6.15.0: + resolution: {integrity: sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==} + + ajv@8.20.0: + resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} + + ansi-regex@5.0.1: + resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} + engines: {node: '>=8'} + + ansi-regex@6.4.0: + resolution: {integrity: sha512-KzTVk2tCWAHtYrvvvaP8bJKJq2pVinhLcGEQdtLIYPbmNGNyYe8QwNaTUYQp2J7/vIsUKt5QCqAfUkYyG9DkOw==} + engines: {node: '>=12'} + + ansi-styles@4.3.0: + resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} + engines: {node: '>=8'} + + ansi-styles@6.2.3: + resolution: {integrity: sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==} + engines: {node: '>=12'} + + append-field@1.0.0: + resolution: {integrity: sha512-klpgFSWLW1ZEs8svjfb7g4qWY0YS5imI82dTg+QahUvJ8YqAY0P10Uk8tTyh9ZGuYEZEMaeJYCF5BFuX552hsw==} + + argparse@2.0.1: + resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + + aria-hidden@1.2.6: + resolution: {integrity: sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==} + engines: {node: '>=10'} + + array-flatten@1.1.1: + resolution: {integrity: sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==} + + assertion-error@2.0.1: + resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} + engines: {node: '>=12'} + + atomic-sleep@1.0.0: + resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} + engines: {node: '>=8.0.0'} + + balanced-match@1.0.2: + resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + + balanced-match@4.0.4: + resolution: {integrity: sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==} + engines: {node: 18 || 20 || >=22} + + baseline-browser-mapping@2.11.27: + resolution: {integrity: sha512-ElY12DaROGuan+lMmZ8Cvo/ZUbXPe7Enc/9VU/b1T3Kp4dwytRcNdR8DoSJN5SNJT/CuvcCA0DHDVmMOCePdRQ==} + engines: {node: '>=6.0.0'} + hasBin: true + + bignumber.js@9.3.1: + resolution: {integrity: sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==} + + body-parser@1.20.4: + resolution: {integrity: sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + + bowser@2.14.1: + resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==} + + brace-expansion@1.1.21: + resolution: {integrity: sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==} + + brace-expansion@2.1.7: + resolution: {integrity: sha512-uZbew1NqdmPDTMJ8ah1y+b+9QEJrfkXFk3RcTQw3X0jW/xRUvFKsg1CfQdSYGdTbXZWExtU3J3ccxtnfw1Fi0g==} + + brace-expansion@5.0.12: + resolution: {integrity: sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==} + engines: {node: 20 || >=22} + + browserslist@4.29.3: + resolution: {integrity: sha512-1R4kiYKXGViqEN0CnoDrXc1StD9niAwu+j2dukWzrD4bJgsD4lDmEp0CRbc6E/vYJIfTHwPmwyaKtVSudICdPA==} + engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} + hasBin: true + + buffer-from@1.1.2: + resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} + + busboy@1.6.0: + resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} + engines: {node: '>=10.16.0'} + + bytes@3.1.2: + resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} + engines: {node: '>= 0.8'} + + cac@6.7.14: + resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} + engines: {node: '>=8'} + + call-bind-apply-helpers@1.0.2: + resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} + engines: {node: '>= 0.4'} + + call-bound@1.0.4: + resolution: {integrity: sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==} + engines: {node: '>= 0.4'} + + callsites@3.1.0: + resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} + engines: {node: '>=6'} + + caniuse-lite@1.0.30001814: + resolution: {integrity: sha512-/Uaf1lAzr59XcMpW0o96WoEfr+VXK2OX4U9AgFoiSHsVJ4HppnIFUjtYzsyDH2+tgANaQb2/oxYGwCPapN1FpA==} + + chai@5.3.3: + resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==} + engines: {node: '>=18'} + + chalk@4.1.2: + resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} + engines: {node: '>=10'} + + check-error@2.1.3: + resolution: {integrity: sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==} + engines: {node: '>= 16'} + + chrome-trace-event@1.0.4: + resolution: {integrity: sha512-rNjApaLzuwaOTjCiT8lSDdGN1APCiqkChLMJxJPWLunPAt5fy8xgU9/jNOchV84wfIxrA0lRQB7oCT8jrn/wrQ==} + engines: {node: '>=6.0'} + + cjs-module-lexer@2.3.0: + resolution: {integrity: sha512-lmNyBzi6iYiqyrExTctfmjd8tInx+dHu9PBomTfH5RXS53c27JMFJgzNv5TyAnm3PgGf+MLy0+Ws41/HlVxdKQ==} + + client-only@0.0.1: + resolution: {integrity: sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==} + + cliui@8.0.1: + resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} + engines: {node: '>=12'} + + cmdk@1.1.1: + resolution: {integrity: sha512-Vsv7kFaXm+ptHDMZ7izaRsP70GgrW9NBNGswt9OZaVBLlE0SNpDq8eu/VGXyF9r7M0azK3Wy7OlYXsuyYLFzHg==} + peerDependencies: + react: ^18 || ^19 || ^19.0.0-rc + react-dom: ^18 || ^19 || ^19.0.0-rc + + color-convert@2.0.1: + resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} + engines: {node: '>=7.0.0'} + + color-name@1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + + commander@2.20.3: + resolution: {integrity: sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==} + + concat-map@0.0.1: + resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} + + concat-stream@2.0.0: + resolution: {integrity: sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==} + engines: {'0': node >= 6.0} + + consola@2.15.3: + resolution: {integrity: sha512-9vAdYbHj6x2fLKC4+oPH0kFzY/orMZyG2Aj+kNylHxKGJ/Ed4dpNyAQYwJOdqO4zdM7XpVHmyejQDcQHrnuXbw==} + + content-disposition@0.5.4: + resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} + engines: {node: '>= 0.6'} + + content-type@1.0.5: + resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} + engines: {node: '>= 0.6'} + + cookie-signature@1.0.7: + resolution: {integrity: sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==} + + cookie@0.7.2: + resolution: {integrity: sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==} + engines: {node: '>= 0.6'} + + cors@2.8.5: + resolution: {integrity: sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==} + engines: {node: '>= 0.10'} + + cross-spawn@7.0.6: + resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} + engines: {node: '>= 8'} + + csstype@3.2.3: + resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + + data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + + debug@2.6.9: + resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + deep-eql@5.0.2: + resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} + engines: {node: '>=6'} + + deep-is@0.1.4: + resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} + + depd@2.0.0: + resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} + engines: {node: '>= 0.8'} + + destroy@1.2.0: + resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + detect-node-es@1.1.0: + resolution: {integrity: sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ==} + + dunder-proto@1.0.1: + resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} + engines: {node: '>= 0.4'} + + eastasianwidth@0.2.0: + resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + + ee-first@1.1.1: + resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} + + electron-to-chromium@1.5.444: + resolution: {integrity: sha512-5ss/uJfoDYDHT0lfJzT6FbcskIzROIOPf0BbbFkGcvDzoJU7i//9GDrwwIHQVmIsrAGiF3ihpADBRIsrEFt1rQ==} + + emoji-regex@8.0.0: + resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + + emoji-regex@9.2.2: + resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} + + encodeurl@2.0.0: + resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} + engines: {node: '>= 0.8'} + + enhanced-resolve@5.26.0: + resolution: {integrity: sha512-9vhedylFonb2YGogzUKX6+Ja72gOJbN1QHAqdrvqLwhdl/QWbKopzoUC9EbQNVsAns/bx/4uyqalrQAoy1IByw==} + engines: {node: '>=10.13.0'} + + es-define-property@1.0.1: + resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} + engines: {node: '>= 0.4'} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + + es-module-lexer@1.7.0: + resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} + + es-module-lexer@2.3.2: + resolution: {integrity: sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==} + + es-module-lexer@3.0.3: + resolution: {integrity: sha512-yWGe1O/j+almnV4hOYrmZo8u58JdnmXEMneRQAYkWtqSiT86yoMIEKF4rpqEDj52tbYl3xceBD3DQIBwcsC9Pw==} + + es-object-atoms@1.1.2: + resolution: {integrity: sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==} + engines: {node: '>= 0.4'} + + esbuild@0.21.5: + resolution: {integrity: sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==} + engines: {node: '>=12'} + hasBin: true + + esbuild@0.28.2: + resolution: {integrity: sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==} + engines: {node: '>=18'} + hasBin: true + + escalade@3.2.0: + resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} + engines: {node: '>=6'} + + escape-html@1.0.3: + resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} + + escape-string-regexp@4.0.0: + resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} + engines: {node: '>=10'} + + eslint-scope@8.4.0: + resolution: {integrity: sha512-sNXOfKCn74rt8RICKMvJS7XKV/Xk9kA7DyJr8mJik3S7Cwgy3qlkkmyS2uQB3jiJg6VNdZd/pDBJu0nvG2NlTg==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + eslint-visitor-keys@3.4.3: + resolution: {integrity: sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==} + engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} + + eslint-visitor-keys@4.2.1: + resolution: {integrity: sha512-Uhdk5sfqcee/9H/rCOJikYz67o0a2Tw2hGRPOG2Y1R2dg7brRe1uG0yaNQDHu+TO/uQPF/5eCapvYSmHUjt7JQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + eslint-visitor-keys@5.0.1: + resolution: {integrity: sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==} + engines: {node: ^20.19.0 || ^22.13.0 || >=24} + + eslint@9.39.5: + resolution: {integrity: sha512-DgZS62aPLXKlnxILS/AYCoRvHaZeXceIzlXPkkGGzJWSow1aEk0lbTlxUSlyjC8jcaKxAdOnTDz+o1JFSBsyjw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + deprecated: This version is no longer supported. Please see https://eslint.org/version-support for other options. + hasBin: true + peerDependencies: + jiti: '*' + peerDependenciesMeta: + jiti: + optional: true + + espree@10.4.0: + resolution: {integrity: sha512-j6PAQ2uUr79PZhBjP5C5fhl8e39FmRnOjsD5lGnWrFU8i2G776tBK7+nP8KuQUTTyAZUwfQqXAgrVH5MbH9CYQ==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + + esquery@1.7.0: + resolution: {integrity: sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==} + engines: {node: '>=0.10'} + + esrecurse@4.3.0: + resolution: {integrity: sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==} + engines: {node: '>=4.0'} + + estraverse@5.3.0: + resolution: {integrity: sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==} + engines: {node: '>=4.0'} + + estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + + esutils@2.0.3: + resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} + engines: {node: '>=0.10.0'} + + etag@1.8.1: + resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} + engines: {node: '>= 0.6'} + + event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + + events@3.3.0: + resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} + engines: {node: '>=0.8.x'} + + expect-type@1.4.0: + resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} + engines: {node: '>=12.0.0'} + + express@4.22.1: + resolution: {integrity: sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==} + engines: {node: '>= 0.10.0'} + + extend@3.0.2: + resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} + + fast-deep-equal@3.1.3: + resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} + + fast-equals@5.4.3: + resolution: {integrity: sha512-8unmvA0qrfpFWBYUate9BTGtQdKHWBYt+KdAIkag7Vvxe7wz8Q2nJg7fLMC0bJqe0bjP31kca7Ckaawrs4Istg==} + engines: {node: '>=6.0.0'} + + fast-json-stable-stringify@2.1.0: + resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} + + fast-levenshtein@2.0.6: + resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} + + fast-safe-stringify@2.1.1: + resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==} + + fast-uri@3.1.8: + resolution: {integrity: sha512-GZMtZUTNRpOVIECoXwLNZS5xUGE+mVNbTB8h/7Rwh2TFWcBQiPzTgyZi05BF9UMZKkLJv8XBRJTlU7zg8+ZfMg==} + + fdir@6.5.0: + resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} + engines: {node: '>=12.0.0'} + peerDependencies: + picomatch: ^3 || ^4 + peerDependenciesMeta: + picomatch: + optional: true + + fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + + fflate@0.8.3: + resolution: {integrity: sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==} + + file-entry-cache@8.0.0: + resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} + engines: {node: '>=16.0.0'} + + file-type@20.4.1: + resolution: {integrity: sha512-hw9gNZXUfZ02Jo0uafWLaFVPter5/k2rfcrjFJJHX/77xtSDOfJuEFb6oKlFV86FLP1SuyHMW1PSk0U9M5tKkQ==} + engines: {node: '>=18'} + + finalhandler@1.3.2: + resolution: {integrity: sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==} + engines: {node: '>= 0.8'} + + find-up@5.0.0: + resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} + engines: {node: '>=10'} + + flat-cache@4.0.1: + resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==} + engines: {node: '>=16'} + + flatted@3.4.4: + resolution: {integrity: sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==} + + foreground-child@3.3.1: + resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} + engines: {node: '>=14'} + + formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + + forwarded-parse@2.1.2: + resolution: {integrity: sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==} + + forwarded@0.2.0: + resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} + engines: {node: '>= 0.6'} + + fresh@0.5.2: + resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} + engines: {node: '>= 0.6'} + + fs-monkey@1.1.0: + resolution: {integrity: sha512-QMUezzXWII9EV5aTFXW1UBVUO77wYPpjqIF8/AviUCThNeSYZykpoTixUeaNNBwmCev0AMDWMAni+f8Hxb1IFw==} + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + + gaxios@7.1.3: + resolution: {integrity: sha512-YGGyuEdVIjqxkxVH1pUTMY/XtmmsApXrCVv5EU25iX6inEPbV+VakJfLealkBtJN69AQmh1eGOdCl9Sm1UP6XQ==} + engines: {node: '>=18'} + + gcp-metadata@8.1.4: + resolution: {integrity: sha512-iJ9KMsiu+xKtNRX0PmGLSaIU3bUBAyzWTyqKemKPzNPsmmsBCQYmlNg+brEbES7IHSXtdVwzBPzx1vz3FAaipw==} + engines: {node: '>=18'} + + get-caller-file@2.0.5: + resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} + engines: {node: 6.* || 8.* || >= 10.*} + + get-intrinsic@1.3.0: + resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} + engines: {node: '>= 0.4'} + + get-nonce@1.0.1: + resolution: {integrity: sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==} + engines: {node: '>=6'} + + get-proto@1.0.1: + resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} + engines: {node: '>= 0.4'} + + glob-parent@6.0.2: + resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} + engines: {node: '>=10.13.0'} + + glob-to-regex.js@1.3.1: + resolution: {integrity: sha512-zhWhMRsgnOym+1txhUmYJGD6eUsbUdrZv55gCYKuJX8t2p3+M8My23PD09bEIsreGUs5pnSa/idmcfp2/IlqGg==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + glob@10.5.0: + resolution: {integrity: sha512-DfXN8DfhJ7NH3Oe7cFmu3NCu1wKbkReJ8TorzSAFbSKrlNaQSKfIzqYqVY8zlbs2NLBbWpRiU52GX2PbaBVNkg==} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me + hasBin: true + + globals@14.0.0: + resolution: {integrity: sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==} + engines: {node: '>=18'} + + google-logging-utils@1.1.3: + resolution: {integrity: sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==} + engines: {node: '>=14'} + + gopd@1.2.0: + resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} + engines: {node: '>= 0.4'} + + graceful-fs@4.2.11: + resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + + has-flag@4.0.0: + resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} + engines: {node: '>=8'} + + has-symbols@1.1.0: + resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} + engines: {node: '>= 0.4'} + + hasown@2.0.4: + resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==} + engines: {node: '>= 0.4'} + + heap-js@2.7.1: + resolution: {integrity: sha512-EQfezRg0NCZGNlhlDR3Evrw1FVL2G3LhU7EgPoxufQKruNBSYA8MiRPHeWbU+36o+Fhel0wMwM+sLEiBAlNLJA==} + engines: {node: '>=10.0.0'} + + http-errors@2.0.1: + resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} + engines: {node: '>= 0.8'} + + https-proxy-agent@7.0.6: + resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} + engines: {node: '>= 14'} + + hyperdyperid@1.2.0: + resolution: {integrity: sha512-Y93lCzHYgGWdrJ66yIktxiaGULYc6oGiABxhcO5AufBeOyoIdZF7bIfLaOrbM0iGIOXQQgxxRrFEnb+Y6w1n4A==} + engines: {node: '>=10.18'} + + iconv-lite@0.4.24: + resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} + engines: {node: '>=0.10.0'} + + iconv-lite@0.6.3: + resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} + engines: {node: '>=0.10.0'} + + ieee754@1.2.1: + resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + + ignore@5.3.2: + resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} + engines: {node: '>= 4'} + + ignore@7.0.12: + resolution: {integrity: sha512-/8UvqAPU9DGTI9k4mxtf49U37Isfwr8Uts96+SBHIkFxPJnHS0Ew4f00sM4Scd8V8EjM0jUNtVDdv1kPdt35lg==} + engines: {node: '>= 4'} + + import-fresh@3.3.1: + resolution: {integrity: sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==} + engines: {node: '>=6'} + + import-in-the-middle@3.5.2: + resolution: {integrity: sha512-WseIA/4o56+GYO3RmlOPvejYi771OFeJEeK/ejROsr9bdjW+E5NXTOD1Di8TL40Qgg3JJX9aa97Hf8iiSVIKZw==} + engines: {node: '>=18'} + + imurmurhash@0.1.4: + resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} + engines: {node: '>=0.8.19'} + + inherits@2.0.4: + resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + + ipaddr.js@1.9.1: + resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} + engines: {node: '>= 0.10'} + + is-extglob@2.1.1: + resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} + engines: {node: '>=0.10.0'} + + is-fullwidth-code-point@3.0.0: + resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} + engines: {node: '>=8'} + + is-glob@4.0.3: + resolution: {integrity: sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==} + engines: {node: '>=0.10.0'} + + isexe@2.0.0: + resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + + iterare@1.2.1: + resolution: {integrity: sha512-RKYVTCjAnRthyJes037NX/IiqeidgN1xc3j1RjFfECFp28A1GVwK9nA+i0rJPaHqSZwygLzRnFlzUuHFoWWy+Q==} + engines: {node: '>=6'} + + jackspeak@3.4.3: + resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} + + jest-worker@27.5.1: + resolution: {integrity: sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==} + engines: {node: '>= 10.13.0'} + + jiti@2.7.0: + resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} + hasBin: true + + jose@6.2.12: + resolution: {integrity: sha512-9NiFmJEex0sy2Dk58j2UGBSHgUs2ypF9eZSu4L6vjOX3Dp96Sw1F3uL+H+D1sx02jZZdzUT0HgvCy59CuvXcWw==} + + js-tokens@4.0.0: + resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + + js-yaml@4.3.2: + resolution: {integrity: sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==} + hasBin: true + + json-bigint@1.0.0: + resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==} + + json-buffer@3.0.1: + resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} + + json-schema-traverse@0.4.1: + resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} + + json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + + json-stable-stringify-without-jsonify@1.0.1: + resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} + + keyv@4.5.4: + resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} + + levn@0.4.1: + resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==} + engines: {node: '>= 0.8.0'} + + lightningcss-android-arm64@1.32.0: + resolution: {integrity: sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [android] + + lightningcss-darwin-arm64@1.32.0: + resolution: {integrity: sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [darwin] + + lightningcss-darwin-x64@1.32.0: + resolution: {integrity: sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [darwin] + + lightningcss-freebsd-x64@1.32.0: + resolution: {integrity: sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [freebsd] + + lightningcss-linux-arm-gnueabihf@1.32.0: + resolution: {integrity: sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==} + engines: {node: '>= 12.0.0'} + cpu: [arm] + os: [linux] + + lightningcss-linux-arm64-gnu@1.32.0: + resolution: {integrity: sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + + lightningcss-linux-arm64-musl@1.32.0: + resolution: {integrity: sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [linux] + + lightningcss-linux-x64-gnu@1.32.0: + resolution: {integrity: sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + + lightningcss-linux-x64-musl@1.32.0: + resolution: {integrity: sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [linux] + + lightningcss-win32-arm64-msvc@1.32.0: + resolution: {integrity: sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==} + engines: {node: '>= 12.0.0'} + cpu: [arm64] + os: [win32] + + lightningcss-win32-x64-msvc@1.32.0: + resolution: {integrity: sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==} + engines: {node: '>= 12.0.0'} + cpu: [x64] + os: [win32] + + lightningcss@1.32.0: + resolution: {integrity: sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==} + engines: {node: '>= 12.0.0'} + + linkifyjs@4.3.3: + resolution: {integrity: sha512-P8aEP5U/D1/IlTY2OeYsErdwh9bGuLE30NcXtKEjgdHcahveQoQwM2yZNsioQHsWFz0P7KKudisbrzCgR0sDHg==} + + locate-path@6.0.0: + resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} + engines: {node: '>=10'} + + lodash.camelcase@4.3.0: + resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} + + lodash.merge@4.6.2: + resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} + + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + + loose-envify@1.4.0: + resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} + hasBin: true + + loupe@3.2.1: + resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} + + lru-cache@10.4.3: + resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + + magic-string@0.30.21: + resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + + math-intrinsics@1.1.0: + resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} + engines: {node: '>= 0.4'} + + media-typer@0.3.0: + resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} + engines: {node: '>= 0.6'} + + memfs@4.80.0: + resolution: {integrity: sha512-N70W2isi44XdGn7uPo9NJ4zRukcdkwSFHlnYx42SWOH/+hzqNc8P7+tVoA2up4z7faHvNi1RSNCFjxoYQ71VPQ==} + + merge-descriptors@1.0.3: + resolution: {integrity: sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==} + + merge-stream@2.0.0: + resolution: {integrity: sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==} + + methods@1.1.2: + resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==} + engines: {node: '>= 0.6'} + + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-db@1.54.0: + resolution: {integrity: sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + + mime@1.6.0: + resolution: {integrity: sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==} + engines: {node: '>=4'} + hasBin: true + + minimatch@10.2.6: + resolution: {integrity: sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==} + engines: {node: 18 || 20 || >=22} + + minimatch@3.1.5: + resolution: {integrity: sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==} + + minimatch@9.0.9: + resolution: {integrity: sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==} + engines: {node: '>=16 || 14 >=14.17'} + + minimist@1.2.8: + resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + + minimizer-webpack-plugin@5.13.0: + resolution: {integrity: sha512-OO1NF2zHj6sf4CNv2AsUWYcS8WEwkVKQDXxxKzlOxx1Xl+WlqYtVC+oNlGNwGkLAOnpj277BSHypWwRtd9w4JQ==} + engines: {node: '>= 10.13.0'} + peerDependencies: + '@minify-html/node': '*' + '@napi-rs/image': '*' + '@swc/core': '*' + '@swc/css': '*' + '@swc/html': '*' + clean-css: '*' + cssnano: '*' + csso: '*' + esbuild: '*' + html-minifier-terser: '*' + imagemin: '*' + lightningcss: '*' + postcss: '*' + sharp: '*' + svgo: '*' + uglify-js: '*' + webpack: ^5.1.0 + peerDependenciesMeta: + '@minify-html/node': + optional: true + '@napi-rs/image': + optional: true + '@swc/core': + optional: true + '@swc/css': + optional: true + '@swc/html': + optional: true + clean-css: + optional: true + cssnano: + optional: true + csso: + optional: true + esbuild: + optional: true + html-minifier-terser: + optional: true + imagemin: + optional: true + lightningcss: + optional: true + postcss: + optional: true + sharp: + optional: true + svgo: + optional: true + uglify-js: + optional: true + + minipass@7.1.3: + resolution: {integrity: sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==} + engines: {node: '>=16 || 14 >=14.17'} + + mkdirp@0.5.6: + resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==} + hasBin: true + + module-details-from-path@1.0.4: + resolution: {integrity: sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==} + + ms@2.0.0: + resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + ms@3.0.0-canary.1: + resolution: {integrity: sha512-kh8ARjh8rMN7Du2igDRO9QJnqCb2xYTJxyQYK7vJJS4TvLLmsbyhiKpSW+t+y26gyOyMd0riphX0GeWKU3ky5g==} + engines: {node: '>=12.13'} + + multer@2.0.2: + resolution: {integrity: sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==} + engines: {node: '>= 10.16.0'} + + nanoid@3.3.20: + resolution: {integrity: sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + natural-compare@1.4.0: + resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} + + negotiator@0.6.3: + resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} + engines: {node: '>= 0.6'} + + next@14.2.35: + resolution: {integrity: sha512-KhYd2Hjt/O1/1aZVX3dCwGXM1QmOV4eNM2UTacK5gipDdPN/oHHK/4oVGy7X8GMfPMsUTUEmGlsy0EY1YGAkig==} + engines: {node: '>=18.17.0'} + hasBin: true + peerDependencies: + '@opentelemetry/api': ^1.1.0 + '@playwright/test': ^1.41.2 + react: ^18.2.0 + react-dom: ^18.2.0 + sass: ^1.3.0 + peerDependenciesMeta: + '@opentelemetry/api': + optional: true + '@playwright/test': + optional: true + sass: + optional: true + + nexus-rpc@0.0.3: + resolution: {integrity: sha512-ksIBeYHDv6FHAR1h3Vwk0kQ6REhdo1STqDHXMzu/yn/VN/XzgOor2IUjewG6paoA7IgxlIPZoOU/PD78PC6HTg==} + engines: {node: '>= 20.0.0'} + + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead + + node-fetch@2.7.0: + resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + + node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + + node-releases@2.0.57: + resolution: {integrity: sha512-kQK9LGGFiHtrWiNhZtA7Qbw17AQz+dmsEKODRIVTXA9+e5MS/2gZEBhYJt13GrAz5/IOZKddH/0Z3TP/Zgo+yw==} + engines: {node: '>=18'} + + object-assign@4.1.1: + resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} + engines: {node: '>=0.10.0'} + + object-inspect@1.13.4: + resolution: {integrity: sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==} + engines: {node: '>= 0.4'} + + on-exit-leak-free@2.1.2: + resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==} + engines: {node: '>=14.0.0'} + + on-finished@2.4.1: + resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} + engines: {node: '>= 0.8'} + + optionator@0.9.4: + resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} + engines: {node: '>= 0.8.0'} + + orderedmap@2.1.1: + resolution: {integrity: sha512-TvAWxi0nDe1j/rtMcWcIj94+Ffe6n7zhow33h40SKxmsmozs6dz/e+EajymfoFcHd7sxNn8yHM8839uixMOV6g==} + + p-limit@3.1.0: + resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} + engines: {node: '>=10'} + + p-locate@5.0.0: + resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} + engines: {node: '>=10'} + + package-json-from-dist@1.0.1: + resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} + + parent-module@1.0.1: + resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} + engines: {node: '>=6'} + + parseurl@1.3.3: + resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} + engines: {node: '>= 0.8'} + + path-exists@4.0.0: + resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} + engines: {node: '>=8'} + + path-key@3.1.1: + resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} + engines: {node: '>=8'} + + path-scurry@1.11.1: + resolution: {integrity: sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==} + engines: {node: '>=16 || 14 >=14.18'} + + path-to-regexp@0.1.13: + resolution: {integrity: sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA==} + + path-to-regexp@3.3.0: + resolution: {integrity: sha512-qyCH421YQPS2WFDxDjftfc1ZR5WKQzVzqsp4n9M2kQhVOo/ByahFoUNJfl58kOcEGfQ//7weFTDhm+ss8Ecxgw==} + + pathe@1.1.2: + resolution: {integrity: sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==} + + pathval@2.0.1: + resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} + engines: {node: '>= 14.16'} + + pg-cloudflare@1.4.1: + resolution: {integrity: sha512-6PQbsFWZcp9EmJEwy5cGQ2La+AMWpP46lgbb8X+U/XsHIUweYDNCpeuKck5RxL2MdVFi7krbbEi5nX4Zh7JhrQ==} + + pg-connection-string@2.14.1: + resolution: {integrity: sha512-qR3kGNPBLpCNtz0evbKA0Y/MRFXwSSdT+pTJvYp/bXTcReZbvX1kzF0IyTc1QnxqF7AZbOeBhNL8R5mYQZV/MA==} + + pg-int8@1.0.1: + resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} + engines: {node: '>=4.0.0'} + + pg-pool@3.14.0: + resolution: {integrity: sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==} + peerDependencies: + pg: '>=8.0' + + pg-protocol@1.16.1: + resolution: {integrity: sha512-p9VOFMiHB/ZbJATetbg+99PxssTVSQRnyuPSQ67mN1+1KBOjZaZ83ZQzltnxPhJwSsC3nwVjJ10DVJlerbFzLg==} + + pg-types@2.2.0: + resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} + engines: {node: '>=4'} + + pg@8.23.1: + resolution: {integrity: sha512-aL96AHANtWjPLDOLqnhx+ngp9+UK7ETEU8VJrDCGvsSSi/mGLcWYsS6Herg7lmaBJe4uwrfqsa7gTEFaSizDoQ==} + engines: {node: '>= 16.0.0'} + peerDependencies: + pg-native: '>=3.0.1' + peerDependenciesMeta: + pg-native: + optional: true + + pgpass@1.0.5: + resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + picomatch@4.0.7: + resolution: {integrity: sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==} + engines: {node: '>=12'} + + pino-abstract-transport@2.0.0: + resolution: {integrity: sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw==} + + pino-std-serializers@7.1.0: + resolution: {integrity: sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==} + + pino@9.14.0: + resolution: {integrity: sha512-8OEwKp5juEvb/MjpIc4hjqfgCNysrS94RIOMXYvpYCdm/jglrKEiAYmiumbmGhCvs+IcInsphYDFwqrjr7398w==} + hasBin: true + + playwright-core@1.63.0: + resolution: {integrity: sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==} + engines: {node: '>=20'} + hasBin: true + + playwright@1.63.0: + resolution: {integrity: sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==} + engines: {node: '>=20'} + hasBin: true + + postcss@8.4.31: + resolution: {integrity: sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ==} + engines: {node: ^10 || ^12 || >=14} + + postcss@8.5.29: + resolution: {integrity: sha512-49cGhUbXj8Qenv0iTMxA1cFBzxXoctpC9Ujd77t1WcbJIr6nF/eI7g/8MgxrYldFRuAXvja7xQRwavoW7kgrxQ==} + engines: {node: ^10 || ^12 || >=14} + + postgres-array@2.0.0: + resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} + engines: {node: '>=4'} + + postgres-bytea@1.0.1: + resolution: {integrity: sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==} + engines: {node: '>=0.10.0'} + + postgres-date@1.0.7: + resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} + engines: {node: '>=0.10.0'} + + postgres-interval@1.2.0: + resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} + engines: {node: '>=0.10.0'} + + prelude-ls@1.2.1: + resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} + engines: {node: '>= 0.8.0'} + + process-warning@5.1.0: + resolution: {integrity: sha512-jQSaVHsPgtyw60e1rQ/A+/ArPEj/S8pS/vFnyGa/gYFXrKk/6RuDkoqVDQ5NI5MmS01698ltlAk0NoDBNLujRw==} + + prosemirror-changeset@2.4.4: + resolution: {integrity: sha512-Gyuz9jYdrAeA9I7AholyOes9KUmv/lEwsC5O9s5xF5VB3EgTPsXMGUxz4+H1B+wKVPdZbvRcg1Cp3x8qX+8ofA==} + + prosemirror-commands@1.7.2: + resolution: {integrity: sha512-q6Q6szxqdu9Xd6EcdKsqXghu5nQdZTpB4Q9yd04WRc7/jt763e/rT60Owh0L1GYY+T46o5rD+9lEN36dZS43tw==} + + prosemirror-dropcursor@1.8.4: + resolution: {integrity: sha512-VmKxhcA6a+h5oPSYmfF1ZNeQqPbyYaIxqZsKqeYCyTlPtrjUgNaHL3P8vVaO19oqAV0zuaM/axkowZHrB1P3Tg==} + + prosemirror-gapcursor@1.4.1: + resolution: {integrity: sha512-pMdYaEnjNMSwl11yjEGtgTmLkR08m/Vl+Jj443167p9eB3HVQKhYCc4gmHVDsLPODfZfjr/MmirsdyZziXbQKw==} + + prosemirror-history@1.5.1: + resolution: {integrity: sha512-onlwnqKvgtFg6srZOmDOAhlAy1FIHQpOExORybX0S+DvJepE5tPALrB9iGet/2sqqqrelSnB10OoiOJqAvu2DA==} + + prosemirror-inputrules@1.5.1: + resolution: {integrity: sha512-7wj4uMjKaXWAQ1CDgxNzNtR9AlsuwzHfdFH1ygEHA2KHF2DOEaXl1CJfNPAKCg9qNEh4rum975QLaCiQPyY6Fw==} + + prosemirror-keymap@1.2.3: + resolution: {integrity: sha512-4HucRlpiLd1IPQQXNqeo81BGtkY8Ai5smHhKW9jjPKRc2wQIxksg7Hl1tTI2IfT2B/LgX6bfYvXxEpJl7aKYKw==} + + prosemirror-model@1.25.12: + resolution: {integrity: sha512-Ue2gTmXMa7EhpLNhC7J+h4+ykD8ha12K6rrZFFKKJHBForfIStw5gJ6Zrf1mqqAa9NmpmEB4wp9bKmA4eUVjWg==} + + prosemirror-schema-list@1.5.1: + resolution: {integrity: sha512-927lFx/uwyQaGwJxLWCZRkjXG0p48KpMj6ueoYiu4JX05GGuGcgzAy62dfiV8eFZftgyBUvLx76RsMe20fJl+Q==} + + prosemirror-state@1.4.4: + resolution: {integrity: sha512-6jiYHH2CIGbCfnxdHbXZ12gySFY/fz/ulZE333G6bPqIZ4F+TXo9ifiR86nAHpWnfoNjOb3o5ESi7J8Uz1jXHw==} + + prosemirror-tables@1.8.5: + resolution: {integrity: sha512-V/0cDCsHKHe/tfWkeCmthNUcEp1IVO3p6vwN8XtwE9PZQLAZJigbw3QoraAdfJPir4NKJtNvOB8oYGKRl+t0Dw==} + + prosemirror-transform@1.12.2: + resolution: {integrity: sha512-PE/aY0HEY4zczvmqrilgkUK/WautF0chvMqkmM/iN5/aPHvwrCWaQZjS6CIZp+K84YrvPFqrL+YArrfJX8xX7g==} + + prosemirror-view@1.42.6: + resolution: {integrity: sha512-3g25f6hKV2bBxo3puH3EcYWmmVNSH0iWmNsMVT7sugDSpA025jgpOLarcDWr39SdzPjYMaiYtRdNtXialPSjEQ==} + + protobufjs@7.6.6: + resolution: {integrity: sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==} + engines: {node: '>=12.0.0'} + + protobufjs@8.8.0: + resolution: {integrity: sha512-N3xhQ5yyBx3vQq4gubBfASzYhJGNzeDbjqBpu61g7UVylsN/qyffU96TKWD3GbbLOKF82VGNRNvv1+BFgE31Eg==} + engines: {node: '>=12.0.0'} + + proxy-addr@2.0.8: + resolution: {integrity: sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==} + engines: {node: '>= 0.10'} + + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + + qs@6.14.2: + resolution: {integrity: sha512-V/yCWTTF7VJ9hIh18Ugr2zhJMP01MY7c5kh4J870L7imm6/DIzBsNLTXzMwUA3yZ5b/KBqLx8Kp3uRvd7xSe3Q==} + engines: {node: '>=0.6'} + + quick-format-unescaped@4.0.4: + resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==} + + range-parser@1.2.1: + resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} + engines: {node: '>= 0.6'} + + raw-body@2.5.3: + resolution: {integrity: sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==} + engines: {node: '>= 0.8'} + + react-dom@18.3.1: + resolution: {integrity: sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==} + peerDependencies: + react: ^18.3.1 + + react-remove-scroll-bar@2.3.8: + resolution: {integrity: sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': '*' + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + peerDependenciesMeta: + '@types/react': + optional: true + + react-remove-scroll@2.7.2: + resolution: {integrity: sha512-Iqb9NjCCTt6Hf+vOdNIZGdTiH1QSqr27H/Ek9sv/a97gfueI/5h1s3yRi1nngzMUaOOToin5dI1dXKdXiF+u0Q==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': '*' + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + react-style-singleton@2.2.3: + resolution: {integrity: sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': '*' + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + react@18.3.1: + resolution: {integrity: sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==} + engines: {node: '>=0.10.0'} + + readable-stream@3.6.2: + resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} + engines: {node: '>= 6'} + + real-require@0.2.0: + resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==} + engines: {node: '>= 12.13.0'} + + reflect-metadata@0.2.2: + resolution: {integrity: sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==} + + require-directory@2.1.1: + resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} + engines: {node: '>=0.10.0'} + + require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + + require-in-the-middle@8.0.1: + resolution: {integrity: sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==} + engines: {node: '>=9.3.0 || >=8.10.0 <9.0.0'} + + resolve-from@4.0.0: + resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} + engines: {node: '>=4'} + + rimraf@5.0.10: + resolution: {integrity: sha512-l0OE8wL34P4nJH/H2ffoaniAokM2qSmrtXHmlpvYr5AVVX8msAyW0l8NVJFDxlSK4u3Uh/f41cQheDVdnYijwQ==} + hasBin: true + + rollup@4.64.0: + resolution: {integrity: sha512-gjA/kFeDfffII8COoPsvLPvE16hWDTEc1+0QKQNKUkuV8iJjqsC4xXs8/FzuL0v9EV39kuxTbAgcFoTRyy2TGQ==} + engines: {node: '>=18.0.0', npm: '>=8.0.0'} + hasBin: true + + rope-sequence@1.3.4: + resolution: {integrity: sha512-UT5EDe2cu2E/6O4igUr5PSFs23nvvukicWHx6GnOPlHAiiYbzNuCRQCuiUdHJQcqKalLKlrYJnjY0ySGsXNQXQ==} + + rxjs@7.8.2: + resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==} + + safe-buffer@5.2.1: + resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + + safe-stable-stringify@2.5.0: + resolution: {integrity: sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==} + engines: {node: '>=10'} + + safer-buffer@2.1.2: + resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + + scheduler@0.23.2: + resolution: {integrity: sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==} + + schema-utils@4.5.0: + resolution: {integrity: sha512-zJlMCZ0cAR5p/Y4oVpRoqioDMJcGxaXRrQ/4rP4WyR84vc5z/DolXdbvXeDpTwbtocDFr2rhPqHPErDCUtz2kA==} + engines: {node: '>= 10.13.0'} + + semver@7.8.5: + resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} + engines: {node: '>=10'} + hasBin: true + + send@0.19.2: + resolution: {integrity: sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==} + engines: {node: '>= 0.8.0'} + + serve-static@1.16.3: + resolution: {integrity: sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==} + engines: {node: '>= 0.8.0'} + + setprototypeof@1.2.0: + resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + + shebang-command@2.0.0: + resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} + engines: {node: '>=8'} + + shebang-regex@3.0.0: + resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} + engines: {node: '>=8'} + + side-channel-list@1.0.1: + resolution: {integrity: sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==} + engines: {node: '>= 0.4'} + + side-channel-map@1.0.1: + resolution: {integrity: sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==} + engines: {node: '>= 0.4'} + + side-channel-weakmap@1.0.2: + resolution: {integrity: sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==} + engines: {node: '>= 0.4'} + + side-channel@1.1.1: + resolution: {integrity: sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==} + engines: {node: '>= 0.4'} + + siginfo@2.0.0: + resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + + signal-exit@4.1.0: + resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} + engines: {node: '>=14'} + + sonic-boom@4.2.1: + resolution: {integrity: sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==} + + source-map-js@1.2.2: + resolution: {integrity: sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==} + engines: {node: '>=0.10.0'} + + source-map-loader@5.0.0: + resolution: {integrity: sha512-k2Dur7CbSLcAH73sBcIkV5xjPV4SzqO1NJ7+XaQl8if3VODDUj3FNchNGpqgJSKbvUfJuhVdv8K2Eu8/TNl2eA==} + engines: {node: '>= 18.12.0'} + peerDependencies: + webpack: ^5.72.1 + + source-map-support@0.5.21: + resolution: {integrity: sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==} + + source-map@0.6.1: + resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} + engines: {node: '>=0.10.0'} + + source-map@0.7.6: + resolution: {integrity: sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==} + engines: {node: '>= 12'} + + split2@4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} + engines: {node: '>= 10.x'} + + stackback@0.0.2: + resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + + statuses@2.0.2: + resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} + engines: {node: '>= 0.8'} + + std-env@3.10.0: + resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} + + streamsearch@1.1.0: + resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} + engines: {node: '>=10.0.0'} + + string-width@4.2.3: + resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} + engines: {node: '>=8'} + + string-width@5.1.2: + resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} + engines: {node: '>=12'} + + string_decoder@1.3.0: + resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + + strip-ansi@6.0.1: + resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} + engines: {node: '>=8'} + + strip-ansi@7.2.0: + resolution: {integrity: sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==} + engines: {node: '>=12'} + + strip-json-comments@3.1.1: + resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} + engines: {node: '>=8'} + + strtok3@10.3.5: + resolution: {integrity: sha512-ki4hZQfh5rX0QDLLkOCj+h+CVNkqmp/CMf8v8kZpkNVK6jGQooMytqzLZYUVYIZcFZ6yDB70EfD8POcFXiF5oA==} + engines: {node: '>=18'} + + styled-jsx@5.1.1: + resolution: {integrity: sha512-pW7uC1l4mBZ8ugbiZrcIsiIvVx1UmTfw7UkC3Um2tmfUq9Bhk8IiyEIPl6F8agHgjzku6j0xQEZbfA5uSgSaCw==} + engines: {node: '>= 12.0.0'} + peerDependencies: + '@babel/core': '*' + babel-plugin-macros: '*' + react: '>= 16.8.0 || 17.x.x || ^18.0.0-0' + peerDependenciesMeta: + '@babel/core': + optional: true + babel-plugin-macros: + optional: true + + supports-color@7.2.0: + resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} + engines: {node: '>=8'} + + supports-color@8.1.1: + resolution: {integrity: sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==} + engines: {node: '>=10'} + + swc-loader@0.2.7: + resolution: {integrity: sha512-nwYWw3Fh9ame3Rtm7StS9SBLpHRRnYcK7bnpF3UKZmesAK0gw2/ADvlURFAINmPvKtDLzp+GBiP9yLoEjg6S9w==} + peerDependencies: + '@swc/core': ^1.2.147 + webpack: '>=2' + + systeminformation@5.33.15: + resolution: {integrity: sha512-s5RloL0F7xhN0ZjfHtLGnbf7EW74EI0N8NFL+1xR7Oxz9QA33obNZsl7lyfnJOhN04E/kGFPoJPlvY7TZ5fzmw==} + engines: {node: '>=10.0.0'} + os: [darwin, linux, win32, freebsd, openbsd, netbsd, sunos, android] + hasBin: true + + tailwindcss@4.3.3: + resolution: {integrity: sha512-gOhV3P7ufE62QDGg1zVaTgCR+EtPv92k2nIhVcVKcLmxT1sUBsQGhnZj175j+MqRt4zLF7ic+sCYjfhxMxj7YQ==} + + tapable@2.3.3: + resolution: {integrity: sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==} + engines: {node: '>=6'} + + terser@5.51.2: + resolution: {integrity: sha512-bWnjSNscmuI+GJze6ZupnHP8G/cTcsJF+bXCeQknk2SHQsgbNJnLrqiH9jZ2W4STPVXH2mDKKRX3iwPhc9Cn/Q==} + engines: {node: '>=10'} + hasBin: true + + thingies@2.6.1: + resolution: {integrity: sha512-cV/CMGTK3M4MlnJ/0At6ismOw/A0EEniDNScajjz/Br3c1sqE72YD01rGpPTKwd27wAxI5Pr+6+0w8yofzFRYw==} + engines: {node: '>=10.18'} + peerDependencies: + tslib: ^2 + + thread-stream@3.2.0: + resolution: {integrity: sha512-zLBvqpwr4Esa0kRjcrzGU6zL25lePWaCLMx0RQFrmteozIfeNdaMLpG5U7PeHzvlFkAWaRKA9/KVW4F60iB+qw==} + + tinybench@2.9.0: + resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} + + tinyexec@0.3.2: + resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} + + tinyglobby@0.2.17: + resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} + engines: {node: '>=12.0.0'} + + tinypool@1.1.1: + resolution: {integrity: sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==} + engines: {node: ^18.0.0 || >=20.0.0} + + tinyrainbow@1.2.0: + resolution: {integrity: sha512-weEDEq7Z5eTHPDh4xjX789+fHfF+P8boiFB+0vbWzpbnbsEr/GRaohi/uMKxg8RZMXnl1ItAi/IUHWMsjDV7kQ==} + engines: {node: '>=14.0.0'} + + tinyspy@3.0.2: + resolution: {integrity: sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q==} + engines: {node: '>=14.0.0'} + + toidentifier@1.0.1: + resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} + engines: {node: '>=0.6'} + + token-types@6.1.2: + resolution: {integrity: sha512-dRXchy+C0IgK8WPC6xvCHFRIWYUbqqdEIKPaKo/AcTUNzwLTK6AH7RjdLWsEZcAN/TBdtfUw3PYEgPr5VPr6ww==} + engines: {node: '>=14.16'} + + tr46@0.0.3: + resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + + tree-dump@1.1.0: + resolution: {integrity: sha512-rMuvhU4MCDbcbnleZTFezWsaZXRFemSqAM+7jPnzUl1fo9w3YEKOxAeui0fz3OI4EU4hf23iyA7uQRVko+UaBA==} + engines: {node: '>=10.0'} + peerDependencies: + tslib: '2' + + ts-api-utils@2.5.0: + resolution: {integrity: sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==} + engines: {node: '>=18.12'} + peerDependencies: + typescript: '>=4.8.4' + + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + + tsx@4.23.15: + resolution: {integrity: sha512-Yiex1Ovn8z2xPpOWckIiysV1SSyRMY9BkLF++q0yKiDxCqRhosKfMg3janKkiLBwZ5c/YryloKwGZcrEmtwxKw==} + engines: {node: '>=18.0.0'} + hasBin: true + + turbo@2.11.7: + resolution: {integrity: sha512-RhXHbqfvY6o3xt/zSHCwSNwLrZlXiqHzewi4eAj9PtZ1T1CqgMJ2yIDb7SKPTG9IikTm925mfzL6hldQVdyNWw==} + hasBin: true + + type-check@0.4.0: + resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} + engines: {node: '>= 0.8.0'} + + type-is@1.6.18: + resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} + engines: {node: '>= 0.6'} + + typedarray@0.0.6: + resolution: {integrity: sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==} + + typescript-eslint@8.71.0: + resolution: {integrity: sha512-fBdHYiqQ14RW6mOMXD14Svn82ZsCYAoQSzGRzyEjR59S5A2Krh/l7fGTOQ7iCr8gGy/mHVXtEF7s5fgjEdV0Pw==} + engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + peerDependencies: + eslint: ^8.57.0 || ^9.0.0 || ^10.0.0 + typescript: '>=4.8.4 <6.1.0' + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + uid@2.0.2: + resolution: {integrity: sha512-u3xV3X7uzvi5b1MncmZo3i2Aw222Zk1keqLA1YkHldREkAhAqi65wuPfe7lHx8H/Wzy+8CE7S7uS3jekIM5s8g==} + engines: {node: '>=8'} + + uint8array-extras@1.6.0: + resolution: {integrity: sha512-8iAasVS4wUx0gPLjH8Xtz2PeDzTSiy8QiLGu2DT3X5GU+egFEQhpnbtkehbAwjvDcxIERmCYcysiODFmE3Ud0Q==} + engines: {node: '>=18'} + + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + + unionfs@4.6.0: + resolution: {integrity: sha512-fJAy3gTHjFi5S3TP5EGdjs/OUMFFvI/ady3T8qVuZfkv8Qi8prV/Q8BuFEgODJslhZTT2z2qdD2lGdee9qjEnA==} + + unpipe@1.0.0: + resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} + engines: {node: '>= 0.8'} + + update-browserslist-db@1.3.3: + resolution: {integrity: sha512-pJ2sYawQS0R/WI928Gj5GlPhTGzbMelq0+4INtSYNDV9ErKJcX6xjGWkoG/VnB3dpUm00zALaqkrUD77pO5TDQ==} + hasBin: true + peerDependencies: + browserslist: '>= 4.21.0' + + uri-js@4.4.1: + resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + + use-callback-ref@1.3.3: + resolution: {integrity: sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': '*' + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + use-sidecar@1.1.3: + resolution: {integrity: sha512-Fedw0aZvkhynoPYlA5WXrMCAMm+nSWdZt6lzJQ7Ok8S6Q+VsHmHpRWndVRJ8Be0ZbkfPc5LRYH+5XrzXcEeLRQ==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': '*' + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + use-sync-external-store@1.7.0: + resolution: {integrity: sha512-6L+EeigHMQhdaIPNIFUKwfWJSwWFQ8gJbJ2DLOs5sDIegTwR9fRxvnM3uciHKjIZhFz+KAv2emhWMRvDmMcY8A==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + + util-deprecate@1.0.2: + resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + + utils-merge@1.0.1: + resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==} + engines: {node: '>= 0.4.0'} + + uuid@11.1.1: + resolution: {integrity: sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ==} + hasBin: true + + vary@1.1.2: + resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} + engines: {node: '>= 0.8'} + + vite-node@2.1.9: + resolution: {integrity: sha512-AM9aQ/IPrW/6ENLQg3AGY4K1N2TGZdR5e4gu/MmmR2xR3Ll1+dib+nook92g4TV3PXVyeyxdWwtaCAiUL0hMxA==} + engines: {node: ^18.0.0 || >=20.0.0} + hasBin: true + + vite@5.4.21: + resolution: {integrity: sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==} + engines: {node: ^18.0.0 || >=20.0.0} + hasBin: true + peerDependencies: + '@types/node': ^18.0.0 || >=20.0.0 + less: '*' + lightningcss: ^1.21.0 + sass: '*' + sass-embedded: '*' + stylus: '*' + sugarss: '*' + terser: ^5.4.0 + peerDependenciesMeta: + '@types/node': + optional: true + less: + optional: true + lightningcss: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + + vitest@2.1.9: + resolution: {integrity: sha512-MSmPM9REYqDGBI8439mA4mWhV5sKmDlBKWIYbA3lRb2PTHACE0mgKwA8yQ2xq9vxDTuk4iPrECBAEW2aoFXY0Q==} + engines: {node: ^18.0.0 || >=20.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@types/node': ^18.0.0 || >=20.0.0 + '@vitest/browser': 2.1.9 + '@vitest/ui': 2.1.9 + happy-dom: '*' + jsdom: '*' + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@types/node': + optional: true + '@vitest/browser': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + + w3c-keyname@2.2.8: + resolution: {integrity: sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==} + + watchpack@2.5.2: + resolution: {integrity: sha512-6i/00NBjP4yGPs+caKSyRfpTF/8Torsu0MOW3mMzIbhgISFder8i7xbqgHlLMwJrdiN8ndBV3UA1/AfzPSr+jg==} + engines: {node: '>=10.13.0'} + + web-streams-polyfill@3.3.3: + resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} + engines: {node: '>= 8'} + + webidl-conversions@3.0.1: + resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} + + webpack-sources@3.6.0: + resolution: {integrity: sha512-EIMmPVvNI0CYXyRPp34F2Qk7W37/BZVZIofS6LAUnrNjCVahS+hO4mxwFJDjQjODTGZDtVzLYoN2+1dYkhk9qA==} + engines: {node: '>=10.13.0'} + + webpack@5.111.1: + resolution: {integrity: sha512-cNypaz0RP+S4cvQVi1M/3bRUkvNP0xZpL0TjFx+c6jg64VbxD+2weWDgY9UnjRI6dhZgtaj8DU76GGFcJ79xPQ==} + engines: {node: '>=10.13.0'} + hasBin: true + peerDependencies: + webpack-cli: '*' + peerDependenciesMeta: + webpack-cli: + optional: true + + whatwg-url@5.0.0: + resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} + + which@2.0.2: + resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} + engines: {node: '>= 8'} + hasBin: true + + why-is-node-running@2.3.0: + resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} + engines: {node: '>=8'} + hasBin: true + + word-wrap@1.2.5: + resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} + engines: {node: '>=0.10.0'} + + wrap-ansi@7.0.0: + resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} + engines: {node: '>=10'} + + wrap-ansi@8.1.0: + resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} + engines: {node: '>=12'} + + xtend@4.0.2: + resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} + engines: {node: '>=0.4'} + + y18n@5.0.8: + resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} + engines: {node: '>=10'} + + yaml@2.9.1: + resolution: {integrity: sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==} + engines: {node: '>= 14.6'} + hasBin: true + + yargs-parser@21.1.1: + resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} + engines: {node: '>=12'} + + yargs@17.7.3: + resolution: {integrity: sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g==} + engines: {node: '>=12'} + + yocto-queue@0.1.0: + resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} + engines: {node: '>=10'} + +snapshots: + + '@alloc/quick-lru@5.3.0': {} + + '@aws-sdk/checksums@3.1001.1': + dependencies: + '@aws-sdk/core': 3.978.1 + '@aws-sdk/types': 3.974.6 + '@smithy/core': 3.35.1 + '@smithy/types': 4.19.0 + tslib: 2.8.1 + + '@aws-sdk/client-s3@3.1146.0': + dependencies: + '@aws-sdk/checksums': 3.1001.1 + '@aws-sdk/core': 3.978.1 + '@aws-sdk/credential-provider-node': 3.972.84 + '@aws-sdk/middleware-sdk-s3': 3.972.77 + '@aws-sdk/signature-v4-multi-region': 3.996.47 + '@aws-sdk/types': 3.974.6 + '@smithy/core': 3.35.1 + '@smithy/fetch-http-handler': 5.8.0 + '@smithy/node-http-handler': 4.12.1 + '@smithy/types': 4.19.0 + tslib: 2.8.1 + + '@aws-sdk/core@3.978.1': + dependencies: + '@aws-sdk/types': 3.974.6 + '@aws-sdk/xml-builder': 3.972.41 + '@aws/lambda-invoke-store': 0.3.0 + '@smithy/core': 3.35.1 + '@smithy/signature-v4': 5.7.4 + '@smithy/types': 4.19.0 + bowser: 2.14.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-env@3.972.72': + dependencies: + '@aws-sdk/core': 3.978.1 + '@aws-sdk/types': 3.974.6 + '@smithy/core': 3.35.1 + '@smithy/types': 4.19.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-http@3.972.74': + dependencies: + '@aws-sdk/core': 3.978.1 + '@aws-sdk/types': 3.974.6 + '@smithy/core': 3.35.1 + '@smithy/fetch-http-handler': 5.8.0 + '@smithy/node-http-handler': 4.12.1 + '@smithy/types': 4.19.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-ini@3.973.17': + dependencies: + '@aws-sdk/core': 3.978.1 + '@aws-sdk/credential-provider-env': 3.972.72 + '@aws-sdk/credential-provider-http': 3.972.74 + '@aws-sdk/credential-provider-login': 3.972.79 + '@aws-sdk/credential-provider-process': 3.972.72 + '@aws-sdk/credential-provider-sso': 3.973.16 + '@aws-sdk/credential-provider-web-identity': 3.972.78 + '@aws-sdk/nested-clients': 3.997.46 + '@aws-sdk/types': 3.974.6 + '@smithy/core': 3.35.1 + '@smithy/credential-provider-imds': 4.5.2 + '@smithy/types': 4.19.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-login@3.972.79': + dependencies: + '@aws-sdk/core': 3.978.1 + '@aws-sdk/nested-clients': 3.997.46 + '@aws-sdk/types': 3.974.6 + '@smithy/core': 3.35.1 + '@smithy/types': 4.19.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-node@3.972.84': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.72 + '@aws-sdk/credential-provider-http': 3.972.74 + '@aws-sdk/credential-provider-ini': 3.973.17 + '@aws-sdk/credential-provider-process': 3.972.72 + '@aws-sdk/credential-provider-sso': 3.973.16 + '@aws-sdk/credential-provider-web-identity': 3.972.78 + '@aws-sdk/types': 3.974.6 + '@smithy/core': 3.35.1 + '@smithy/credential-provider-imds': 4.5.2 + '@smithy/types': 4.19.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-process@3.972.72': + dependencies: + '@aws-sdk/core': 3.978.1 + '@aws-sdk/types': 3.974.6 + '@smithy/core': 3.35.1 + '@smithy/types': 4.19.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-sso@3.973.16': + dependencies: + '@aws-sdk/core': 3.978.1 + '@aws-sdk/nested-clients': 3.997.46 + '@aws-sdk/token-providers': 3.1138.0 + '@aws-sdk/types': 3.974.6 + '@smithy/core': 3.35.1 + '@smithy/types': 4.19.0 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-web-identity@3.972.78': + dependencies: + '@aws-sdk/core': 3.978.1 + '@aws-sdk/nested-clients': 3.997.46 + '@aws-sdk/types': 3.974.6 + '@smithy/core': 3.35.1 + '@smithy/types': 4.19.0 + tslib: 2.8.1 + + '@aws-sdk/middleware-sdk-s3@3.972.77': + dependencies: + '@aws-sdk/core': 3.978.1 + '@aws-sdk/signature-v4-multi-region': 3.996.47 + '@aws-sdk/types': 3.974.6 + '@smithy/core': 3.35.1 + '@smithy/types': 4.19.0 + tslib: 2.8.1 + + '@aws-sdk/nested-clients@3.997.46': + dependencies: + '@aws-sdk/core': 3.978.1 + '@aws-sdk/signature-v4-multi-region': 3.996.47 + '@aws-sdk/types': 3.974.6 + '@smithy/core': 3.35.1 + '@smithy/fetch-http-handler': 5.8.0 + '@smithy/node-http-handler': 4.12.1 + '@smithy/types': 4.19.0 + tslib: 2.8.1 + + '@aws-sdk/s3-request-presigner@3.1146.0': + dependencies: + '@aws-sdk/core': 3.978.1 + '@aws-sdk/signature-v4-multi-region': 3.996.47 + '@aws-sdk/types': 3.974.6 + '@smithy/core': 3.35.1 + '@smithy/types': 4.19.0 + tslib: 2.8.1 + + '@aws-sdk/signature-v4-multi-region@3.996.47': + dependencies: + '@aws-sdk/types': 3.974.6 + '@smithy/signature-v4': 5.7.4 + '@smithy/types': 4.19.0 + tslib: 2.8.1 + + '@aws-sdk/token-providers@3.1138.0': + dependencies: + '@aws-sdk/core': 3.978.1 + '@aws-sdk/nested-clients': 3.997.46 + '@aws-sdk/types': 3.974.6 + '@smithy/core': 3.35.1 + '@smithy/types': 4.19.0 + tslib: 2.8.1 + + '@aws-sdk/types@3.974.6': + dependencies: + '@smithy/types': 4.19.0 + tslib: 2.8.1 + + '@aws-sdk/xml-builder@3.972.41': + dependencies: + '@smithy/types': 4.19.0 + tslib: 2.8.1 + + '@aws/lambda-invoke-store@0.3.0': {} + + '@borewit/text-codec@0.2.2': {} + + '@esbuild/aix-ppc64@0.21.5': + optional: true + + '@esbuild/aix-ppc64@0.28.2': + optional: true + + '@esbuild/android-arm64@0.21.5': + optional: true + + '@esbuild/android-arm64@0.28.2': + optional: true + + '@esbuild/android-arm@0.21.5': + optional: true + + '@esbuild/android-arm@0.28.2': + optional: true + + '@esbuild/android-x64@0.21.5': + optional: true + + '@esbuild/android-x64@0.28.2': + optional: true + + '@esbuild/darwin-arm64@0.21.5': + optional: true + + '@esbuild/darwin-arm64@0.28.2': + optional: true + + '@esbuild/darwin-x64@0.21.5': + optional: true + + '@esbuild/darwin-x64@0.28.2': + optional: true + + '@esbuild/freebsd-arm64@0.21.5': + optional: true + + '@esbuild/freebsd-arm64@0.28.2': + optional: true + + '@esbuild/freebsd-x64@0.21.5': + optional: true + + '@esbuild/freebsd-x64@0.28.2': + optional: true + + '@esbuild/linux-arm64@0.21.5': + optional: true + + '@esbuild/linux-arm64@0.28.2': + optional: true + + '@esbuild/linux-arm@0.21.5': + optional: true + + '@esbuild/linux-arm@0.28.2': + optional: true + + '@esbuild/linux-ia32@0.21.5': + optional: true + + '@esbuild/linux-ia32@0.28.2': + optional: true + + '@esbuild/linux-loong64@0.21.5': + optional: true + + '@esbuild/linux-loong64@0.28.2': + optional: true + + '@esbuild/linux-mips64el@0.21.5': + optional: true + + '@esbuild/linux-mips64el@0.28.2': + optional: true + + '@esbuild/linux-ppc64@0.21.5': + optional: true + + '@esbuild/linux-ppc64@0.28.2': + optional: true + + '@esbuild/linux-riscv64@0.21.5': + optional: true + + '@esbuild/linux-riscv64@0.28.2': + optional: true + + '@esbuild/linux-s390x@0.21.5': + optional: true + + '@esbuild/linux-s390x@0.28.2': + optional: true + + '@esbuild/linux-x64@0.21.5': + optional: true + + '@esbuild/linux-x64@0.28.2': + optional: true + + '@esbuild/netbsd-arm64@0.28.2': + optional: true + + '@esbuild/netbsd-x64@0.21.5': + optional: true + + '@esbuild/netbsd-x64@0.28.2': + optional: true + + '@esbuild/openbsd-arm64@0.28.2': + optional: true + + '@esbuild/openbsd-x64@0.21.5': + optional: true + + '@esbuild/openbsd-x64@0.28.2': + optional: true + + '@esbuild/openharmony-arm64@0.28.2': + optional: true + + '@esbuild/sunos-x64@0.21.5': + optional: true + + '@esbuild/sunos-x64@0.28.2': + optional: true + + '@esbuild/win32-arm64@0.21.5': + optional: true + + '@esbuild/win32-arm64@0.28.2': + optional: true + + '@esbuild/win32-ia32@0.21.5': + optional: true + + '@esbuild/win32-ia32@0.28.2': + optional: true + + '@esbuild/win32-x64@0.21.5': + optional: true + + '@esbuild/win32-x64@0.28.2': + optional: true + + '@eslint-community/eslint-utils@4.10.1(eslint@9.39.5(jiti@2.7.0))': + dependencies: + eslint: 9.39.5(jiti@2.7.0) + eslint-visitor-keys: 3.4.3 + + '@eslint-community/regexpp@4.12.2': {} + + '@eslint/config-array@0.21.2': + dependencies: + '@eslint/object-schema': 2.1.7 + debug: 4.4.3 + minimatch: 3.1.5 + transitivePeerDependencies: + - supports-color + + '@eslint/config-helpers@0.4.2': + dependencies: + '@eslint/core': 0.17.0 + + '@eslint/core@0.17.0': + dependencies: + '@types/json-schema': 7.0.15 + + '@eslint/eslintrc@3.3.7': + dependencies: + ajv: 6.15.0 + debug: 4.4.3 + espree: 10.4.0 + globals: 14.0.0 + ignore: 5.3.2 + import-fresh: 3.3.1 + js-yaml: 4.3.2 + minimatch: 3.1.5 + strip-json-comments: 3.1.1 + transitivePeerDependencies: + - supports-color + + '@eslint/js@9.39.5': {} + + '@eslint/object-schema@2.1.7': {} + + '@eslint/plugin-kit@0.4.1': + dependencies: + '@eslint/core': 0.17.0 + levn: 0.4.1 + + '@floating-ui/core@1.8.0': + dependencies: + '@floating-ui/utils': 0.2.12 + optional: true + + '@floating-ui/dom@1.8.0': + dependencies: + '@floating-ui/core': 1.8.0 + '@floating-ui/utils': 0.2.12 + optional: true + + '@floating-ui/utils@0.2.12': + optional: true + + '@grpc/grpc-js@1.14.5': + dependencies: + '@grpc/proto-loader': 0.8.1 + '@js-sdsl/ordered-map': 4.4.2 + + '@grpc/proto-loader@0.8.1': + dependencies: + lodash.camelcase: 4.3.0 + long: 5.3.2 + protobufjs: 7.6.6 + yargs: 17.7.3 + + '@humanfs/core@0.19.2': + dependencies: + '@humanfs/types': 0.15.0 + + '@humanfs/node@0.16.8': + dependencies: + '@humanfs/core': 0.19.2 + '@humanfs/types': 0.15.0 + '@humanwhocodes/retry': 0.4.3 + + '@humanfs/types@0.15.0': {} + + '@humanwhocodes/module-importer@1.0.1': {} + + '@humanwhocodes/retry@0.4.3': {} + + '@isaacs/cliui@8.0.2': + dependencies: + string-width: 5.1.2 + string-width-cjs: string-width@4.2.3 + strip-ansi: 7.2.0 + strip-ansi-cjs: strip-ansi@6.0.1 + wrap-ansi: 8.1.0 + wrap-ansi-cjs: wrap-ansi@7.0.0 + + '@jridgewell/gen-mapping@0.3.13': + dependencies: + '@jridgewell/sourcemap-codec': 1.6.0 + '@jridgewell/trace-mapping': 0.3.31 + + '@jridgewell/remapping@2.3.5': + dependencies: + '@jridgewell/gen-mapping': 0.3.13 + '@jridgewell/trace-mapping': 0.3.31 + + '@jridgewell/resolve-uri@3.1.2': {} + + '@jridgewell/source-map@0.3.11': + dependencies: + '@jridgewell/gen-mapping': 0.3.13 + '@jridgewell/trace-mapping': 0.3.31 + + '@jridgewell/sourcemap-codec@1.6.0': {} + + '@jridgewell/trace-mapping@0.3.31': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.6.0 + + '@js-sdsl/ordered-map@4.4.2': {} + + '@jsonjoy.com/base64@1.1.2(tslib@2.8.1)': + dependencies: + tslib: 2.8.1 + + '@jsonjoy.com/base64@17.67.0(tslib@2.8.1)': + dependencies: + tslib: 2.8.1 + + '@jsonjoy.com/buffers@1.2.1(tslib@2.8.1)': + dependencies: + tslib: 2.8.1 + + '@jsonjoy.com/buffers@17.67.0(tslib@2.8.1)': + dependencies: + tslib: 2.8.1 + + '@jsonjoy.com/codegen@1.0.0(tslib@2.8.1)': + dependencies: + tslib: 2.8.1 + + '@jsonjoy.com/codegen@17.67.0(tslib@2.8.1)': + dependencies: + tslib: 2.8.1 + + '@jsonjoy.com/fs-core@4.80.0(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/fs-node-builtins': 4.80.0(tslib@2.8.1) + '@jsonjoy.com/fs-node-utils': 4.80.0(tslib@2.8.1) + thingies: 2.6.1(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/fs-fsa@4.80.0(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/fs-core': 4.80.0(tslib@2.8.1) + '@jsonjoy.com/fs-node-builtins': 4.80.0(tslib@2.8.1) + '@jsonjoy.com/fs-node-utils': 4.80.0(tslib@2.8.1) + thingies: 2.6.1(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/fs-node-builtins@4.80.0(tslib@2.8.1)': + dependencies: + tslib: 2.8.1 + + '@jsonjoy.com/fs-node-to-fsa@4.80.0(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/fs-fsa': 4.80.0(tslib@2.8.1) + '@jsonjoy.com/fs-node-builtins': 4.80.0(tslib@2.8.1) + '@jsonjoy.com/fs-node-utils': 4.80.0(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/fs-node-utils@4.80.0(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/fs-node-builtins': 4.80.0(tslib@2.8.1) + glob-to-regex.js: 1.3.1(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/fs-node@4.80.0(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/fs-core': 4.80.0(tslib@2.8.1) + '@jsonjoy.com/fs-node-builtins': 4.80.0(tslib@2.8.1) + '@jsonjoy.com/fs-node-utils': 4.80.0(tslib@2.8.1) + '@jsonjoy.com/fs-print': 4.80.0(tslib@2.8.1) + '@jsonjoy.com/fs-snapshot': 4.80.0(tslib@2.8.1) + glob-to-regex.js: 1.3.1(tslib@2.8.1) + thingies: 2.6.1(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/fs-print@4.80.0(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/fs-node-utils': 4.80.0(tslib@2.8.1) + tree-dump: 1.1.0(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/fs-snapshot@4.80.0(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/buffers': 17.67.0(tslib@2.8.1) + '@jsonjoy.com/fs-node-utils': 4.80.0(tslib@2.8.1) + '@jsonjoy.com/json-pack': 17.67.0(tslib@2.8.1) + '@jsonjoy.com/util': 17.67.0(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/json-pack@1.21.0(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/base64': 1.1.2(tslib@2.8.1) + '@jsonjoy.com/buffers': 1.2.1(tslib@2.8.1) + '@jsonjoy.com/codegen': 1.0.0(tslib@2.8.1) + '@jsonjoy.com/json-pointer': 1.0.2(tslib@2.8.1) + '@jsonjoy.com/util': 1.9.0(tslib@2.8.1) + hyperdyperid: 1.2.0 + thingies: 2.6.1(tslib@2.8.1) + tree-dump: 1.1.0(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/json-pack@17.67.0(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/base64': 17.67.0(tslib@2.8.1) + '@jsonjoy.com/buffers': 17.67.0(tslib@2.8.1) + '@jsonjoy.com/codegen': 17.67.0(tslib@2.8.1) + '@jsonjoy.com/json-pointer': 17.67.0(tslib@2.8.1) + '@jsonjoy.com/util': 17.67.0(tslib@2.8.1) + hyperdyperid: 1.2.0 + thingies: 2.6.1(tslib@2.8.1) + tree-dump: 1.1.0(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/json-pointer@1.0.2(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/codegen': 1.0.0(tslib@2.8.1) + '@jsonjoy.com/util': 1.9.0(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/json-pointer@17.67.0(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/util': 17.67.0(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/util@1.9.0(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/buffers': 1.2.1(tslib@2.8.1) + '@jsonjoy.com/codegen': 1.0.0(tslib@2.8.1) + tslib: 2.8.1 + + '@jsonjoy.com/util@17.67.0(tslib@2.8.1)': + dependencies: + '@jsonjoy.com/buffers': 17.67.0(tslib@2.8.1) + '@jsonjoy.com/codegen': 17.67.0(tslib@2.8.1) + tslib: 2.8.1 + + '@lukeed/csprng@1.1.0': {} + + '@napi-rs/lzma-linux-x64-gnu@1.5.1': + optional: true + + '@nestjs/common@10.4.22(reflect-metadata@0.2.2)(rxjs@7.8.2)': + dependencies: + file-type: 20.4.1 + iterare: 1.2.1 + reflect-metadata: 0.2.2 + rxjs: 7.8.2 + tslib: 2.8.1 + uid: 2.0.2 + transitivePeerDependencies: + - supports-color + + '@nestjs/core@10.4.22(@nestjs/common@10.4.22(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/platform-express@10.4.22)(reflect-metadata@0.2.2)(rxjs@7.8.2)': + dependencies: + '@nestjs/common': 10.4.22(reflect-metadata@0.2.2)(rxjs@7.8.2) + '@nuxtjs/opencollective': 0.3.2 + fast-safe-stringify: 2.1.1 + iterare: 1.2.1 + path-to-regexp: 3.3.0 + reflect-metadata: 0.2.2 + rxjs: 7.8.2 + tslib: 2.8.1 + uid: 2.0.2 + optionalDependencies: + '@nestjs/platform-express': 10.4.22(@nestjs/common@10.4.22(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@10.4.22) + transitivePeerDependencies: + - encoding + + '@nestjs/platform-express@10.4.22(@nestjs/common@10.4.22(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@10.4.22)': + dependencies: + '@nestjs/common': 10.4.22(reflect-metadata@0.2.2)(rxjs@7.8.2) + '@nestjs/core': 10.4.22(@nestjs/common@10.4.22(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/platform-express@10.4.22)(reflect-metadata@0.2.2)(rxjs@7.8.2) + body-parser: 1.20.4 + cors: 2.8.5 + express: 4.22.1 + multer: 2.0.2 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@next/env@14.2.35': {} + + '@next/swc-darwin-arm64@14.2.33': + optional: true + + '@next/swc-darwin-x64@14.2.33': + optional: true + + '@next/swc-linux-arm64-gnu@14.2.33': + optional: true + + '@next/swc-linux-arm64-musl@14.2.33': + optional: true + + '@next/swc-linux-x64-gnu@14.2.33': + optional: true + + '@next/swc-linux-x64-musl@14.2.33': + optional: true + + '@next/swc-win32-arm64-msvc@14.2.33': + optional: true + + '@next/swc-win32-ia32-msvc@14.2.33': + optional: true + + '@next/swc-win32-x64-msvc@14.2.33': + optional: true + + '@nuxtjs/opencollective@0.3.2': + dependencies: + chalk: 4.1.2 + consola: 2.15.3 + node-fetch: 2.7.0 + transitivePeerDependencies: + - encoding + + '@opentelemetry/api-logs@0.222.0': + dependencies: + '@opentelemetry/api': 1.9.1 + + '@opentelemetry/api@1.9.1': {} + + '@opentelemetry/auto-instrumentations-node@0.80.0(@opentelemetry/api@1.9.1)(@opentelemetry/core@2.11.0(@opentelemetry/api@1.9.1))': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-amqplib': 0.69.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-aws-lambda': 0.74.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-aws-sdk': 0.77.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-bunyan': 0.67.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-cassandra-driver': 0.67.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-connect': 0.65.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-cucumber': 0.38.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-dataloader': 0.39.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-dns': 0.65.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-express': 0.70.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-fs': 0.41.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-generic-pool': 0.65.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-graphql': 0.70.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-grpc': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-hapi': 0.68.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-host-metrics': 0.5.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-http': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-ioredis': 0.70.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-kafkajs': 0.31.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-knex': 0.66.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-koa': 0.70.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-lru-memoizer': 0.66.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-memcached': 0.65.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-mongodb': 0.75.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-mongoose': 0.68.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-mysql': 0.68.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-mysql2': 0.68.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-nestjs-core': 0.68.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-net': 0.66.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-openai': 0.20.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-oracledb': 0.47.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-pg': 0.74.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-pino': 0.68.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-redis': 0.70.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-restify': 0.67.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-router': 0.66.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-runtime-node': 0.35.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-socket.io': 0.69.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-tedious': 0.41.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-undici': 0.32.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-winston': 0.66.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resource-detector-alibaba-cloud': 0.37.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resource-detector-aws': 2.22.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resource-detector-azure': 0.30.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resource-detector-container': 0.8.13(@opentelemetry/api@1.9.1) + '@opentelemetry/resource-detector-gcp': 0.57.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-node': 0.222.0(@opentelemetry/api@1.9.1) + transitivePeerDependencies: + - supports-color + + '@opentelemetry/configuration@0.222.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + yaml: 2.9.1 + + '@opentelemetry/context-async-hooks@2.11.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + + '@opentelemetry/core@2.11.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/exporter-logs-otlp-grpc@0.222.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/otlp-exporter-base': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-grpc-exporter-base': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-logs': 0.222.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/exporter-logs-otlp-http@0.222.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/otlp-exporter-base': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-logs': 0.222.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/exporter-logs-otlp-proto@0.222.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/otlp-exporter-base': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-logs': 0.222.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/exporter-metrics-otlp-grpc@0.222.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/exporter-metrics-otlp-http': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-grpc-exporter-base': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.222.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/exporter-metrics-otlp-http@0.222.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-exporter-base': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-metrics': 2.11.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/exporter-metrics-otlp-proto@0.222.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/exporter-metrics-otlp-http': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-exporter-base': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.222.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/exporter-prometheus@0.222.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-metrics': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/exporter-trace-otlp-grpc@0.222.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/otlp-exporter-base': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-grpc-exporter-base': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace': 2.11.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/exporter-trace-otlp-http@0.222.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/otlp-exporter-base': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace': 2.11.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/exporter-trace-otlp-proto@0.222.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/otlp-exporter-base': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace': 2.11.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/exporter-zipkin@2.11.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/instrumentation-amqplib@0.69.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-aws-lambda@0.74.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/propagator-aws-xray': 2.2.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + '@types/aws-lambda': 8.10.164 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-aws-sdk@0.77.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-bunyan@0.67.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.222.0 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-cassandra-driver@0.67.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-connect@0.65.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + '@types/connect': 3.4.38 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-cucumber@0.38.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-dataloader@0.39.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-dns@0.65.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-express@0.70.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-fs@0.41.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-generic-pool@0.65.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-graphql@0.70.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-grpc@0.222.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-hapi@0.68.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-host-metrics@0.5.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + systeminformation: 5.33.15 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-http@0.222.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + forwarded-parse: 2.1.2 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-ioredis@0.70.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/redis-common': 0.38.3 + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-kafkajs@0.31.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-knex@0.66.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-koa@0.70.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-lru-memoizer@0.66.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-memcached@0.65.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + '@types/memcached': 2.2.10 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-mongodb@0.75.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-mongoose@0.68.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-mysql2@0.68.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + '@opentelemetry/sql-common': 0.42.0(@opentelemetry/api@1.9.1) + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-mysql@0.68.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-nestjs-core@0.68.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-net@0.66.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-openai@0.20.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.222.0 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-oracledb@0.47.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-pg@0.74.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + '@opentelemetry/sql-common': 0.42.0(@opentelemetry/api@1.9.1) + '@types/pg': 8.15.6 + '@types/pg-pool': 2.0.7 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-pino@0.68.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.222.0 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-redis@0.70.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/redis-common': 0.38.3 + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-restify@0.67.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-router@0.66.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-runtime-node@0.35.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.222.0 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-socket.io@0.69.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-tedious@0.41.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-undici@0.32.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-winston@0.66.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.222.0 + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation@0.222.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.222.0 + import-in-the-middle: 3.5.2 + require-in-the-middle: 8.0.1 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/otlp-exporter-base@0.222.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.222.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/otlp-grpc-exporter-base@0.222.0(@opentelemetry/api@1.9.1)': + dependencies: + '@grpc/grpc-js': 1.14.5 + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-exporter-base': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.222.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/otlp-transformer@0.222.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.222.0 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-logs': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-metrics': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace': 2.11.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/propagator-aws-xray@2.2.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + + '@opentelemetry/propagator-b3@2.11.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/propagator-jaeger@2.11.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/redis-common@0.38.3': {} + + '@opentelemetry/resource-detector-alibaba-cloud@0.37.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.11.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/resource-detector-aws@2.22.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/resource-detector-azure@0.30.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/resource-detector-container@0.8.13(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.11.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/resource-detector-gcp@0.57.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.11.0(@opentelemetry/api@1.9.1) + gcp-metadata: 8.1.4 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/resources@2.11.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/sdk-logs@0.222.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.222.0 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/sdk-metrics@2.11.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.11.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/sdk-node@0.222.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.222.0 + '@opentelemetry/configuration': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/context-async-hooks': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/exporter-logs-otlp-grpc': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/exporter-logs-otlp-http': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/exporter-logs-otlp-proto': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/exporter-metrics-otlp-grpc': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/exporter-metrics-otlp-http': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/exporter-metrics-otlp-proto': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/exporter-prometheus': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/exporter-trace-otlp-grpc': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/exporter-trace-otlp-http': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/exporter-trace-otlp-proto': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/exporter-zipkin': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-exporter-base': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-grpc-exporter-base': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/propagator-b3': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/propagator-jaeger': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-logs': 0.222.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-metrics': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-node': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/sdk-trace-base@2.11.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/sdk-trace-node@2.11.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/context-async-hooks': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': 2.11.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/sdk-trace@2.11.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.11.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/semantic-conventions@1.43.0': {} + + '@opentelemetry/sql-common@0.42.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.11.0(@opentelemetry/api@1.9.1) + + '@pinojs/redact@0.4.0': {} + + '@pkgjs/parseargs@0.11.0': + optional: true + + '@protobufjs/aspromise@1.1.2': {} + + '@protobufjs/base64@1.1.2': {} + + '@protobufjs/codegen@2.0.5': {} + + '@protobufjs/eventemitter@1.1.1': {} + + '@protobufjs/fetch@1.1.1': + dependencies: + '@protobufjs/aspromise': 1.1.2 + + '@protobufjs/float@1.0.2': {} + + '@protobufjs/path@1.1.2': {} + + '@protobufjs/pool@1.1.0': {} + + '@protobufjs/utf8@1.1.2': {} + + '@radix-ui/primitive@1.1.7': {} + + '@radix-ui/react-compose-refs@1.1.5(@types/react@18.3.31)(react@18.3.1)': + dependencies: + react: 18.3.1 + optionalDependencies: + '@types/react': 18.3.31 + + '@radix-ui/react-context@1.2.2(@types/react@18.3.31)(react@18.3.1)': + dependencies: + react: 18.3.1 + optionalDependencies: + '@types/react': 18.3.31 + + '@radix-ui/react-dialog@1.2.0(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': + dependencies: + '@radix-ui/primitive': 1.1.7 + '@radix-ui/react-compose-refs': 1.1.5(@types/react@18.3.31)(react@18.3.1) + '@radix-ui/react-context': 1.2.2(@types/react@18.3.31)(react@18.3.1) + '@radix-ui/react-dismissable-layer': 1.1.20(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@radix-ui/react-focus-guards': 1.1.6(@types/react@18.3.31)(react@18.3.1) + '@radix-ui/react-focus-scope': 1.2.0(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@radix-ui/react-id': 1.1.4(@types/react@18.3.31)(react@18.3.1) + '@radix-ui/react-portal': 1.1.18(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@radix-ui/react-presence': 1.1.11(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@radix-ui/react-primitive': 2.1.11(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@radix-ui/react-slot': 1.4.0(@types/react@18.3.31)(react@18.3.1) + '@radix-ui/react-use-controllable-state': 1.2.6(@types/react@18.3.31)(react@18.3.1) + '@radix-ui/react-use-layout-effect': 1.1.4(@types/react@18.3.31)(react@18.3.1) + aria-hidden: 1.2.6 + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + react-remove-scroll: 2.7.2(@types/react@18.3.31)(react@18.3.1) + optionalDependencies: + '@types/react': 18.3.31 + '@types/react-dom': 18.3.7(@types/react@18.3.31) + + '@radix-ui/react-dismissable-layer@1.1.20(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': + dependencies: + '@radix-ui/primitive': 1.1.7 + '@radix-ui/react-compose-refs': 1.1.5(@types/react@18.3.31)(react@18.3.1) + '@radix-ui/react-primitive': 2.1.11(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@radix-ui/react-use-callback-ref': 1.1.4(@types/react@18.3.31)(react@18.3.1) + '@radix-ui/react-use-effect-event': 0.0.5(@types/react@18.3.31)(react@18.3.1) + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + optionalDependencies: + '@types/react': 18.3.31 + '@types/react-dom': 18.3.7(@types/react@18.3.31) + + '@radix-ui/react-focus-guards@1.1.6(@types/react@18.3.31)(react@18.3.1)': + dependencies: + react: 18.3.1 + optionalDependencies: + '@types/react': 18.3.31 + + '@radix-ui/react-focus-scope@1.2.0(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': + dependencies: + '@radix-ui/react-compose-refs': 1.1.5(@types/react@18.3.31)(react@18.3.1) + '@radix-ui/react-primitive': 2.1.11(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@radix-ui/react-use-callback-ref': 1.1.4(@types/react@18.3.31)(react@18.3.1) + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + optionalDependencies: + '@types/react': 18.3.31 + '@types/react-dom': 18.3.7(@types/react@18.3.31) + + '@radix-ui/react-id@1.1.4(@types/react@18.3.31)(react@18.3.1)': + dependencies: + '@radix-ui/react-use-layout-effect': 1.1.4(@types/react@18.3.31)(react@18.3.1) + react: 18.3.1 + optionalDependencies: + '@types/react': 18.3.31 + + '@radix-ui/react-portal@1.1.18(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': + dependencies: + '@radix-ui/react-primitive': 2.1.11(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@radix-ui/react-use-layout-effect': 1.1.4(@types/react@18.3.31)(react@18.3.1) + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + optionalDependencies: + '@types/react': 18.3.31 + '@types/react-dom': 18.3.7(@types/react@18.3.31) + + '@radix-ui/react-presence@1.1.11(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': + dependencies: + '@radix-ui/react-use-layout-effect': 1.1.4(@types/react@18.3.31)(react@18.3.1) + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + optionalDependencies: + '@types/react': 18.3.31 + '@types/react-dom': 18.3.7(@types/react@18.3.31) + + '@radix-ui/react-primitive@2.1.11(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': + dependencies: + '@radix-ui/react-slot': 1.4.0(@types/react@18.3.31)(react@18.3.1) + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + optionalDependencies: + '@types/react': 18.3.31 + '@types/react-dom': 18.3.7(@types/react@18.3.31) + + '@radix-ui/react-slot@1.4.0(@types/react@18.3.31)(react@18.3.1)': + dependencies: + '@radix-ui/react-compose-refs': 1.1.5(@types/react@18.3.31)(react@18.3.1) + react: 18.3.1 + optionalDependencies: + '@types/react': 18.3.31 + + '@radix-ui/react-use-callback-ref@1.1.4(@types/react@18.3.31)(react@18.3.1)': + dependencies: + react: 18.3.1 + optionalDependencies: + '@types/react': 18.3.31 + + '@radix-ui/react-use-controllable-state@1.2.6(@types/react@18.3.31)(react@18.3.1)': + dependencies: + '@radix-ui/primitive': 1.1.7 + '@radix-ui/react-use-effect-event': 0.0.5(@types/react@18.3.31)(react@18.3.1) + '@radix-ui/react-use-layout-effect': 1.1.4(@types/react@18.3.31)(react@18.3.1) + react: 18.3.1 + optionalDependencies: + '@types/react': 18.3.31 + + '@radix-ui/react-use-effect-event@0.0.5(@types/react@18.3.31)(react@18.3.1)': + dependencies: + '@radix-ui/react-use-layout-effect': 1.1.4(@types/react@18.3.31)(react@18.3.1) + react: 18.3.1 + optionalDependencies: + '@types/react': 18.3.31 + + '@radix-ui/react-use-layout-effect@1.1.4(@types/react@18.3.31)(react@18.3.1)': + dependencies: + react: 18.3.1 + optionalDependencies: + '@types/react': 18.3.31 + + '@rollup/rollup-android-arm-eabi@4.64.0': + optional: true + + '@rollup/rollup-android-arm64@4.64.0': + optional: true + + '@rollup/rollup-darwin-arm64@4.64.0': + optional: true + + '@rollup/rollup-darwin-x64@4.64.0': + optional: true + + '@rollup/rollup-freebsd-arm64@4.64.0': + optional: true + + '@rollup/rollup-freebsd-x64@4.64.0': + optional: true + + '@rollup/rollup-linux-arm-gnueabihf@4.64.0': + optional: true + + '@rollup/rollup-linux-arm-musleabihf@4.64.0': + optional: true + + '@rollup/rollup-linux-arm64-gnu@4.64.0': + optional: true + + '@rollup/rollup-linux-arm64-musl@4.64.0': + optional: true + + '@rollup/rollup-linux-loong64-gnu@4.64.0': + optional: true + + '@rollup/rollup-linux-loong64-musl@4.64.0': + optional: true + + '@rollup/rollup-linux-ppc64-gnu@4.64.0': + optional: true + + '@rollup/rollup-linux-ppc64-musl@4.64.0': + optional: true + + '@rollup/rollup-linux-riscv64-gnu@4.64.0': + optional: true + + '@rollup/rollup-linux-riscv64-musl@4.64.0': + optional: true + + '@rollup/rollup-linux-s390x-gnu@4.64.0': + optional: true + + '@rollup/rollup-linux-x64-gnu@4.64.0': + optional: true + + '@rollup/rollup-linux-x64-musl@4.64.0': + optional: true + + '@rollup/rollup-openbsd-x64@4.64.0': + optional: true + + '@rollup/rollup-openharmony-arm64@4.64.0': + optional: true + + '@rollup/rollup-win32-arm64-msvc@4.64.0': + optional: true + + '@rollup/rollup-win32-ia32-msvc@4.64.0': + optional: true + + '@rollup/rollup-win32-x64-gnu@4.64.0': + optional: true + + '@rollup/rollup-win32-x64-msvc@4.64.0': + optional: true + + '@smithy/core@3.35.1': + dependencies: + '@smithy/types': 4.19.0 + tslib: 2.8.1 + + '@smithy/credential-provider-imds@4.5.2': + dependencies: + '@smithy/core': 3.35.1 + '@smithy/types': 4.19.0 + tslib: 2.8.1 + + '@smithy/fetch-http-handler@5.8.0': + dependencies: + '@smithy/core': 3.35.1 + '@smithy/types': 4.19.0 + tslib: 2.8.1 + + '@smithy/node-http-handler@4.12.1': + dependencies: + '@smithy/core': 3.35.1 + '@smithy/types': 4.19.0 + tslib: 2.8.1 + + '@smithy/signature-v4@5.7.4': + dependencies: + '@smithy/core': 3.35.1 + '@smithy/types': 4.19.0 + tslib: 2.8.1 + + '@smithy/types@4.19.0': + dependencies: + tslib: 2.8.1 + + '@swc/core-darwin-arm64@1.16.13': + optional: true + + '@swc/core-darwin-x64@1.16.13': + optional: true + + '@swc/core-linux-arm-gnueabihf@1.16.13': + optional: true + + '@swc/core-linux-arm64-gnu@1.16.13': + optional: true + + '@swc/core-linux-arm64-musl@1.16.13': + optional: true + + '@swc/core-linux-ppc64-gnu@1.16.13': + optional: true + + '@swc/core-linux-s390x-gnu@1.16.13': + optional: true + + '@swc/core-linux-x64-gnu@1.16.13': + optional: true + + '@swc/core-linux-x64-musl@1.16.13': + optional: true + + '@swc/core-win32-arm64-msvc@1.16.13': + optional: true + + '@swc/core-win32-ia32-msvc@1.16.13': + optional: true + + '@swc/core-win32-x64-msvc@1.16.13': + optional: true + + '@swc/core@1.16.13': + dependencies: + '@swc/counter': 0.1.3 + '@swc/types': 0.1.28 + optionalDependencies: + '@swc/core-darwin-arm64': 1.16.13 + '@swc/core-darwin-x64': 1.16.13 + '@swc/core-linux-arm-gnueabihf': 1.16.13 + '@swc/core-linux-arm64-gnu': 1.16.13 + '@swc/core-linux-arm64-musl': 1.16.13 + '@swc/core-linux-ppc64-gnu': 1.16.13 + '@swc/core-linux-s390x-gnu': 1.16.13 + '@swc/core-linux-x64-gnu': 1.16.13 + '@swc/core-linux-x64-musl': 1.16.13 + '@swc/core-win32-arm64-msvc': 1.16.13 + '@swc/core-win32-ia32-msvc': 1.16.13 + '@swc/core-win32-x64-msvc': 1.16.13 + + '@swc/counter@0.1.3': {} + + '@swc/helpers@0.5.5': + dependencies: + '@swc/counter': 0.1.3 + tslib: 2.8.1 + + '@swc/types@0.1.28': + dependencies: + '@swc/counter': 0.1.3 + + '@tailwindcss/node@4.3.3': + dependencies: + '@jridgewell/remapping': 2.3.5 + enhanced-resolve: 5.26.0 + jiti: 2.7.0 + lightningcss: 1.32.0 + magic-string: 0.30.21 + source-map-js: 1.2.2 + tailwindcss: 4.3.3 + + '@tailwindcss/oxide-android-arm64@4.3.3': + optional: true + + '@tailwindcss/oxide-darwin-arm64@4.3.3': + optional: true + + '@tailwindcss/oxide-darwin-x64@4.3.3': + optional: true + + '@tailwindcss/oxide-freebsd-x64@4.3.3': + optional: true + + '@tailwindcss/oxide-linux-arm-gnueabihf@4.3.3': + optional: true + + '@tailwindcss/oxide-linux-arm64-gnu@4.3.3': + optional: true + + '@tailwindcss/oxide-linux-arm64-musl@4.3.3': + optional: true + + '@tailwindcss/oxide-linux-x64-gnu@4.3.3': + optional: true + + '@tailwindcss/oxide-linux-x64-musl@4.3.3': + optional: true + + '@tailwindcss/oxide-wasm32-wasi@4.3.3': + optional: true + + '@tailwindcss/oxide-win32-arm64-msvc@4.3.3': + optional: true + + '@tailwindcss/oxide-win32-x64-msvc@4.3.3': + optional: true + + '@tailwindcss/oxide@4.3.3': + optionalDependencies: + '@tailwindcss/oxide-android-arm64': 4.3.3 + '@tailwindcss/oxide-darwin-arm64': 4.3.3 + '@tailwindcss/oxide-darwin-x64': 4.3.3 + '@tailwindcss/oxide-freebsd-x64': 4.3.3 + '@tailwindcss/oxide-linux-arm-gnueabihf': 4.3.3 + '@tailwindcss/oxide-linux-arm64-gnu': 4.3.3 + '@tailwindcss/oxide-linux-arm64-musl': 4.3.3 + '@tailwindcss/oxide-linux-x64-gnu': 4.3.3 + '@tailwindcss/oxide-linux-x64-musl': 4.3.3 + '@tailwindcss/oxide-wasm32-wasi': 4.3.3 + '@tailwindcss/oxide-win32-arm64-msvc': 4.3.3 + '@tailwindcss/oxide-win32-x64-msvc': 4.3.3 + + '@tailwindcss/postcss@4.3.3': + dependencies: + '@alloc/quick-lru': 5.3.0 + '@tailwindcss/node': 4.3.3 + '@tailwindcss/oxide': 4.3.3 + postcss: 8.5.29 + tailwindcss: 4.3.3 + + '@tanstack/react-virtual@3.14.13(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': + dependencies: + '@tanstack/virtual-core': 3.17.11 + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + + '@tanstack/virtual-core@3.17.11': {} + + '@temporalio/activity@1.24.0': + dependencies: + '@temporalio/client': 1.24.0 + '@temporalio/common': 1.24.0 + + '@temporalio/client@1.24.0': + dependencies: + '@grpc/grpc-js': 1.14.5 + '@temporalio/common': 1.24.0 + '@temporalio/proto': 1.24.0 + abort-controller: 3.0.0 + long: 5.3.2 + nexus-rpc: 0.0.3 + uuid: 11.1.1 + + '@temporalio/common@1.24.0': + dependencies: + '@temporalio/proto': 1.24.0 + long: 5.3.2 + ms: 3.0.0-canary.1 + nexus-rpc: 0.0.3 + protobufjs: 8.8.0 + + '@temporalio/core-bridge@1.24.0': + dependencies: + '@grpc/grpc-js': 1.14.5 + '@temporalio/common': 1.24.0 + + '@temporalio/nexus@1.24.0': + dependencies: + '@temporalio/client': 1.24.0 + '@temporalio/common': 1.24.0 + '@temporalio/proto': 1.24.0 + long: 5.3.2 + nexus-rpc: 0.0.3 + + '@temporalio/proto@1.24.0': + dependencies: + long: 5.3.2 + protobufjs: 8.8.0 + + '@temporalio/worker@1.24.0(lightningcss@1.32.0)': + dependencies: + '@grpc/grpc-js': 1.14.5 + '@swc/core': 1.16.13 + '@temporalio/activity': 1.24.0 + '@temporalio/client': 1.24.0 + '@temporalio/common': 1.24.0 + '@temporalio/core-bridge': 1.24.0 + '@temporalio/nexus': 1.24.0 + '@temporalio/proto': 1.24.0 + '@temporalio/workflow': 1.24.0 + heap-js: 2.7.1 + memfs: 4.80.0 + nexus-rpc: 0.0.3 + protobufjs: 8.8.0 + rxjs: 7.8.2 + source-map: 0.7.6 + source-map-loader: 5.0.0(webpack@5.111.1(@swc/core@1.16.13)(lightningcss@1.32.0)) + supports-color: 8.1.1 + swc-loader: 0.2.7(@swc/core@1.16.13)(webpack@5.111.1(@swc/core@1.16.13)(lightningcss@1.32.0)) + unionfs: 4.6.0 + webpack: 5.111.1(@swc/core@1.16.13)(lightningcss@1.32.0) + transitivePeerDependencies: + - '@minify-html/node' + - '@napi-rs/image' + - '@swc/css' + - '@swc/helpers' + - '@swc/html' + - clean-css + - cssnano + - csso + - esbuild + - html-minifier-terser + - imagemin + - lightningcss + - postcss + - sharp + - svgo + - uglify-js + - webpack-cli + + '@temporalio/workflow@1.24.0': + dependencies: + '@temporalio/common': 1.24.0 + '@temporalio/proto': 1.24.0 + nexus-rpc: 0.0.3 + + '@tiptap/core@3.31.4(@tiptap/pm@3.31.4)': + dependencies: + '@tiptap/pm': 3.31.4 + + '@tiptap/extension-blockquote@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4)': + dependencies: + '@tiptap/core': 3.31.4(@tiptap/pm@3.31.4) + '@tiptap/pm': 3.31.4 + + '@tiptap/extension-bold@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))': + dependencies: + '@tiptap/core': 3.31.4(@tiptap/pm@3.31.4) + + '@tiptap/extension-bubble-menu@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4)': + dependencies: + '@floating-ui/dom': 1.8.0 + '@tiptap/core': 3.31.4(@tiptap/pm@3.31.4) + '@tiptap/pm': 3.31.4 + optional: true + + '@tiptap/extension-bullet-list@3.31.4(@tiptap/extension-list@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4))': + dependencies: + '@tiptap/extension-list': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4) + + '@tiptap/extension-character-count@3.31.4(@tiptap/extensions@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4))': + dependencies: + '@tiptap/extensions': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4) + + '@tiptap/extension-code-block@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4)': + dependencies: + '@tiptap/core': 3.31.4(@tiptap/pm@3.31.4) + '@tiptap/pm': 3.31.4 + + '@tiptap/extension-code@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))': + dependencies: + '@tiptap/core': 3.31.4(@tiptap/pm@3.31.4) + + '@tiptap/extension-document@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))': + dependencies: + '@tiptap/core': 3.31.4(@tiptap/pm@3.31.4) + + '@tiptap/extension-dropcursor@3.31.4(@tiptap/extensions@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4))': + dependencies: + '@tiptap/extensions': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4) + + '@tiptap/extension-floating-menu@3.31.4(@floating-ui/dom@1.8.0)(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4)': + dependencies: + '@floating-ui/dom': 1.8.0 + '@tiptap/core': 3.31.4(@tiptap/pm@3.31.4) + '@tiptap/pm': 3.31.4 + optional: true + + '@tiptap/extension-gapcursor@3.31.4(@tiptap/extensions@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4))': + dependencies: + '@tiptap/extensions': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4) + + '@tiptap/extension-hard-break@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))': + dependencies: + '@tiptap/core': 3.31.4(@tiptap/pm@3.31.4) + + '@tiptap/extension-heading@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))': + dependencies: + '@tiptap/core': 3.31.4(@tiptap/pm@3.31.4) + + '@tiptap/extension-horizontal-rule@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4)': + dependencies: + '@tiptap/core': 3.31.4(@tiptap/pm@3.31.4) + '@tiptap/pm': 3.31.4 + + '@tiptap/extension-italic@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))': + dependencies: + '@tiptap/core': 3.31.4(@tiptap/pm@3.31.4) + + '@tiptap/extension-link@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4)': + dependencies: + '@tiptap/core': 3.31.4(@tiptap/pm@3.31.4) + '@tiptap/pm': 3.31.4 + linkifyjs: 4.3.3 + + '@tiptap/extension-list-item@3.31.4(@tiptap/extension-list@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4))': + dependencies: + '@tiptap/extension-list': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4) + + '@tiptap/extension-list-keymap@3.31.4(@tiptap/extension-list@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4))': + dependencies: + '@tiptap/extension-list': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4) + + '@tiptap/extension-list@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4)': + dependencies: + '@tiptap/core': 3.31.4(@tiptap/pm@3.31.4) + '@tiptap/pm': 3.31.4 + + '@tiptap/extension-ordered-list@3.31.4(@tiptap/extension-list@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4))': + dependencies: + '@tiptap/extension-list': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4) + + '@tiptap/extension-paragraph@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))': + dependencies: + '@tiptap/core': 3.31.4(@tiptap/pm@3.31.4) + + '@tiptap/extension-placeholder@3.31.4(@tiptap/extensions@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4))': + dependencies: + '@tiptap/extensions': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4) + + '@tiptap/extension-strike@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))': + dependencies: + '@tiptap/core': 3.31.4(@tiptap/pm@3.31.4) + + '@tiptap/extension-text@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))': + dependencies: + '@tiptap/core': 3.31.4(@tiptap/pm@3.31.4) + + '@tiptap/extension-underline@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))': + dependencies: + '@tiptap/core': 3.31.4(@tiptap/pm@3.31.4) + + '@tiptap/extensions@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4)': + dependencies: + '@tiptap/core': 3.31.4(@tiptap/pm@3.31.4) + '@tiptap/pm': 3.31.4 + + '@tiptap/pm@3.31.4': + dependencies: + prosemirror-changeset: 2.4.4 + prosemirror-commands: 1.7.2 + prosemirror-dropcursor: 1.8.4 + prosemirror-gapcursor: 1.4.1 + prosemirror-history: 1.5.1 + prosemirror-inputrules: 1.5.1 + prosemirror-keymap: 1.2.3 + prosemirror-model: 1.25.12 + prosemirror-schema-list: 1.5.1 + prosemirror-state: 1.4.4 + prosemirror-tables: 1.8.5 + prosemirror-transform: 1.12.2 + prosemirror-view: 1.42.6 + + '@tiptap/react@3.31.4(@floating-ui/dom@1.8.0)(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4)(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': + dependencies: + '@tiptap/core': 3.31.4(@tiptap/pm@3.31.4) + '@tiptap/pm': 3.31.4 + '@types/react': 18.3.31 + '@types/react-dom': 18.3.7(@types/react@18.3.31) + '@types/use-sync-external-store': 0.0.6 + fast-equals: 5.4.3 + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + use-sync-external-store: 1.7.0(react@18.3.1) + optionalDependencies: + '@tiptap/extension-bubble-menu': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4) + '@tiptap/extension-floating-menu': 3.31.4(@floating-ui/dom@1.8.0)(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4) + transitivePeerDependencies: + - '@floating-ui/dom' + + '@tiptap/starter-kit@3.31.4': + dependencies: + '@tiptap/core': 3.31.4(@tiptap/pm@3.31.4) + '@tiptap/extension-blockquote': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4) + '@tiptap/extension-bold': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4)) + '@tiptap/extension-bullet-list': 3.31.4(@tiptap/extension-list@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4)) + '@tiptap/extension-code': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4)) + '@tiptap/extension-code-block': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4) + '@tiptap/extension-document': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4)) + '@tiptap/extension-dropcursor': 3.31.4(@tiptap/extensions@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4)) + '@tiptap/extension-gapcursor': 3.31.4(@tiptap/extensions@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4)) + '@tiptap/extension-hard-break': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4)) + '@tiptap/extension-heading': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4)) + '@tiptap/extension-horizontal-rule': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4) + '@tiptap/extension-italic': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4)) + '@tiptap/extension-link': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4) + '@tiptap/extension-list': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4) + '@tiptap/extension-list-item': 3.31.4(@tiptap/extension-list@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4)) + '@tiptap/extension-list-keymap': 3.31.4(@tiptap/extension-list@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4)) + '@tiptap/extension-ordered-list': 3.31.4(@tiptap/extension-list@3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4)) + '@tiptap/extension-paragraph': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4)) + '@tiptap/extension-strike': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4)) + '@tiptap/extension-text': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4)) + '@tiptap/extension-underline': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4)) + '@tiptap/extensions': 3.31.4(@tiptap/core@3.31.4(@tiptap/pm@3.31.4))(@tiptap/pm@3.31.4) + '@tiptap/pm': 3.31.4 + + '@tokenizer/inflate@0.2.7': + dependencies: + debug: 4.4.3 + fflate: 0.8.3 + token-types: 6.1.2 + transitivePeerDependencies: + - supports-color + + '@tokenizer/token@0.3.0': {} + + '@turbo/darwin-64@2.11.7': + optional: true + + '@turbo/darwin-arm64@2.11.7': + optional: true + + '@turbo/linux-64@2.11.7': + optional: true + + '@turbo/linux-arm64@2.11.7': + optional: true + + '@turbo/windows-64@2.11.7': + optional: true + + '@turbo/windows-arm64@2.11.7': + optional: true + + '@types/aws-lambda@8.10.164': {} + + '@types/body-parser@1.19.6': + dependencies: + '@types/connect': 3.4.38 + '@types/node': 22.20.5 + + '@types/connect@3.4.38': + dependencies: + '@types/node': 22.20.5 + + '@types/estree@1.0.9': {} + + '@types/express-serve-static-core@4.19.9': + dependencies: + '@types/node': 22.20.5 + '@types/qs': 6.15.1 + '@types/range-parser': 1.2.7 + '@types/send': 1.2.1 + + '@types/express@4.17.25': + dependencies: + '@types/body-parser': 1.19.6 + '@types/express-serve-static-core': 4.19.9 + '@types/qs': 6.15.1 + '@types/serve-static': 1.15.10 + + '@types/http-errors@2.0.5': {} + + '@types/json-schema@7.0.15': {} + + '@types/memcached@2.2.10': + dependencies: + '@types/node': 22.20.5 + + '@types/mime@1.3.5': {} + + '@types/node@22.20.5': + dependencies: + undici-types: 6.21.0 + + '@types/pg-pool@2.0.7': + dependencies: + '@types/pg': 8.23.1 + + '@types/pg@8.15.6': + dependencies: + '@types/node': 22.20.5 + pg-protocol: 1.16.1 + pg-types: 2.2.0 + + '@types/pg@8.23.1': + dependencies: + '@types/node': 22.20.5 + pg-protocol: 1.16.1 + pg-types: 2.2.0 + + '@types/prop-types@15.7.15': {} + + '@types/qs@6.15.1': {} + + '@types/range-parser@1.2.7': {} + + '@types/react-dom@18.3.7(@types/react@18.3.31)': + dependencies: + '@types/react': 18.3.31 + + '@types/react@18.3.31': + dependencies: + '@types/prop-types': 15.7.15 + csstype: 3.2.3 + + '@types/send@0.17.6': + dependencies: + '@types/mime': 1.3.5 + '@types/node': 22.20.5 + + '@types/send@1.2.1': + dependencies: + '@types/node': 22.20.5 + + '@types/serve-static@1.15.10': + dependencies: + '@types/http-errors': 2.0.5 + '@types/node': 22.20.5 + '@types/send': 0.17.6 + + '@types/use-sync-external-store@0.0.6': {} + + '@typescript-eslint/eslint-plugin@8.71.0(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@5.9.3))(eslint@9.39.5(jiti@2.7.0))(typescript@5.9.3)': + dependencies: + '@eslint-community/regexpp': 4.12.2 + '@typescript-eslint/parser': 8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@5.9.3) + '@typescript-eslint/scope-manager': 8.71.0 + '@typescript-eslint/type-utils': 8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@5.9.3) + '@typescript-eslint/utils': 8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@5.9.3) + '@typescript-eslint/visitor-keys': 8.71.0 + eslint: 9.39.5(jiti@2.7.0) + ignore: 7.0.12 + natural-compare: 1.4.0 + ts-api-utils: 2.5.0(typescript@5.9.3) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@5.9.3)': + dependencies: + '@typescript-eslint/scope-manager': 8.71.0 + '@typescript-eslint/types': 8.71.0 + '@typescript-eslint/typescript-estree': 8.71.0(typescript@5.9.3) + '@typescript-eslint/visitor-keys': 8.71.0 + debug: 4.4.3 + eslint: 9.39.5(jiti@2.7.0) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/project-service@8.71.0(typescript@5.9.3)': + dependencies: + '@typescript-eslint/tsconfig-utils': 8.71.0(typescript@5.9.3) + '@typescript-eslint/types': 8.71.0 + debug: 4.4.3 + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/scope-manager@8.71.0': + dependencies: + '@typescript-eslint/types': 8.71.0 + '@typescript-eslint/visitor-keys': 8.71.0 + + '@typescript-eslint/tsconfig-utils@8.71.0(typescript@5.9.3)': + dependencies: + typescript: 5.9.3 + + '@typescript-eslint/type-utils@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@5.9.3)': + dependencies: + '@typescript-eslint/types': 8.71.0 + '@typescript-eslint/typescript-estree': 8.71.0(typescript@5.9.3) + '@typescript-eslint/utils': 8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@5.9.3) + debug: 4.4.3 + eslint: 9.39.5(jiti@2.7.0) + ts-api-utils: 2.5.0(typescript@5.9.3) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/types@8.71.0': {} + + '@typescript-eslint/typescript-estree@8.71.0(typescript@5.9.3)': + dependencies: + '@typescript-eslint/project-service': 8.71.0(typescript@5.9.3) + '@typescript-eslint/tsconfig-utils': 8.71.0(typescript@5.9.3) + '@typescript-eslint/types': 8.71.0 + '@typescript-eslint/visitor-keys': 8.71.0 + debug: 4.4.3 + minimatch: 10.2.6 + semver: 7.8.5 + tinyglobby: 0.2.17 + ts-api-utils: 2.5.0(typescript@5.9.3) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/utils@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@5.9.3)': + dependencies: + '@eslint-community/eslint-utils': 4.10.1(eslint@9.39.5(jiti@2.7.0)) + '@typescript-eslint/scope-manager': 8.71.0 + '@typescript-eslint/types': 8.71.0 + '@typescript-eslint/typescript-estree': 8.71.0(typescript@5.9.3) + eslint: 9.39.5(jiti@2.7.0) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + '@typescript-eslint/visitor-keys@8.71.0': + dependencies: + '@typescript-eslint/types': 8.71.0 + eslint-visitor-keys: 5.0.1 + + '@vitest/expect@2.1.9': + dependencies: + '@vitest/spy': 2.1.9 + '@vitest/utils': 2.1.9 + chai: 5.3.3 + tinyrainbow: 1.2.0 + + '@vitest/mocker@2.1.9(vite@5.4.21(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2))': + dependencies: + '@vitest/spy': 2.1.9 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 5.4.21(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + + '@vitest/pretty-format@2.1.9': + dependencies: + tinyrainbow: 1.2.0 + + '@vitest/runner@2.1.9': + dependencies: + '@vitest/utils': 2.1.9 + pathe: 1.1.2 + + '@vitest/snapshot@2.1.9': + dependencies: + '@vitest/pretty-format': 2.1.9 + magic-string: 0.30.21 + pathe: 1.1.2 + + '@vitest/spy@2.1.9': + dependencies: + tinyspy: 3.0.2 + + '@vitest/utils@2.1.9': + dependencies: + '@vitest/pretty-format': 2.1.9 + loupe: 3.2.1 + tinyrainbow: 1.2.0 + + '@webassemblyjs/ast@1.14.1': + dependencies: + '@webassemblyjs/helper-numbers': 1.13.2 + '@webassemblyjs/helper-wasm-bytecode': 1.13.2 + + '@webassemblyjs/floating-point-hex-parser@1.13.2': {} + + '@webassemblyjs/helper-api-error@1.13.2': {} + + '@webassemblyjs/helper-buffer@1.14.1': {} + + '@webassemblyjs/helper-numbers@1.13.2': + dependencies: + '@webassemblyjs/floating-point-hex-parser': 1.13.2 + '@webassemblyjs/helper-api-error': 1.13.2 + '@xtuc/long': 4.2.2 + + '@webassemblyjs/helper-wasm-bytecode@1.13.2': {} + + '@webassemblyjs/helper-wasm-section@1.14.1': + dependencies: + '@webassemblyjs/ast': 1.14.1 + '@webassemblyjs/helper-buffer': 1.14.1 + '@webassemblyjs/helper-wasm-bytecode': 1.13.2 + '@webassemblyjs/wasm-gen': 1.14.1 + + '@webassemblyjs/ieee754@1.13.2': + dependencies: + '@xtuc/ieee754': 1.2.0 + + '@webassemblyjs/leb128@1.13.2': + dependencies: + '@xtuc/long': 4.2.2 + + '@webassemblyjs/utf8@1.13.2': {} + + '@webassemblyjs/wasm-edit@1.14.1': + dependencies: + '@webassemblyjs/ast': 1.14.1 + '@webassemblyjs/helper-buffer': 1.14.1 + '@webassemblyjs/helper-wasm-bytecode': 1.13.2 + '@webassemblyjs/helper-wasm-section': 1.14.1 + '@webassemblyjs/wasm-gen': 1.14.1 + '@webassemblyjs/wasm-opt': 1.14.1 + '@webassemblyjs/wasm-parser': 1.14.1 + '@webassemblyjs/wast-printer': 1.14.1 + + '@webassemblyjs/wasm-gen@1.14.1': + dependencies: + '@webassemblyjs/ast': 1.14.1 + '@webassemblyjs/helper-wasm-bytecode': 1.13.2 + '@webassemblyjs/ieee754': 1.13.2 + '@webassemblyjs/leb128': 1.13.2 + '@webassemblyjs/utf8': 1.13.2 + + '@webassemblyjs/wasm-opt@1.14.1': + dependencies: + '@webassemblyjs/ast': 1.14.1 + '@webassemblyjs/helper-buffer': 1.14.1 + '@webassemblyjs/wasm-gen': 1.14.1 + '@webassemblyjs/wasm-parser': 1.14.1 + + '@webassemblyjs/wasm-parser@1.14.1': + dependencies: + '@webassemblyjs/ast': 1.14.1 + '@webassemblyjs/helper-api-error': 1.13.2 + '@webassemblyjs/helper-wasm-bytecode': 1.13.2 + '@webassemblyjs/ieee754': 1.13.2 + '@webassemblyjs/leb128': 1.13.2 + '@webassemblyjs/utf8': 1.13.2 + + '@webassemblyjs/wast-printer@1.14.1': + dependencies: + '@webassemblyjs/ast': 1.14.1 + '@xtuc/long': 4.2.2 + + '@xtuc/ieee754@1.2.0': {} + + '@xtuc/long@4.2.2': {} + + abort-controller@3.0.0: + dependencies: + event-target-shim: 5.0.1 + + accepts@1.3.8: + dependencies: + mime-types: 2.1.35 + negotiator: 0.6.3 + + acorn-jsx@5.3.2(acorn@8.19.0): + dependencies: + acorn: 8.19.0 + + acorn@8.19.0: {} + + agent-base@7.1.4: {} + + ajv-formats@3.0.1(ajv@8.20.0): + optionalDependencies: + ajv: 8.20.0 + + ajv-keywords@5.1.0(ajv@8.20.0): + dependencies: + ajv: 8.20.0 + fast-deep-equal: 3.1.3 + + ajv@6.15.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-json-stable-stringify: 2.1.0 + json-schema-traverse: 0.4.1 + uri-js: 4.4.1 + + ajv@8.20.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-uri: 3.1.8 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + + ansi-regex@5.0.1: {} + + ansi-regex@6.4.0: {} + + ansi-styles@4.3.0: + dependencies: + color-convert: 2.0.1 + + ansi-styles@6.2.3: {} + + append-field@1.0.0: {} + + argparse@2.0.1: {} + + aria-hidden@1.2.6: + dependencies: + tslib: 2.8.1 + + array-flatten@1.1.1: {} + + assertion-error@2.0.1: {} + + atomic-sleep@1.0.0: {} + + balanced-match@1.0.2: {} + + balanced-match@4.0.4: {} + + baseline-browser-mapping@2.11.27: {} + + bignumber.js@9.3.1: {} + + body-parser@1.20.4: + dependencies: + bytes: 3.1.2 + content-type: 1.0.5 + debug: 2.6.9 + depd: 2.0.0 + destroy: 1.2.0 + http-errors: 2.0.1 + iconv-lite: 0.4.24 + on-finished: 2.4.1 + qs: 6.14.2 + raw-body: 2.5.3 + type-is: 1.6.18 + unpipe: 1.0.0 + transitivePeerDependencies: + - supports-color + + bowser@2.14.1: {} + + brace-expansion@1.1.21: + dependencies: + balanced-match: 1.0.2 + concat-map: 0.0.1 + + brace-expansion@2.1.7: + dependencies: + balanced-match: 1.0.2 + + brace-expansion@5.0.12: + dependencies: + balanced-match: 4.0.4 + + browserslist@4.29.3: + dependencies: + baseline-browser-mapping: 2.11.27 + caniuse-lite: 1.0.30001814 + electron-to-chromium: 1.5.444 + node-releases: 2.0.57 + update-browserslist-db: 1.3.3(browserslist@4.29.3) + + buffer-from@1.1.2: {} + + busboy@1.6.0: + dependencies: + streamsearch: 1.1.0 + + bytes@3.1.2: {} + + cac@6.7.14: {} + + call-bind-apply-helpers@1.0.2: + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + + call-bound@1.0.4: + dependencies: + call-bind-apply-helpers: 1.0.2 + get-intrinsic: 1.3.0 + + callsites@3.1.0: {} + + caniuse-lite@1.0.30001814: {} + + chai@5.3.3: + dependencies: + assertion-error: 2.0.1 + check-error: 2.1.3 + deep-eql: 5.0.2 + loupe: 3.2.1 + pathval: 2.0.1 + + chalk@4.1.2: + dependencies: + ansi-styles: 4.3.0 + supports-color: 7.2.0 + + check-error@2.1.3: {} + + chrome-trace-event@1.0.4: {} + + cjs-module-lexer@2.3.0: {} + + client-only@0.0.1: {} + + cliui@8.0.1: + dependencies: + string-width: 4.2.3 + strip-ansi: 6.0.1 + wrap-ansi: 7.0.0 + + cmdk@1.1.1(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + dependencies: + '@radix-ui/react-compose-refs': 1.1.5(@types/react@18.3.31)(react@18.3.1) + '@radix-ui/react-dialog': 1.2.0(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@radix-ui/react-id': 1.1.4(@types/react@18.3.31)(react@18.3.1) + '@radix-ui/react-primitive': 2.1.11(@types/react-dom@18.3.7(@types/react@18.3.31))(@types/react@18.3.31)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' + + color-convert@2.0.1: + dependencies: + color-name: 1.1.4 + + color-name@1.1.4: {} + + commander@2.20.3: {} + + concat-map@0.0.1: {} + + concat-stream@2.0.0: + dependencies: + buffer-from: 1.1.2 + inherits: 2.0.4 + readable-stream: 3.6.2 + typedarray: 0.0.6 + + consola@2.15.3: {} + + content-disposition@0.5.4: + dependencies: + safe-buffer: 5.2.1 + + content-type@1.0.5: {} + + cookie-signature@1.0.7: {} + + cookie@0.7.2: {} + + cors@2.8.5: + dependencies: + object-assign: 4.1.1 + vary: 1.1.2 + + cross-spawn@7.0.6: + dependencies: + path-key: 3.1.1 + shebang-command: 2.0.0 + which: 2.0.2 + + csstype@3.2.3: {} + + data-uri-to-buffer@4.0.1: {} + + debug@2.6.9: + dependencies: + ms: 2.0.0 + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + deep-eql@5.0.2: {} + + deep-is@0.1.4: {} + + depd@2.0.0: {} + + destroy@1.2.0: {} + + detect-libc@2.1.2: {} + + detect-node-es@1.1.0: {} + + dunder-proto@1.0.1: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-errors: 1.3.0 + gopd: 1.2.0 + + eastasianwidth@0.2.0: {} + + ee-first@1.1.1: {} + + electron-to-chromium@1.5.444: {} + + emoji-regex@8.0.0: {} + + emoji-regex@9.2.2: {} + + encodeurl@2.0.0: {} + + enhanced-resolve@5.26.0: + dependencies: + graceful-fs: 4.2.11 + tapable: 2.3.3 + + es-define-property@1.0.1: {} + + es-errors@1.3.0: {} + + es-module-lexer@1.7.0: {} + + es-module-lexer@2.3.2: {} + + es-module-lexer@3.0.3: {} + + es-object-atoms@1.1.2: + dependencies: + es-errors: 1.3.0 + + esbuild@0.21.5: + optionalDependencies: + '@esbuild/aix-ppc64': 0.21.5 + '@esbuild/android-arm': 0.21.5 + '@esbuild/android-arm64': 0.21.5 + '@esbuild/android-x64': 0.21.5 + '@esbuild/darwin-arm64': 0.21.5 + '@esbuild/darwin-x64': 0.21.5 + '@esbuild/freebsd-arm64': 0.21.5 + '@esbuild/freebsd-x64': 0.21.5 + '@esbuild/linux-arm': 0.21.5 + '@esbuild/linux-arm64': 0.21.5 + '@esbuild/linux-ia32': 0.21.5 + '@esbuild/linux-loong64': 0.21.5 + '@esbuild/linux-mips64el': 0.21.5 + '@esbuild/linux-ppc64': 0.21.5 + '@esbuild/linux-riscv64': 0.21.5 + '@esbuild/linux-s390x': 0.21.5 + '@esbuild/linux-x64': 0.21.5 + '@esbuild/netbsd-x64': 0.21.5 + '@esbuild/openbsd-x64': 0.21.5 + '@esbuild/sunos-x64': 0.21.5 + '@esbuild/win32-arm64': 0.21.5 + '@esbuild/win32-ia32': 0.21.5 + '@esbuild/win32-x64': 0.21.5 + + esbuild@0.28.2: + optionalDependencies: + '@esbuild/aix-ppc64': 0.28.2 + '@esbuild/android-arm': 0.28.2 + '@esbuild/android-arm64': 0.28.2 + '@esbuild/android-x64': 0.28.2 + '@esbuild/darwin-arm64': 0.28.2 + '@esbuild/darwin-x64': 0.28.2 + '@esbuild/freebsd-arm64': 0.28.2 + '@esbuild/freebsd-x64': 0.28.2 + '@esbuild/linux-arm': 0.28.2 + '@esbuild/linux-arm64': 0.28.2 + '@esbuild/linux-ia32': 0.28.2 + '@esbuild/linux-loong64': 0.28.2 + '@esbuild/linux-mips64el': 0.28.2 + '@esbuild/linux-ppc64': 0.28.2 + '@esbuild/linux-riscv64': 0.28.2 + '@esbuild/linux-s390x': 0.28.2 + '@esbuild/linux-x64': 0.28.2 + '@esbuild/netbsd-arm64': 0.28.2 + '@esbuild/netbsd-x64': 0.28.2 + '@esbuild/openbsd-arm64': 0.28.2 + '@esbuild/openbsd-x64': 0.28.2 + '@esbuild/openharmony-arm64': 0.28.2 + '@esbuild/sunos-x64': 0.28.2 + '@esbuild/win32-arm64': 0.28.2 + '@esbuild/win32-ia32': 0.28.2 + '@esbuild/win32-x64': 0.28.2 + + escalade@3.2.0: {} + + escape-html@1.0.3: {} + + escape-string-regexp@4.0.0: {} + + eslint-scope@8.4.0: + dependencies: + esrecurse: 4.3.0 + estraverse: 5.3.0 + + eslint-visitor-keys@3.4.3: {} + + eslint-visitor-keys@4.2.1: {} + + eslint-visitor-keys@5.0.1: {} + + eslint@9.39.5(jiti@2.7.0): + dependencies: + '@eslint-community/eslint-utils': 4.10.1(eslint@9.39.5(jiti@2.7.0)) + '@eslint-community/regexpp': 4.12.2 + '@eslint/config-array': 0.21.2 + '@eslint/config-helpers': 0.4.2 + '@eslint/core': 0.17.0 + '@eslint/eslintrc': 3.3.7 + '@eslint/js': 9.39.5 + '@eslint/plugin-kit': 0.4.1 + '@humanfs/node': 0.16.8 + '@humanwhocodes/module-importer': 1.0.1 + '@humanwhocodes/retry': 0.4.3 + '@types/estree': 1.0.9 + ajv: 6.15.0 + chalk: 4.1.2 + cross-spawn: 7.0.6 + debug: 4.4.3 + escape-string-regexp: 4.0.0 + eslint-scope: 8.4.0 + eslint-visitor-keys: 4.2.1 + espree: 10.4.0 + esquery: 1.7.0 + esutils: 2.0.3 + fast-deep-equal: 3.1.3 + file-entry-cache: 8.0.0 + find-up: 5.0.0 + glob-parent: 6.0.2 + ignore: 5.3.2 + imurmurhash: 0.1.4 + is-glob: 4.0.3 + json-stable-stringify-without-jsonify: 1.0.1 + lodash.merge: 4.6.2 + minimatch: 3.1.5 + natural-compare: 1.4.0 + optionator: 0.9.4 + optionalDependencies: + jiti: 2.7.0 + transitivePeerDependencies: + - supports-color + + espree@10.4.0: + dependencies: + acorn: 8.19.0 + acorn-jsx: 5.3.2(acorn@8.19.0) + eslint-visitor-keys: 4.2.1 + + esquery@1.7.0: + dependencies: + estraverse: 5.3.0 + + esrecurse@4.3.0: + dependencies: + estraverse: 5.3.0 + + estraverse@5.3.0: {} + + estree-walker@3.0.3: + dependencies: + '@types/estree': 1.0.9 + + esutils@2.0.3: {} + + etag@1.8.1: {} + + event-target-shim@5.0.1: {} + + events@3.3.0: {} + + expect-type@1.4.0: {} + + express@4.22.1: + dependencies: + accepts: 1.3.8 + array-flatten: 1.1.1 + body-parser: 1.20.4 + content-disposition: 0.5.4 + content-type: 1.0.5 + cookie: 0.7.2 + cookie-signature: 1.0.7 + debug: 2.6.9 + depd: 2.0.0 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + finalhandler: 1.3.2 + fresh: 0.5.2 + http-errors: 2.0.1 + merge-descriptors: 1.0.3 + methods: 1.1.2 + on-finished: 2.4.1 + parseurl: 1.3.3 + path-to-regexp: 0.1.13 + proxy-addr: 2.0.8 + qs: 6.14.2 + range-parser: 1.2.1 + safe-buffer: 5.2.1 + send: 0.19.2 + serve-static: 1.16.3 + setprototypeof: 1.2.0 + statuses: 2.0.2 + type-is: 1.6.18 + utils-merge: 1.0.1 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + + extend@3.0.2: {} + + fast-deep-equal@3.1.3: {} + + fast-equals@5.4.3: {} + + fast-json-stable-stringify@2.1.0: {} + + fast-levenshtein@2.0.6: {} + + fast-safe-stringify@2.1.1: {} + + fast-uri@3.1.8: {} + + fdir@6.5.0(picomatch@4.0.7): + optionalDependencies: + picomatch: 4.0.7 + + fetch-blob@3.2.0: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.3.3 + + fflate@0.8.3: {} + + file-entry-cache@8.0.0: + dependencies: + flat-cache: 4.0.1 + + file-type@20.4.1: + dependencies: + '@tokenizer/inflate': 0.2.7 + strtok3: 10.3.5 + token-types: 6.1.2 + uint8array-extras: 1.6.0 + transitivePeerDependencies: + - supports-color + + finalhandler@1.3.2: + dependencies: + debug: 2.6.9 + encodeurl: 2.0.0 + escape-html: 1.0.3 + on-finished: 2.4.1 + parseurl: 1.3.3 + statuses: 2.0.2 + unpipe: 1.0.0 + transitivePeerDependencies: + - supports-color + + find-up@5.0.0: + dependencies: + locate-path: 6.0.0 + path-exists: 4.0.0 + + flat-cache@4.0.1: + dependencies: + flatted: 3.4.4 + keyv: 4.5.4 + + flatted@3.4.4: {} + + foreground-child@3.3.1: + dependencies: + cross-spawn: 7.0.6 + signal-exit: 4.1.0 + + formdata-polyfill@4.0.10: + dependencies: + fetch-blob: 3.2.0 + + forwarded-parse@2.1.2: {} + + forwarded@0.2.0: {} + + fresh@0.5.2: {} + + fs-monkey@1.1.0: {} + + fsevents@2.3.3: + optional: true + + function-bind@1.1.2: {} + + gaxios@7.1.3: + dependencies: + extend: 3.0.2 + https-proxy-agent: 7.0.6 + node-fetch: 3.3.2 + rimraf: 5.0.10 + transitivePeerDependencies: + - supports-color + + gcp-metadata@8.1.4: + dependencies: + gaxios: 7.1.3 + google-logging-utils: 1.1.3 + json-bigint: 1.0.0 + transitivePeerDependencies: + - supports-color + + get-caller-file@2.0.5: {} + + get-intrinsic@1.3.0: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + es-errors: 1.3.0 + es-object-atoms: 1.1.2 + function-bind: 1.1.2 + get-proto: 1.0.1 + gopd: 1.2.0 + has-symbols: 1.1.0 + hasown: 2.0.4 + math-intrinsics: 1.1.0 + + get-nonce@1.0.1: {} + + get-proto@1.0.1: + dependencies: + dunder-proto: 1.0.1 + es-object-atoms: 1.1.2 + + glob-parent@6.0.2: + dependencies: + is-glob: 4.0.3 + + glob-to-regex.js@1.3.1(tslib@2.8.1): + dependencies: + tslib: 2.8.1 + + glob@10.5.0: + dependencies: + foreground-child: 3.3.1 + jackspeak: 3.4.3 + minimatch: 9.0.9 + minipass: 7.1.3 + package-json-from-dist: 1.0.1 + path-scurry: 1.11.1 + + globals@14.0.0: {} + + google-logging-utils@1.1.3: {} + + gopd@1.2.0: {} + + graceful-fs@4.2.11: {} + + has-flag@4.0.0: {} + + has-symbols@1.1.0: {} + + hasown@2.0.4: + dependencies: + function-bind: 1.1.2 + + heap-js@2.7.1: {} + + http-errors@2.0.1: + dependencies: + depd: 2.0.0 + inherits: 2.0.4 + setprototypeof: 1.2.0 + statuses: 2.0.2 + toidentifier: 1.0.1 + + https-proxy-agent@7.0.6: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + hyperdyperid@1.2.0: {} + + iconv-lite@0.4.24: + dependencies: + safer-buffer: 2.1.2 + + iconv-lite@0.6.3: + dependencies: + safer-buffer: 2.1.2 + + ieee754@1.2.1: {} + + ignore@5.3.2: {} + + ignore@7.0.12: {} + + import-fresh@3.3.1: + dependencies: + parent-module: 1.0.1 + resolve-from: 4.0.0 + + import-in-the-middle@3.5.2: + dependencies: + cjs-module-lexer: 2.3.0 + es-module-lexer: 3.0.3 + module-details-from-path: 1.0.4 + + imurmurhash@0.1.4: {} + + inherits@2.0.4: {} + + ipaddr.js@1.9.1: {} + + is-extglob@2.1.1: {} + + is-fullwidth-code-point@3.0.0: {} + + is-glob@4.0.3: + dependencies: + is-extglob: 2.1.1 + + isexe@2.0.0: {} + + iterare@1.2.1: {} + + jackspeak@3.4.3: + dependencies: + '@isaacs/cliui': 8.0.2 + optionalDependencies: + '@pkgjs/parseargs': 0.11.0 + + jest-worker@27.5.1: + dependencies: + '@types/node': 22.20.5 + merge-stream: 2.0.0 + supports-color: 8.1.1 + + jiti@2.7.0: {} + + jose@6.2.12: {} + + js-tokens@4.0.0: {} + + js-yaml@4.3.2: + dependencies: + argparse: 2.0.1 + + json-bigint@1.0.0: + dependencies: + bignumber.js: 9.3.1 + + json-buffer@3.0.1: {} + + json-schema-traverse@0.4.1: {} + + json-schema-traverse@1.0.0: {} + + json-stable-stringify-without-jsonify@1.0.1: {} + + keyv@4.5.4: + dependencies: + json-buffer: 3.0.1 + + levn@0.4.1: + dependencies: + prelude-ls: 1.2.1 + type-check: 0.4.0 + + lightningcss-android-arm64@1.32.0: + optional: true + + lightningcss-darwin-arm64@1.32.0: + optional: true + + lightningcss-darwin-x64@1.32.0: + optional: true + + lightningcss-freebsd-x64@1.32.0: + optional: true + + lightningcss-linux-arm-gnueabihf@1.32.0: + optional: true + + lightningcss-linux-arm64-gnu@1.32.0: + optional: true + + lightningcss-linux-arm64-musl@1.32.0: + optional: true + + lightningcss-linux-x64-gnu@1.32.0: + optional: true + + lightningcss-linux-x64-musl@1.32.0: + optional: true + + lightningcss-win32-arm64-msvc@1.32.0: + optional: true + + lightningcss-win32-x64-msvc@1.32.0: + optional: true + + lightningcss@1.32.0: + dependencies: + detect-libc: 2.1.2 + optionalDependencies: + lightningcss-android-arm64: 1.32.0 + lightningcss-darwin-arm64: 1.32.0 + lightningcss-darwin-x64: 1.32.0 + lightningcss-freebsd-x64: 1.32.0 + lightningcss-linux-arm-gnueabihf: 1.32.0 + lightningcss-linux-arm64-gnu: 1.32.0 + lightningcss-linux-arm64-musl: 1.32.0 + lightningcss-linux-x64-gnu: 1.32.0 + lightningcss-linux-x64-musl: 1.32.0 + lightningcss-win32-arm64-msvc: 1.32.0 + lightningcss-win32-x64-msvc: 1.32.0 + + linkifyjs@4.3.3: {} + + locate-path@6.0.0: + dependencies: + p-locate: 5.0.0 + + lodash.camelcase@4.3.0: {} + + lodash.merge@4.6.2: {} + + long@5.3.2: {} + + loose-envify@1.4.0: + dependencies: + js-tokens: 4.0.0 + + loupe@3.2.1: {} + + lru-cache@10.4.3: {} + + magic-string@0.30.21: + dependencies: + '@jridgewell/sourcemap-codec': 1.6.0 + + math-intrinsics@1.1.0: {} + + media-typer@0.3.0: {} + + memfs@4.80.0: + dependencies: + '@jsonjoy.com/fs-core': 4.80.0(tslib@2.8.1) + '@jsonjoy.com/fs-fsa': 4.80.0(tslib@2.8.1) + '@jsonjoy.com/fs-node': 4.80.0(tslib@2.8.1) + '@jsonjoy.com/fs-node-builtins': 4.80.0(tslib@2.8.1) + '@jsonjoy.com/fs-node-to-fsa': 4.80.0(tslib@2.8.1) + '@jsonjoy.com/fs-node-utils': 4.80.0(tslib@2.8.1) + '@jsonjoy.com/fs-print': 4.80.0(tslib@2.8.1) + '@jsonjoy.com/fs-snapshot': 4.80.0(tslib@2.8.1) + '@jsonjoy.com/json-pack': 1.21.0(tslib@2.8.1) + '@jsonjoy.com/util': 1.9.0(tslib@2.8.1) + glob-to-regex.js: 1.3.1(tslib@2.8.1) + thingies: 2.6.1(tslib@2.8.1) + tree-dump: 1.1.0(tslib@2.8.1) + tslib: 2.8.1 + + merge-descriptors@1.0.3: {} + + merge-stream@2.0.0: {} + + methods@1.1.2: {} + + mime-db@1.52.0: {} + + mime-db@1.54.0: {} + + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + + mime@1.6.0: {} + + minimatch@10.2.6: + dependencies: + brace-expansion: 5.0.12 + + minimatch@3.1.5: + dependencies: + brace-expansion: 1.1.21 + + minimatch@9.0.9: + dependencies: + brace-expansion: 2.1.7 + + minimist@1.2.8: {} + + minimizer-webpack-plugin@5.13.0(@swc/core@1.16.13)(lightningcss@1.32.0)(webpack@5.111.1(@swc/core@1.16.13)(lightningcss@1.32.0)): + dependencies: + '@jridgewell/trace-mapping': 0.3.31 + jest-worker: 27.5.1 + schema-utils: 4.5.0 + terser: 5.51.2 + webpack: 5.111.1(@swc/core@1.16.13)(lightningcss@1.32.0) + optionalDependencies: + '@swc/core': 1.16.13 + lightningcss: 1.32.0 + + minipass@7.1.3: {} + + mkdirp@0.5.6: + dependencies: + minimist: 1.2.8 + + module-details-from-path@1.0.4: {} + + ms@2.0.0: {} + + ms@2.1.3: {} + + ms@3.0.0-canary.1: {} + + multer@2.0.2: + dependencies: + append-field: 1.0.0 + busboy: 1.6.0 + concat-stream: 2.0.0 + mkdirp: 0.5.6 + object-assign: 4.1.1 + type-is: 1.6.18 + xtend: 4.0.2 + + nanoid@3.3.20: {} + + natural-compare@1.4.0: {} + + negotiator@0.6.3: {} + + next@14.2.35(@opentelemetry/api@1.9.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + dependencies: + '@next/env': 14.2.35 + '@swc/helpers': 0.5.5 + busboy: 1.6.0 + caniuse-lite: 1.0.30001814 + graceful-fs: 4.2.11 + postcss: 8.4.31 + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + styled-jsx: 5.1.1(react@18.3.1) + optionalDependencies: + '@next/swc-darwin-arm64': 14.2.33 + '@next/swc-darwin-x64': 14.2.33 + '@next/swc-linux-arm64-gnu': 14.2.33 + '@next/swc-linux-arm64-musl': 14.2.33 + '@next/swc-linux-x64-gnu': 14.2.33 + '@next/swc-linux-x64-musl': 14.2.33 + '@next/swc-win32-arm64-msvc': 14.2.33 + '@next/swc-win32-ia32-msvc': 14.2.33 + '@next/swc-win32-x64-msvc': 14.2.33 + '@opentelemetry/api': 1.9.1 + transitivePeerDependencies: + - '@babel/core' + - babel-plugin-macros + + nexus-rpc@0.0.3: {} + + node-domexception@1.0.0: {} + + node-fetch@2.7.0: + dependencies: + whatwg-url: 5.0.0 + + node-fetch@3.3.2: + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + + node-releases@2.0.57: {} + + object-assign@4.1.1: {} + + object-inspect@1.13.4: {} + + on-exit-leak-free@2.1.2: {} + + on-finished@2.4.1: + dependencies: + ee-first: 1.1.1 + + optionator@0.9.4: + dependencies: + deep-is: 0.1.4 + fast-levenshtein: 2.0.6 + levn: 0.4.1 + prelude-ls: 1.2.1 + type-check: 0.4.0 + word-wrap: 1.2.5 + + orderedmap@2.1.1: {} + + p-limit@3.1.0: + dependencies: + yocto-queue: 0.1.0 + + p-locate@5.0.0: + dependencies: + p-limit: 3.1.0 + + package-json-from-dist@1.0.1: {} + + parent-module@1.0.1: + dependencies: + callsites: 3.1.0 + + parseurl@1.3.3: {} + + path-exists@4.0.0: {} + + path-key@3.1.1: {} + + path-scurry@1.11.1: + dependencies: + lru-cache: 10.4.3 + minipass: 7.1.3 + + path-to-regexp@0.1.13: {} + + path-to-regexp@3.3.0: {} + + pathe@1.1.2: {} + + pathval@2.0.1: {} + + pg-cloudflare@1.4.1: + optional: true + + pg-connection-string@2.14.1: {} + + pg-int8@1.0.1: {} + + pg-pool@3.14.0(pg@8.23.1): + dependencies: + pg: 8.23.1 + + pg-protocol@1.16.1: {} + + pg-types@2.2.0: + dependencies: + pg-int8: 1.0.1 + postgres-array: 2.0.0 + postgres-bytea: 1.0.1 + postgres-date: 1.0.7 + postgres-interval: 1.2.0 + + pg@8.23.1: + dependencies: + pg-connection-string: 2.14.1 + pg-pool: 3.14.0(pg@8.23.1) + pg-protocol: 1.16.1 + pg-types: 2.2.0 + pgpass: 1.0.5 + optionalDependencies: + pg-cloudflare: 1.4.1 + + pgpass@1.0.5: + dependencies: + split2: 4.2.0 + + picocolors@1.1.1: {} + + picomatch@4.0.7: {} + + pino-abstract-transport@2.0.0: + dependencies: + split2: 4.2.0 + + pino-std-serializers@7.1.0: {} + + pino@9.14.0: + dependencies: + '@pinojs/redact': 0.4.0 + atomic-sleep: 1.0.0 + on-exit-leak-free: 2.1.2 + pino-abstract-transport: 2.0.0 + pino-std-serializers: 7.1.0 + process-warning: 5.1.0 + quick-format-unescaped: 4.0.4 + real-require: 0.2.0 + safe-stable-stringify: 2.5.0 + sonic-boom: 4.2.1 + thread-stream: 3.2.0 + + playwright-core@1.63.0: {} + + playwright@1.63.0: + dependencies: + playwright-core: 1.63.0 + + postcss@8.4.31: + dependencies: + nanoid: 3.3.20 + picocolors: 1.1.1 + source-map-js: 1.2.2 + + postcss@8.5.29: + dependencies: + nanoid: 3.3.20 + picocolors: 1.1.1 + source-map-js: 1.2.2 + + postgres-array@2.0.0: {} + + postgres-bytea@1.0.1: {} + + postgres-date@1.0.7: {} + + postgres-interval@1.2.0: + dependencies: + xtend: 4.0.2 + + prelude-ls@1.2.1: {} + + process-warning@5.1.0: {} + + prosemirror-changeset@2.4.4: + dependencies: + prosemirror-transform: 1.12.2 + + prosemirror-commands@1.7.2: + dependencies: + prosemirror-model: 1.25.12 + prosemirror-state: 1.4.4 + prosemirror-transform: 1.12.2 + + prosemirror-dropcursor@1.8.4: + dependencies: + prosemirror-state: 1.4.4 + prosemirror-transform: 1.12.2 + prosemirror-view: 1.42.6 + + prosemirror-gapcursor@1.4.1: + dependencies: + prosemirror-keymap: 1.2.3 + prosemirror-model: 1.25.12 + prosemirror-state: 1.4.4 + prosemirror-view: 1.42.6 + + prosemirror-history@1.5.1: + dependencies: + prosemirror-state: 1.4.4 + prosemirror-transform: 1.12.2 + prosemirror-view: 1.42.6 + rope-sequence: 1.3.4 + + prosemirror-inputrules@1.5.1: + dependencies: + prosemirror-state: 1.4.4 + prosemirror-transform: 1.12.2 + + prosemirror-keymap@1.2.3: + dependencies: + prosemirror-state: 1.4.4 + w3c-keyname: 2.2.8 + + prosemirror-model@1.25.12: + dependencies: + orderedmap: 2.1.1 + + prosemirror-schema-list@1.5.1: + dependencies: + prosemirror-model: 1.25.12 + prosemirror-state: 1.4.4 + prosemirror-transform: 1.12.2 + + prosemirror-state@1.4.4: + dependencies: + prosemirror-model: 1.25.12 + prosemirror-transform: 1.12.2 + prosemirror-view: 1.42.6 + + prosemirror-tables@1.8.5: + dependencies: + prosemirror-keymap: 1.2.3 + prosemirror-model: 1.25.12 + prosemirror-state: 1.4.4 + prosemirror-transform: 1.12.2 + prosemirror-view: 1.42.6 + + prosemirror-transform@1.12.2: + dependencies: + prosemirror-model: 1.25.12 + + prosemirror-view@1.42.6: + dependencies: + prosemirror-model: 1.25.12 + prosemirror-state: 1.4.4 + prosemirror-transform: 1.12.2 + + protobufjs@7.6.6: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.5 + '@protobufjs/eventemitter': 1.1.1 + '@protobufjs/fetch': 1.1.1 + '@protobufjs/float': 1.0.2 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.2 + '@types/node': 22.20.5 + long: 5.3.2 + + protobufjs@8.8.0: + dependencies: + long: 5.3.2 + + proxy-addr@2.0.8: + dependencies: + forwarded: 0.2.0 + ipaddr.js: 1.9.1 + + punycode@2.3.1: {} + + qs@6.14.2: + dependencies: + side-channel: 1.1.1 + + quick-format-unescaped@4.0.4: {} + + range-parser@1.2.1: {} + + raw-body@2.5.3: + dependencies: + bytes: 3.1.2 + http-errors: 2.0.1 + iconv-lite: 0.4.24 + unpipe: 1.0.0 + + react-dom@18.3.1(react@18.3.1): + dependencies: + loose-envify: 1.4.0 + react: 18.3.1 + scheduler: 0.23.2 + + react-remove-scroll-bar@2.3.8(@types/react@18.3.31)(react@18.3.1): + dependencies: + react: 18.3.1 + react-style-singleton: 2.2.3(@types/react@18.3.31)(react@18.3.1) + tslib: 2.8.1 + optionalDependencies: + '@types/react': 18.3.31 + + react-remove-scroll@2.7.2(@types/react@18.3.31)(react@18.3.1): + dependencies: + react: 18.3.1 + react-remove-scroll-bar: 2.3.8(@types/react@18.3.31)(react@18.3.1) + react-style-singleton: 2.2.3(@types/react@18.3.31)(react@18.3.1) + tslib: 2.8.1 + use-callback-ref: 1.3.3(@types/react@18.3.31)(react@18.3.1) + use-sidecar: 1.1.3(@types/react@18.3.31)(react@18.3.1) + optionalDependencies: + '@types/react': 18.3.31 + + react-style-singleton@2.2.3(@types/react@18.3.31)(react@18.3.1): + dependencies: + get-nonce: 1.0.1 + react: 18.3.1 + tslib: 2.8.1 + optionalDependencies: + '@types/react': 18.3.31 + + react@18.3.1: + dependencies: + loose-envify: 1.4.0 + + readable-stream@3.6.2: + dependencies: + inherits: 2.0.4 + string_decoder: 1.3.0 + util-deprecate: 1.0.2 + + real-require@0.2.0: {} + + reflect-metadata@0.2.2: {} + + require-directory@2.1.1: {} + + require-from-string@2.0.2: {} + + require-in-the-middle@8.0.1: + dependencies: + debug: 4.4.3 + module-details-from-path: 1.0.4 + transitivePeerDependencies: + - supports-color + + resolve-from@4.0.0: {} + + rimraf@5.0.10: + dependencies: + glob: 10.5.0 + + rollup@4.64.0: + dependencies: + '@types/estree': 1.0.9 + optionalDependencies: + '@napi-rs/lzma-linux-x64-gnu': 1.5.1 + '@rollup/rollup-android-arm-eabi': 4.64.0 + '@rollup/rollup-android-arm64': 4.64.0 + '@rollup/rollup-darwin-arm64': 4.64.0 + '@rollup/rollup-darwin-x64': 4.64.0 + '@rollup/rollup-freebsd-arm64': 4.64.0 + '@rollup/rollup-freebsd-x64': 4.64.0 + '@rollup/rollup-linux-arm-gnueabihf': 4.64.0 + '@rollup/rollup-linux-arm-musleabihf': 4.64.0 + '@rollup/rollup-linux-arm64-gnu': 4.64.0 + '@rollup/rollup-linux-arm64-musl': 4.64.0 + '@rollup/rollup-linux-loong64-gnu': 4.64.0 + '@rollup/rollup-linux-loong64-musl': 4.64.0 + '@rollup/rollup-linux-ppc64-gnu': 4.64.0 + '@rollup/rollup-linux-ppc64-musl': 4.64.0 + '@rollup/rollup-linux-riscv64-gnu': 4.64.0 + '@rollup/rollup-linux-riscv64-musl': 4.64.0 + '@rollup/rollup-linux-s390x-gnu': 4.64.0 + '@rollup/rollup-linux-x64-gnu': 4.64.0 + '@rollup/rollup-linux-x64-musl': 4.64.0 + '@rollup/rollup-openbsd-x64': 4.64.0 + '@rollup/rollup-openharmony-arm64': 4.64.0 + '@rollup/rollup-win32-arm64-msvc': 4.64.0 + '@rollup/rollup-win32-ia32-msvc': 4.64.0 + '@rollup/rollup-win32-x64-gnu': 4.64.0 + '@rollup/rollup-win32-x64-msvc': 4.64.0 + fsevents: 2.3.3 + + rope-sequence@1.3.4: {} + + rxjs@7.8.2: + dependencies: + tslib: 2.8.1 + + safe-buffer@5.2.1: {} + + safe-stable-stringify@2.5.0: {} + + safer-buffer@2.1.2: {} + + scheduler@0.23.2: + dependencies: + loose-envify: 1.4.0 + + schema-utils@4.5.0: + dependencies: + '@types/json-schema': 7.0.15 + ajv: 8.20.0 + ajv-formats: 3.0.1(ajv@8.20.0) + ajv-keywords: 5.1.0(ajv@8.20.0) + + semver@7.8.5: {} + + send@0.19.2: + dependencies: + debug: 2.6.9 + depd: 2.0.0 + destroy: 1.2.0 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + fresh: 0.5.2 + http-errors: 2.0.1 + mime: 1.6.0 + ms: 2.1.3 + on-finished: 2.4.1 + range-parser: 1.2.1 + statuses: 2.0.2 + transitivePeerDependencies: + - supports-color + + serve-static@1.16.3: + dependencies: + encodeurl: 2.0.0 + escape-html: 1.0.3 + parseurl: 1.3.3 + send: 0.19.2 + transitivePeerDependencies: + - supports-color + + setprototypeof@1.2.0: {} + + shebang-command@2.0.0: + dependencies: + shebang-regex: 3.0.0 + + shebang-regex@3.0.0: {} + + side-channel-list@1.0.1: + dependencies: + es-errors: 1.3.0 + object-inspect: 1.13.4 + + side-channel-map@1.0.1: + dependencies: + call-bound: 1.0.4 + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + object-inspect: 1.13.4 + + side-channel-weakmap@1.0.2: + dependencies: + call-bound: 1.0.4 + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + object-inspect: 1.13.4 + side-channel-map: 1.0.1 + + side-channel@1.1.1: + dependencies: + es-errors: 1.3.0 + object-inspect: 1.13.4 + side-channel-list: 1.0.1 + side-channel-map: 1.0.1 + side-channel-weakmap: 1.0.2 + + siginfo@2.0.0: {} + + signal-exit@4.1.0: {} + + sonic-boom@4.2.1: + dependencies: + atomic-sleep: 1.0.0 + + source-map-js@1.2.2: {} + + source-map-loader@5.0.0(webpack@5.111.1(@swc/core@1.16.13)(lightningcss@1.32.0)): + dependencies: + iconv-lite: 0.6.3 + source-map-js: 1.2.2 + webpack: 5.111.1(@swc/core@1.16.13)(lightningcss@1.32.0) + + source-map-support@0.5.21: + dependencies: + buffer-from: 1.1.2 + source-map: 0.6.1 + + source-map@0.6.1: {} + + source-map@0.7.6: {} + + split2@4.2.0: {} + + stackback@0.0.2: {} + + statuses@2.0.2: {} + + std-env@3.10.0: {} + + streamsearch@1.1.0: {} + + string-width@4.2.3: + dependencies: + emoji-regex: 8.0.0 + is-fullwidth-code-point: 3.0.0 + strip-ansi: 6.0.1 + + string-width@5.1.2: + dependencies: + eastasianwidth: 0.2.0 + emoji-regex: 9.2.2 + strip-ansi: 7.2.0 + + string_decoder@1.3.0: + dependencies: + safe-buffer: 5.2.1 + + strip-ansi@6.0.1: + dependencies: + ansi-regex: 5.0.1 + + strip-ansi@7.2.0: + dependencies: + ansi-regex: 6.4.0 + + strip-json-comments@3.1.1: {} + + strtok3@10.3.5: + dependencies: + '@tokenizer/token': 0.3.0 + + styled-jsx@5.1.1(react@18.3.1): + dependencies: + client-only: 0.0.1 + react: 18.3.1 + + supports-color@7.2.0: + dependencies: + has-flag: 4.0.0 + + supports-color@8.1.1: + dependencies: + has-flag: 4.0.0 + + swc-loader@0.2.7(@swc/core@1.16.13)(webpack@5.111.1(@swc/core@1.16.13)(lightningcss@1.32.0)): + dependencies: + '@swc/core': 1.16.13 + '@swc/counter': 0.1.3 + webpack: 5.111.1(@swc/core@1.16.13)(lightningcss@1.32.0) + + systeminformation@5.33.15: {} + + tailwindcss@4.3.3: {} + + tapable@2.3.3: {} + + terser@5.51.2: + dependencies: + '@jridgewell/source-map': 0.3.11 + acorn: 8.19.0 + commander: 2.20.3 + source-map-support: 0.5.21 + + thingies@2.6.1(tslib@2.8.1): + dependencies: + tslib: 2.8.1 + + thread-stream@3.2.0: + dependencies: + real-require: 0.2.0 + + tinybench@2.9.0: {} + + tinyexec@0.3.2: {} + + tinyglobby@0.2.17: + dependencies: + fdir: 6.5.0(picomatch@4.0.7) + picomatch: 4.0.7 + + tinypool@1.1.1: {} + + tinyrainbow@1.2.0: {} + + tinyspy@3.0.2: {} + + toidentifier@1.0.1: {} + + token-types@6.1.2: + dependencies: + '@borewit/text-codec': 0.2.2 + '@tokenizer/token': 0.3.0 + ieee754: 1.2.1 + + tr46@0.0.3: {} + + tree-dump@1.1.0(tslib@2.8.1): + dependencies: + tslib: 2.8.1 + + ts-api-utils@2.5.0(typescript@5.9.3): + dependencies: + typescript: 5.9.3 + + tslib@2.8.1: {} + + tsx@4.23.15: + dependencies: + esbuild: 0.28.2 + optionalDependencies: + fsevents: 2.3.3 + + turbo@2.11.7: + optionalDependencies: + '@turbo/darwin-64': 2.11.7 + '@turbo/darwin-arm64': 2.11.7 + '@turbo/linux-64': 2.11.7 + '@turbo/linux-arm64': 2.11.7 + '@turbo/windows-64': 2.11.7 + '@turbo/windows-arm64': 2.11.7 + + type-check@0.4.0: + dependencies: + prelude-ls: 1.2.1 + + type-is@1.6.18: + dependencies: + media-typer: 0.3.0 + mime-types: 2.1.35 + + typedarray@0.0.6: {} + + typescript-eslint@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@5.9.3): + dependencies: + '@typescript-eslint/eslint-plugin': 8.71.0(@typescript-eslint/parser@8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@5.9.3))(eslint@9.39.5(jiti@2.7.0))(typescript@5.9.3) + '@typescript-eslint/parser': 8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@5.9.3) + '@typescript-eslint/typescript-estree': 8.71.0(typescript@5.9.3) + '@typescript-eslint/utils': 8.71.0(eslint@9.39.5(jiti@2.7.0))(typescript@5.9.3) + eslint: 9.39.5(jiti@2.7.0) + typescript: 5.9.3 + transitivePeerDependencies: + - supports-color + + typescript@5.9.3: {} + + uid@2.0.2: + dependencies: + '@lukeed/csprng': 1.1.0 + + uint8array-extras@1.6.0: {} + + undici-types@6.21.0: {} + + unionfs@4.6.0: + dependencies: + fs-monkey: 1.1.0 + + unpipe@1.0.0: {} + + update-browserslist-db@1.3.3(browserslist@4.29.3): + dependencies: + browserslist: 4.29.3 + escalade: 3.2.0 + picocolors: 1.1.1 + + uri-js@4.4.1: + dependencies: + punycode: 2.3.1 + + use-callback-ref@1.3.3(@types/react@18.3.31)(react@18.3.1): + dependencies: + react: 18.3.1 + tslib: 2.8.1 + optionalDependencies: + '@types/react': 18.3.31 + + use-sidecar@1.1.3(@types/react@18.3.31)(react@18.3.1): + dependencies: + detect-node-es: 1.1.0 + react: 18.3.1 + tslib: 2.8.1 + optionalDependencies: + '@types/react': 18.3.31 + + use-sync-external-store@1.7.0(react@18.3.1): + dependencies: + react: 18.3.1 + + util-deprecate@1.0.2: {} + + utils-merge@1.0.1: {} + + uuid@11.1.1: {} + + vary@1.1.2: {} + + vite-node@2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2): + dependencies: + cac: 6.7.14 + debug: 4.4.3 + es-module-lexer: 1.7.0 + pathe: 1.1.2 + vite: 5.4.21(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + transitivePeerDependencies: + - '@types/node' + - less + - lightningcss + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + + vite@5.4.21(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2): + dependencies: + esbuild: 0.21.5 + postcss: 8.5.29 + rollup: 4.64.0 + optionalDependencies: + '@types/node': 22.20.5 + fsevents: 2.3.3 + lightningcss: 1.32.0 + terser: 5.51.2 + + vitest@2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2): + dependencies: + '@vitest/expect': 2.1.9 + '@vitest/mocker': 2.1.9(vite@5.4.21(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2)) + '@vitest/pretty-format': 2.1.9 + '@vitest/runner': 2.1.9 + '@vitest/snapshot': 2.1.9 + '@vitest/spy': 2.1.9 + '@vitest/utils': 2.1.9 + chai: 5.3.3 + debug: 4.4.3 + expect-type: 1.4.0 + magic-string: 0.30.21 + pathe: 1.1.2 + std-env: 3.10.0 + tinybench: 2.9.0 + tinyexec: 0.3.2 + tinypool: 1.1.1 + tinyrainbow: 1.2.0 + vite: 5.4.21(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + vite-node: 2.1.9(@types/node@22.20.5)(lightningcss@1.32.0)(terser@5.51.2) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 22.20.5 + transitivePeerDependencies: + - less + - lightningcss + - msw + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + + w3c-keyname@2.2.8: {} + + watchpack@2.5.2: + dependencies: + graceful-fs: 4.2.11 + + web-streams-polyfill@3.3.3: {} + + webidl-conversions@3.0.1: {} + + webpack-sources@3.6.0: {} + + webpack@5.111.1(@swc/core@1.16.13)(lightningcss@1.32.0): + dependencies: + '@types/estree': 1.0.9 + '@types/json-schema': 7.0.15 + '@webassemblyjs/ast': 1.14.1 + '@webassemblyjs/wasm-edit': 1.14.1 + '@webassemblyjs/wasm-parser': 1.14.1 + browserslist: 4.29.3 + chrome-trace-event: 1.0.4 + enhanced-resolve: 5.26.0 + es-module-lexer: 2.3.2 + events: 3.3.0 + graceful-fs: 4.2.11 + mime-db: 1.54.0 + minimizer-webpack-plugin: 5.13.0(@swc/core@1.16.13)(lightningcss@1.32.0)(webpack@5.111.1(@swc/core@1.16.13)(lightningcss@1.32.0)) + schema-utils: 4.5.0 + tapable: 2.3.3 + watchpack: 2.5.2 + webpack-sources: 3.6.0 + transitivePeerDependencies: + - '@minify-html/node' + - '@napi-rs/image' + - '@swc/core' + - '@swc/css' + - '@swc/html' + - clean-css + - cssnano + - csso + - esbuild + - html-minifier-terser + - imagemin + - lightningcss + - postcss + - sharp + - svgo + - uglify-js + + whatwg-url@5.0.0: + dependencies: + tr46: 0.0.3 + webidl-conversions: 3.0.1 + + which@2.0.2: + dependencies: + isexe: 2.0.0 + + why-is-node-running@2.3.0: + dependencies: + siginfo: 2.0.0 + stackback: 0.0.2 + + word-wrap@1.2.5: {} + + wrap-ansi@7.0.0: + dependencies: + ansi-styles: 4.3.0 + string-width: 4.2.3 + strip-ansi: 6.0.1 + + wrap-ansi@8.1.0: + dependencies: + ansi-styles: 6.2.3 + string-width: 5.1.2 + strip-ansi: 7.2.0 + + xtend@4.0.2: {} + + y18n@5.0.8: {} + + yaml@2.9.1: {} + + yargs-parser@21.1.1: {} + + yargs@17.7.3: + dependencies: + cliui: 8.0.1 + escalade: 3.2.0 + get-caller-file: 2.0.5 + require-directory: 2.1.1 + string-width: 4.2.3 + y18n: 5.0.8 + yargs-parser: 21.1.1 + + yocto-queue@0.1.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 0000000..3ff5faa --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1,3 @@ +packages: + - "apps/*" + - "packages/*" diff --git a/scripts/build-manual.py b/scripts/build-manual.py new file mode 100644 index 0000000..b39d3ed --- /dev/null +++ b/scripts/build-manual.py @@ -0,0 +1,430 @@ +#!/usr/bin/env python3 +"""SuperPoster User Manual PDF — with embedded screenshots from the running app.""" +import os +from reportlab.lib.pagesizes import A4 +from reportlab.lib.units import inch, mm +from reportlab.lib.colors import HexColor +from reportlab.lib.styles import ParagraphStyle +from reportlab.lib.enums import TA_LEFT, TA_CENTER +from reportlab.platypus import ( + SimpleDocTemplate, Paragraph, Spacer, PageBreak, Image, Table, TableStyle, + HRFlowable, KeepTogether, NextPageTemplate, PageTemplate, Frame, +) +from reportlab.lib.utils import ImageReader + +SHOTS = "apps/web/runs/manual/shots" +OUT = "docs/SuperPoster-User-Manual.pdf" + +# Colors (match the app's dark-first identity) +BG = HexColor("#0a0a0b") +SURFACE = HexColor("#1a1a1e") +ACCENT = HexColor("#6c7cff") +TEXT = HexColor("#e0e0e6") +MUTED = HexColor("#8b8b96") +WHITE = HexColor("#ffffff") +DARK_TEXT = HexColor("#1a1a2e") +DARK_MUTED = HexColor("#555566") +BRAND_INDIGO = HexColor("#5b6cff") +BRAND_DARK = HexColor("#111118") +LIGHT_BG = HexColor("#f5f5fa") + +# Styles +title_s = ParagraphStyle("title", fontName="Helvetica-Bold", fontSize=32, textColor=BRAND_INDIGO, spaceAfter=6, leading=38) +subtitle_s = ParagraphStyle("subtitle", fontName="Helvetica", fontSize=14, textColor=DARK_MUTED, spaceAfter=20) +h1_s = ParagraphStyle("h1", fontName="Helvetica-Bold", fontSize=22, textColor=BRAND_INDIGO, spaceBefore=30, spaceAfter=10, leading=26) +h2_s = ParagraphStyle("h2", fontName="Helvetica-Bold", fontSize=16, textColor=DARK_TEXT, spaceBefore=20, spaceAfter=8) +body_s = ParagraphStyle("body", fontName="Helvetica", fontSize=10.5, textColor=DARK_TEXT, leading=16, spaceAfter=8) +step_s = ParagraphStyle("step", fontName="Helvetica-Bold", fontSize=11, textColor=BRAND_INDIGO, spaceBefore=14, spaceAfter=6) +note_s = ParagraphStyle("note", fontName="Helvetica-Oblique", fontSize=9.5, textColor=DARK_MUTED, leading=14, spaceAfter=8) +caption_s = ParagraphStyle("caption", fontName="Helvetica-Oblique", fontSize=9, textColor=DARK_MUTED, spaceBefore=4, spaceAfter=12) +kbd_s = ParagraphStyle("kbd", fontName="Courier-Bold", fontSize=10, textColor=BRAND_INDIGO, backColor=LIGHT_BG, borderPadding=4, spaceAfter=8) +toc_s = ParagraphStyle("toc", fontName="Helvetica", fontSize=11, textColor=DARK_TEXT, leading=22) + +def img(name, width=6.5*inch): + path = os.path.join(SHOTS, name) + if not os.path.exists(path): + return Paragraph(f"[screenshot: {name}]", note_s) + img = ImageReader(path) + iw, ih = img.getSize() + h = width * ih / iw + return Image(path, width=width, height=h) + +def screenshot(name, caption): + return KeepTogether([img(name), Paragraph(caption, caption_s)]) + +doc = SimpleDocTemplate(OUT, pagesize=A4, + leftMargin=25*mm, rightMargin=25*mm, topMargin=20*mm, bottomMargin=20*mm, + title="SuperPoster User Manual", author="SuperPoster") + +story = [] + +# ══════════════════════════════════════ COVER ══════════════════════════════════════ +story.append(Spacer(1, 80)) +story.append(Paragraph("SuperPoster", title_s)) +story.append(Paragraph("User Manual", subtitle_s)) +story.append(HRFlowable(width="100%", color=BRAND_INDIGO, thickness=2, spaceAfter=20)) +story.append(Paragraph(""" +Version 1.0 · October 2026

+This guide covers everything you need to get started: creating your workspace, +inviting your team, connecting social accounts, monitoring streams, composing +and scheduling posts, and using the audit trail.

+Design identity reference: Canva Visual Kit · Canva Post Kit · Canva Graphic Resources +""", body_s)) +story.append(Spacer(1, 40)) +story.append(screenshot("streams-01-overview.png", "SuperPoster — Streams view, the monitoring hub")) +story.append(PageBreak()) + +# ══════════════════════════════════════ CONTENTS ══════════════════════════════════════ +story.append(Paragraph("Contents", h1_s)) +story.append(HRFlowable(width="100%", color=BRAND_INDIGO, thickness=1, spaceAfter=14)) +toc_items = [ + ("1.", "Getting Started — Onboarding"), + ("2.", "Understanding Streams"), + ("3.", "Composing and Scheduling Posts"), + ("4.", "Managing Connected Accounts"), + ("5.", "Team, Groups and Permissions"), + ("6.", "Inbox — Direct Messages"), + ("7.", "Audit Trail — Who Did What"), + ("8.", "Keyboard Shortcuts"), + ("9.", "Troubleshooting"), +] +for num, title in toc_items: + story.append(Paragraph(f"{num}  {title}", toc_s)) +story.append(PageBreak()) + +# ══════════════════════════════════════ 1. GETTING STARTED ══════════════════════════════════════ +story.append(Paragraph("1. Getting Started", h1_s)) +story.append(HRFlowable(width="100%", color=BRAND_INDIGO, thickness=1, spaceAfter=14)) + +story.append(Paragraph("Creating Your Workspace", h2_s)) +story.append(Paragraph(""" +The first time you use SuperPoster, you'll see the onboarding wizard. A workspace +is your team's tenant boundary — everything (accounts, drafts, posts, audit) +belongs to exactly one workspace and is isolated from all others. +""", body_s)) + +story.append(Paragraph("Step 1 — Name your workspace", step_s)) +story.append(screenshot("onboard-01-start.png", "The onboarding wizard starts here")) +story.append(Paragraph("Type a name that identifies your team or organisation.", body_s)) +story.append(screenshot("onboard-02-named.png", "Enter a descriptive name — you can change it later in Settings")) + +story.append(Paragraph("Step 2 — Invite your team", step_s)) +story.append(screenshot("onboard-03-invite.png", "Enter email addresses separated by commas")) +story.append(Paragraph(""" +Add teammates by email. They'll receive an invitation and can join with the role +you assign them (or the default role). You can always change roles later from +the Team screen. +""", body_s)) +story.append(screenshot("onboard-04-emails-filled.png", "Multiple invitations entered")) + +story.append(Paragraph("Step 3 — Connect your first accounts", step_s)) +story.append(screenshot("onboard-05-connect.png", "Choose platforms to connect — OAuth only, no passwords stored")) +story.append(Paragraph(""" +Click Connect on any platform. You'll be redirected to that platform's own +OAuth page where you log in securely. SuperPoster never sees or stores your +platform password — only an encrypted access token. +""", body_s)) +story.append(Paragraph(""" +Important: SuperPoster stores OAuth tokens only. Your account password never +transits or rests in this system. You can revoke access at any time from either +SuperPoster or the platform itself. +""", note_s)) +story.append(PageBreak()) + +# ══════════════════════════════════════ 2. STREAMS ══════════════════════════════════════ +story.append(Paragraph("2. Understanding Streams", h1_s)) +story.append(HRFlowable(width="100%", color=BRAND_INDIGO, thickness=1, spaceAfter=14)) + +story.append(Paragraph("Your Monitoring Hub", h2_s)) +story.append(screenshot("streams-01-overview.png", "Streams — three columns, each tracking a different feed")) +story.append(Paragraph(""" +The Streams screen is your command centre. Each column tracks a feed from a +connected platform. Cards show the author, age, text, and engagement. Hover over +any card to reveal its amplify bar. +""", body_s)) + +story.append(Paragraph("Selecting Items", h2_s)) +story.append(screenshot("streams-02-item-selected.png", "An item is selected (highlighted in blue) — its amplify bar is now visible")) +story.append(Paragraph(""" +Press j to move to the next item or k to go back. You can also click any +card. The selected card shows a blue highlight and reveals its amplify buttons. +""", body_s)) + +story.append(Paragraph("Amplifying — Boost, Quote and Share", h2_s)) +story.append(screenshot("streams-03-repost-toast.png", "Pressing r boosts the selected item — a toast confirms the action")) +story.append(Paragraph(""" +Each platform's amplify bar shows only what that API truly supports: +""", body_s)) +story.append(Paragraph(""" +Mastodon: Boost (native reblog) and Quote (native quote-post)
+Telegram: Forward
+Bluesky: Repost and Quote
+Instagram: No native reshare — but Quote… with attribution always works
+Facebook: Share link (approximation)
+Reddit: Crosspost +""", body_s)) + +story.append(screenshot("streams-04-quote-popover.png", "Pressing q opens the quote composer")) +story.append(Paragraph(""" +Type your commentary. The character limit matches the platform you're quoting +on. The original post is shown above for reference. +""", body_s)) +story.append(screenshot("streams-05-quote-typed.png", "Your commentary appears above the quoted content")) + +story.append(Paragraph("Quote with Attribution", h2_s)) +story.append(Paragraph(""" +On platforms without a native quote API (like Instagram or Facebook), SuperPoster +offers Quote… with attribution — this composes a new original post that +references the source with proper attribution. It's indicated by a dashed-border +button. This isn't a native API call; it's your own post with attribution text, +which every platform allows. +""", body_s)) +story.append(PageBreak()) + +# ══════════════════════════════════════ 3. COMPOSING ══════════════════════════════════════ +story.append(Paragraph("3. Composing and Scheduling Posts", h1_s)) +story.append(HRFlowable(width="100%", color=BRAND_INDIGO, thickness=1, spaceAfter=14)) + +story.append(Paragraph("Writing a Post", h2_s)) +story.append(screenshot("compose-01-blank.png", "The Compose screen — a rich editor with markdown support")) +story.append(Paragraph(""" +Type your post in the rich editor. Markdown formatting is supported (bold with +**text**, headings with #, etc.). As you type, the character counter updates. +""", body_s)) + +story.append(screenshot("compose-02-typed.png", "A composed post — note the character counter and platform previews")) + +story.append(Paragraph("Targeting Platforms", h2_s)) +story.append(Paragraph(""" +Click platform chips (Mastodon, Telegram, Facebook, Instagram) to toggle targets. +Each target shows its own character limit. The overall counter reflects the +tightest limit among your selected targets. +""", body_s)) + +story.append(Paragraph("The Approval Flow", h2_s)) +story.append(Paragraph(""" +If your workspace uses approvals, the flow is:

+Draft → Submit for Review → In Review → Approve/Reject → Approved → Schedule/Publish +""", body_s)) +story.append(Paragraph(""" +Every transition is permission-checked and recorded in the audit trail with a +full diff of what changed. +""", note_s)) + +story.append(Paragraph("Scheduling", h2_s)) +story.append(Paragraph(""" +Set a date and time in the Schedule field. Your post enters the Temporal-backed +scheduling pipeline — durable, idempotent, and survives restarts. If Temporal +is briefly unavailable, your scheduled post sits safely as "pending" until the +worker picks it up. +""", body_s)) +story.append(PageBreak()) + +# ══════════════════════════════════════ 4. CONNECTIONS ══════════════════════════════════════ +story.append(Paragraph("4. Managing Connected Accounts", h1_s)) +story.append(HRFlowable(width="100%", color=BRAND_INDIGO, thickness=1, spaceAfter=14)) + +story.append(screenshot("connections-01-overview.png", "The Channels screen — each card shows platform, handle, capabilities and health")) +story.append(Paragraph(""" +Every connected account appears as a card showing: +""", body_s)) +story.append(Paragraph(""" +Platform and handle — which account this is
+Capability chips — what this platform's API actually supports
+Health badge — healthy (green) or token expiring (amber)
+Capacity chip — how many posts you have left in your rolling 24h window +""", body_s)) + +story.append(Paragraph("Post Capacity", h2_s)) +story.append(Paragraph(""" +Click the capacity chip (e.g., 17/25 posts left) to see your usage bar, +when the next slot frees, and any tier notes. Provider caps are rolling 24-hour +windows — slots free up continuously as older posts age out, not at midnight. +""", body_s)) + +story.append(Paragraph("Connecting a New Account", h2_s)) +story.append(Paragraph(""" +1. Click Connect account (or the Connect button on a disconnected card)
+2. You'll be redirected to the platform's OAuth page
+3. Log in on the platform's own page and authorize SuperPoster
+4. You're redirected back — the connection appears with its capabilities +""", body_s)) +story.append(Paragraph(""" +Security guarantee: OAuth tokens are envelope-encrypted in the vault. +Nobody — not even workspace Owners — can read raw tokens through any interface. +""", note_s)) +story.append(PageBreak()) + +# ══════════════════════════════════════ 5. TEAM ══════════════════════════════════════ +story.append(Paragraph("5. Team, Groups and Permissions", h1_s)) +story.append(HRFlowable(width="100%", color=BRAND_INDIGO, thickness=1, spaceAfter=14)) + +story.append(screenshot("team-01-overview.png", "The Team screen — members, roles and the shared asset library")) + +story.append(Paragraph("Roles", h2_s)) +story.append(Paragraph(""" +SuperPoster ships with 10 seeded roles. Each role is a bundle of permissions — +the system checks permissions, never role names, so you can customise roles +without breaking anything: +""", body_s)) + +role_table = Table([ + ["Role", "Can Do"], + ["Owner", "Everything, including credential management"], + ["Admin", "Everything except credential management"], + ["Publisher", "Compose, schedule, publish, manage media"], + ["Editor", "Compose and edit drafts, manage media"], + ["Approver", "Review and approve/reject submitted drafts"], + ["Analyst", "View analytics and audit trail"], + ["Moderator", "Manage inbox, live chat moderation"], + ["Video Publisher", "Compose, schedule and publish video content"], + ["Stream Operator", "Manage livestreams"], + ["Broadcast Operator", "Virtual channel control, director actions, live chat"], +], colWidths=[1.6*inch, 4.9*inch]) +role_table.setStyle(TableStyle([ + ("FONTNAME", (0,0), (-1,0), "Helvetica-Bold"), + ("FONTSIZE", (0,0), (-1,-1), 9), + ("TEXTCOLOR", (0,0), (0,-1), BRAND_INDIGO), + ("TEXTCOLOR", (1,0), (1,-1), DARK_TEXT), + ("VALIGN", (0,0), (-1,-1), "TOP"), + ("BOTTOMPADDING", (0,0), (-1,-1), 6), + ("TOPPADDING", (0,0), (-1,-1), 6), + ("LINEBELOW", (0,0), (-1,-2), 0.5, HexColor("#e0e0e8")), + ("BACKGROUND", (0,0), (-1,0), LIGHT_BG), +])) +story.append(role_table) + +story.append(Paragraph("Changing Roles and Removing Members", h2_s)) +story.append(Paragraph(""" +Use the role dropdown on any member row. Removing a member takes effect +immediately — their active sessions fail closed on their next request thanks +to the authorization epoch system. They are logged out instantly; no stale +access lingers. +""", body_s)) + +story.append(Paragraph("Groups (Coming Soon)", h2_s)) +story.append(Paragraph(""" +Groups let you scope which connected accounts and media assets each sub-team +can use. For example: the "Breaking News" group gets access to the main +Mastodon and Bluesky accounts, while the "Lifestyle" group only sees the +Instagram and Facebook pages. A member's publish targets filter automatically +to show only the accounts their groups permit. +""", body_s)) + +story.append(Paragraph("Asset Library", h2_s)) +story.append(Paragraph(""" +The shared asset library holds your team's images, videos and documents. +Assets can be scoped to groups (shown with a lock icon) — members outside the +group see the lock but cannot access the file. +""", body_s)) +story.append(PageBreak()) + +# ══════════════════════════════════════ 6. INBOX ══════════════════════════════════════ +story.append(Paragraph("6. Inbox — Direct Messages", h1_s)) +story.append(HRFlowable(width="100%", color=BRAND_INDIGO, thickness=1, spaceAfter=14)) + +story.append(screenshot("inbox-01-overview.png", "The Inbox — unified conversations across all connected accounts")) +story.append(Paragraph(""" +All direct-message conversations from your connected accounts appear in one list. +Unread badges show new messages; assignee initials show who's handling each thread. +""", body_s)) +story.append(Paragraph(""" +Reply windows: Messaging APIs allow replies within each platform's window +(typically 24 hours for Meta platforms). Outside the window, only pre-approved +template messages are allowed. SuperPoster shows you exactly what's possible +rather than failing silently. +""", note_s)) +story.append(PageBreak()) + +# ══════════════════════════════════════ 7. AUDIT ══════════════════════════════════════ +story.append(Paragraph("7. Audit Trail — Who Did What", h1_s)) +story.append(HRFlowable(width="100%", color=BRAND_INDIGO, thickness=1, spaceAfter=14)) + +story.append(screenshot("audit-01-overview.png", "The Audit Trail — filterable, hash-chained, tamper-evident")) +story.append(Paragraph(""" +Every meaningful action is recorded: posts published, roles changed, credentials +rotated, connections added. The chain is hash-linked — each entry's hash +incorporates the previous entry's hash, so any tampering is mathematically +detectable. +""", body_s)) + +story.append(screenshot("audit-02-diff-open.png", "Click any row to see the exact field-by-field diff")) +story.append(Paragraph(""" +The diff viewer shows before → after values with red/green highlighting. +Secret values always appear as [REDACTED] — they are stripped before the +audit entry is created, not hidden by the UI. This is enforced at the database +function level. +""", body_s)) +story.append(Paragraph(""" +Use the filter bar to narrow by actor or action. Export to CSV for compliance +reporting. +""", body_s)) +story.append(PageBreak()) + +# ══════════════════════════════════════ 8. SHORTCUTS ══════════════════════════════════════ +story.append(Paragraph("8. Keyboard Shortcuts", h1_s)) +story.append(HRFlowable(width="100%", color=BRAND_INDIGO, thickness=1, spaceAfter=14)) + +keys = Table([ + ["Shortcut", "Action", "Where"], + ["j / ↓", "Next item", "Streams"], + ["k / ↑", "Previous item", "Streams"], + ["r", "Repost/boost selected item", "Streams"], + ["q", "Quote selected item", "Streams"], + ["Esc", "Close popover/dialog", "Everywhere"], + ["⌘K / Ctrl+K", "Command palette", "Everywhere"], +], colWidths=[1.4*inch, 3.3*inch, 1.8*inch]) +keys.setStyle(TableStyle([ + ("FONTNAME", (0,0), (-1,0), "Helvetica-Bold"), + ("FONTSIZE", (0,0), (-1,-1), 10), + ("FONTNAME", (0,1), (0,-1), "Courier-Bold"), + ("TEXTCOLOR", (0,1), (0,-1), BRAND_INDIGO), + ("TEXTCOLOR", (1,0), (1,-1), DARK_TEXT), + ("VALIGN", (0,0), (-1,-1), "TOP"), + ("BOTTOMPADDING", (0,0), (-1,-1), 8), + ("TOPPADDING", (0,0), (-1,-1), 8), + ("LINEBELOW", (0,0), (-1,-2), 0.5, HexColor("#e0e0e8")), + ("BACKGROUND", (0,0), (-1,0), LIGHT_BG), +])) +story.append(keys) +story.append(Spacer(1, 20)) +story.append(Paragraph(""" +The command palette (⌘K) lets you jump to any screen, compose a new post, or +trigger actions without touching the mouse. +""", body_s)) +story.append(PageBreak()) + +# ══════════════════════════════════════ 9. TROUBLESHOOTING ══════════════════════════════════════ +story.append(Paragraph("9. Troubleshooting", h1_s)) +story.append(HRFlowable(width="100%", color=BRAND_INDIGO, thickness=1, spaceAfter=14)) + +faq = Table([ + ["Question", "Answer"], + ["Why can't I boost on some columns?", "That platform's API doesn't expose resharing. The button is simply not there — we never fake capabilities. Use Quote… with attribution instead."], + ["Why did everyone get logged out?", "A role or membership change bumped the workspace's authorization epoch. Stale sessions fail closed immediately. One re-login fixes it."], + ["Where are my passwords?", "Nowhere. SuperPoster stores OAuth tokens only, envelope-encrypted in the vault. Revoke access from the platform or in SuperPoster at any time."], + ["Who can see my stream keys?", "Nobody through any interface. Keys are redacted from every screen, log, API response, and audit entry by database-level enforcement."], + ["Why is my capacity at 0?", "You've hit your API tier's rolling 24h cap. The capacity chip shows when the next slot frees — as your oldest post exits the window."], + ["Posts stuck as 'pending'?", "The scheduler worker picks them up automatically. Check that the Temporal service is running (your admin can verify via /readyz)."], +], colWidths=[1.8*inch, 4.7*inch]) +faq.setStyle(TableStyle([ + ("FONTNAME", (0,0), (-1,0), "Helvetica-Bold"), + ("FONTSIZE", (0,0), (-1,-1), 9.5), + ("TEXTCOLOR", (0,1), (0,-1), BRAND_INDIGO), + ("TEXTCOLOR", (1,1), (1,-1), DARK_TEXT), + ("VALIGN", (0,0), (-1,-1), "TOP"), + ("BOTTOMPADDING", (0,0), (-1,-1), 10), + ("TOPPADDING", (0,0), (-1,-1), 10), + ("LINEBELOW", (0,0), (-1,-2), 0.5, HexColor("#e0e0e8")), + ("BACKGROUND", (0,0), (-1,0), LIGHT_BG), +])) +story.append(faq) + +story.append(Spacer(1, 30)) +story.append(HRFlowable(width="100%", color=BRAND_INDIGO, thickness=1, spaceAfter=10)) +story.append(Paragraph("SuperPoster · Version 1.0 · October 2026", ParagraphStyle( + "footer", fontName="Helvetica", fontSize=9, textColor=DARK_MUTED, alignment=TA_CENTER +))) + +doc.build(story) +print(f"PDF: {OUT} ({os.path.getsize(OUT):,} bytes)") diff --git a/tsconfig.base.json b/tsconfig.base.json new file mode 100644 index 0000000..b9493bc --- /dev/null +++ b/tsconfig.base.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "strict": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, + "noFallthroughCasesInSwitch": true, + "forceConsistentCasingInFileNames": true, + "esModuleInterop": true, + "skipLibCheck": true, + "resolveJsonModule": true, + "declaration": true, + "sourceMap": true, + "outDir": "dist", + "baseUrl": "." + } +} diff --git a/turbo.json b/turbo.json new file mode 100644 index 0000000..bb6c842 --- /dev/null +++ b/turbo.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://turbo.build/schema.json", + "agentGuidance": false, + "tasks": { + "build": { + "dependsOn": ["^build"], + "outputs": ["dist/**", ".next/**", "!.next/cache/**"] + }, + "lint": {}, + "typecheck": { + "dependsOn": ["^build"] + }, + "test": {}, + "dev": { + "cache": false, + "persistent": true + } + } +}