glass beadian
aa5e262f3a
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
230 lines
13 KiB
SQL
230 lines
13 KiB
SQL
-- 026: columns P1 — persistence + RBAC (WP3 §1.2/§3, §6 P1 row).
|
|
--
|
|
-- (025 was taken by WP15 archive search; this file is the columns slice.)
|
|
--
|
|
-- Three things land here, in 024's many-tables-one-file style:
|
|
--
|
|
-- 1. The resolved-source / ingest / fan-out tables 021 deliberately left
|
|
-- for their own slice: column_sources, feed_items, feed_item_columns.
|
|
-- (provider_budget_ledger is P2b/P5 territory — no ledger caller exists
|
|
-- yet, so the table is not created here.)
|
|
-- 2. host_budgets — the shared per-host egress budget table owned by
|
|
-- @tactical-media-broadcast-hub/egress (its migrations/001_host_budgets.sql, applied
|
|
-- verbatim with credit; the egress package never migrates at runtime and
|
|
-- expects the application to apply and ledger it). The columns scheduler
|
|
-- is the first app-side caller of HostBudgets that does not go through
|
|
-- the whole egressFetch client, so the table must exist before P1's
|
|
-- poll loop draws from it. IF NOT EXISTS keeps this idempotent if a
|
|
-- later egress-integration migration also carries it.
|
|
-- 3. The `streams.columns.manage` RBAC seed (design §1.4/§0.5): write
|
|
-- permission on columns, granted to Owner/Admin/Editor by default.
|
|
-- Reads stay on `draft.view`.
|
|
--
|
|
-- RLS / NULLIF / reject_workspace_change / sp_app grants follow the
|
|
-- 001/014/021/024 conventions: no app.workspace_id => NULL => no rows; FORCE
|
|
-- binds sp_app too. `spec`/`refresh_policy` stay validated app-side by
|
|
-- @tactical-media-broadcast-hub/core (021 note); the CHECKs here are coarse backstops.
|
|
|
|
-- ---------------------------------------------------------------------------
|
|
-- column_sources — what the scheduler actually polls for a column (§1.2)
|
|
-- ---------------------------------------------------------------------------
|
|
CREATE TABLE column_sources (
|
|
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT,
|
|
column_id uuid NOT NULL REFERENCES stream_columns(id) ON DELETE CASCADE ON UPDATE RESTRICT,
|
|
-- Identity of the thing polled (manifest id of a connector).
|
|
provider text NOT NULL CHECK (btrim(provider) <> '' AND length(provider) <= 64),
|
|
-- NULL for keyless sources (YouTube RSS, public Bluesky/Mastodon) — those
|
|
-- draw from the per-host budget only (§1.2 "Keyless sources").
|
|
connection_id uuid,
|
|
source_kind text NOT NULL CHECK (source_kind IN ('home','mentions','search','author')),
|
|
-- Platform-adapted query (post-adaptation, what gets sent); NULL for
|
|
-- home/mentions/author sources.
|
|
query text,
|
|
-- Canonical account ref for source_kind='author': '{provider}:{handle@host}'.
|
|
actor_ref text,
|
|
state text NOT NULL DEFAULT 'pending'
|
|
CHECK (state IN ('pending','ready','stale','error','unsupported','rate_limited','budget_paused','disabled')),
|
|
last_error text,
|
|
last_polled_at timestamptz,
|
|
next_poll_at timestamptz,
|
|
consecutive_failures integer NOT NULL DEFAULT 0 CHECK (consecutive_failures >= 0),
|
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
updated_at timestamptz NOT NULL DEFAULT now(),
|
|
UNIQUE (workspace_id, id),
|
|
FOREIGN KEY (connection_id, workspace_id) REFERENCES connections (id, workspace_id)
|
|
ON DELETE NO ACTION ON UPDATE RESTRICT,
|
|
-- One source row per (column, polled identity); NULLs equal NULLs so a
|
|
-- keyless search source dedupes against itself (024 NULLS NOT DISTINCT
|
|
-- style). The SAME identity in two different columns is intentional (§3.3:
|
|
-- following the same account in two columns costs one poll) — the due
|
|
-- source scan collapses those rows into one poll and fans items out to
|
|
-- every column sharing the identity.
|
|
UNIQUE NULLS NOT DISTINCT (workspace_id, column_id, provider, source_kind, connection_id, query, actor_ref)
|
|
);
|
|
|
|
CREATE TRIGGER trg_column_sources_ws_immutable BEFORE UPDATE ON column_sources
|
|
FOR EACH ROW EXECUTE FUNCTION reject_workspace_change();
|
|
|
|
ALTER TABLE column_sources ENABLE ROW LEVEL SECURITY;
|
|
ALTER TABLE column_sources FORCE ROW LEVEL SECURITY;
|
|
|
|
CREATE POLICY tenant_isolation ON column_sources
|
|
USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid)
|
|
WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid);
|
|
|
|
GRANT SELECT, INSERT, UPDATE, DELETE ON column_sources TO sp_app;
|
|
|
|
-- Due-source scan (§3.4): workspace + next_poll_at with the disabled rows
|
|
-- partial-indexed away. Design §1.2 names this column_sources_due_idx on
|
|
-- (next_poll_at); workspace leads here because every scan is tenant-scoped
|
|
-- under RLS and the composite keeps each workspace's due window contiguous.
|
|
CREATE INDEX column_sources_due_idx
|
|
ON column_sources (workspace_id, next_poll_at)
|
|
WHERE state <> 'disabled';
|
|
|
|
-- Column fan-out listing (ColumnView.sources) and per-column refresh.
|
|
CREATE INDEX column_sources_column_idx ON column_sources (column_id);
|
|
|
|
-- Connection → sources (the degenerate platform_feed lookup in the scheduler
|
|
-- and the /feeds per-connection store fallback).
|
|
CREATE INDEX column_sources_connection_idx
|
|
ON column_sources (connection_id)
|
|
WHERE connection_id IS NOT NULL;
|
|
|
|
-- ---------------------------------------------------------------------------
|
|
-- feed_items — ingested items (§1.2): persisted depth for keyset pagination
|
|
-- and cross-column dedup; Redis stays the hot read-through cache in front.
|
|
-- ---------------------------------------------------------------------------
|
|
CREATE TABLE feed_items (
|
|
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT,
|
|
-- sha256(workspace_id | provider | canonical_remote_id) — see §3.3; the
|
|
-- canonicalization rule is app-side, the key is opaque to the DB.
|
|
dedup_key text NOT NULL CHECK (dedup_key <> ''),
|
|
provider text NOT NULL CHECK (btrim(provider) <> '' AND length(provider) <= 64),
|
|
remote_id text NOT NULL CHECK (remote_id <> ''),
|
|
-- Connector-reported FeedItem.id; first writer wins, kept stable so a
|
|
-- cursor minted from the hot cache still tie-breaks identically when the
|
|
-- same page is later served from the store.
|
|
item_id text NOT NULL CHECK (item_id <> ''),
|
|
author_handle text NOT NULL CHECK (author_handle <> ''),
|
|
-- Canonical '{provider}:{handle@host}' when resolvable (P0c feedAuthorRef).
|
|
author_ref text,
|
|
-- Provider permalink; absolute credential-free http(s) only (P0c parseSafeHttpUrl).
|
|
url text CHECK (url IS NULL OR url ~ '^https?://'),
|
|
-- Provider timestamp — the deterministic keyset sort key.
|
|
created_at_remote timestamptz NOT NULL,
|
|
fetched_at timestamptz NOT NULL DEFAULT now(),
|
|
-- safe-content typed rich-text AST. NEVER raw HTML (§5.1) — the app layer
|
|
-- produces this through normalizeFeedItem; the CHECK only backstops shape.
|
|
content jsonb NOT NULL CHECK (jsonb_typeof(content) = 'object'),
|
|
-- Derived flat text (astToPlainText) for saved-search filtering/previews.
|
|
plain_text text NOT NULL,
|
|
-- [{kind:'image'|'video', proxyUrl, ...}] — proxy-minted only (§5.3).
|
|
media jsonb NOT NULL DEFAULT '[]'::jsonb CHECK (jsonb_typeof(media) = 'array'),
|
|
-- Finite numbers only (validated at ingest).
|
|
metrics jsonb NOT NULL DEFAULT '{}'::jsonb CHECK (jsonb_typeof(metrics) = 'object'),
|
|
source_id uuid NOT NULL,
|
|
-- ToS expiry (YouTube Data-API items: fetched_at + 30d); sweeper is P2c.
|
|
expires_at timestamptz,
|
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
UNIQUE (workspace_id, dedup_key),
|
|
UNIQUE (id, workspace_id),
|
|
FOREIGN KEY (source_id, workspace_id) REFERENCES column_sources (id, workspace_id)
|
|
ON DELETE NO ACTION ON UPDATE RESTRICT
|
|
);
|
|
|
|
CREATE TRIGGER trg_feed_items_ws_immutable BEFORE UPDATE ON feed_items
|
|
FOR EACH ROW EXECUTE FUNCTION reject_workspace_change();
|
|
|
|
ALTER TABLE feed_items ENABLE ROW LEVEL SECURITY;
|
|
ALTER TABLE feed_items FORCE ROW LEVEL SECURITY;
|
|
|
|
CREATE POLICY tenant_isolation ON feed_items
|
|
USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid)
|
|
WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid);
|
|
|
|
GRANT SELECT, INSERT, UPDATE, DELETE ON feed_items TO sp_app;
|
|
|
|
-- Keyset serving: GET /feeds/columns/:id/items pages (created_at_remote DESC,
|
|
-- dedup_key DESC) over this index (§1.4 "Cursor rule").
|
|
CREATE INDEX feed_items_ws_time_idx
|
|
ON feed_items (workspace_id, created_at_remote DESC, dedup_key DESC);
|
|
|
|
-- Store backfill for /feeds per connection and per-source item listing.
|
|
CREATE INDEX feed_items_source_idx ON feed_items (source_id, created_at_remote DESC);
|
|
|
|
-- Expiry sweeper (P2c) pre-indexed here so the later slice adds no DDL.
|
|
CREATE INDEX feed_items_expiry_idx ON feed_items (expires_at) WHERE expires_at IS NOT NULL;
|
|
|
|
-- ---------------------------------------------------------------------------
|
|
-- feed_item_columns — which columns show an item (§1.2 table 4). An item
|
|
-- ingested for a source identity is visible to every column sharing that
|
|
-- identity; rows are written by poll-time fan-out with ON CONFLICT DO NOTHING.
|
|
-- ---------------------------------------------------------------------------
|
|
-- stream_columns (021) has only its surrogate PK; migrations < 025 are frozen,
|
|
-- so the tenant-composite keys the FKs below need are added here — additive
|
|
-- unique indexes on supersets of existing keys, so no uniqueness changes.
|
|
CREATE UNIQUE INDEX IF NOT EXISTS stream_columns_id_workspace_key
|
|
ON stream_columns (id, workspace_id);
|
|
|
|
CREATE TABLE feed_item_columns (
|
|
workspace_id uuid NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE ON UPDATE RESTRICT,
|
|
item_id uuid NOT NULL,
|
|
column_id uuid NOT NULL,
|
|
created_at timestamptz NOT NULL DEFAULT now(),
|
|
PRIMARY KEY (item_id, column_id),
|
|
FOREIGN KEY (item_id, workspace_id) REFERENCES feed_items (id, workspace_id)
|
|
ON DELETE CASCADE ON UPDATE RESTRICT,
|
|
FOREIGN KEY (column_id, workspace_id) REFERENCES stream_columns (id, workspace_id)
|
|
ON DELETE CASCADE ON UPDATE RESTRICT
|
|
);
|
|
|
|
CREATE TRIGGER trg_feed_item_columns_ws_immutable BEFORE UPDATE ON feed_item_columns
|
|
FOR EACH ROW EXECUTE FUNCTION reject_workspace_change();
|
|
|
|
ALTER TABLE feed_item_columns ENABLE ROW LEVEL SECURITY;
|
|
ALTER TABLE feed_item_columns FORCE ROW LEVEL SECURITY;
|
|
|
|
CREATE POLICY tenant_isolation ON feed_item_columns
|
|
USING (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid)
|
|
WITH CHECK (workspace_id = NULLIF(current_setting('app.workspace_id', true), '')::uuid);
|
|
|
|
GRANT SELECT, INSERT, DELETE ON feed_item_columns TO sp_app;
|
|
|
|
-- Serving index: the items endpoint joins column_id → item_id before the
|
|
-- keyset filter; the PK alone would force an item_id-ordered probe.
|
|
CREATE INDEX feed_item_columns_column_idx ON feed_item_columns (column_id, item_id);
|
|
|
|
-- ---------------------------------------------------------------------------
|
|
-- host_budgets — applied verbatim from @tactical-media-broadcast-hub/egress
|
|
-- migrations/001_host_budgets.sql (WP7/WP8). Infrastructure state shared by
|
|
-- all egress callers; deliberately NOT tenant/RLS scoped (one IDNA lowercase
|
|
-- host:port key across callers and host classes — the whole point is that
|
|
-- monitor, feeds and any other caller of the same host share one bucket).
|
|
-- ---------------------------------------------------------------------------
|
|
CREATE TABLE IF NOT EXISTS host_budgets (
|
|
host text PRIMARY KEY CHECK (host <> '' AND host = lower(host)),
|
|
capacity double precision NOT NULL CHECK (capacity >= 1 AND capacity < 'Infinity'::double precision),
|
|
-- Tokens per fixed 10-second interval (fractional tokens support continuous refill).
|
|
refill_per_interval double precision NOT NULL CHECK (refill_per_interval > 0 AND refill_per_interval < 'Infinity'::double precision),
|
|
tokens double precision NOT NULL CHECK (tokens >= 0 AND tokens <= capacity),
|
|
-- A future timestamp parks the bucket: no refill/debit until this deadline.
|
|
updated_at timestamptz NOT NULL DEFAULT clock_timestamp() CHECK (isfinite(updated_at))
|
|
);
|
|
|
|
COMMENT ON TABLE host_budgets IS 'Infrastructure egress budgets; one IDNA lowercase host:effective-port key across callers and host classes. Retain while host is in use.';
|
|
|
|
GRANT SELECT, INSERT, UPDATE ON host_budgets TO sp_app;
|
|
|
|
-- ---------------------------------------------------------------------------
|
|
-- RBAC seed: streams.columns.manage (design §1.4 — granted to Owner/Admin/
|
|
-- Editor by default; reads stay on draft.view). 002's unnest + ON CONFLICT
|
|
-- pattern so the seed is re-runnable and never fights a later role edit.
|
|
-- ---------------------------------------------------------------------------
|
|
INSERT INTO role_permissions (role, permission)
|
|
SELECT 'Owner', 'streams.columns.manage'
|
|
UNION ALL SELECT 'Admin', 'streams.columns.manage'
|
|
UNION ALL SELECT 'Editor', 'streams.columns.manage'
|
|
ON CONFLICT DO NOTHING;
|