hacklab/T-HUB
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Tactical Media Broadcast Hub — initial release

Browse source 
Self-hosted multi-platform broadcast newsroom for activist collectives:
publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style
column wall with hostile-content-safe rendering, live control room with Auto
Director, bridged chat governance with polls and quorum, distributed sandboxed
transcode workers, encrypted content-addressed archive, monthly analytics.

MIT licensed. See README.md and docs/.
This commit is contained in:
glass beadian 2026-10-10 11:56:42 +00:00
commit aa5e262f3a
880 changed files with 111042 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

114
apps/api/src/intake/adapters/adapter.ts Normal file
View file

@ -0,0 +1,114 @@
/**
* Intake adapter contracts.
*
* One shape in, one shape out:
*
* - `IntakeAdapter` consumes a source row (kind + params + carried adapter
* state) and emits `RawIntakeItem`s — provider-shaped, pre-normalize.
* - `RawIntakeItem` is deliberately the `FeedItem` shape from
* @tactical-media-broadcast-hub/core plus the P0c options, because the task mandate is
* that intake items flow through feeds/feed-normalize.ts unchanged: the
* adapter builds a `ConnectorConnection` stand-in for its (provider, host)
* pair, `normalizeFeedItem` parses the text/html through the safe-content
* allowlist, mints the authorRef and proxied media refs, and the service
* derives `text` (the FTS corpus) from `astToPlainText(content)`.
*
* Fetch surface: `IntakeFetch` is the injectable egress client seam. Tests
* install a fake that records (url, hostClass) draws so budget accounting is
* asserted; production binds `intakeEgress()`.
*/
import type { ConnectorConnection, FeedItem } from "@tactical-media-broadcast-hub/core";
import type { NormalizeFeedItemOptions } from "../../feeds/feed-normalize.js";
import { HOST_CLASS_BY_KIND, PROVIDER_BY_KIND, type IntakeKind } from "../intake-kinds.js";
import type { IntakeParams } from "../intake-params.js";
import { IntakeHttpError } from "../intake-egress.js";
/** The egress fetch surface an adapter draws on. */
export interface IntakeFetch {
/**
* One budgeted outbound GET. `hostClass` selects the budget row class;
* non-2xx resolves to a thrown `IntakeHttpError` carrying status/headers,
* except 304 which the caller classifies (free revalidation, not an error
* path for the scheduler).
*/
fetch(url: string | URL, options: { headers?: Record<string, string>; hostClass: string }): Promise<Response>;
}
/** Per-source adapter state carried in intake_sources.state (etag etc.). */
export type AdapterState = Record<string, unknown>;
/** Provider-shaped item an adapter emits, pre-normalization. */
export interface RawIntakeItem extends FeedItem {
/** P0c options: content kind (html for Mastodon/RSS summaries) + permalink. */
normalize: NormalizeFeedItemOptions;
/** Platform-native id (guid, status id, yt:video:…, url#hash). Defaults to remoteId. */
remoteRef?: string;
/** attribution jsonb written to intake_items (author, url, title, extras). */
attribution: Record<string, unknown>;
/** Harvestable media: direct file URLs, fetched+stored when policy allows. */
media?: Array<{ url: string; kind: "image" | "video" }>;
}
export interface AdapterPollResult {
items: RawIntakeItem[];
/** Next carried state (etag/lastModified/lastContentHash/lastRemoteRef). */
state: AdapterState;
/** True when upstream returned 304/unchanged — a clean, cheap poll. */
unchanged?: boolean;
}
export interface AdapterContext {
workspaceId: string;
sourceId: string;
params: IntakeParams["params"];
kind: IntakeKind;
state: AdapterState;
fetch: IntakeFetch;
}
export interface IntakeAdapter {
readonly kind: IntakeKind;
poll(context: AdapterContext): Promise<AdapterPollResult>;
}
/**
* The ConnectorConnection stand-in handed to normalizeFeedItem: provider is
* the per-kind platform id (so authorRef renders `{provider}:{handle@host}`)
* and `fields.instanceUrl` carries the federated instance origin when the
* kind has one — exactly the field feed-normalize already reads.
*/
export function connectionFor(kind: IntakeKind, params: IntakeParams["params"], connectionId: string, workspaceId: string): ConnectorConnection {
const fields: Record<string, string> = {};
if ("instance" in params && typeof params.instance === "string") fields.instanceUrl = params.instance;
if ("url" in params && typeof params.url === "string") fields.instanceUrl = new URL(params.url).origin;
return {
id: connectionId,
workspaceId,
provider: PROVIDER_BY_KIND[kind],
channelId: connectionId,
credentialId: { kind: "credential-ref", id: connectionId },
fields,
createdAt: new Date(0),
};
}
/** The host class an adapter's own feed fetch draws on. */
export function hostClassFor(kind: IntakeKind): string {
return HOST_CLASS_BY_KIND[kind];
}
/** Budgeted GET with JSON parsing; throws IntakeHttpError on bad status. */
export async function fetchJson<T>(fetcher: IntakeFetch, url: string | URL, hostClass: string, headers?: Record<string, string>): Promise<{ body: T; response: Response }> {
const response = await fetcher.fetch(url, { headers: { accept: "application/json", ...headers }, hostClass });
if (!response.ok) throw new IntakeHttpError(response.status, response.headers, String(url));
const body = JSON.parse(await response.text()) as T;
return { body, response };
}
/** Budgeted GET with text body; 304 resolves as `{ unchanged: true }`. */
export async function fetchText(fetcher: IntakeFetch, url: string | URL, hostClass: string, headers?: Record<string, string>): Promise<{ text: string; response: Response; unchanged: boolean }> {
const response = await fetcher.fetch(url, { headers: { accept: "application/xml, text/xml, text/html;q=0.9, */*;q=0.8", ...headers }, hostClass });
if (response.status === 304) return { text: "", response, unchanged: true };
if (!response.ok) throw new IntakeHttpError(response.status, response.headers, String(url));
return { text: await response.text(), response, unchanged: false };
}

415
apps/api/src/intake/adapters/adapters.test.ts Normal file
View file

@ -0,0 +1,415 @@
/**
* Intake adapter unit tests — fixtures per source kind (task requirement 7).
* No DB, no network: every adapter runs against a recording fake egress
* client so per-request budget draws are asserted, and the RSS/Atom/Mastodon
* fixtures are the payloads the research appendix verified live.
*/
import { describe, expect, it } from "vitest";
import { createHash } from "node:crypto";
import { parseFeed } from "./rss-parse.js";
import { feedToItems, RssAdapter } from "./rss.adapter.js";
import { MastodonAccountAdapter, MastodonTagAdapter, carryState, freshItems, statusesToItems } from "./mastodon.adapter.js";
import { PeerTubeChannelAdapter, videosToItems } from "./peertube.adapter.js";
import { YouTubeChannelAdapter } from "./youtube.adapter.js";
import { WebWatchAdapter, normalizePageText, pageHash } from "./web-watch.adapter.js";
import { BlueskyKeywordAdapter, TelegramChannelAdapter, matchesKeywords } from "./push-seams.js";
import { adapterFor } from "./index.js";
import type { AdapterContext, IntakeFetch } from "./adapter.js";
import { itemContentHash, collapseText } from "../intake-ingest.js";
import { parseIntakeParams, paramsHash, canonicalParams, egressOrigin } from "../intake-params.js";
import { isIntakeKind, KIND_TELEGRAM_CHANNEL, INTAKE_KINDS } from "../intake-kinds.js";
import { DEFAULT_POLL_INTERVAL_SEC, HOST_CLASS_BY_KIND, PROVIDER_BY_KIND } from "../intake-kinds.js";
import { backoffDelayMs, jitteredIntervalMs } from "../intake.service.js";
import { intakeRetryAfterMs, IntakeHttpError } from "../intake-egress.js";
/** Recording fake: every draw + its host class, fixture-served. */
class FakeEgress implements IntakeFetch {
readonly draws: Array<{ url: string; hostClass: string }> = [];
constructor(private readonly routes: Record<string, () => Response | { status: number; headers: Record<string, string>; body?: string }>) {}
async fetch(url: string | URL, options: { headers?: Record<string, string>; hostClass: string }): Promise<Response> {
const key = String(url);
this.draws.push({ url: key, hostClass: options.hostClass });
const route = Object.entries(this.routes).find(([prefix]) => key.startsWith(prefix));
if (route === undefined) return new Response("not found", { status: 404 });
const result = route[1]();
if (result instanceof Response) return result;
if (result.status === 204 || result.status === 304) return new Response(null, { status: 304, headers: result.headers });
return new Response(result.body ?? "", { status: result.status, headers: result.headers });
}
}
function context(kind: string, params: Record<string, unknown>, state: Record<string, unknown> = {}, fetcher: IntakeFetch): AdapterContext {
return {
workspaceId: "ws", sourceId: "src",
kind: kind as AdapterContext["kind"],
params: params as unknown as AdapterContext["params"],
state, fetch: fetcher,
};
}
const RSS_FIXTURE = `<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:media="http://search.yahoo.com/mrss/">
<channel>
<title>Water Working Group</title>
<link>https://example.org/water</link>
<item>
<title>Water rights are a community responsibility</title>
<link>https://example.org/water/2026-10-09</link>
<guid isPermaLink="true">https://example.org/water/2026-10-09</guid>
<pubDate>Fri, 09 Oct 2026 12:00:00 GMT</pubDate>
<dc:creator xmlns:dc="http://purl.org/dc/elements/1.1/">Water WG</dc:creator>
<description><![CDATA[<p>The hearing is postponed to February.</p>]]></description>
<media:thumbnail url="https://example.org/water.jpg" />
</item>
<item>
<title>Repair caf&eacute; this Saturday</title>
<link>https://example.org/water/repair</link>
<guid>repair-2</guid>
<pubDate>Thu, 08 Oct 2026 09:00:00 GMT</pubDate>
<description>Bring your bikes and sewing machines.</description>
<enclosure url="https://example.org/repair.jpg" type="image/jpeg" length="1000" />
</item>
</channel>
</rss>`;
const MASTODON_TAG_FIXTURE = JSON.stringify([
{
id: "110001", uri: "https://social.example/users/alice/statuses/110001", url: "https://social.example/@alice/110001",
created_at: "2026-10-09T11:00:00.000Z",
content: "<p>Solidarity with the <a href=\"https://social.example/tags/water\" class=\"mention hashtag\" rel=\"tag\">#<span>water</span></a> strikers &amp; their families</p>",
account: { id: "1", acct: "alice", display_name: "Alice", username: "alice" },
media_attachments: [{ id: "9", url: "https://files.social.example/media/9.png", preview_url: "https://files.social.example/media/9_small.png", type: "image", mime_type: "image/png" }],
visibility: "public",
},
{ id: "110002", created_at: "2026-10-09T10:00:00.000Z", content: "<p>unrelated post</p>", account: { acct: "bob@example.social", username: "bob" }, visibility: "public" },
]);
const PEERTUBE_FIXTURE = JSON.stringify({
total: 2,
data: [
{ uuid: "9c1de6de-8df0-4e3a-97a6-7e3a1d6e9a11", name: "General assembly recording", publishedAt: "2026-10-08T18:00:00.000Z", channel: { displayName: "Collective Media", name: "collective" }, thumbnailPath: "/static/thumbnails/a11.jpg", duration: 5400 },
{ uuid: "0b2ef7ef-9e01-4f4b-88b7-8f4b2e7f9b22", name: "Street medics training", publishedAt: "2026-10-07T18:00:00.000Z", channel: { displayName: "Collective Media" }, isLive: false },
],
});
const YOUTUBE_FIXTURE = `<?xml version="1.0" encoding="UTF-8"?>
<feed xmlns:yt="http://www.youtube.com/xml/schemas/2015" xmlns="http://www.w3.org/2005/Atom">
<link rel="self" href="https://www.youtube.com/feeds/videos.xml?channel_id=UCtest0000000000000000"/>
<title>Collective Channel</title>
<entry>
<id>yt:video:AAAAAAAAAAA</id>
<yt:videoId>AAAAAAAAAAA</yt:videoId>
<yt:channelId>UCtest0000000000000000</yt:channelId>
<title>Community assembly livestream</title>
<link rel="alternate" href="https://www.youtube.com/watch?v=AAAAAAAAAAA"/>
<published>2026-10-09T08:00:00.000Z</published>
<updated>2026-10-09T08:00:00.000Z</updated>
<author><name>Collective Channel</name></author>
<media:group xmlns:media="http://search.yahoo.com/mrss/">
<media:thumbnail url="https://i.ytimg.com/vi/AAAAAAAAAAA/hqdefault.jpg"/>
</media:group>
</entry>
</feed>`;
const WEB_PAGE_A = `<!doctype html><html><head><style>.x{color:red}</style></head>
<body><div id="main"><p>Next hearing scheduled for March 2027 at the courthouse.</p>
<span data-token="9f8e7d6c5b4a3f2e1d0c9b8a7f6e5d4c"></span></div></body></html>`;
const WEB_PAGE_B = `<!doctype html><html><body><div id="main"><p>Next hearing scheduled for April 2027 at the courthouse.</p>
<span data-token="1a2b3c4d5e6f708192a3b4c5d6e7f809"></span></div></body></html>`;
describe("intake params validation + canonicalization", () => {
it("validates and canonicalizes per kind", () => {
const a = parseIntakeParams("rss", { url: "https://example.org/feed.xml" });
const b = parseIntakeParams("rss", { url: "https://example.org/feed.xml#" });
expect(paramsHash(a)).toBe(paramsHash(b)); // hash-stripped
const c = parseIntakeParams("mastodon_tag", { instance: "Mastodon.Social", tag: "Water" });
expect(c.params).toEqual({ instance: "https://mastodon.social", tag: "water" });
const d = parseIntakeParams("mastodon_account", { instance: "https://mastodon.social", accountId: "000123", handle: "@Alice@mastodon.social" });
expect(d.params).toEqual({ instance: "https://mastodon.social", accountId: "000123", handle: "alice@mastodon.social" });
expect(() => parseIntakeParams("mastodon_account", { instance: "https://mastodon.social", accountId: "abc" })).toThrow();
expect(() => parseIntakeParams("rss", { url: "file:///etc/passwd" })).toThrow();
expect(() => parseIntakeParams("rss", { url: "https://user:pass@example.org/f" })).toThrow();
expect(() => parseIntakeParams("youtube_channel", { channelId: "not-a-channel" })).toThrow();
expect(() => parseIntakeParams("nope", {})).toThrow();
expect(() => parseIntakeParams("telegram_channel", { channelId: "@durov" })).toThrow();
const e = parseIntakeParams("peertube_channel", { instance: "https://video.example", handle: "@collective@video.example" });
expect(e.params).toEqual({ instance: "https://video.example", handle: "collective@video.example" });
const w = parseIntakeParams("web_watch", { url: "https://example.org/page", selector: "#main" });
expect(w.params).toEqual({ url: "https://example.org/page", selector: "#main" });
});
it("canonical form is key-order independent", () => {
const left = parseIntakeParams("mastodon_tag", { tag: "water", instance: "https://mastodon.social" });
const right = parseIntakeParams("mastodon_tag", { instance: "https://mastodon.social", tag: "water" });
expect(canonicalParams(left)).toBe(canonicalParams(right));
});
it("maps kinds to poll defaults, host classes, and providers", () => {
expect(DEFAULT_POLL_INTERVAL_SEC.telegram_channel).toBe(0);
expect(DEFAULT_POLL_INTERVAL_SEC.bluesky_keyword).toBe(0);
expect(DEFAULT_POLL_INTERVAL_SEC.rss).toBeGreaterThan(0);
expect(HOST_CLASS_BY_KIND.mastodon_tag).toBe("intake-social");
expect(PROVIDER_BY_KIND.mastodon_tag).toBe("mastodon");
expect(PROVIDER_BY_KIND.web_watch).toBe("web");
for (const kind of INTAKE_KINDS) expect(isIntakeKind(kind)).toBe(true);
expect(egressOrigin(parseIntakeParams("youtube_channel", { channelId: "UCtest0000000000000000" })))
.toBe("https://www.youtube.com");
expect(egressOrigin(parseIntakeParams("rss", { url: "https://example.org/f.xml" }))).toBe("https://example.org");
});
});
describe("rss parse + adapter", () => {
it("parses RSS 2.0 with CDATA, entities, dc:creator, media:thumbnail", () => {
const feed = parseFeed(RSS_FIXTURE);
expect(feed.type).toBe("rss");
expect(feed.channelTitle).toBe("Water Working Group");
expect(feed.entries).toHaveLength(2);
const first = feed.entries[0]!;
expect(first.title).toBe("Water rights are a community responsibility");
expect(first.link).toBe("https://example.org/water/2026-10-09");
expect(first.publishedAt?.toISOString()).toBe("2026-10-09T12:00:00.000Z");
expect(first.author).toBe("Water WG");
expect(first.contentHtml).toContain("postponed to February");
expect(first.mediaThumbnails).toEqual(["https://example.org/water.jpg"]);
expect(feed.entries[1]!.title).toBe("Repair café this Saturday");
expect(feed.entries[1]!.enclosures[0]?.type).toBe("image/jpeg");
});
it("parses Atom (YouTube shape) with yt:videoId", () => {
const feed = parseFeed(YOUTUBE_FIXTURE);
expect(feed.type).toBe("atom");
expect(feed.channelTitle).toBe("Collective Channel");
expect(feed.entries[0]!.videoId).toBe("AAAAAAAAAAA");
expect(feed.entries[0]!.link).toBe("https://www.youtube.com/watch?v=AAAAAAAAAAA");
});
it("classifies enclosure + thumbnail media kinds", () => {
const items = feedToItems(parseFeed(RSS_FIXTURE), new Date("2026-10-09T12:00:00Z"));
expect(items[0]?.media).toEqual([{ url: "https://example.org/water.jpg", kind: "image" }]);
expect(items[1]?.media?.some((entry) => entry.url === "https://example.org/repair.jpg")).toBe(true);
expect(items[0]?.remoteId).toBe("https://example.org/water/2026-10-09");
expect(items).toHaveLength(2);
});
it("conditional GET: 304 is free and emits nothing; etag carries", async () => {
let etag = "\"v1\"";
const egress = new FakeEgress({
"https://example.org/feed.xml": () => ({ status: 200, headers: { etag, "content-type": "application/xml" }, body: RSS_FIXTURE }),
});
const adapter = new RssAdapter();
const first = await adapter.poll(context("rss", { url: "https://example.org/feed.xml" }, {}, egress));
expect(first.items).toHaveLength(2);
expect(first.state.etag).toBe(etag);
expect(egress.draws[0]!.hostClass).toBe("intake-rss");
etag = "\"v2\"";
const second = await adapter.poll(context("rss", { url: "https://example.org/feed.xml" }, { etag: "\"v1\"" },
new FakeEgress({ "https://example.org/feed.xml": () => ({ status: 304, headers: {} }) })));
expect(second.unchanged).toBe(true);
expect(second.items).toHaveLength(0);
});
it("is total over malformed XML", () => {
const feed = parseFeed("<rss><channel><item><title>only");
expect(feed.entries).toHaveLength(1);
expect(parseFeed("").entries).toHaveLength(0);
expect(parseFeed("not xml at all").entries).toHaveLength(0);
});
});
describe("mastodon adapters", () => {
it("tag timeline: html status → items with attachments, budget class intake-social", async () => {
const egress = new FakeEgress({
"https://mastodon.social/api/v1/timelines/tag/water": () => new Response(MASTODON_TAG_FIXTURE, { headers: { "content-type": "application/json", "x-ratelimit-limit": "300" } }),
});
const adapter = new MastodonTagAdapter();
const result = await adapter.poll(context("mastodon_tag", { instance: "https://mastodon.social", tag: "water" }, {}, egress));
expect(result.items).toHaveLength(2);
expect(egress.draws[0]!.hostClass).toBe("intake-social");
expect(egress.draws[0]!.url).toContain("/api/v1/timelines/tag/water?limit=40");
const first = result.items[0]!;
expect(first.remoteId).toBe("110001");
expect(first.media![0]).toEqual({ url: "https://files.social.example/media/9.png", kind: "image" });
expect(first.attribution.authorHandle).toBe("alice");
// remote federation: acct carries @host
expect(result.items[1]!.attribution.authorHandle).toBe("bob@example.social");
const state = carryState(result.items, {});
expect(state.lastRemoteRef).toBe("110001");
});
it("account timeline: exclude_replies + freshness window from carried state", async () => {
const egress = new FakeEgress({
"https://mastodon.social/api/v1/accounts/000123/statuses": () => new Response(MASTODON_TAG_FIXTURE, { headers: { "content-type": "application/json" } }),
});
const adapter = new MastodonAccountAdapter();
const first = await adapter.poll(context("mastodon_account", { instance: "https://mastodon.social", accountId: "000123" }, {}, egress));
expect(first.items).toHaveLength(2);
expect(egress.draws[0]!.url).toContain("exclude_replies=true");
// Second delivery of the same window: the high-water mark (110001, the
// newest) is present, so nothing newer exists and 110002 was already
// delivered in the first poll — zero new items.
const second = await adapter.poll(context("mastodon_account", { instance: "https://mastodon.social", accountId: "000123" },
{ lastRemoteRef: "110001", lastSeenIds: ["110001", "110002"] }, egress));
expect(second.items).toHaveLength(0);
});
it("freshItems cuts at the high-water mark and dedups seen ids", () => {
const items = statusesToItems([
{ id: "3", created_at: "2026-10-09T11:00:00Z", content: "c" },
{ id: "2", created_at: "2026-10-09T10:00:00Z", content: "b" },
{ id: "1", created_at: "2026-10-09T09:00:00Z", content: "a" },
], new Date());
expect(freshItems(items, "2", ["1"]).map((i) => i.remoteId)).toEqual(["3"]);
expect(freshItems(items, undefined, ["1", "2"]).map((i) => i.remoteId)).toEqual(["3"]);
});
it("drops private/direct statuses", () => {
const items = statusesToItems([
{ id: "1", visibility: "private", content: "x" },
{ id: "2", visibility: "direct", content: "x" },
{ id: "3", visibility: "public", content: "x" },
], new Date());
expect(items).toHaveLength(1);
});
});
describe("peertube adapter", () => {
it("channel videos via public REST, thumbnail media, duration metric", async () => {
const egress = new FakeEgress({
"https://video.example/api/v1/video-channels/collective%40video.example/videos": () => new Response(PEERTUBE_FIXTURE, { headers: { "content-type": "application/json" } }),
});
const adapter = new PeerTubeChannelAdapter();
const result = await adapter.poll(context("peertube_channel", { instance: "https://video.example", handle: "collective@video.example" }, {}, egress));
expect(result.items).toHaveLength(2);
expect(egress.draws[0]!.hostClass).toBe("intake-peertube");
expect(result.items[0]!.remoteId).toBe("9c1de6de-8df0-4e3a-97a6-7e3a1d6e9a11");
expect(result.items[0]!.media![0]!.url).toBe("https://video.example/static/thumbnails/a11.jpg");
expect(result.items[0]!.metrics.durationSec).toBe(5400);
const second = await adapter.poll(context("peertube_channel", { instance: "https://video.example", handle: "collective@video.example" }, result.state, egress));
expect(second.items).toHaveLength(0); // seen-set window
});
it("videosToItems tolerates missing uuid/dates", () => {
expect(videosToItems([{ name: "no id" }], "https://video.example", new Date())).toHaveLength(0);
const one = videosToItems([{ uuid: "x", name: "n" }], "https://video.example", new Date("2026-10-09T00:00:00Z"));
expect(one[0]!.createdAt).toEqual(new Date("2026-10-09T00:00:00Z"));
});
});
describe("youtube adapter", () => {
it("channel Atom: videoId remote ref, watch URL, thumbnail; seen window suppresses re-emission", async () => {
const egress = new FakeEgress({
"https://www.youtube.com/feeds/videos.xml": () => new Response(YOUTUBE_FIXTURE, { headers: { "content-type": "application/atom+xml", etag: "\"yt1\"" } }),
});
const adapter = new YouTubeChannelAdapter();
const result = await adapter.poll(context("youtube_channel", { channelId: "UCtest0000000000000000" }, {}, egress));
expect(result.items).toHaveLength(1);
const item = result.items[0]!;
expect(item.remoteRef).toBe("yt:video:AAAAAAAAAAA");
expect(item.attribution.url).toBe("https://www.youtube.com/watch?v=AAAAAAAAAAA");
expect(item.media![0]!.kind).toBe("image");
expect(result.state.etag).toBe("\"yt1\"");
const second = await adapter.poll(context("youtube_channel", { channelId: "UCtest0000000000000000" }, result.state, egress));
expect(second.items).toHaveLength(0);
});
});
describe("web watch adapter", () => {
it("diff normalization strips churn and hash only moves on real change", () => {
const a = normalizePageText(WEB_PAGE_A);
const b = normalizePageText(WEB_PAGE_B);
const churn = normalizePageText(WEB_PAGE_A.replace("9f8e7d6c5b4a3f2e1d0c9b8a7f6e5d4c", "ffffffffffffffffffffffffffffffff").replace("March", "March"));
expect(pageHash(a)).toBe(pageHash(churn)); // token + identical text → same hash
expect(pageHash(a)).not.toBe(pageHash(b)); // March → April
expect(a).toContain("courthouse");
expect(a).not.toContain("csrf");
expect(a).not.toContain("color:red");
});
it("first fetch seeds the baseline without emitting; change emits one item", async () => {
let body = WEB_PAGE_A;
const egress = new FakeEgress({ "https://example.org/hearing": () => new Response(body, { headers: { "content-type": "text/html" } }) });
const adapter = new WebWatchAdapter();
const params = { url: "https://example.org/hearing", selector: "#main" };
const seed = await adapter.poll(context("web_watch", params, {}, egress));
expect(seed.items).toHaveLength(0);
expect(typeof seed.state.contentHash).toBe("string");
body = WEB_PAGE_B;
const changed = await adapter.poll(context("web_watch", params, seed.state, egress));
expect(changed.items).toHaveLength(1);
expect(changed.items[0]!.attribution.diffHash).toBe(changed.state.contentHash);
const stable = await adapter.poll(context("web_watch", params, changed.state, egress));
expect(stable.unchanged).toBe(true);
});
});
describe("push-only seams", () => {
it("bluesky keyword gate is exact-substring, multi-term AND", () => {
expect(matchesKeywords("Water rights now", "water rights")).toBe(true);
expect(matchesKeywords("Water rights now", "water luxury")).toBe(false);
expect(matchesKeywords("WATER RIGHTS", "water")).toBe(true);
expect(matchesKeywords("anything", "")).toBe(false);
const adapter = new BlueskyKeywordAdapter();
expect(adapter.gate("solidarity with water strikers", { keywords: ["water strikers"] })).toBe(true);
expect(adapter.gate("unrelated", { keywords: ["water strikers"] })).toBe(false);
});
it("telegram + bluesky poll() are clean no-ops", async () => {
for (const Adapter of [TelegramChannelAdapter, BlueskyKeywordAdapter]) {
const egress = new FakeEgress({});
const result = await new Adapter().poll(context("telegram_channel", { channelId: "-100123" }, {}, egress));
expect(result.items).toHaveLength(0);
expect(result.unchanged).toBe(true);
expect(egress.draws).toHaveLength(0); // never fetches
}
});
});
describe("adapter registry", () => {
it("is total over kinds", () => {
for (const kind of INTAKE_KINDS) {
const adapter = adapterFor(kind);
expect(adapter.kind).toBe(kind);
}
});
});
describe("content hash + text collapse", () => {
it("collapses whitespace and NFKC before hashing; media identity is sorted+unique", () => {
expect(collapseText(" a\u00a0 b ")).toBe("a b");
const h1 = itemContentHash("same text", ["https://b.example/x", "https://a.example/y"]);
const h2 = itemContentHash("same text", ["https://a.example/y", "https://b.example/x", "https://a.example/y"]);
expect(h1).toBe(h2); // whitespace + order + duplicates do not matter
expect(h1).not.toBe(itemContentHash("same text", []));
expect(h1).toMatch(/^[0-9a-f]{64}$/);
});
});
describe("scheduler math + error classification helpers", () => {
it("backoff curve: 2^n capped at 15 min", () => {
expect(backoffDelayMs(1, 60_000)).toBe(60_000);
expect(backoffDelayMs(2, 60_000)).toBe(120_000);
expect(backoffDelayMs(7, 60_000)).toBe(900_000); // 60s·2^6 = 64 min → capped at 15 min
expect(backoffDelayMs(1, 60_000)).toBeLessThanOrEqual(15 * 60_000);
});
it("jitter stays within ±20 % and validates inputs", () => {
expect(jitteredIntervalMs(100, 0)).toBe(80);
expect(jitteredIntervalMs(100, 1)).toBe(120);
expect(jitteredIntervalMs(100, 0.5)).toBe(100);
expect(() => jitteredIntervalMs(0, 0.5)).toThrow();
expect(() => jitteredIntervalMs(100, 1.5)).toThrow();
});
it("retry-after parses seconds and HTTP-date", () => {
const seconds = new Headers({ "retry-after": "30" });
expect(intakeRetryAfterMs(seconds, 0)).toBe(30_000);
const past = new Headers({ "retry-after": new Date(0).toUTCString() });
expect(intakeRetryAfterMs(past, 10_000)).toBe(0);
expect(intakeRetryAfterMs(new Headers(), 0)).toBe(0);
const err = new IntakeHttpError(429, new Headers({ "retry-after": "5" }), "https://x");
expect(err.rateLimited && err.transient).toBe(true);
expect(new IntakeHttpError(410, new Headers(), "https://x").gone).toBe(true);
});
});

29
apps/api/src/intake/adapters/index.ts Normal file
View file

@ -0,0 +1,29 @@
/**
* The per-kind adapter registry — one entry per intake kind, mirroring the
* feed-registry composition boundary. Adding a source kind = one adapter +
* one registry line; the scheduler and service never branch on kind.
*/
import type { IntakeAdapter } from "./adapter.js";
import { RssAdapter } from "./rss.adapter.js";
import { MastodonAccountAdapter, MastodonTagAdapter } from "./mastodon.adapter.js";
import { PeerTubeChannelAdapter } from "./peertube.adapter.js";
import { YouTubeChannelAdapter } from "./youtube.adapter.js";
import { WebWatchAdapter } from "./web-watch.adapter.js";
import { BlueskyKeywordAdapter, TelegramChannelAdapter } from "./push-seams.js";
import type { IntakeKind } from "../intake-kinds.js";
export const INTAKE_ADAPTERS: Readonly<Record<IntakeKind, () => IntakeAdapter>> = {
rss: () => new RssAdapter(),
mastodon_tag: () => new MastodonTagAdapter(),
mastodon_account: () => new MastodonAccountAdapter(),
bluesky_keyword: () => new BlueskyKeywordAdapter(),
telegram_channel: () => new TelegramChannelAdapter(),
peertube_channel: () => new PeerTubeChannelAdapter(),
youtube_channel: () => new YouTubeChannelAdapter(),
web_watch: () => new WebWatchAdapter(),
};
/** A fresh adapter per source poll (adapters are stateless; context carries state). */
export function adapterFor(kind: IntakeKind): IntakeAdapter {
return INTAKE_ADAPTERS[kind]();
}

141
apps/api/src/intake/adapters/mastodon.adapter.ts Normal file
View file

@ -0,0 +1,141 @@
/**
* Mastodon adapters — public anon tag + account timelines.
*
* Verified facts (research §1.3, 2026-10-09): `/api/v1/timelines/tag/{tag}`
* and `/api/v1/accounts/{id}/statuses` answer 200 anonymously; the federated
* public/local timelines are 422 for anon users, so this adapter never builds
* on those. Budget: `x-ratelimit-limit: 300` per 5 min per IP — modeled by the
* intake-social host class (1 req/s ceiling), and `X-RateLimit-Remaining` /
* `Retry-After` headers are already honored by the shared egress bucket's
* observe() path, which parks the host on 429/0-remaining.
*
* Status HTML runs through the P0c allowlist parser (`contentKind: "html"`);
* `media_attachments[].url` (directly fetchable, no auth) become harvest
* candidates alongside the proxied preview refs.
*/
import { createHash } from "node:crypto";
import { fetchJson, hostClassFor, type AdapterContext, type AdapterPollResult, type IntakeAdapter, type RawIntakeItem } from "./adapter.js";
import { KIND_MASTODON_ACCOUNT, KIND_MASTODON_TAG } from "../intake-kinds.js";
import type { MastodonAccountParams, MastodonTagParams } from "../intake-params.js";
export const MASTODON_PAGE_LIMIT = 40;
interface MastodonStatus {
id: string;
uri?: string;
url?: string | null;
created_at?: string;
content?: string;
account?: { id?: string; acct?: string; display_name?: string; url?: string; username?: string };
media_attachments?: Array<{ id?: string; url?: string | null; preview_url?: string | null; type?: string; mime_type?: string }>;
reblog?: MastodonStatus | null;
visibility?: string;
}
const isStatus = (value: unknown): value is MastodonStatus =>
typeof value === "object" && value !== null && typeof (value as MastodonStatus).id === "string";
const MEDIA_KIND: Record<string, "image" | "video"> = { image: "image", gifv: "image", video: "video", audio: "video" };
export function statusesToItems(statuses: MastodonStatus[], now: Date): RawIntakeItem[] {
const out: RawIntakeItem[] = [];
for (const status of statuses) {
if (!isStatus(status) || status.visibility === "private" || status.visibility === "direct") continue;
const author = status.account?.acct ?? status.account?.username ?? "unknown";
const host = author.includes("@") ? author.split("@").at(-1) : null;
const media: Array<{ url: string; kind: "image" | "video" }> = [];
for (const attachment of status.media_attachments ?? []) {
const url = typeof attachment.url === "string" ? attachment.url : attachment.preview_url;
if (typeof url === "string" && url !== "") {
media.push({ url, kind: MEDIA_KIND[attachment.type ?? ""] ?? (/^video\//i.test(attachment.mime_type ?? "") ? "video" : "image") });
}
}
const url = status.url ?? status.uri ?? null;
out.push({
id: status.id,
remoteId: status.id,
remoteRef: status.id,
provider: "mastodon",
authorHandle: author,
text: status.content ?? "",
mediaUrls: media.map((entry) => entry.url),
createdAt: parseDate(status.created_at) ?? now,
metrics: {},
normalize: { contentKind: "html", url: url ?? undefined },
attribution: {
authorHandle: author,
authorRef: host ? `mastodon:${author}` : `mastodon:${author}`,
url,
provider: "mastodon",
statusId: status.id,
},
media,
});
}
return out;
}
function parseDate(raw: string | undefined): Date | null {
if (typeof raw !== "string") return null;
const date = new Date(raw);
return Number.isFinite(date.getTime()) ? date : null;
}
/** Only statuses newer than the carried high-water mark are emitted. */
export function freshItems(items: RawIntakeItem[], lastRemoteRef: string | undefined, lastSeenIds: string[] | undefined): RawIntakeItem[] {
const seen = new Set(lastSeenIds ?? []);
const refOf = (item: RawIntakeItem): string => item.remoteRef ?? item.remoteId;
if (lastRemoteRef !== undefined && lastRemoteRef !== "") {
const cut = items.findIndex((item) => refOf(item) === lastRemoteRef);
if (cut >= 0) return items.slice(0, cut).filter((item) => !seen.has(refOf(item)));
}
return items.filter((item) => !seen.has(refOf(item)));
}
export function carryState(items: RawIntakeItem[], previous: Record<string, unknown>): Record<string, unknown> {
const priorIds = Array.isArray(previous.lastSeenIds) ? previous.lastSeenIds.filter((id): id is string => typeof id === "string") : [];
const priorRef = typeof previous.lastRemoteRef === "string" ? previous.lastRemoteRef : undefined;
const ids = [...priorIds, ...items.map((item) => item.remoteRef ?? item.remoteId)].slice(-80);
return { lastRemoteRef: items[0] !== undefined ? (items[0].remoteRef ?? items[0].remoteId) : priorRef, lastSeenIds: ids };
}
export class MastodonTagAdapter implements IntakeAdapter {
readonly kind = KIND_MASTODON_TAG;
async poll(context: AdapterContext): Promise<AdapterPollResult> {
const params = context.params as MastodonTagParams;
const { body } = await fetchJson<unknown>(
context.fetch,
`${params.instance}/api/v1/timelines/tag/${encodeURIComponent(params.tag)}?limit=${MASTODON_PAGE_LIMIT}&local=false`,
hostClassFor(KIND_MASTODON_TAG),
);
const statuses = Array.isArray(body) ? body.filter(isStatus) : [];
const items = freshItems(statusesToItems(statuses, new Date()),
typeof context.state.lastRemoteRef === "string" ? context.state.lastRemoteRef : undefined,
Array.isArray(context.state.lastSeenIds) ? context.state.lastSeenIds.filter((id): id is string => typeof id === "string") : undefined);
return { items, state: carryState(items, context.state) };
}
}
export class MastodonAccountAdapter implements IntakeAdapter {
readonly kind = KIND_MASTODON_ACCOUNT;
async poll(context: AdapterContext): Promise<AdapterPollResult> {
const params = context.params as MastodonAccountParams;
const { body } = await fetchJson<unknown>(
context.fetch,
`${params.instance}/api/v1/accounts/${encodeURIComponent(params.accountId)}/statuses?limit=${MASTODON_PAGE_LIMIT}&exclude_replies=true`,
hostClassFor(KIND_MASTODON_ACCOUNT),
);
const statuses = Array.isArray(body) ? body.filter(isStatus) : [];
const items = freshItems(statusesToItems(statuses, new Date()),
typeof context.state.lastRemoteRef === "string" ? context.state.lastRemoteRef : undefined,
Array.isArray(context.state.lastSeenIds) ? context.state.lastSeenIds.filter((id): id is string => typeof id === "string") : undefined);
return { items, state: carryState(items, context.state) };
}
}
/** Deterministic per-status scratch id (tests + dedup helpers). */
export function statusScratchId(status: MastodonStatus): string {
return createHash("sha256").update(status.id).digest("hex").slice(0, 16);
}

96
apps/api/src/intake/adapters/peertube.adapter.ts Normal file
View file

@ -0,0 +1,96 @@
/**
* PeerTube channel adapter — public channel videos over the same REST the
* connector and the live-monitor use: `GET /api/v1/video-channels/{handle}/videos`
* (anonymous, per research §1.5; instance default 50 req/10 s per IP, our
* intake-peertube host class runs stricter at 5/10 s). No webhooks exist
* upstream (verified), so polling is the only path.
*
* Media policy is thumbnail-first (research §2.5): the thumbnail is a harvest
* candidate; the full webseed file is only a candidate when the source opted
* into full-harvest, which the media policy gate in the service enforces.
*/
import { fetchJson, hostClassFor, type AdapterContext, type AdapterPollResult, type IntakeAdapter, type RawIntakeItem } from "./adapter.js";
import { KIND_PEERTUBE_CHANNEL } from "../intake-kinds.js";
import type { PeerTubeChannelParams } from "../intake-params.js";
interface PtVideo {
uuid?: string;
shortUUID?: string;
name?: string;
createdAt?: string;
publishedAt?: string;
account?: { name?: string; displayName?: string };
channel?: { displayName?: string; name?: string };
thumbnailPath?: string;
previewPath?: string;
isLive?: boolean;
duration?: number;
}
interface PtList { total?: number; data?: PtVideo[] }
function dateOf(...values: Array<string | undefined>): Date | null {
for (const value of values) {
if (typeof value !== "string") continue;
const date = new Date(value);
if (Number.isFinite(date.getTime())) return date;
}
return null;
}
export function videosToItems(videos: PtVideo[], instance: string, now: Date): RawIntakeItem[] {
const out: RawIntakeItem[] = [];
for (const video of videos) {
const uuid = video.uuid ?? video.shortUUID;
if (typeof uuid !== "string" || uuid === "") continue;
const url = `${instance}/videos/watch/${uuid}`;
const thumb = typeof video.thumbnailPath === "string" && video.thumbnailPath !== ""
? `${instance}${video.thumbnailPath}` : typeof video.previewPath === "string" && video.previewPath !== ""
? `${instance}${video.previewPath}` : null;
const handle = video.channel?.displayName ?? video.channel?.name ?? video.account?.name ?? "channel";
out.push({
id: uuid,
remoteId: uuid,
remoteRef: uuid,
provider: "peertube",
authorHandle: handle,
text: video.name ?? "",
mediaUrls: thumb ? [thumb] : [],
createdAt: dateOf(video.publishedAt, video.createdAt) ?? now,
metrics: typeof video.duration === "number" ? { durationSec: video.duration } : {},
normalize: { contentKind: "text", url },
attribution: {
title: video.name ?? null,
url,
channelTitle: video.channel?.displayName ?? null,
videoId: uuid,
provider: "peertube",
...(video.isLive === true ? { live: true } : {}),
},
media: thumb ? [{ url: thumb, kind: "image" }] : [],
});
}
return out;
}
export class PeerTubeChannelAdapter implements IntakeAdapter {
readonly kind = KIND_PEERTUBE_CHANNEL;
async poll(context: AdapterContext): Promise<AdapterPollResult> {
const params = context.params as PeerTubeChannelParams;
const { body } = await fetchJson<PtList>(
context.fetch,
`${params.instance}/api/v1/video-channels/${encodeURIComponent(params.handle)}/videos?count=25&sort=-publishedAt`,
hostClassFor(KIND_PEERTUBE_CHANNEL),
);
const videos = Array.isArray(body.data) ? body.data : [];
const seen = new Set(Array.isArray(context.state.seenVideoIds)
? context.state.seenVideoIds.filter((id): id is string => typeof id === "string") : []);
const items = videosToItems(videos, params.instance, new Date()).filter((item) => !seen.has(item.remoteRef ?? item.remoteId));
for (const item of items) seen.add(item.remoteRef ?? item.remoteId);
return {
items,
state: { ...context.state, seenVideoIds: [...seen].slice(-50) },
};
}
}

69
apps/api/src/intake/adapters/push-seams.ts Normal file
View file

@ -0,0 +1,69 @@
/**
* Push-only seams (research §1.2/§1.4): these kinds are never poll-due
* (DEFAULT_POLL_INTERVAL_SEC = 0). The adapters exist so the scheduler has a
* complete registry and so the ingest halves are testable now:
*
* - bluesky_keyword: the keyword gate consumed by the Jetstream WS worker.
* `matchesKeywords(text, keywords)` is the exact predicate the future
* consumer applies to every firehose event (~52 posts/s measured); it is
* exported and unit-tested so the seam is honest, not a stub that always
* returns true.
*
* - telegram_channel: intake rows for the EXISTING archive webhook path.
* The webhook already normalizes posts, harvests media and upserts into
* archived_feed_items (apps/api/src/archive/normalizer.ts + service);
* this adapter's poll() is unreachable by design, and the service's
* ingest path bridges archive items into intake rows (see
* IntakeService.ingestArchiveItem).
*/
import type { AdapterContext, AdapterPollResult, IntakeAdapter } from "./adapter.js";
import { KIND_BLUESKY_KEYWORD, KIND_TELEGRAM_CHANNEL } from "../intake-kinds.js";
import type { BlueskyKeywordParams } from "../intake-params.js";
/**
* Case/diacritic-folded substring gate. Multi-term keywords require BOTH
* terms (AND) — the research noise mitigation: one common word alone must
* not pull a trending meme into intake.
*/
export function matchesKeywords(text: string, keyword: string): boolean {
const needle = keyword.trim().toLowerCase();
if (needle === "") return false;
const terms = needle.split(/\s+/).filter((term) => term !== "");
if (terms.length === 0) return false;
const haystack = text.toLowerCase();
return terms.every((term) => haystack.includes(term));
}
export function filterByKeywords<T>(items: T[], textOf: (item: T) => string, keywords: readonly string[]): T[] {
return items.filter((item) => {
const text = textOf(item);
return keywords.some((keyword) => matchesKeywords(text, keyword));
});
}
export class BlueskyKeywordAdapter implements IntakeAdapter {
readonly kind = KIND_BLUESKY_KEYWORD;
async poll(_context: AdapterContext): Promise<AdapterPollResult> {
// Push-only: the Jetstream WS consumer owns this kind's write path; poll()
// exists so the registry is total over kinds and always answers a
// misconfigured cadence with a clean no-op instead of a fetch.
return { items: [], state: _context.state, unchanged: true };
}
/** The gate the WS worker applies per firehose event. */
gate(text: string, params: BlueskyKeywordParams): boolean {
return params.keywords.some((keyword) => matchesKeywords(text, keyword));
}
}
export class TelegramChannelAdapter implements IntakeAdapter {
readonly kind = KIND_TELEGRAM_CHANNEL;
async poll(_context: AdapterContext): Promise<AdapterPollResult> {
// Push-only: telegram rows arrive through the existing archive webhook
// (bot-admin channels) and the future MTProto sidecar; see the service's
// ingestArchiveItem bridge.
return { items: [], state: _context.state, unchanged: true };
}
}

276
apps/api/src/intake/adapters/rss-parse.ts Normal file
View file

@ -0,0 +1,276 @@
/**
* Minimal RSS 2.0 / Atom / RSS-in-HTML-entity parser for the intake adapters.
*
* In-process by the research verdict ("rss-parser in-process is ~100 lines;
* Miniflux only if feed edge-cases bite"). No external dependency: feeds are
* a closed, well-known XML subset, and the extractor below is total over
* arbitrary input — unknown elements are ignored, missing dates fall back to
* fetch time, and the caller (feed-normalize) owns all safety-critical
* parsing of text/URLs.
*
* XML entity decoding covers the five predefined entities plus numeric
* references; unknown entities pass through verbatim (a literal "&foo;" in a
* title is not a decoding failure).
*/
const ENTITY: Record<string, string> = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'",
nbsp: "\u00a0", eacute: "é", egrave: "è", agrave: "à", ccedil: "ç", auml: "ä", ouml: "ö", uuml: "ü",
laquo: "«", raquo: "»", ldquo: "\u201c", rdquo: "\u201d", lsquo: "\u2018", rsquo: "\u2019",
ndash: "–", mdash: "—", hellip: "…", rsquo2: "’" };
export function decodeEntities(text: string): string {
return text.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*);/g, (whole, entity: string) => {
if (entity.startsWith("#x") || entity.startsWith("#X")) {
const code = Number.parseInt(entity.slice(2), 16);
return Number.isInteger(code) && code > 0 && code <= 0x10ffff ? safeCodePoint(code) : whole;
}
if (entity.startsWith("#")) {
const code = Number.parseInt(entity.slice(1), 10);
return Number.isInteger(code) && code > 0 && code <= 0x10ffff ? safeCodePoint(code) : whole;
}
return ENTITY[entity] ?? whole;
});
}
function safeCodePoint(code: number): string {
try { return String.fromCodePoint(code); } catch { return "\uFFFD"; }
}
interface XmlNode {
tag: string;
attrs: Record<string, string>;
children: XmlNode[];
text: string;
}
const TAG_TOKEN = /<!--[\s\S]*?-->|<!\[CDATA\[([\s\S]*?)\]\]>|<\/( [a-zA-Z][a-zA-Z0-9:_-]* )\s*>|<([a-zA-Z][a-zA-Z0-9:_-]*)((?:\s+[^<>]*?)?)(\/?)>/g;
const NAME_RE = /^([a-zA-Z][a-zA-Z0-9:_-]*)\s*$/;
/** Strip any namespace prefix: "dc:creator" -> "creator". */
function localName(tag: string): string {
const at = tag.indexOf(":");
return at > 0 ? tag.slice(at + 1) : tag;
}
function parseAttrs(raw: string | undefined): Record<string, string> {
const attrs: Record<string, string> = {};
if (!raw) return attrs;
for (const match of raw.matchAll(/([a-zA-Z_:][a-zA-Z0-9_:.-]*)\s*=\s*(?:"([^"]*)"|'([^']*)')/g)) {
attrs[match[1]!.toLowerCase()] = decodeEntities(match[2] ?? match[3] ?? "");
}
return attrs;
}
/**
* Tolerant single-pass scan into a flat tree: one regex walks every tag token
* in document order, a stack tracks open elements, text between tags
* accumulates on the current node. Malformed markup (unterminated tags,
* mismatched closes) degrades to whatever structure is recoverable — never
* an exception, because feeds in the wild are routinely broken.
*/
export function parseXml(input: string): XmlNode {
const root: XmlNode = { tag: "#root", attrs: {}, children: [], text: "" };
const stack: XmlNode[] = [root];
let cursor = 0;
let textChunk = "";
const flush = (): void => {
if (textChunk !== "") {
stack[stack.length - 1]!.text += decodeEntities(textChunk);
textChunk = "";
}
};
const token = /<!--[\s\S]*?-->|<!\[CDATA\[([\s\S]*?)\]\]>|<\/([a-zA-Z][a-zA-Z0-9:_-]*)\s*>|<([a-zA-Z][a-zA-Z0-9:_-]*)((?:\s+[^<>]*?)?)(\/?)>/g;
for (const match of input.matchAll(token)) {
textChunk += input.slice(cursor, match.index);
cursor = match.index! + match[0].length;
if (match[1] !== undefined) { textChunk += match[1]; continue; } // CDATA: raw text
if (match[0].startsWith("<!")) continue; // comment / doctype
if (match[2] !== undefined) { // closing tag
flush();
const close = localName(match[2].toLowerCase());
const at = stack.map((node) => node.tag).lastIndexOf(close);
if (at > 0) stack.length = at; // pop to the matching open; stray closes are ignored
continue;
}
// Local-name only (dc:creator -> creator, yt:videoId -> videoid,
// media:thumbnail -> thumbnail): feed namespaces are decorative for the
// fields this extractor reads, and matching on the local name keeps every
// lookup below prefix-agnostic.
const tag = localName(match[3]!.toLowerCase());
flush();
const node: XmlNode = { tag, attrs: parseAttrs(match[4]), children: [], text: "" };
stack[stack.length - 1]!.children.push(node);
if (match[5] !== "/") stack.push(node); // not self-closing: descend
}
textChunk += input.slice(cursor);
flush();
return root;
}
function findFirst(node: XmlNode, tag: string): XmlNode | undefined {
for (const child of node.children) {
if (child.tag === tag) return child;
const nested = findFirst(child, tag);
if (nested) return nested;
}
return undefined;
}
function findAll(node: XmlNode, tag: string): XmlNode[] {
const out: XmlNode[] = [];
const walk = (current: XmlNode): void => {
for (const child of current.children) {
if (child.tag === tag) out.push(child);
walk(child);
}
};
walk(node);
return out;
}
/** text of a child tag by name, "" when absent — keeps author lookups flat. */
function childText(node: XmlNode, tag: string): string {
return textOf(childTag(node, tag));
}
/** Raw inner text (entities decoded, whitespace preserved) — CDATA HTML. */
function rawText(node: XmlNode): string {
return decodeEntities(node.text);
}
/** Inner text of a node across nested markup, whitespace-collapsed. */
function textOf(node: XmlNode | undefined): string {
if (!node) return "";
let out = node.text;
for (const child of node.children) out += " " + textOf(child);
return out.replace(/\s+/g, " ").trim();
}
export interface FeedEntry {
id: string;
title: string;
link: string | null;
contentHtml: string | null;
contentText: string | null;
publishedAt: Date | null;
updatedAt: Date | null;
author: string | null;
enclosures: Array<{ url: string; type: string }>;
mediaThumbnails: string[];
/** youtube atom: yt:videoId */
videoId: string | null;
}
export interface ParsedFeed {
type: "rss" | "atom";
channelTitle: string | null;
channelLink: string | null;
entries: FeedEntry[];
}
function childTag(node: XmlNode, tag: string): XmlNode | undefined {
return node.children.find((child) => child.tag === tag);
}
/** RSS <author><name>…</name></author> / Atom author/name text. */
function authorNameOf(node: XmlNode): string {
const author = childTag(node, "author");
return author === undefined ? "" : textOf(childTag(author, "name") ?? author);
}
function linkOf(item: XmlNode, channelLink: string | null): string | null {
// Atom: <link rel="alternate" href="…"/> wins; rel absent is alternate too.
// Self/service links (rel="self") are skipped. RSS: <link>text</link>.
const links = item.children.filter((child) => child.tag === "link");
for (const link of links) {
const rel = link.attrs.rel;
const href = link.attrs.href;
if (href !== undefined && (rel === undefined || rel === "alternate")) return href;
}
const plain = links.map((link) => textOf(link)).find((text) => text !== "");
if (plain) return plain;
const guid = childTag(item, "guid");
if (guid && /^https?:\/\//i.test(guid.text.trim())) return guid.text.trim();
return channelLink;
}
function dateOf(...candidates: Array<XmlNode | undefined>): Date | null {
for (const node of candidates) {
const raw = textOf(node);
if (raw === "") continue;
const parsed = new Date(raw);
if (Number.isFinite(parsed.getTime())) return parsed;
// RFC822 without timezone ("Thu, 09 Oct 2026 12:00:00") is common in the wild
const zoned = new Date(`${raw} GMT`);
if (Number.isFinite(zoned.getTime())) return zoned;
}
return null;
}
export function parseFeed(xml: string): ParsedFeed {
const root = parseXml(xml);
const rssChannel = findFirst(root, "channel");
const atomFeed = root.children.find((child) => child.tag === "feed");
if (rssChannel) {
const channelTitle = textOf(childTag(rssChannel, "title")) || null;
const channelLink = textOf(childTag(rssChannel, "link")) || null;
const entries = findAll(rssChannel, "item").map((item): FeedEntry => {
const encoded = childTag(item, "encoded"); // content:encoded (local name)
const description = childTag(item, "description");
// A description carrying markup IS provider HTML (CDATA-wrapped in
// practice); plain descriptions stay text so P0c parses them as text.
const descriptionHtml = encoded !== undefined ? textOf(encoded)
: description !== undefined && /<\s*[a-zA-Z]/.test(rawText(description)) ? rawText(description) : null;
const enclosures = item.children.filter((child) => child.tag === "enclosure")
.map((enclosure) => ({ url: enclosure.attrs.url ?? "", type: enclosure.attrs.type ?? "" }))
.filter((entry) => entry.url !== "");
const group = findFirst(item, "group"); // media:group
const thumbnails = [...findAll(item, "thumbnail"), ...(group ? findAll(group, "thumbnail") : [])]
.map((thumb) => thumb.attrs.url ?? "").filter((url) => url !== "");
return {
id: textOf(childTag(item, "guid")) || textOf(childTag(item, "link")) || "",
title: textOf(childTag(item, "title")),
link: linkOf(item, channelLink),
contentHtml: descriptionHtml,
contentText: descriptionHtml === null && description !== undefined ? textOf(description) : null,
publishedAt: dateOf(childTag(item, "pubdate"), childTag(item, "date"), childTag(item, "published")),
updatedAt: dateOf(childTag(item, "updated")),
author: textOf(childTag(item, "creator")) || childText(item, "author") || authorNameOf(item) || null,
enclosures,
mediaThumbnails: thumbnails,
videoId: null,
};
});
return { type: "rss", channelTitle, channelLink, entries };
}
if (atomFeed) {
const channelTitle = textOf(childTag(atomFeed, "title")) || null;
const channelLink = linkOf(atomFeed, null);
const entries = findAll(atomFeed, "entry").map((entry): FeedEntry => {
const contentHtml = textOf(childTag(entry, "content"));
const summary = textOf(childTag(entry, "summary"));
const enclosures = entry.children.filter((child) => child.tag === "link" && child.attrs.rel === "enclosure")
.map((link) => ({ url: link.attrs.href ?? "", type: link.attrs.type ?? "" }))
.filter((item) => item.url !== "");
const videoIdNode = entry.children.find((child) => child.tag === "videoid");
// yt:videoId is a leaf: raw text only (no nested aggregation).
const group = findFirst(entry, "group");
const thumbnails = [...findAll(entry, "thumbnail"), ...(group ? findAll(group, "thumbnail") : [])]
.map((thumb) => thumb.attrs.url ?? "").filter((url) => url !== "");
return {
id: textOf(childTag(entry, "id")) || "",
title: textOf(childTag(entry, "title")),
link: linkOf(entry, channelLink),
contentHtml: contentHtml !== "" ? contentHtml : null,
contentText: contentHtml === "" && summary !== "" ? summary : null,
publishedAt: dateOf(childTag(entry, "published"), childTag(entry, "updated")),
updatedAt: dateOf(childTag(entry, "updated")),
author: authorNameOf(entry) || childText(entry, "name") || null,
enclosures,
mediaThumbnails: thumbnails,
videoId: videoIdNode ? videoIdNode.text.trim() || null : null,
};
});
return { type: "atom", channelTitle, channelLink, entries };
}
return { type: "rss", channelTitle: null, channelLink: null, entries: [] };
}

78
apps/api/src/intake/adapters/rss.adapter.ts Normal file
View file

@ -0,0 +1,78 @@
/**
* RSS/Atom adapter — conditional GET via ETag / Last-Modified.
*
* The feed is fetched once per poll through the egress client (one budget
* draw); when upstream answers 304 the poll is free and emits nothing, and
* the carried ETag/Last-Modified state rides in intake_sources.state. Text
* lands as sanitized HTML through the P0c allowlist parser (feed summaries
* are HTML in practice), enclosures/thumbnails become harvest candidates.
*/
import { createHash } from "node:crypto";
import { fetchText, type AdapterContext, type AdapterPollResult, type IntakeAdapter, type RawIntakeItem } from "./adapter.js";
import { parseFeed, type ParsedFeed } from "./rss-parse.js";
import { hostClassFor } from "./adapter.js";
import { KIND_RSS } from "../intake-kinds.js";
import type { RssParams } from "../intake-params.js";
export const MAX_FEED_ITEMS = 100;
function stateHeaders(state: Record<string, unknown>): Record<string, string> {
const headers: Record<string, string> = {};
if (typeof state.etag === "string" && state.etag !== "") headers["if-none-match"] = state.etag;
if (typeof state.lastModified === "string" && state.lastModified !== "") headers["if-modified-since"] = state.lastModified;
return headers;
}
export function feedToItems(feed: ParsedFeed, publishedFallback: Date): RawIntakeItem[] {
return feed.entries.slice(0, MAX_FEED_ITEMS).map((entry) => {
const media: Array<{ url: string; kind: "image" | "video" }> = [];
for (const enclosure of entry.enclosures) {
if (/^video\//i.test(enclosure.type)) media.push({ url: enclosure.url, kind: "video" });
else if (/^image\//i.test(enclosure.type) || /\.(?:png|jpe?g|webp|gif)(?:[?#]|$)/i.test(enclosure.url)) media.push({ url: enclosure.url, kind: "image" });
else if (/^audio\//i.test(enclosure.type)) media.push({ url: enclosure.url, kind: "video" });
}
for (const thumb of entry.mediaThumbnails.slice(0, 4)) media.push({ url: thumb, kind: "image" });
const text = entry.contentHtml ?? entry.contentText ?? entry.title;
const scratch = createHash("sha256").update(text).digest("hex").slice(0, 32);
const ref = entry.id || entry.link || `sha256:${scratch}`;
return {
id: entry.id || entry.link || scratch,
remoteId: ref,
remoteRef: ref,
provider: "rss",
authorHandle: entry.author ?? feed.channelTitle ?? "feed",
text,
mediaUrls: media.map((entry_media) => entry_media.url),
createdAt: entry.publishedAt ?? entry.updatedAt ?? publishedFallback,
metrics: {},
normalize: { contentKind: entry.contentHtml !== null ? "html" : "text", url: entry.link ?? undefined },
attribution: {
title: entry.title || null,
url: entry.link ?? null,
author: entry.author ?? null,
channelTitle: feed.channelTitle,
provider: "rss",
...(entry.publishedAt ? { publishedAt: entry.publishedAt.toISOString() } : {}),
},
media,
};
});
}
export class RssAdapter implements IntakeAdapter {
readonly kind = KIND_RSS;
async poll(context: AdapterContext): Promise<AdapterPollResult> {
const params = context.params as RssParams;
const hostClass = hostClassFor(KIND_RSS);
const { text, response, unchanged } = await fetchText(context.fetch, params.url, hostClass, stateHeaders(context.state));
const state: Record<string, unknown> = { ...context.state };
const etag = response.headers.get("etag");
const lastModified = response.headers.get("last-modified");
if (etag !== null) state.etag = etag;
if (lastModified !== null) state.lastModified = lastModified;
if (unchanged) return { items: [], state, unchanged: true };
const feed = parseFeed(text);
return { items: feedToItems(feed, new Date()), state };
}
}

100
apps/api/src/intake/adapters/web-watch.adapter.ts Normal file
View file

@ -0,0 +1,100 @@
/**
* Page-watch adapter — the changedetection.io pattern in ~100 lines
* (research §1.8): fetch the page through the egress budget, normalize the
* text (strip scripts/styles, collapse whitespace, strip rotating
* timestamps/CSRF nonces), hash, and emit an item only when the hash moves.
*
* Plain fetch only — no headless browser (per the task's "no headless"
* bound); a page that needs JS renders as its static DOM, which is the right
* default for press pages and gazettes.
*/
import { createHash } from "node:crypto";
import { fetchText, hostClassFor, type AdapterContext, type AdapterPollResult, type IntakeAdapter, type RawIntakeItem } from "./adapter.js";
import { KIND_WEB_WATCH } from "../intake-kinds.js";
import type { WebWatchParams } from "../intake-params.js";
/**
* Diff normalization: what changes on a page without changing its content —
* numbers (counts, timestamps), long hex/base64 tokens (CSRF, cache busters).
* Hashing the normalized text is what keeps phantom diffs out of the archive.
*/
export function normalizePageText(html: string, selector?: string): string {
let work = html
.replace(/<script[\s\S]*?<\/script>/gi, " ")
.replace(/<style[\s\S]*?<\/style>/gi, " ")
.replace(/<noscript[\s\S]*?<\/noscript>/gi, " ")
.replace(/<!--[\s\S]*?-->/g, " ")
.replace(/<[^>]+>/g, " ")
.replace(/&nbsp;/gi, " ")
.replace(/&amp;/gi, "&")
.replace(/&lt;/gi, "<")
.replace(/&gt;/gi, ">")
.replace(/&quot;/gi, '"')
.replace(/&#0?39;/g, "'");
if (selector !== undefined && selector.trim() !== "") {
// Cheap id/class containment: keep only lines containing the selector's
// leaf token when the selector looks like #id or .class (a full CSS
// engine is out of scope for phase 1; a wrong selector degrades to the
// whole page's diff, which is still correct — just noisier).
const leaf = selector.trim().split(/[ >]+/).at(-1) ?? "";
const match = /^([#.])([\w-]+)$/.exec(leaf);
if (match) {
// Extract the region around the marker token from the raw HTML instead.
const marker = match[1] === "#" ? `id="${match[2]}"` : `class="${match[2]}"`;
const at = html.toLowerCase().indexOf(marker.toLowerCase());
if (at >= 0) {
const from = html.lastIndexOf("<", at);
const to = html.indexOf("</body>", at);
work = normalizePageText(html.slice(from > 0 ? from : 0, to > 0 ? to : undefined));
}
}
}
return work
.replace(/\b[0-9]{2,}\b/g, " ") // counts, epochs, timestamps
.replace(/\b[0-9a-f]{16,}\b/gi, " ") // CSRF tokens, digests
.replace(/\s+/g, " ")
.trim();
}
export function pageHash(normalized: string): string {
return createHash("sha256").update(normalized).digest("hex");
}
export class WebWatchAdapter implements IntakeAdapter {
readonly kind = KIND_WEB_WATCH;
async poll(context: AdapterContext): Promise<AdapterPollResult> {
const params = context.params as WebWatchParams;
const { text, unchanged } = await fetchText(context.fetch, params.url, hostClassFor(KIND_WEB_WATCH), {
"cache-control": "no-cache",
});
if (unchanged) return { items: [], state: context.state, unchanged: true };
const normalized = normalizePageText(text, params.selector);
const hash = pageHash(normalized);
if (typeof context.state.contentHash === "string" && context.state.contentHash === hash) {
return { items: [], state: context.state, unchanged: true };
}
const now = new Date();
const items: RawIntakeItem[] = [];
if (typeof context.state.contentHash === "string") {
// First fetch seeds the baseline; only a CHANGED page (second fetch
// onward) emits — a new subscription never manufactures an item for a
// page that was already in that state.
items.push({
id: hash.slice(0, 24),
remoteId: hash.slice(0, 24),
remoteRef: `${params.url}#${hash.slice(0, 16)}`,
provider: "web",
authorHandle: new URL(params.url).hostname,
text: normalized.slice(0, 8_000),
mediaUrls: [],
createdAt: now,
metrics: {},
normalize: { contentKind: "text", url: params.url },
attribution: { url: params.url, provider: "web", changedAt: now.toISOString(), diffHash: hash },
media: [],
});
}
return { items, state: { ...context.state, contentHash: hash, lastChangedAt: now.toISOString() } };
}
}

59
apps/api/src/intake/adapters/youtube.adapter.ts Normal file
View file

@ -0,0 +1,59 @@
/**
* YouTube channel adapter — the channel Atom feed, no key, no quota
* (verified live, research §1.1/§1.6: `feeds/videos.xml?channel_id=…`
* returns Atom with yt:videoId, media thumbnails; 15-entry window, ~15 min
* freshness → poll ≤ every 10 min). The Data API is deliberately NOT used
* here: search.list costs 100 units against a 10k/day project budget while
* the feed is free.
*/
import { fetchText, hostClassFor, type AdapterContext, type AdapterPollResult, type IntakeAdapter, type RawIntakeItem } from "./adapter.js";
import { parseFeed } from "./rss-parse.js";
import { KIND_YOUTUBE_CHANNEL, YOUTUBE_FEED_URL } from "../intake-kinds.js";
import type { YouTubeChannelParams } from "../intake-params.js";
export class YouTubeChannelAdapter implements IntakeAdapter {
readonly kind = KIND_YOUTUBE_CHANNEL;
async poll(context: AdapterContext): Promise<AdapterPollResult> {
const params = context.params as YouTubeChannelParams;
const { text, response, unchanged } = await fetchText(context.fetch, YOUTUBE_FEED_URL(params.channelId), hostClassFor(KIND_YOUTUBE_CHANNEL));
const state: Record<string, unknown> = { ...context.state };
const etag = response.headers.get("etag");
if (etag !== null) state.etag = etag;
if (unchanged) return { items: [], state, unchanged: true };
const feed = parseFeed(text);
const seen = new Set(Array.isArray(state.seenVideoIds)
? state.seenVideoIds.filter((id): id is string => typeof id === "string") : []);
const items: RawIntakeItem[] = [];
for (const entry of feed.entries) {
const videoId = entry.videoId ?? (entry.id.startsWith("yt:video:") ? entry.id.slice("yt:video:".length) : null);
if (videoId === null || seen.has(videoId)) continue;
const watchUrl = `https://www.youtube.com/watch?v=${videoId}`;
items.push({
id: videoId,
remoteId: `yt:video:${videoId}`,
remoteRef: `yt:video:${videoId}`,
provider: "youtube",
authorHandle: entry.author ?? feed.channelTitle ?? params.channelId,
text: entry.title,
mediaUrls: entry.mediaThumbnails.slice(0, 1),
createdAt: entry.publishedAt ?? entry.updatedAt ?? new Date(),
metrics: {},
normalize: { contentKind: "text", url: watchUrl },
attribution: {
title: entry.title || null,
url: watchUrl,
channelTitle: feed.channelTitle,
channelId: params.channelId,
videoId,
provider: "youtube",
},
// metadata-only default: the thumbnail is harvested, never the video.
media: entry.mediaThumbnails.slice(0, 1).map((url) => ({ url, kind: "image" as const })),
});
seen.add(videoId);
}
state.seenVideoIds = [...seen].slice(-30);
return { items, state };
}
}

108
apps/api/src/intake/intake-egress.ts Normal file
View file

@ -0,0 +1,108 @@
/**
* Intake egress binding — THE single outbound surface for every intake
* adapter (task requirement 2: all outbound through packages/egress per-host
* budgets).
*
* Same shape as live-monitor/poller/egress.ts and virtual-channels/
* import-egress.ts: the egress package is ESM-only while the API emits
* CommonJS, so it is type-imported (erased at build) and loaded natively at
* runtime. Every request drawn by an adapter goes through `egressFetch`,
* which means SSRF policy (private-address denial, scheme/port policy), the
* global timeout, the response-size cap and — the point — the shared
* `host_budgets` rows all apply. Two replicas polling mastodon.social contend
* for one bucket; a 429 parks that host for every caller.
*
* `IntakeHttpError` carries the status + headers so adapters can classify
* 304-not-modified (free), 429/Retry-After (budget/park) and 5xx (backoff)
* the way the monitor poller's BackoffPollDriver does.
*/
import { existsSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { pathToFileURL } from "node:url";
import { pool } from "../db/client.js";
import { INTAKE_HOST_CLASSES } from "./intake-kinds.js";
// Type-only (erased in the CommonJS build); loaded natively below.
import type { createEgressClient, EgressError, EgressPolicy } from "../../../../packages/egress/src/index.js";
type EgressPackage = typeof import("../../../../packages/egress/src/index.js");
export interface IntakeEgress {
egressFetch: ReturnType<typeof createEgressClient>["egressFetch"];
policy: EgressPolicy;
isEgressError(error: unknown): error is EgressError;
}
/** Response body cap for feed/page/JSON fetches (not media downloads). */
export const INTAKE_MAX_BYTES = 8 * 1024 * 1024;
export const INTAKE_TIMEOUT_MS = 20_000;
/** Media harvest ceiling per file (research §2.5 default image cap). */
export const INTAKE_MEDIA_MAX_BYTES = 50 * 1024 * 1024;
const nativeImport = new Function("specifier", "return import(specifier)") as (specifier: string) => Promise<EgressPackage>;
let shared: Promise<IntakeEgress> | undefined;
function resolveEgressPackage(): string {
for (const start of [process.cwd(), __dirname]) {
for (let root = start; ; root = dirname(root)) {
const directory = resolve(root, "packages/egress");
const files = process.env.NODE_ENV === "production" ? ["dist/index.js"] : ["src/index.ts", "dist/index.js"];
for (const file of files) {
const candidate = resolve(directory, file);
if (existsSync(candidate)) return candidate;
}
if (dirname(root) === root) break;
}
}
throw new Error("Build packages/egress before starting intake");
}
/** Process-wide egress client shared by every intake adapter. */
export function intakeEgress(): Promise<IntakeEgress> {
shared ??= (async () => {
const pkg = await nativeImport(pathToFileURL(resolveEgressPackage()).href);
const client = pkg.createEgressClient({
pool,
timeoutMs: INTAKE_TIMEOUT_MS,
maxResponseBytes: INTAKE_MAX_BYTES,
hostClasses: INTAKE_HOST_CLASSES,
});
return { ...client, isEgressError: (error: unknown): error is EgressError => error instanceof pkg.EgressError };
})();
return shared;
}
/** Non-2xx/3xx egress response surfaced with its headers (Retry-After, ETag…). */
export class IntakeHttpError extends Error {
constructor(readonly status: number, readonly headers: Headers, readonly url: string) {
super(`intake HTTP ${status}`);
this.name = "IntakeHttpError";
}
get notModified(): boolean { return this.status === 304; }
get gone(): boolean { return this.status === 404 || this.status === 410; }
get rateLimited(): boolean { return this.status === 429; }
get transient(): boolean { return this.status === 429 || this.status >= 500; }
}
/** Retry-After as a lower-bound delay in ms, including HTTP-date form. */
export function intakeRetryAfterMs(headers: Headers, now: number): number {
const raw = headers.get("retry-after");
if (raw === null) return 0;
if (/^\d+(?:\.\d+)?$/.test(raw.trim())) {
const delay = Number(raw) * 1000;
return Number.isFinite(delay) ? delay : 0;
}
const deadline = Date.parse(raw);
return Number.isFinite(deadline) ? Math.max(0, deadline - now) : 0;
}
/** Reads a response body to a string under the shared size cap. */
export async function readBodyText(response: Response): Promise<string> {
return response.text();
}
/** Reads a response body to a Buffer under the media cap, before S3 put. */
export async function readBodyBytes(response: Response): Promise<Buffer> {
const bytes = Buffer.from(await response.arrayBuffer());
if (bytes.byteLength > INTAKE_MEDIA_MAX_BYTES) throw new Error("intake media exceeds size cap");
return bytes;
}

122
apps/api/src/intake/intake-ingest.ts Normal file
View file

@ -0,0 +1,122 @@
/**
* The intake ingest funnel: raw adapter items → P0c normalize → content-hash
* → idempotent upsert into intake_items (027).
*
* Every item passes `normalizeFeedItem` (task requirement 3), so `content`
* lands as the closed safe-content AST — never raw provider HTML — and media
* as proxy-minted refs. `text` (the FTS corpus) is derived from the AST via
* `astToPlainText`, never the raw provider string.
*
* `content_hash` is the schema's dedup key — UNIQUE (workspace_id,
* content_hash): sha256 of the NFKC + whitespace-collapsed AST plain text
* plus the sorted media origin identity. The same statement arriving via RSS
* and a Mastodon boost (or the same URL watched twice) hashes equal and
* stores once; first writer wins, later deliveries no-op.
*/
import { createHash } from "node:crypto";
import type { PoolClient } from "pg";
import { normalizeFeedItem, astToPlainText, type MediaRef, type NormalizedFeedItem } from "../feeds/feed-normalize.js";
import { connectionFor, type RawIntakeItem } from "./adapters/adapter.js";
import { PROVIDER_BY_KIND, type IntakeKind } from "./intake-kinds.js";
import type { IntakeParams } from "./intake-params.js";
import type { HarvestedMedia } from "./media-harvest.js";
export type IntakeMediaRef = MediaRef & {
/** S3 content-addressed key when the harvest stored the origin file. */
storageKey?: string;
hash?: string;
sizeBytes?: number;
contentType?: string;
harvestStatus?: "stored" | "skipped" | "too_large" | "failed";
originUrl?: string;
};
export interface NormalizeOutcome {
normalized: NormalizedFeedItem | null;
plainText: string;
contentHash: string;
}
/** NFKC + whitespace-collapse, per the 027 content_hash note. */
export function collapseText(plainText: string): string {
return plainText.normalize("NFKC").replace(/\s+/g, " ").trim();
}
/** sha256(collapsed plain text + sorted unique media origin URLs). */
export function itemContentHash(plainText: string, mediaUrls: readonly string[]): string {
const origins = [...new Set(mediaUrls.filter((url) => typeof url === "string" && url !== ""))].sort();
return createHash("sha256")
.update(JSON.stringify([collapseText(plainText), origins]))
.digest("hex");
}
/** Normalize one raw item; normalized = null when the funnel's validation drops it. */
export function normalizeRawItem(raw: RawIntakeItem, kind: IntakeKind, params: IntakeParams["params"],
workspaceId: string, sourceId: string): NormalizeOutcome {
const connection = connectionFor(kind, params, sourceId, workspaceId);
const normalized = normalizeFeedItem(raw, connection, raw.normalize);
if (normalized === null) return { normalized: null, plainText: "", contentHash: "" };
const plainText = astToPlainText(normalized.content);
return { normalized, plainText, contentHash: itemContentHash(plainText, raw.mediaUrls) };
}
/** Merge the P0c proxied media refs with the harvest outcome per origin URL. */
export function mergeMediaRefs(normalized: NormalizedFeedItem, harvested: HarvestedMedia[]): IntakeMediaRef[] {
const byUrl = new Map(harvested.map((entry) => [entry.url, entry]));
return normalized.media.map((ref): IntakeMediaRef => {
const origin = originOfProxyUrl(ref.proxyUrl);
const hit = origin === null ? undefined : byUrl.get(origin);
if (!hit) return ref;
return {
...ref,
originUrl: hit.url,
...(hit.status === "stored" ? {
storageKey: hit.storageKey,
hash: hit.hash,
sizeBytes: hit.sizeBytes,
contentType: hit.contentType,
harvestStatus: "stored" as const,
} : { harvestStatus: hit.status }),
};
});
}
/** Extracts the origin URL back out of a media-proxy URL. */
export function originOfProxyUrl(proxyUrl: string): string | null {
const marker = "url=";
const at = proxyUrl.indexOf(marker);
if (at < 0) return null;
try { return decodeURIComponent(proxyUrl.slice(at + marker.length)); } catch { return null; }
}
export interface UpsertResult {
inserted: number;
unchanged: number;
}
/**
* Idempotent upsert inside the caller's tenant transaction. The conflict
* target is the content-addressed key — (workspace_id, content_hash) — so a
* re-poll that yields the same content writes nothing, and cross-source
* duplicates (RSS + Mastodon boost of the same statement) collapse too.
*/
export async function upsertItems(client: PoolClient, workspaceId: string, sourceId: string, kind: IntakeKind,
rows: Array<{ raw: RawIntakeItem; outcome: NormalizeOutcome; harvested: HarvestedMedia[] }>): Promise<UpsertResult> {
const result: UpsertResult = { inserted: 0, unchanged: 0 };
for (const entry of rows) {
const { normalized, plainText, contentHash } = entry.outcome;
if (normalized === null) continue;
const mediaRefs = mergeMediaRefs(normalized, entry.harvested);
const attribution: Record<string, unknown> = { ...entry.raw.attribution, provider: PROVIDER_BY_KIND[kind] };
const inserted = await client.query(
`INSERT INTO intake_items (workspace_id, source_id, remote_ref, content_hash, published_at,
text, content, media_refs, attribution)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)
ON CONFLICT (workspace_id, content_hash) DO NOTHING`,
[workspaceId, sourceId, entry.raw.remoteRef, contentHash, normalized.createdAt.toISOString(),
plainText, JSON.stringify(normalized.content), JSON.stringify(mediaRefs), JSON.stringify(attribution)]);
if ((inserted.rowCount ?? 0) > 0) result.inserted += 1;
else result.unchanged += 1;
}
return result;
}

128
apps/api/src/intake/intake-kinds.ts Normal file
View file

@ -0,0 +1,128 @@
/**
* Intake source kinds (migration 027 CHECK) and the per-kind maps the rest of
* the module keys off.
*
* This file is the ONE place provider-named string literals live outside the
* connector packages — the same standing as archive's ARCHIVE_PROVIDER
* boundary: the intake adapters ARE the provider integration surface for
* keyless public sources, so naming the platform they adapt is the boundary,
* not provider branching. Everything else in apps/api/src/intake references
* these constants; nothing else declares the literals.
*/
/* eslint-disable no-restricted-syntax -- intake kinds name the platform they adapt; this file is the provider boundary (archive ARCHIVE_PROVIDER pattern). */
export const INTAKE_KINDS = [
"rss", "mastodon_tag", "mastodon_account", "bluesky_keyword",
"telegram_channel", "peertube_channel", "youtube_channel", "web_watch",
] as const;
export type IntakeKind = (typeof INTAKE_KINDS)[number];
export function isIntakeKind(value: unknown): value is IntakeKind {
return typeof value === "string" && (INTAKE_KINDS as readonly string[]).includes(value);
}
export const KIND_RSS: IntakeKind = "rss";
export const KIND_MASTODON_TAG: IntakeKind = "mastodon_tag";
export const KIND_MASTODON_ACCOUNT: IntakeKind = "mastodon_account";
export const KIND_BLUESKY_KEYWORD: IntakeKind = "bluesky_keyword";
export const KIND_TELEGRAM_CHANNEL: IntakeKind = "telegram_channel";
export const KIND_PEERTUBE_CHANNEL: IntakeKind = "peertube_channel";
export const KIND_YOUTUBE_CHANNEL: IntakeKind = "youtube_channel";
export const KIND_WEB_WATCH: IntakeKind = "web_watch";
/** Kinds whose default posture is push-only (poll_interval_sec = 0, never due). */
export const PUSH_ONLY_KINDS: readonly IntakeKind[] = [KIND_BLUESKY_KEYWORD, KIND_TELEGRAM_CHANNEL];
/**
* The provider id handed to the P0c `normalizeFeedItem` funnel, so intake
* items carry the same canonical `{provider}:{handle@host}` authorRef shape
* every other feed item does.
*/
export const PROVIDER_BY_KIND: Record<IntakeKind, string> = {
rss: "rss",
mastodon_tag: "mastodon",
mastodon_account: "mastodon",
bluesky_keyword: "bluesky",
telegram_channel: "telegram",
peertube_channel: "peertube",
youtube_channel: "youtube",
web_watch: "web",
};
/**
* Default poll cadence per kind (seconds), from the firehose research source
* inventory: RSS 5–15 min, YouTube channel RSS ≤10 min (15-entry window),
* Mastodon 300/5min budget comfortably supports 60 s hashtags, PeerTube 10 min,
* page-watch hourly. 0 = push-only (never due; the telegram webhook bridge and
* the Jetstream seam park there).
*/
export const DEFAULT_POLL_INTERVAL_SEC: Record<IntakeKind, number> = {
rss: 900,
mastodon_tag: 300,
mastodon_account: 300,
bluesky_keyword: 0,
telegram_channel: 0,
peertube_channel: 600,
youtube_channel: 600,
web_watch: 3_600,
};
/**
* Egress host class per kind — the budget row class every outbound request of
* that adapter draws from (packages/egress HostBudgets, shared host_budgets
* rows deployment-wide). intake-social runs at capacity 3 / refill 10 per 10 s
* = 1 req/s ceiling = exactly the verified Mastodon 300/5min-per-IP budget.
*/
export const HOST_CLASS_BY_KIND: Record<IntakeKind, string> = {
rss: "intake-rss",
mastodon_tag: "intake-social",
mastodon_account: "intake-social",
bluesky_keyword: "intake-jetstream",
telegram_channel: "intake-telegram",
peertube_channel: "intake-peertube",
youtube_channel: "intake-rss",
web_watch: "intake-web",
};
/** Host-class budget shapes handed to createEgressClient. */
export const INTAKE_HOST_CLASSES = {
"intake-rss": { capacity: 3, refillPerInterval: 10 },
"intake-social": { capacity: 3, refillPerInterval: 10 },
"intake-peertube": { capacity: 3, refillPerInterval: 5 },
"intake-web": { capacity: 2, refillPerInterval: 2 },
"intake-telegram": { capacity: 3, refillPerInterval: 10 },
"intake-jetstream": { capacity: 1, refillPerInterval: 1 },
} as const;
/**
* Media policy defaults per kind (research §2.5): images are cheap and safe to
* harvest; full video files are opt-in per source because YouTube-class full
* harvest is the storage wildcard.
*/
export type MediaPolicy = "metadata" | "images" | "full";
export const DEFAULT_MEDIA_POLICY: Record<IntakeKind, MediaPolicy> = {
rss: "images",
mastodon_tag: "images",
mastodon_account: "images",
bluesky_keyword: "metadata",
telegram_channel: "metadata",
peertube_channel: "images",
youtube_channel: "metadata",
web_watch: "images",
};
/**
* Public endpoints the adapters poll. Verified keyless/live in the firehose
* research appendix (2026-10-09): YouTube channel Atom feed (no key, no
* quota), the Telegram Bot API base.
*/
export const YOUTUBE_FEED_URL = (channelId: string): string =>
`https://www.youtube.com/feeds/videos.xml?channel_id=${encodeURIComponent(channelId)}`;
export const TELEGRAM_API_ORIGIN = "https://api.telegram.org";
export const YOUTUBE_ORIGIN = "https://www.youtube.com";
/* eslint-enable no-restricted-syntax */

169
apps/api/src/intake/intake-params.ts Normal file
View file

@ -0,0 +1,169 @@
/**
* Per-kind source params: validation, canonicalization, and the params_hash
* identity half of UNIQUE (workspace_id, kind, params_hash) (migration 027).
*
* The DB only backstops "params is an object" — everything shape-related is
* validated here so a bad subscription is a 400/422 at the API edge, never a
* CHECK violation at insert time. Canonicalization is deterministic and total
* over the validated shape (sorted keys, trimmed strings, normalized URLs), so
* two subscriptions that differ only by `https://x/` vs `https://x` collide on
* the unique key instead of double-polling one feed.
*/
import { createHash } from "node:crypto";
import { BadRequestException } from "@nestjs/common";
import {
isIntakeKind, KIND_MASTODON_ACCOUNT, KIND_MASTODON_TAG, KIND_PEERTUBE_CHANNEL,
KIND_RSS, KIND_TELEGRAM_CHANNEL, KIND_WEB_WATCH, KIND_YOUTUBE_CHANNEL,
TELEGRAM_API_ORIGIN, YOUTUBE_ORIGIN, type IntakeKind,
} from "./intake-kinds.js";
export interface RssParams { url: string }
export interface MastodonTagParams { instance: string; tag: string }
export interface MastodonAccountParams { instance: string; accountId: string; handle?: string }
export interface BlueskyKeywordParams { keywords: string[] }
export interface TelegramChannelParams { channelId: string }
export interface PeerTubeChannelParams { instance: string; handle: string }
export interface YouTubeChannelParams { channelId: string }
export interface WebWatchParams { url: string; selector?: string }
export type IntakeParams =
| { kind: typeof KIND_RSS; params: RssParams }
| { kind: typeof KIND_MASTODON_TAG; params: MastodonTagParams }
| { kind: typeof KIND_MASTODON_ACCOUNT; params: MastodonAccountParams }
| { kind: "bluesky_keyword"; params: BlueskyKeywordParams }
| { kind: typeof KIND_TELEGRAM_CHANNEL; params: TelegramChannelParams }
| { kind: typeof KIND_PEERTUBE_CHANNEL; params: PeerTubeChannelParams }
| { kind: typeof KIND_YOUTUBE_CHANNEL; params: YouTubeChannelParams }
| { kind: typeof KIND_WEB_WATCH; params: WebWatchParams };
const URL_MAX = 2_048;
const STRING_MAX = 512;
const TAG = /^[\p{L}\p{N}_]{1,64}$/u;
const ACCOUNT_ID = /^[0-9]{1,19}$/;
const TELEGRAM_CHANNEL_ID = /^-?\d{3,}$/;
const YOUTUBE_CHANNEL = /^UC[\p{L}\p{N}_-]{20,24}$/u;
function isObject(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function requireString(value: unknown, field: string, max = STRING_MAX): string {
if (typeof value !== "string") throw new BadRequestException(`${field} must be a string`);
const trimmed = value.trim();
if (trimmed === "") throw new BadRequestException(`${field} must not be empty`);
if (trimmed.length > max) throw new BadRequestException(`${field} must be at most ${max} characters`);
return trimmed;
}
/** Origin-only instance URL (`https://mastodon.social`) — never a path/query. */
function requireInstance(value: unknown, field: string): string {
const raw = requireString(value, field, 2_048);
const withScheme = /^https?:\/\//i.test(raw) ? raw : `https://${raw}`;
let url: URL;
try { url = new URL(withScheme); } catch { throw new BadRequestException(`${field} must be a valid host or URL`); }
if (url.protocol !== "http:" && url.protocol !== "https:") throw new BadRequestException(`${field} must be http(s)`);
if (!url.hostname.includes(".")) throw new BadRequestException(`${field} must be a public host`);
return `${url.protocol}//${url.hostname.toLowerCase()}${url.port ? `:${url.port}` : ""}`;
}
function requireHttpUrl(value: unknown, field: string): string {
const raw = requireString(value, field, URL_MAX);
let url: URL;
try { url = new URL(raw); } catch { throw new BadRequestException(`${field} must be an absolute URL`); }
if (url.protocol !== "http:" && url.protocol !== "https:") throw new BadRequestException(`${field} must be an http(s) URL`);
if (url.username !== "" || url.password !== "") throw new BadRequestException(`${field} must not carry credentials`);
if (!url.hostname.includes(".")) throw new BadRequestException(`${field} must be a public host`);
url.hash = "";
return url.toString();
}
/** Parse + canonicalize the per-kind params object. Throws BadRequest on shape. */
export function parseIntakeParams(kind: unknown, raw: unknown): IntakeParams {
if (!isIntakeKind(kind)) throw new BadRequestException("kind must be a supported intake kind");
if (!isObject(raw)) throw new BadRequestException("params must be an object");
switch (kind) {
case KIND_RSS:
return { kind, params: { url: requireHttpUrl(raw.url, "params.url") } };
case KIND_MASTODON_TAG: {
const tag = requireString(raw.tag, "params.tag", 64);
if (!TAG.test(tag)) throw new BadRequestException("params.tag must be 1-64 word characters");
return { kind, params: { instance: requireInstance(raw.instance, "params.instance"), tag: tag.toLowerCase() } };
}
case KIND_MASTODON_ACCOUNT: {
const accountId = requireString(raw.accountId, "params.accountId", 19);
if (!ACCOUNT_ID.test(accountId)) throw new BadRequestException("params.accountId must be the numeric account id");
const params: MastodonAccountParams = { instance: requireInstance(raw.instance, "params.instance"), accountId };
const handle = raw.handle;
if (handle !== undefined && handle !== null && handle !== "") {
params.handle = requireString(handle, "params.handle", 64).replace(/^@+/, "").toLowerCase();
}
return { kind, params };
}
case "bluesky_keyword": {
const keywords = raw.keywords;
if (!Array.isArray(keywords) || keywords.length === 0 || keywords.length > 32) {
throw new BadRequestException("params.keywords must be a non-empty array (max 32)");
}
return { kind, params: { keywords: keywords.map((k) => requireString(k, "params.keywords[]", 100).toLowerCase()) } };
}
case KIND_TELEGRAM_CHANNEL: {
const channelId = requireString(raw.channelId, "params.channelId", 32);
if (!TELEGRAM_CHANNEL_ID.test(channelId)) throw new BadRequestException("params.channelId must be a numeric Telegram chat id");
return { kind, params: { channelId } };
}
case KIND_PEERTUBE_CHANNEL: {
const handle = requireString(raw.handle, "params.handle", 253).replace(/^@+/, "");
if (!/^[\p{L}\p{N}_.-]+(@[\p{L}\p{N}.-]+)?$/u.test(handle)) throw new BadRequestException("params.handle must be a channel handle");
return { kind, params: { instance: requireInstance(raw.instance, "params.instance"), handle } };
}
case KIND_YOUTUBE_CHANNEL: {
const channelId = requireString(raw.channelId, "params.channelId", 32);
if (!YOUTUBE_CHANNEL.test(channelId)) throw new BadRequestException("params.channelId must be a UC… channel id");
return { kind, params: { channelId } };
}
case KIND_WEB_WATCH: {
const params: WebWatchParams = { url: requireHttpUrl(raw.url, "params.url") };
if (raw.selector !== undefined && raw.selector !== null && raw.selector !== "") {
params.selector = requireString(raw.selector, "params.selector", 512);
}
return { kind, params };
}
default: {
throw new BadRequestException("kind must be a supported intake kind");
}
}
}
/** Deterministic key-sorted JSON — the canonical form hashing reads. */
export function canonicalParams(input: IntakeParams): string {
const sorted = (value: unknown): unknown => {
if (Array.isArray(value)) return value.map(sorted);
if (isObject(value)) {
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, sorted(value[key])]));
}
return value;
};
return JSON.stringify(sorted({ kind: input.kind, params: input.params }));
}
/** sha256 of the canonical serialization — migration 027's params_hash. */
export function paramsHash(input: IntakeParams): string {
return createHash("sha256").update(canonicalParams(input)).digest("hex");
}
/** Parse params already stored on a row (trusted shape, still coerced). */
export function storedParams(kind: IntakeKind, raw: unknown): IntakeParams["params"] {
return parseIntakeParams(kind, isObject(raw) ? raw : {}).params;
}
/** The outbound host a kind+params pair polls — the egress budget key base. */
export function egressOrigin(input: IntakeParams): string {
const params = input.params;
if ("instance" in params && typeof params.instance === "string") return params.instance;
if ("url" in params && typeof params.url === "string") return originOf(params.url);
return input.kind === KIND_YOUTUBE_CHANNEL ? YOUTUBE_ORIGIN : TELEGRAM_API_ORIGIN;
}
function originOf(url: string): string {
try { return new URL(url).origin; } catch { return "https://invalid.example"; }
}

193
apps/api/src/intake/intake-search.ts Normal file
View file

@ -0,0 +1,193 @@
/**
* Intake search + listing (task requirement 5) — the 025 archive-search
* pattern applied to intake_items: Postgres FTS `websearch_to_tsquery`
* ORed with pg_trgm similarity per token for typo tolerance, RLS-scoped via
* withTenant (FORCE binds — there is no system path), keyset pagination with
* a base64url JSON cursor and an N+1 fetch.
*
* Ranking: ts_rank dominates (weight-A title hits outrank body hits via the
* 027 trigger's setweight), trigram similarity breaks ties. The no-query
* path keysets on (published_at, id).
*/
import { BadRequestException } from "@nestjs/common";
import { withTenant } from "../db/client.js";
import type { StorageService } from "../storage/storage.js";
import { searchTokens } from "../archive/archive-search.js";
export interface IntakeQueryFilters {
q: string | null;
sourceId: string | null;
kind: string | null;
cursor: string | undefined;
limit: number;
}
export const SEARCH_DEFAULT_LIMIT = 20;
export const SEARCH_MAX_LIMIT = 100;
export interface IntakeItemRow {
id: string;
source_id: string;
remote_ref: string;
content_hash: string;
published_at: Date;
text: string;
content: Record<string, unknown>;
media_refs: Array<{ storageKey?: string; proxyUrl?: string; kind?: string }>;
attribution: Record<string, unknown>;
fetched_at: Date;
created_at: Date;
source_kind: string;
rank?: number;
}
const UUID_SHAPE = /^[0-9a-f-]{36}$/i;
const ISO_TIMESTAMP = /^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?)?$/;
interface Params {
values: unknown[];
bind(value: unknown): number;
}
function params(): Params {
const values: unknown[] = [];
return {
values,
bind(value: unknown): number { values.push(value); return values.length; },
};
}
function decodeCursor<T>(cursor: string | undefined, what: string, shape: (v: Record<string, unknown>) => T): T {
if (cursor === undefined) throw new BadRequestException(`invalid cursor: ${what}`);
let value: unknown;
try { value = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8")); }
catch { throw new BadRequestException(`invalid cursor: ${what}`); }
if (typeof value !== "object" || value === null || Array.isArray(value)) {
throw new BadRequestException(`invalid cursor: ${what}`);
}
return shape(value as Record<string, unknown>);
}
const ITEM_FIELDS = "intake_items.id, intake_items.source_id, intake_items.remote_ref, intake_items.content_hash, intake_items.published_at, intake_items.text, intake_items.content, intake_items.media_refs, intake_items.attribution, intake_items.fetched_at, intake_items.created_at";
function filterConds(p: Params, filters: IntakeQueryFilters): string[] {
const conds: string[] = [];
if (filters.sourceId !== null) conds.push(`source_id = $${p.bind(filters.sourceId)}::uuid`);
if (filters.kind !== null) conds.push(`s.kind = $${p.bind(filters.kind)}`);
return conds;
}
/**
* Word-level fuzzy predicate. `text % $token` compares the WHOLE corpus
* string to the token, so a single misspelled word inside a long post scores
* ~0.1 and never clears pg_trgm's 0.3 threshold. Splitting first makes the
* comparison word-vs-word, where a one-letter typo still scores ~0.67 —
* that is the typo tolerance the spec asks pg_trgm for.
*/
function fuzzyPredicate(p: Params, tokens: string[]): string {
if (tokens.length === 0) return "false";
return tokens.map((token) => {
const ti = p.bind(token);
return `(EXISTS (SELECT 1 FROM regexp_split_to_table(intake_items.text, ' ') w WHERE w % $${ti})`
+ ` OR coalesce(intake_items.attribution->>'title', '') % $${ti})`;
}).join(" OR ");
}
function rankExpression(p: Params, q: string, tokens: string[]): string {
const qi = p.bind(q);
const textSims = tokens.map((token) => `(SELECT max(similarity(w, $${p.bind(token)})) FROM regexp_split_to_table(intake_items.text, ' ') w)`);
const titleSims = tokens.map((token) => `similarity(coalesce(intake_items.attribution->>'title',''), $${p.bind(token)})`);
return `(ts_rank(intake_items.search_vector, websearch_to_tsquery('simple', $${qi})) * 2`
+ (textSims.length > 0 ? ` + GREATEST(${textSims.join(", ")})` : "")
+ (titleSims.length > 0 ? ` + GREATEST(${titleSims.join(", ")})` : "")
+ `)`;
}
export function searchIntake(workspaceId: string, filters: IntakeQueryFilters,
storage: StorageService): Promise<{ items: Record<string, unknown>[]; nextCursor: string | null }> {
return runQuery(workspaceId, filters, storage, true);
}
export function listIntakeItems(workspaceId: string, filters: IntakeQueryFilters,
storage: StorageService): Promise<{ items: Record<string, unknown>[]; nextCursor: string | null }> {
// Listing without q is the plain keyset path; a q present upgrades it to search.
return runQuery(workspaceId, filters, storage, filters.q !== null);
}
async function runQuery(workspaceId: string, filters: IntakeQueryFilters,
storage: StorageService, ranked: boolean): Promise<{ items: Record<string, unknown>[]; nextCursor: string | null }> {
const p = params();
const tokens = filters.q !== null ? searchTokens(filters.q) : [];
let sql: string;
let cursorValue: { rk: number; id: string } | null = null;
let cursorPosted: { at: string; id: string } | null = null;
if (ranked && filters.q !== null) {
const rank = rankExpression(p, filters.q, tokens);
const conds = filterConds(p, filters);
const qi = p.bind(filters.q);
conds.push(`(intake_items.search_vector @@ websearch_to_tsquery('simple', $${qi}) OR ${fuzzyPredicate(p, tokens)})`);
if (filters.cursor !== undefined) {
cursorValue = decodeCursor(filters.cursor, "rank", (v) => {
if (typeof v.rk !== "number" || !Number.isFinite(v.rk) || typeof v.id !== "string" || !UUID_SHAPE.test(v.id)) {
throw new BadRequestException("invalid cursor: rank");
}
return { rk: v.rk, id: v.id };
});
conds.push(`(${rank}, intake_items.id) < ($${p.bind(cursorValue.rk)}::real, $${p.bind(cursorValue.id)}::uuid)`);
}
sql = `SELECT ranked.*, s.kind AS source_kind FROM
(SELECT ${ITEM_FIELDS}, ${rank} AS rank
FROM intake_items JOIN intake_sources s ON s.id = intake_items.source_id
WHERE ${conds.join(" AND ")}) ranked
JOIN intake_sources s ON s.id = ranked.source_id
ORDER BY ranked.rank DESC, ranked.id ASC LIMIT $${p.bind(filters.limit + 1)}`;
} else {
const conds = filterConds(p, filters);
if (filters.cursor !== undefined) {
cursorPosted = decodeCursor(filters.cursor, "published", (v) => {
if (typeof v.at !== "string" || !ISO_TIMESTAMP.test(v.at) || !Number.isFinite(Date.parse(v.at))
|| typeof v.id !== "string" || !UUID_SHAPE.test(v.id)) {
throw new BadRequestException("invalid cursor: published");
}
return { at: v.at, id: v.id };
});
conds.push(`(intake_items.published_at, intake_items.id) < ($${p.bind(cursorPosted.at)}::timestamptz, $${p.bind(cursorPosted.id)}::uuid)`);
}
sql = `SELECT ${ITEM_FIELDS}, s.kind AS source_kind FROM intake_items
JOIN intake_sources s ON s.id = intake_items.source_id
WHERE ${conds.length > 0 ? conds.join(" AND ") : "true"}
ORDER BY intake_items.published_at DESC, intake_items.id DESC LIMIT $${p.bind(filters.limit + 1)}`;
}
const result = await withTenant(workspaceId, (client) => client.query<IntakeItemRow>(sql, p.values));
const page = result.rows.slice(0, filters.limit);
const last = page.at(-1);
const items = await Promise.all(page.map(async (row) => {
const media = await Promise.all((row.media_refs ?? []).map(async (media) => ({
...media,
thumbUrl: media.storageKey ? await storage.presign(media.storageKey, 900) : null,
})));
return {
id: row.id,
sourceId: row.source_id,
remoteRef: row.remote_ref,
url: typeof row.attribution.url === "string" ? row.attribution.url : null,
contentHash: row.content_hash,
publishedAt: row.published_at.toISOString(),
text: row.text,
content: row.content,
media,
attribution: row.attribution,
kind: row.source_kind,
fetchedAt: row.fetched_at.toISOString(),
...(row.rank !== undefined ? { rank: row.rank } : {}),
};
}));
const nextCursor = result.rows.length > filters.limit && last
? Buffer.from(JSON.stringify(cursorShape(last, ranked && filters.q !== null))).toString("base64url")
: null;
return { items, nextCursor };
}
function cursorShape(last: IntakeItemRow, ranked: boolean): Record<string, string | number> {
return ranked ? { rk: last.rank ?? 0, id: last.id } : { at: last.published_at.toISOString(), id: last.id };
}

85
apps/api/src/intake/intake.controller.ts Normal file
View file

@ -0,0 +1,85 @@
/**
* Intake API (task requirement 5).
*
* POST /intake/sources subscribe (kind + params)
* GET /intake/sources list subscriptions
* GET /intake/sources/:id one subscription (+ state)
* PATCH /intake/sources/:id pause / resume
* DELETE /intake/sources/:id unsubscribe
* GET /intake/search?q=… FTS + pg_trgm fuzzy, keyset
* GET /intake/items?sourceId=… per-source listing, keyset
*
* Writes gate on `connection.manage` (subscribing an outbound poller is
* connection-shaped: it spends a shared host budget); reads stay on
* `draft.view` like /feeds and /archive, so every current reader keeps
* reading. RLS scoping is in the service layer (withTenant, FORCE).
*/
import { BadRequestException, Body, Controller, Delete, Get, Param, Patch, Post, Query, Req, UseGuards } from "@nestjs/common";
import { parseId, requireObject, requireSession } from "../http/validation.js";
import { RbacGuard, RequirePermission, type AuthenticatedRequest } from "../rbac/rbac.guard.js";
import { IntakeService } from "./intake.service.js";
import { INTAKE_KINDS } from "./intake-kinds.js";
const KINDS_DOC = INTAKE_KINDS.join(", ");
@Controller("intake")
@UseGuards(RbacGuard)
export class IntakeController {
constructor(private readonly intake: IntakeService) {}
@Post("sources")
@RequirePermission("connection.manage")
async create(@Req() request: AuthenticatedRequest, @Body() body: unknown): Promise<{ id: string; kind: string; health: string }> {
const input = requireObject(body, "body");
const kind = input.kind;
if (typeof kind !== "string" || kind === "") throw new BadRequestException(`kind must be one of: ${KINDS_DOC}`);
return this.intake.createSubscription(requireSession(request).workspaceId, kind,
input.params ?? {}, {
pollIntervalSec: input.pollIntervalSec,
mediaPolicy: input.mediaPolicy,
});
}
@Get("sources")
@RequirePermission("draft.view")
async list(@Req() request: AuthenticatedRequest): Promise<{ sources: Record<string, unknown>[] }> {
return { sources: await this.intake.listSources(requireSession(request).workspaceId) };
}
@Get("sources/:id")
@RequirePermission("draft.view")
async get(@Req() request: AuthenticatedRequest, @Param("id") id: string): Promise<Record<string, unknown>> {
return this.intake.getSource(requireSession(request).workspaceId, parseId(id));
}
@Patch("sources/:id")
@RequirePermission("connection.manage")
async patch(@Req() request: AuthenticatedRequest, @Param("id") id: string, @Body() body: unknown): Promise<{ ok: true }> {
const input = requireObject(body, "body");
const health = input.health;
if (health !== "active" && health !== "paused") throw new BadRequestException("health must be active or paused");
await this.intake.setSourceHealth(requireSession(request).workspaceId, parseId(id), health);
return { ok: true };
}
@Delete("sources/:id")
@RequirePermission("connection.manage")
async remove(@Req() request: AuthenticatedRequest, @Param("id") id: string): Promise<{ ok: true }> {
await this.intake.deleteSource(requireSession(request).workspaceId, parseId(id));
return { ok: true };
}
@Get("search")
@RequirePermission("draft.view")
async search(@Req() request: AuthenticatedRequest,
@Query() query: Record<string, unknown>): Promise<{ items: Record<string, unknown>[]; nextCursor: string | null }> {
return this.intake.search(requireSession(request).workspaceId, query);
}
@Get("items")
@RequirePermission("draft.view")
async items(@Req() request: AuthenticatedRequest,
@Query() query: Record<string, unknown>): Promise<{ items: Record<string, unknown>[]; nextCursor: string | null }> {
return this.intake.listItems(requireSession(request).workspaceId, query);
}
}

467
apps/api/src/intake/intake.integration.test.ts Normal file
View file

@ -0,0 +1,467 @@
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { join } from "node:path";
import { Pool, type PoolClient } from "pg";
/**
* Intake phase-1 integration against a live PostgreSQL, as sp_app (FORCE RLS
* binds both tables). Own scratch database (INTAKE_IT_DATABASE_URL +
* INTAKE_IT_SUPER_DATABASE_URL) per the local-db-sol pattern, because the
* shared-DB suites wipe every workspace and vitest runs files in parallel.
* Skipped when the URLs are unset.
*
* Covers the task's integration list: content-hash dedup, RLS cross-tenant
* denial, budget draws via fake egress, FTS + fuzzy matching, keyset
* pagination (both the ranked and the time-ordered path), the scheduler's
* poll/write-back cycle, and the source lifecycle (subscribe → pause →
* delete). Storage is a fake S3 that content-addresses in memory.
*/
const appUrl = process.env.INTAKE_IT_DATABASE_URL;
const ownerUrl = process.env.INTAKE_IT_SUPER_DATABASE_URL;
type Modules = {
service: typeof import("./intake.service.js");
ingest: typeof import("./intake-ingest.js");
search: typeof import("./intake-search.js");
kinds: typeof import("./intake-kinds.js");
adapter: typeof import("./adapters/adapter.js");
rss: typeof import("./adapters/rss.adapter.js");
db: typeof import("../db/client.js");
};
class FakeStorage {
readonly objects = new Map<string, { body: Buffer; contentType: string }>();
put(buffer: Buffer, contentType: string): { key: string; hash: string; sizeBytes: number } {
const { createHash } = require("node:crypto") as typeof import("node:crypto");
const hash = createHash("sha256").update(buffer).digest("hex");
const ext = contentType.split("/")[1]?.split("+")[0] ?? "bin";
const key = `sha256/${hash}.${ext}`;
this.objects.set(key, { body: buffer, contentType });
return { key, hash, sizeBytes: buffer.byteLength };
}
async putContent(buffer: Buffer, contentType: string) {
const info = this.put(buffer, contentType);
const deduplicated = this.objects.has(info.key);
return { ...info, contentType, deduplicated };
}
async presign(key: string): Promise<string> { return `https://fake-s3/${key}?sig=test`; }
}
/** Recording fake egress with fixture routes + Retry-After budget behavior. */
class FakeEgress {
readonly draws: Array<{ url: string; hostClass: string }> = [];
constructor(private readonly routes: Record<string, () => { status: number; headers?: Record<string, string>; body?: string }>) {}
async egressFetch(url: string | URL, options: { headers?: Record<string, string> }, config: { hostClass: string }): Promise<Response> {
const key = String(url);
this.draws.push({ url: key, hostClass: config.hostClass });
const route = Object.entries(this.routes).find(([prefix]) => key.startsWith(prefix));
if (route === undefined) return new Response("not found", { status: 404 });
const r = route[1]();
return new Response(r.body ?? "", { status: r.status, headers: r.headers ?? {} });
}
readonly policy = { timeoutMs: 1000, maxResponseBytes: 1024 * 1024 } as never;
isEgressError(): boolean { return false; }
}
const RSS_BODY = `<?xml version="1.0"?><rss version="2.0" xmlns:media="http://search.yahoo.com/mrss/"><channel><title>Water WG</title>
<item><title>Water rights are a community responsibility</title><link>https://example.org/w1</link><guid>w1</guid>
<pubDate>Fri, 09 Oct 2026 12:00:00 GMT</pubDate><description>Water rights are a community responsibility, not a commodity.</description>
<media:thumbnail url="https://cdn.example.org/water.jpg" /></item>
<item><title>Bicycle repair night</title><link>https://example.org/w2</link><guid>w2</guid>
<pubDate>Thu, 08 Oct 2026 09:00:00 GMT</pubDate><description>Bring your bicycles to the tool library Thursday.</description></item>
</channel></rss>`;
const MASTODON_BODY = JSON.stringify([
{ id: "s1", created_at: "2026-10-09T11:00:00.000Z",
content: "<p>Water rights update: the hearing is postponed</p>",
account: { acct: "alice", username: "alice" }, url: "https://social.example/@alice/s1",
media_attachments: [{ url: "https://files.social.example/s1.png", type: "image", mime_type: "image/png" }], visibility: "public" },
{ id: "s2", created_at: "2026-10-08T10:00:00.000Z",
content: "<p>Water rights are a community responsibility, not a commodity for markets.</p>",
account: { acct: "bob@example.social", username: "bob" }, visibility: "public" },
]);
const IMAGE_BYTES = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3, 4]);
describe.skipIf(appUrl === undefined || ownerUrl === undefined)("intake phase 1 (live DB)", () => {
let m: Modules;
let ownerPool: Pool;
let storage: FakeStorage;
const suffix = `${Date.now()}${Math.floor(Math.random() * 1e6)}`;
let wsA: string;
let wsB: string;
async function asOwner<T>(fn: (client: PoolClient) => Promise<T>): Promise<T> {
const client = await ownerPool.connect();
try { return await fn(client); } finally { client.release(); }
}
let service: InstanceType<Modules["service"]["IntakeService"]>;
beforeAll(async () => {
process.env.DATABASE_URL = appUrl;
process.env.SUPER_DATABASE_URL = ownerUrl;
const { runMigrations } = await import("../db/migrate.js");
await runMigrations(join(process.cwd(), "migrations"));
m = {
service: await import("./intake.service.js"),
ingest: await import("./intake-ingest.js"),
search: await import("./intake-search.js"),
kinds: await import("./intake-kinds.js"),
adapter: await import("./adapters/adapter.js"),
rss: await import("./adapters/rss.adapter.js"),
db: await import("../db/client.js"),
};
ownerPool = new Pool({ connectionString: ownerUrl });
await asOwner(async (c) => {
const mk = async (name: string): Promise<string> =>
String((await c.query("INSERT INTO workspaces (name, slug) VALUES ($1, $2) RETURNING id", [name, `${name}-${suffix}`])).rows[0].id);
wsA = await mk("intake-a");
wsB = await mk("intake-b");
});
storage = new FakeStorage();
process.env.S3_BUCKET = "intake-test";
// Silence the sweep timer: this suite drives polls explicitly.
service = new m.service.IntakeService({
egress: new FakeEgress({}) as never,
storage: storage as never,
random: () => 0.5,
now: () => new Date(),
sweepMs: 3_600_000,
});
service.onApplicationShutdown().catch(() => undefined);
});
afterAll(async () => {
if (ownerPool !== undefined) {
await asOwner(async (c) => { await c.query("DELETE FROM workspaces WHERE id = ANY($1::uuid[])", [[wsA, wsB]]); });
await ownerPool.end();
}
await m?.db.pool.end();
});
// -----------------------------------------------------------------------
// Source lifecycle + params identity
// -----------------------------------------------------------------------
it("subscribes with canonical params identity; duplicates collapse; push-only parks unsupported", async () => {
const first = await service.createSubscription(wsA, "rss", { url: "https://example.org/feed.xml" }, { mediaPolicy: "images" });
expect(first.kind).toBe("rss");
expect(first.health).toBe("active");
// hash-stripped variant collides on (workspace, kind, params_hash)
const dupe = await service.createSubscription(wsA, "rss", { url: "https://example.org/feed.xml#fragment" });
expect(dupe.id).toBe(first.id);
const other = await service.createSubscription(wsA, "rss", { url: "https://other.example/feed.xml" });
expect(other.id).not.toBe(first.id);
const tg = await service.createSubscription(wsA, "telegram_channel", { channelId: "-1001234567890" });
expect(tg.health).toBe("unsupported"); // 027: push-only kinds park
const list = await service.listSources(wsA);
expect(list).toHaveLength(3);
// push-only kinds cannot opt into polling
await expect(service.createSubscription(wsA, "bluesky_keyword", { keywords: ["water"] }, { pollIntervalSec: 60 }))
.rejects.toThrow(/push-only/);
});
it("validates params shapes at the API edge", async () => {
await expect(service.createSubscription(wsA, "mastodon_account", { instance: "https://mastodon.social", accountId: "not-numeric" })).rejects.toThrow();
await expect(service.createSubscription(wsA, "youtube_channel", { channelId: "XD" })).rejects.toThrow();
await expect(service.createSubscription(wsA, "rss", { url: "file:///etc/passwd" })).rejects.toThrow();
await expect(service.createSubscription(wsA, "nope", {})).rejects.toThrow();
});
it("pauses, resumes and deletes sources", async () => {
const src = await service.createSubscription(wsA, "web_watch", { url: "https://example.org/watched" });
await service.setSourceHealth(wsA, src.id, "paused");
let row = await service.getSource(wsA, src.id) as { health: string };
expect(row.health).toBe("paused");
await service.setSourceHealth(wsA, src.id, "active");
row = await service.getSource(wsA, src.id) as { health: string };
expect(row.health).toBe("active");
await service.deleteSource(wsA, src.id);
await expect(service.getSource(wsA, src.id)).rejects.toThrow();
});
// -----------------------------------------------------------------------
// Poll + ingest + dedup + media harvest
// -----------------------------------------------------------------------
it("polls an rss source end-to-end: budget draws, P0c AST, media to S3, dedup on re-poll", async () => {
const egress = new FakeEgress({
"https://example.org/feed.xml": () => ({ status: 200, headers: { "content-type": "application/xml", etag: "\"v1\"" }, body: RSS_BODY }),
"https://cdn.example.org/water.jpg": () => ({ status: 200, headers: { "content-type": "image/png" }, body: IMAGE_BYTES.toString("latin1") }),
});
const poller = new m.service.IntakeService({
egress: egress as never, storage: storage as never,
random: () => 0.5, now: () => new Date(), sweepMs: 3_600_000,
});
const src = await service.createSubscription(wsA, "rss", { url: "https://example.org/feed.xml" });
const due = await poller.pollWorkspaceDue(wsA);
expect(due.polled).toBeGreaterThanOrEqual(1);
// Feed draw went through the egress client with the intake-rss host class.
expect(egress.draws.some((d) => d.url.startsWith("https://example.org/feed.xml") && d.hostClass === "intake-rss")).toBe(true);
// Media draw billed to the shared media class.
expect(egress.draws.some((d) => d.url.startsWith("https://cdn.example.org/water.jpg") && d.hostClass === "intake-web")).toBe(true);
const items = await service.listItems(wsA, { sourceId: src.id, limit: 50 });
expect(items.items).toHaveLength(2);
const first = items.items[0] as Record<string, unknown>;
// P0c AST landed — never raw HTML.
const content = first.content as { type: string; children: unknown[] };
expect(content.type).toBe("doc");
// text derived from the AST is the searchable corpus
expect(String(first.text)).toMatch(/community responsibility/);
// media ref carries the S3 storage key (content-addressed) + proxy URL
const media = first.media as Array<Record<string, unknown>>;
expect(media.length).toBeGreaterThanOrEqual(1);
expect(String(media[0]!.storageKey)).toMatch(/^sha256\/[0-9a-f]{64}\./);
// state carries the etag for conditional GET
const row = await service.getSource(wsA, src.id) as { state: { etag?: string }; nextPollAt: string | null };
expect(row.state.etag).toBe("\"v1\"");
expect(row.nextPollAt).not.toBeNull();
// Re-poll with identical content: zero inserts (content-hash dedup).
const before = await service.listItems(wsA, { sourceId: src.id, limit: 50 });
await poller.pollWorkspaceDue(wsA);
const after = await service.listItems(wsA, { sourceId: src.id, limit: 50 });
expect(after.items).toHaveLength(before.items.length);
});
it("cross-source dedup: the same statement via rss and mastodon stores once", async () => {
const statement = "Water rights are a community responsibility, not a commodity for markets.";
const rssBody = `<?xml version="1.0"?><rss version="2.0"><channel><title>x</title>
<item><title>Statement</title><link>https://example.org/x</link><guid>x1</guid>
<pubDate>Fri, 09 Oct 2026 12:00:00 GMT</pubDate><description>${statement}</description></item></channel></rss>`;
const mastodonBody = JSON.stringify([
{ id: "m1", created_at: "2026-10-09T13:00:00.000Z", content: `<p>${statement}</p>`,
account: { acct: "relay", username: "relay" }, visibility: "public" },
]);
const egress = new FakeEgress({
"https://dedup.example.org/rss": () => ({ status: 200, headers: { "content-type": "application/xml" }, body: rssBody }),
"https://mastodon.social/api/v1/timelines/tag/water": () => ({ status: 200, headers: { "content-type": "application/json" }, body: mastodonBody }),
});
const poller = new m.service.IntakeService({
egress: egress as never, storage: storage as never,
random: () => 0.5, now: () => new Date(), sweepMs: 3_600_000,
});
const rssSrc = await service.createSubscription(wsA, "rss", { url: "https://dedup.example.org/rss" });
const mdSrc = await service.createSubscription(wsA, "mastodon_tag", { instance: "https://mastodon.social", tag: "water" }, { mediaPolicy: "metadata" });
await poller.pollWorkspaceDue(wsA);
const rssItems = await service.listItems(wsA, { sourceId: rssSrc.id, limit: 10 });
const mdItems = await service.listItems(wsA, { sourceId: mdSrc.id, limit: 10 });
// First writer wins: the row is attributed to whichever source landed it.
expect(rssItems.items.length + mdItems.items.length).toBe(1);
// and the mastodon poll drew from the intake-social class
expect(egress.draws.some((d) => d.hostClass === "intake-social")).toBe(true);
});
it("mastodon items carry html-parsed AST + authorRef attribution", async () => {
const egress = new FakeEgress({
"https://mastodon.social/api/v1/timelines/tag/hearing": () => ({ status: 200, headers: { "content-type": "application/json" }, body: MASTODON_BODY }),
"https://files.social.example/s1.png": () => ({ status: 200, headers: { "content-type": "image/png" }, body: IMAGE_BYTES.toString("latin1") }),
});
const poller = new m.service.IntakeService({
egress: egress as never, storage: storage as never,
random: () => 0.5, now: () => new Date(), sweepMs: 3_600_000,
});
const src = await service.createSubscription(wsA, "mastodon_tag", { instance: "https://mastodon.social", tag: "hearing" });
await poller.pollWorkspaceDue(wsA);
const items = await service.listItems(wsA, { sourceId: src.id, limit: 10 });
expect(items.items.length).toBeGreaterThanOrEqual(1);
const newest = items.items.find((i) => (i as { remoteRef: string }).remoteRef === "s1") as Record<string, unknown>;
const attribution = newest.attribution as Record<string, string>;
expect(attribution.authorHandle).toBe("alice");
expect(attribution.authorRef).toBe("mastodon:alice");
expect(String(newest.url ?? "")).toContain("/@alice/s1");
});
it("poll failure: 429 parks the source in backoff with retry-after honored, then recovers", async () => {
let fail = true;
const body = `<?xml version="1.0"?><rss version="2.0"><channel><title>x</title>
<item><title>Recovered</title><link>https://example.org/r1</link><guid>r1</guid>
<pubDate>Fri, 09 Oct 2026 15:00:00 GMT</pubDate><description>recovered content</description></item></channel></rss>`;
const egress = new FakeEgress({
"https://backoff.example.org/feed": (): { status: number; headers: Record<string, string>; body?: string } => fail
? { status: 429, headers: { "retry-after": "1" } }
: { status: 200, headers: { "content-type": "application/xml" }, body },
});
const poller = new m.service.IntakeService({
egress: egress as never, storage: storage as never,
random: () => 0.5, now: () => new Date(), sweepMs: 3_600_000,
});
const src = await service.createSubscription(wsA, "rss", { url: "https://backoff.example.org/feed" });
const firstPass = await poller.pollWorkspaceDue(wsA);
expect(firstPass.failed).toBeGreaterThanOrEqual(1);
let row = await service.getSource(wsA, src.id) as { health: string; lastError: string | null; consecutiveFailures: number };
expect(row.health).toBe("backoff");
expect(row.lastError).toMatch(/transient: http 429/);
expect(row.consecutiveFailures).toBe(1);
// Retry-After (1 s) becomes the next_poll_at floor — not due immediately.
const again = await poller.pollWorkspaceDue(wsA);
expect(again.polled).toBe(0);
// force-recover: clear backoff + due immediately
await asOwner(async (c) => { await c.query("UPDATE intake_sources SET next_poll_at = now(), health='active' WHERE id = $1", [src.id]); });
fail = false;
const third = await poller.pollWorkspaceDue(wsA);
expect(third.polled).toBeGreaterThanOrEqual(1);
row = await service.getSource(wsA, src.id) as { health: string; consecutiveFailures: number; lastError: string | null };
expect(row.health).toBe("active");
expect(row.consecutiveFailures).toBe(0);
});
it("permanent 410 parks the source in error, not backoff", async () => {
const egress = new FakeEgress({ "https://gone.example.org/feed": () => ({ status: 410 }) });
const poller = new m.service.IntakeService({
egress: egress as never, storage: storage as never,
random: () => 0.5, now: () => new Date(), sweepMs: 3_600_000,
});
const src = await service.createSubscription(wsA, "rss", { url: "https://gone.example.org/feed" });
await poller.pollWorkspaceDue(wsA);
const row = await service.getSource(wsA, src.id) as { health: string; lastError: string | null };
expect(row.health).toBe("error");
expect(row.lastError).toMatch(/permanent: http 410/);
});
// -----------------------------------------------------------------------
// RLS
// -----------------------------------------------------------------------
it("RLS: cross-tenant reads see nothing; writes are scoped", async () => {
// Distinct content from earlier feeds: dedup is content-addressed per
// workspace, so a re-delivery of RSS_BODY would (correctly) no-op.
const body = `<?xml version="1.0"?><rss version="2.0"><channel><title>rls</title>
<item><title>RLS probe</title><link>https://example.org/rls1</link><guid>rls1</guid>
<pubDate>Fri, 09 Oct 2026 16:00:00 GMT</pubDate><description>Tenant isolation probe statement.</description></item></channel></rss>`;
const egress = new FakeEgress({
"https://rls.example.org/feed": () => ({ status: 200, headers: { "content-type": "application/xml" }, body }),
});
const poller = new m.service.IntakeService({
egress: egress as never, storage: storage as never,
random: () => 0.5, now: () => new Date(), sweepMs: 3_600_000,
});
const srcA = await service.createSubscription(wsA, "rss", { url: "https://rls.example.org/feed" });
await poller.pollWorkspaceDue(wsA);
// wsB never subscribed; its listing + search are empty even though wsA has rows.
const inB = await service.listItems(wsB, { limit: 100 });
expect(inB.items).toHaveLength(0);
const searchB = await service.search(wsB, { q: "community responsibility" });
expect(searchB.items).toHaveLength(0);
await expect(service.getSource(wsB, srcA.id)).rejects.toThrow();
await expect(service.deleteSource(wsB, srcA.id)).rejects.toThrow();
const inA = await service.listItems(wsA, { sourceId: srcA.id, limit: 10 });
expect(inA.items.length).toBeGreaterThan(0);
});
// -----------------------------------------------------------------------
// Search: FTS, fuzzy, keyset
// -----------------------------------------------------------------------
// Seeds the corpus the three suites below read: a timestamped feed so each
// run's rows are unique (content-hash dedup is workspace-wide and this
// database is recreated per CI run, but re-runs against a kept database
// stay honest too).
let seeded = false;
it("seeds the search corpus via full polls", async () => {
if (seeded) return;
const stamp = Date.now();
const rssA = `<?xml version="1.0"?><rss version="2.0"><channel><title>Corpus</title>
<item><title>Community responsibility statement ${stamp}</title><link>https://example.org/c1</link><guid>c1-${stamp}</guid>
<pubDate>Fri, 09 Oct 2026 12:00:00 GMT</pubDate><description>Water rights are a community responsibility held in trust for the watershed ${stamp}.</description></item>
<item><title>Hearing update ${stamp}</title><link>https://example.org/c2</link><guid>c2-${stamp}</guid>
<pubDate>Fri, 09 Oct 2026 13:00:00 GMT</pubDate><description>The eviction hearing is postponed until February for the tenants union ${stamp}.</description></item>
<item><title>Tool library ${stamp}</title><link>https://example.org/c3</link><guid>c3-${stamp}</guid>
<pubDate>Fri, 09 Oct 2026 14:00:00 GMT</pubDate><description>Bicycle repair night at the tool library on Thursday evening ${stamp}.</description></item></channel></rss>`;
const egress = new FakeEgress({
"https://corpus.example.org/feed": () => ({ status: 200, headers: { "content-type": "application/xml" }, body: rssA }),
});
const poller = new m.service.IntakeService({
egress: egress as never, storage: storage as never,
random: () => 0.5, now: () => new Date(), sweepMs: 3_600_000,
});
await service.createSubscription(wsA, "rss", { url: "https://corpus.example.org/feed" });
const pass = await poller.pollWorkspaceDue(wsA);
expect(pass.polled).toBeGreaterThanOrEqual(1);
const seededRows = await service.listItems(wsA, { limit: 100 });
expect(seededRows.items.length).toBeGreaterThanOrEqual(3);
seeded = true;
});
it("FTS matches websearch_to_tsquery; fuzzy tolerates typos via pg_trgm", async () => {
const exact = await service.search(wsA, { q: "watershed" });
expect(exact.items.some((item) => /watershed/i.test(String(item.text)))).toBe(true);
// Typo tolerance: "responsability" matches the corpus item whose text has
// "responsibility" (word-level trigram), alongside the FTS hit above.
const fuzzy = await service.search(wsA, { q: "responsability" });
expect(fuzzy.items.some((item) => /responsibility/i.test(String(item.text)))).toBe(true);
const phrase = await service.search(wsA, { q: '"hearing is postponed"' });
expect(phrase.items.length).toBeGreaterThanOrEqual(1);
const nothing = await service.search(wsA, { q: "zzzzqqqq" });
expect(nothing.items).toHaveLength(0);
});
it("search filters by kind and rejects bad cursors", async () => {
const byKind = await service.search(wsA, { q: null, kind: "rss" });
expect(byKind.items.length).toBeGreaterThan(0);
for (const item of byKind.items) expect(item.kind).toBe("rss");
await expect(service.search(wsA, { q: "x", cursor: "not-a-cursor!" })).rejects.toThrow();
await expect(service.search(wsA, { q: "x".repeat(300) })).rejects.toThrow();
await expect(service.search(wsA, { limit: 1000 })).rejects.toThrow();
});
it("keyset pagination walks time-ordered listings without overlap", async () => {
let cursor: string | null = null;
const seen = new Set<string>();
let pages = 0;
for (;;) {
const page = await service.listItems(wsA, { limit: 2, ...(cursor ? { cursor } : {}) });
pages += 1;
for (const item of page.items) expect(seen.add(String(item.id))).toBe(true);
cursor = page.nextCursor;
if (cursor === null) break;
expect(pages).toBeLessThan(20);
}
const all = await service.listItems(wsA, { limit: 100 });
expect(seen.size).toBe(all.items.length);
});
it("keyset pagination on the ranked search path", async () => {
let cursor: string | null = null;
const seen = new Set<string>();
let pages = 0;
for (;;) {
const page = await service.search(wsA, { q: "water rights", limit: 2, ...(cursor ? { cursor } : {}) });
pages += 1;
for (const item of page.items) expect(seen.add(String(item.id))).toBe(true);
cursor = page.nextCursor;
if (cursor === null) break;
expect(pages).toBeLessThan(20);
}
expect(seen.size).toBeGreaterThanOrEqual(2);
});
// -----------------------------------------------------------------------
// Push-only bridge
// -----------------------------------------------------------------------
it("telegram bridge: archive item becomes an intake row for the subscribed channel", async () => {
const channelId = "-1009876543210";
const src = await service.createSubscription(wsA, "telegram_channel", { channelId });
expect(src.health).toBe("unsupported"); // never polled
// Seed an archive item directly (the webhook path owns that write).
let archiveId = "";
await asOwner(async (c) => {
c.query("SELECT set_config('app.workspace_id', $1, true)", [wsA]);
const row = await c.query(
`INSERT INTO archived_feed_items
(workspace_id, remote_ref, channel_id, message_id, text_content, content_hash, posted_at, attribution)
VALUES ($1, $2, $3, 5001, $4, encode(sha256($5::bytea), 'hex'), now(), $6::jsonb) RETURNING id`,
[wsA, `${channelId}:5001`, channelId, "Telegram harvest: water rights hearing moved.",
`tg-${suffix}`, JSON.stringify({ channelUsername: "water_wg", author: "Water WG" })]);
archiveId = String(row.rows[0].id);
});
await service.ingestArchiveItem(wsA, archiveId);
const items = await service.listItems(wsA, { sourceId: src.id, limit: 10 });
expect(items.items).toHaveLength(1);
const found = await service.search(wsA, { q: "harvest water" });
expect(found.items.length).toBeGreaterThanOrEqual(1);
});
});

35
apps/api/src/intake/intake.module.ts Normal file
View file

@ -0,0 +1,35 @@
/**
* Intake module wiring: production providers bind the process-wide egress
* client, the S3 storage service and the real Redis lease surface.
*
* The egress package is ESM-only while the API emits CommonJS, so the client
* is constructed through the native-import binding in intake-egress.ts (same
* pattern as live-monitor/poller/egress.ts) inside a factory — the DI token
* carries the already-loaded surface.
*/
import { Module } from "@nestjs/common";
import { IntakeController } from "./intake.controller.js";
import { INTAKE_DEPENDENCIES, IntakeService } from "./intake.service.js";
import { intakeEgress } from "./intake-egress.js";
import { S3Storage, s3ConfigFromEnv } from "../storage/storage.js";
import { RespControlRedis } from "../media-control/redis.js";
@Module({
controllers: [IntakeController],
providers: [
IntakeService,
{
provide: INTAKE_DEPENDENCIES,
useFactory: async () => ({
egress: await intakeEgress(),
storage: new S3Storage(s3ConfigFromEnv()),
redis: new RespControlRedis(process.env.REDIS_URL ?? "redis://127.0.0.1:6379"),
random: Math.random,
now: () => new Date(),
sweepMs: 15_000,
}),
},
],
exports: [IntakeService],
})
export class IntakeModule {}

432
apps/api/src/intake/intake.service.ts Normal file
View file

@ -0,0 +1,432 @@
/**
* IntakeService — subscriptions, the poll scheduler, ingest, and the search
* surface (firehose phase 1).
*
* Scheduler = the live-monitor pattern (task requirement 1): one serialized
* sweep, per-source state in the DB (`next_poll_at` / `consecutive_failures`
* / `health`), exponential backoff 2^n capped at 15 min, +/−20 % jitter on
* every schedule so a fleet of replicas never thundering-herds one host, and
* a Redis lease + cooldown per source so replicas never double-poll. Sources
* self-schedule via next_poll_at; the sweep only has to notice.
*
* Failures classify exactly like BackoffPollDriver: transient (429/5xx/
* budget/DNS) → backoff with Retry-After as a lower bound, health='backoff';
* policy denials and permanent shapes (404/410, bad params) → health='error'
* with the much longer cap; push-only kinds (poll_interval_sec = 0) are
* never due and park in 'unsupported' when first seen (027 note).
*/
import { BadRequestException, Inject, Injectable, Logger, NotFoundException, Optional, type OnApplicationBootstrap, type OnApplicationShutdown } from "@nestjs/common";
import { randomUUID } from "node:crypto";
import { pool, withTenant } from "../db/client.js";
import { S3Storage, s3ConfigFromEnv, type StorageService } from "../storage/storage.js";
import type { ControlRedis } from "../media-control/redis.js";
import { MemoryFeedRedis } from "../feeds/testing.js";
import { intakeEgress, IntakeHttpError, intakeRetryAfterMs, type IntakeEgress } from "./intake-egress.js";
import { DEFAULT_MEDIA_POLICY, DEFAULT_POLL_INTERVAL_SEC, HOST_CLASS_BY_KIND, isIntakeKind, KIND_TELEGRAM_CHANNEL, PROVIDER_BY_KIND, type IntakeKind, type MediaPolicy } from "./intake-kinds.js";
import { parseIntakeParams, paramsHash, storedParams, type IntakeParams } from "./intake-params.js";
import { adapterFor } from "./adapters/index.js";
import type { AdapterState } from "./adapters/adapter.js";
import { MediaHarvester, type HarvestedMedia } from "./media-harvest.js";
import { itemContentHash, normalizeRawItem, upsertItems, type UpsertResult } from "./intake-ingest.js";
import { searchIntake, listIntakeItems, type IntakeQueryFilters } from "./intake-search.js";
export const INTAKE_DEPENDENCIES = Symbol("INTAKE_DEPENDENCIES");
export interface IntakeDependencies {
egress: IntakeEgress;
storage: StorageService;
redis: Pick<ControlRedis, "get" | "set" | "eval">;
random(): number;
now(): Date;
/** Sweep period; defaults to 15 s (min cadence is 60 s, so this is ample). */
sweepMs?: number;
}
/** Backoff curve caps (live-monitor: 2^n; intake polls are slower, cap 15 min). */
export const BACKOFF_CAP_MS = 15 * 60_000;
export const PERMANENT_ERROR_MS = 6 * 60 * 60_000;
export const LEASE_MS = 60_000;
const RENEW_LEASE = "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('PEXPIRE', KEYS[1], ARGV[2]) else return 0 end";
const RELEASE_LEASE = "if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) else return 0 end";
interface SourceRow {
id: string;
workspace_id: string;
kind: IntakeKind;
params: Record<string, unknown>;
poll_interval_sec: number;
state: AdapterState;
health: string;
consecutive_failures: number;
}
export function backoffDelayMs(failures: number, intervalMs: number): number {
return Math.min(BACKOFF_CAP_MS, Math.max(intervalMs, 1_000) * 2 ** Math.min(6, Math.max(0, failures - 1)));
}
/** +/−20 % jitter (live-monitor jitteredInterval shape). */
export function jitteredIntervalMs(intervalMs: number, random: number): number {
if (!Number.isFinite(intervalMs) || intervalMs <= 0) throw new Error("interval must be positive");
if (!Number.isFinite(random) || random < 0 || random > 1) throw new Error("random must be between 0 and 1");
return Math.round(intervalMs * (0.8 + random * 0.4));
}
function defaults(): IntakeDependencies {
return {
egress: undefined as never, // bound lazily by ensureDeps
storage: new S3Storage(s3ConfigFromEnv()),
redis: new MemoryFeedRedis(),
random: Math.random,
now: () => new Date(),
sweepMs: 15_000,
};
}
@Injectable()
export class IntakeService implements OnApplicationBootstrap, OnApplicationShutdown {
private readonly logger = new Logger(IntakeService.name);
private readonly deps: IntakeDependencies;
private readonly egress: IntakeEgress | undefined;
private running = false;
private timer?: ReturnType<typeof setTimeout>;
private active?: Promise<void>;
private harvester?: MediaHarvester;
constructor(@Optional() @Inject(INTAKE_DEPENDENCIES) dependencies?: Partial<IntakeDependencies>) {
const base = defaults();
this.deps = { ...base, ...dependencies };
// The egress client binds lazily (native ESM import); tests inject a fake.
if (dependencies?.egress !== undefined) this.egress = dependencies.egress;
if (this.egress !== undefined) {
const nowMs = (): number => this.deps.now().getTime();
this.harvester = new MediaHarvester(this.egress, this.deps.storage, nowMs);
}
}
onApplicationBootstrap(): void {
this.running = true;
this.schedule(0);
}
async onApplicationShutdown(): Promise<void> {
this.running = false;
if (this.timer) clearTimeout(this.timer);
await this.active;
}
get isRunning(): boolean { return this.running; }
private schedule(delayMs: number): void {
if (!this.running) return;
const sweep = this.deps.sweepMs ?? 15_000;
this.timer = setTimeout(() => {
this.timer = undefined;
this.active = this.pollDueSources().catch((error: unknown) => {
this.logger.error(`intake sweep failed: ${String(error)}`);
}).then(() => undefined).finally(() => {
this.active = undefined;
this.schedule(jitteredIntervalMs(sweep, this.deps.random()));
});
}, delayMs);
this.timer.unref();
}
// -------------------------------------------------------------------------
// Subscriptions
// -------------------------------------------------------------------------
async createSubscription(workspaceId: string, kind: unknown, paramsRaw: unknown,
options: { pollIntervalSec?: unknown; mediaPolicy?: unknown } = {}): Promise<{ id: string; kind: IntakeKind; health: string }> {
const parsed = parseIntakeParams(kind, paramsRaw);
const hash = paramsHash(parsed);
const kindDefaults = DEFAULT_POLL_INTERVAL_SEC[parsed.kind];
const mediaPolicy = parseMediaPolicy(options.mediaPolicy, DEFAULT_MEDIA_POLICY[parsed.kind]);
let interval = kindDefaults;
const rawInterval: unknown = options.pollIntervalSec;
if (rawInterval !== undefined && rawInterval !== null && rawInterval !== "") {
const n = typeof rawInterval === "number" ? rawInterval : typeof rawInterval === "string" ? Number(rawInterval) : NaN;
if (!Number.isInteger(n) || n < 0 || n > 86_400) throw new BadRequestException("pollIntervalSec must be an integer 0..86400 (0 = push-only)");
if (kindDefaults === 0 && n > 0) throw new BadRequestException("this kind is push-only; pollIntervalSec must be 0");
interval = n > 0 ? Math.max(60, Math.min(n, 86_400)) : 0;
}
return withTenant(workspaceId, async (client) => {
const result = await client.query<{ id: string; health: string }>(
`INSERT INTO intake_sources (workspace_id, kind, params, params_hash, poll_interval_sec, state, health)
VALUES ($1,$2,$3,$4,$5,$6,$7)
ON CONFLICT (workspace_id, kind, params_hash) DO UPDATE SET updated_at = now()
RETURNING id, health`,
[workspaceId, parsed.kind, JSON.stringify({ ...parsed.params, mediaPolicy }), hash, interval, JSON.stringify(mediaPolicy ? { mediaPolicy } : {}),
kindDefaults === 0 ? "unsupported" : "active"]);
const row = result.rows[0];
if (!row) throw new Error("intake source insert failed");
return { id: String(row.id), kind: parsed.kind, health: row.health };
});
}
async listSources(workspaceId: string): Promise<Record<string, unknown>[]> {
return withTenant(workspaceId, async (client) => {
const result = await client.query(
`SELECT id, kind, params, poll_interval_sec, health, last_error, last_polled_at, next_poll_at,
consecutive_failures, created_at, updated_at
FROM intake_sources WHERE workspace_id = $1 ORDER BY created_at`, [workspaceId]);
return result.rows.map((row) => ({
id: String(row.id), kind: row.kind, params: row.params,
pollIntervalSec: Number(row.poll_interval_sec), health: row.health,
lastError: row.last_error, lastPolledAt: row.last_polled_at?.toISOString() ?? null,
nextPollAt: row.next_poll_at?.toISOString() ?? null,
consecutiveFailures: Number(row.consecutive_failures),
createdAt: row.created_at.toISOString(), updatedAt: row.updated_at.toISOString(),
}));
});
}
async getSource(workspaceId: string, sourceId: string): Promise<Record<string, unknown>> {
const rows = await withTenant(workspaceId, async (client) => client.query(
"SELECT * FROM intake_sources WHERE workspace_id = $1 AND id = $2", [workspaceId, sourceId]));
const row = rows.rows[0];
if (!row) throw new NotFoundException("intake source not found");
return {
id: String(row.id), kind: row.kind, params: row.params,
pollIntervalSec: Number(row.poll_interval_sec), health: row.health,
lastError: row.last_error, lastPolledAt: row.last_polled_at?.toISOString() ?? null,
nextPollAt: row.next_poll_at?.toISOString() ?? null, state: row.state,
consecutiveFailures: Number(row.consecutive_failures),
createdAt: row.created_at.toISOString(), updatedAt: row.updated_at.toISOString(),
};
}
async setSourceHealth(workspaceId: string, sourceId: string, health: "active" | "paused"): Promise<void> {
if (health !== "active" && health !== "paused") throw new BadRequestException("health must be active or paused");
await withTenant(workspaceId, async (client) => {
const result = await client.query(
`UPDATE intake_sources SET health = $3, updated_at = now(),
next_poll_at = CASE WHEN $3 = 'active' THEN now() ELSE next_poll_at END
WHERE workspace_id = $1 AND id = $2`, [workspaceId, sourceId, health]);
if (result.rowCount === 0) throw new NotFoundException("intake source not found");
});
}
async deleteSource(workspaceId: string, sourceId: string): Promise<void> {
await withTenant(workspaceId, async (client) => {
const result = await client.query(
"DELETE FROM intake_sources WHERE workspace_id = $1 AND id = $2", [workspaceId, sourceId]);
if (result.rowCount === 0) throw new NotFoundException("intake source not found");
});
}
// -------------------------------------------------------------------------
// Polling
// -------------------------------------------------------------------------
/** One sweep over every workspace's due sources; per-source failures never abort the sweep. */
async pollDueSources(): Promise<{ polled: number; skipped: number; failed: number }> {
if (this.egress === undefined) return { polled: 0, skipped: 0, failed: 0 };
const workspaces = await pool.query<{ id: string }>("SELECT id FROM workspaces ORDER BY id");
const totals = { polled: 0, skipped: 0, failed: 0 };
for (const { id } of workspaces.rows) {
const pass = await this.pollWorkspaceDue(id);
totals.polled += pass.polled;
totals.skipped += pass.skipped;
totals.failed += pass.failed;
}
return totals;
}
async pollWorkspaceDue(workspaceId: string): Promise<{ polled: number; skipped: number; failed: number }> {
const due = await withTenant(workspaceId, async (client) => client.query<SourceRow>(
`SELECT id, workspace_id, kind, params, poll_interval_sec, state, health, consecutive_failures
FROM intake_sources
WHERE workspace_id = $1
AND health NOT IN ('paused','unsupported')
AND poll_interval_sec > 0
AND (next_poll_at IS NULL OR next_poll_at <= now())
ORDER BY COALESCE(next_poll_at, created_at), created_at`, [workspaceId]));
const totals = { polled: 0, skipped: 0, failed: 0 };
for (const row of due.rows) {
const outcome = await this.pollSource(workspaceId, row);
totals[outcome] += 1;
}
return totals;
}
/**
* Poll one source under its Redis lease. The lease makes a multi-replica
* deployment safe (one owner at a time; renewed on a heartbeat, released on
* completion, expires on crash). Spacing between polls is enforced solely
* by next_poll_at in the DB — the single source of truth the scheduler
* reads — so a manual due-reset (or an ops force-poll) is honored rather
* than silently eaten by a side-channel cooldown.
*/
async pollSource(workspaceId: string, row: SourceRow): Promise<"polled" | "skipped" | "failed"> {
if (this.egress === undefined || this.harvester === undefined) return "skipped";
const intervalMs = Math.max(60_000, row.poll_interval_sec * 1_000);
const leaseKey = `intake:${workspaceId}:${row.id}:lease`;
const owner = randomUUID();
if (!await this.deps.redis.set(leaseKey, owner, { nx: true, pxMs: LEASE_MS })) return "skipped";
let renew: Promise<unknown> = Promise.resolve();
let heartbeat: ReturnType<typeof setInterval> | undefined;
try {
heartbeat = setInterval(() => {
renew = this.deps.redis.eval(RENEW_LEASE, [leaseKey], [owner, String(LEASE_MS)]).catch(() => {
this.logger.warn("intake poll lease renewal unavailable");
});
}, LEASE_MS / 3);
heartbeat.unref();
const params = storedParams(row.kind, row.params);
const adapter = adapterFor(row.kind);
const fetcher = {
fetch: (url: string | URL, options: { headers?: Record<string, string>; hostClass: string }) =>
this.egress!.egressFetch(url, { headers: options.headers ?? {} }, { hostClass: options.hostClass }),
};
const result = await adapter.poll({
workspaceId, sourceId: row.id, params, kind: row.kind, state: row.state ?? {}, fetch: fetcher,
});
// Harvest media per the source's policy, then normalize + upsert.
const policy = sourceMediaPolicy(row);
const harvested: HarvestedMedia[][] = [];
for (const item of result.items) {
harvested.push(await this.harvester.harvestAll(policy, item.media ?? []));
}
const rows = result.items.map((raw, index) => ({
raw,
outcome: normalizeRawItem(raw, row.kind, params, workspaceId, row.id),
harvested: harvested[index] ?? [],
}));
const upsert: UpsertResult = await withTenant(workspaceId, async (client) =>
upsertItems(client, workspaceId, row.id, row.kind, rows));
void upsert;
await withTenant(workspaceId, async (client) => client.query(
`UPDATE intake_sources SET state = $3, health = 'active', last_error = NULL, last_polled_at = now(),
consecutive_failures = 0, next_poll_at = now() + ($4::bigint * interval '1 millisecond'), updated_at = now()
WHERE workspace_id = $1 AND id = $2`,
[workspaceId, row.id, JSON.stringify(result.state), jitteredIntervalMs(intervalMs, this.deps.random())]));
return "polled";
} catch (error) {
await this.recordFailure(workspaceId, row, error, intervalMs);
return "failed";
} finally {
if (heartbeat) clearInterval(heartbeat);
await renew;
await this.deps.redis.eval(RELEASE_LEASE, [leaseKey], [owner]).catch(() => undefined);
}
}
private async recordFailure(workspaceId: string, row: SourceRow, error: unknown, intervalMs: number): Promise<void> {
const now = this.deps.now().getTime();
const http = error instanceof IntakeHttpError ? error : undefined;
const egress = this.egress?.isEgressError(error) === true ? error : undefined;
const retryAfter = http !== undefined
? intakeRetryAfterMs(http.headers, now)
: egress !== undefined ? egress.retryAfterMs ?? 0 : 0;
const transient = (http !== undefined && http.transient)
|| egress !== undefined && ["BUDGET_EXHAUSTED", "BUDGET_UNAVAILABLE", "TIMEOUT", "NETWORK_ERROR", "ABORTED", "DNS_FAILED"].includes(egress.code)
|| http === undefined && egress === undefined; // network shape unknown → benefit of the doubt
const permanent = (http !== undefined && http.gone)
|| egress !== undefined && ["DENIED_PRIVATE_ADDRESS", "SCHEME_DENIED", "PORT_DENIED", "ORIGIN_DENIED", "INVALID_URL"].includes(egress.code);
const failures = row.consecutive_failures + 1;
const base = permanent ? PERMANENT_ERROR_MS : transient ? backoffDelayMs(failures, intervalMs) : PERMANENT_ERROR_MS;
const delay = Math.max(retryAfter, jitteredIntervalMs(base, this.deps.random()));
const reason = permanent ? "permanent" : transient ? "transient" : "unknown";
const message = http !== undefined ? `http ${http.status}`
: egress !== undefined ? `egress:${egress.code}`
: error instanceof Error ? error.message.slice(0, 240) : "poll failed";
await withTenant(workspaceId, async (client) => client.query(
`UPDATE intake_sources SET health = $3, last_error = $4, last_polled_at = now(),
consecutive_failures = $5, next_poll_at = now() + ($6::bigint * interval '1 millisecond'), updated_at = now()
WHERE workspace_id = $1 AND id = $2`,
[workspaceId, row.id, permanent ? "error" : "backoff", `${reason}: ${message}`, failures, delay],
)).catch(() => this.logger.warn("intake source state write-back unavailable"));
}
// -------------------------------------------------------------------------
// Telegram archive bridge (task requirement: reuse the existing harvest)
// -------------------------------------------------------------------------
/**
* Bridges an archived_feed_items row into intake_items. The archive webhook
* path (apps/api/src/archive) already normalized the post, harvested media
* into S3 content-addressed keys and enforced revisions — intake stores the
* reference, it never re-fetches.
*/
async ingestArchiveItem(workspaceId: string, archiveItemId: string): Promise<void> {
const found = await withTenant(workspaceId, async (client) => client.query<{
id: string; remote_ref: string; channel_id: string; message_id: string; text_content: string;
media_refs: Array<{ storageKey?: string; contentType?: string; sizeBytes?: number | null }>;
attribution: Record<string, unknown>; posted_at: Date;
}>("SELECT * FROM archived_feed_items WHERE id = $1", [archiveItemId]));
const item = found.rows[0];
if (!item) throw new NotFoundException("archive item not found");
const channelId = String(item.channel_id);
await withTenant(workspaceId, async (client) => {
const source = await client.query<{ id: string; params: Record<string, unknown> }>(
`SELECT id, params FROM intake_sources WHERE workspace_id = $1 AND kind = $2
AND params->>'channelId' = $3`, [workspaceId, KIND_TELEGRAM_CHANNEL, channelId]);
const sourceRow = source.rows[0];
if (!sourceRow) return; // not subscribed in intake — nothing to do
const mediaKeys = (item.media_refs ?? []).map((media) => media.storageKey ?? "").filter((key) => key !== "");
const bridgeHash = itemContentHash(item.text_content, mediaKeys);
await client.query(
`INSERT INTO intake_items (workspace_id, source_id, remote_ref, content_hash, published_at,
text, content, media_refs, attribution)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)
ON CONFLICT (workspace_id, content_hash) DO NOTHING`,
[workspaceId, sourceRow.id, item.remote_ref, bridgeHash,
item.posted_at.toISOString(), item.text_content,
JSON.stringify({ type: "doc", version: 1, children: [{ type: "paragraph", children: [{ type: "text", text: item.text_content }] }], truncated: false }),
JSON.stringify(item.media_refs.map((media) => ({ kind: "image", proxyUrl: "", storageKey: media.storageKey ?? null, contentType: media.contentType ?? null, sizeBytes: media.sizeBytes ?? null }))),
JSON.stringify({ ...item.attribution, provider: PROVIDER_BY_KIND[KIND_TELEGRAM_CHANNEL], archiveItemId: item.id })]);
});
}
// -------------------------------------------------------------------------
// Read surface (search + per-source listing)
// -------------------------------------------------------------------------
search(workspaceId: string, query: Record<string, unknown>): Promise<{ items: Record<string, unknown>[]; nextCursor: string | null }> {
return searchIntake(workspaceId, parseSearchQuery(query), this.deps.storage);
}
listItems(workspaceId: string, query: Record<string, unknown>): Promise<{ items: Record<string, unknown>[]; nextCursor: string | null }> {
return listIntakeItems(workspaceId, parseSearchQuery(query), this.deps.storage);
}
}
function parseMediaPolicy(value: unknown, fallback: MediaPolicy): MediaPolicy {
if (value === undefined || value === null || value === "") return fallback;
if (value !== "metadata" && value !== "images" && value !== "full") {
throw new BadRequestException("mediaPolicy must be metadata, images, or full");
}
return value;
}
function sourceMediaPolicy(row: SourceRow): MediaPolicy {
const fromParams = row.params.mediaPolicy;
if (fromParams === "metadata" || fromParams === "images" || fromParams === "full") return fromParams;
return DEFAULT_MEDIA_POLICY[row.kind];
}
/** Shared query parsing for /intake/search and /intake/items. */
export function parseSearchQuery(query: Record<string, unknown>): IntakeQueryFilters {
const rawQ = query.q;
const q = typeof rawQ === "string" && rawQ.trim() !== "" ? rawQ.trim() : null;
if (q !== null && q.length > 256) throw new BadRequestException("q must be at most 256 characters");
const rawSource = query.sourceId;
const sourceId = rawSource === undefined || rawSource === null || rawSource === "" ? null
: typeof rawSource === "string" && /^[0-9a-f-]{36}$/i.test(rawSource) ? rawSource : (() => { throw new BadRequestException("sourceId must be a UUID"); })();
const rawKind = query.kind;
const kind = rawKind === undefined || rawKind === null || rawKind === "" ? null
: isIntakeKind(rawKind) ? rawKind : (() => { throw new BadRequestException("kind is not a supported intake kind"); })();
const cursor = typeof query.cursor === "string" && query.cursor !== "" ? query.cursor : undefined;
if (cursor !== undefined && (cursor.length > 256 || !/^[A-Za-z0-9_-]+$/.test(cursor))) throw new BadRequestException("invalid cursor");
let limit = 20;
if (query.limit !== undefined && query.limit !== "") {
const n = typeof query.limit === "string" ? Number(query.limit)
: typeof query.limit === "number" ? query.limit : NaN;
if (!Number.isInteger(n) || n < 1 || n > 100) throw new BadRequestException("limit must be an integer between 1 and 100");
limit = n;
}
return { q, sourceId, kind, cursor, limit };
}

107
apps/api/src/intake/media-harvest.ts Normal file
View file

@ -0,0 +1,107 @@
/**
* Intake media harvest — the archive StorageService pattern verbatim
* (task requirement 4): direct GET through the egress budget →
* `putContent(bytes, contentType)` → content-addressed `sha256/<hash>.<ext>`
* key, AES-256-GCM envelope encryption, dedup by plaintext hash. A blob
* already in the bucket is a no-op (`deduplicated: true`), so the same image
* seen via RSS and Mastodon stores once.
*
* Policy gate (research §2.5): `metadata` harvests nothing; `images` accepts
* image candidates only; `full` accepts video/audio too. Size caps are
* enforced twice (declared content-length and actual bytes).
*/
import { HOST_CLASS_BY_KIND, type IntakeKind, type MediaPolicy } from "./intake-kinds.js";
import { INTAKE_MEDIA_MAX_BYTES, IntakeHttpError, intakeRetryAfterMs, type IntakeEgress } from "./intake-egress.js";
import type { StorageService, StoredObject } from "../storage/storage.js";
export interface HarvestRequest {
url: string;
kind: "image" | "video";
}
export interface HarvestedMedia {
url: string;
kind: "image" | "video";
status: "stored" | "skipped" | "too_large" | "failed";
storageKey?: string;
hash?: string;
sizeBytes?: number;
contentType?: string;
deduplicated?: boolean;
error?: string;
}
const IMAGE_TYPES = /^image\//i;
/** Download budget for media files is the shared intake-web class (polite). */
const MEDIA_HOST_CLASS = "intake-web";
export function accepts(policy: MediaPolicy, kind: "image" | "video"): boolean {
if (policy === "metadata") return false;
if (policy === "images") return kind === "image";
return true;
}
export class MediaHarvester {
constructor(
private readonly egress: IntakeEgress,
private readonly storage: StorageService,
private readonly now: () => number = Date.now,
private readonly maxBytes: number = INTAKE_MEDIA_MAX_BYTES,
) {}
async harvestAll(policy: MediaPolicy, requests: HarvestRequest[]): Promise<HarvestedMedia[]> {
const out: HarvestedMedia[] = [];
for (const request of requests) {
out.push(await this.harvest(policy, request));
}
return out;
}
async harvest(policy: MediaPolicy, request: HarvestRequest): Promise<HarvestedMedia> {
if (!accepts(policy, request.kind)) {
return { url: request.url, kind: request.kind, status: "skipped" };
}
try {
const response = await this.egress.egressFetch(request.url, { headers: { accept: "*/*" } }, { hostClass: MEDIA_HOST_CLASS });
if (!response.ok) throw new IntakeHttpError(response.status, response.headers, request.url);
const declared = Number(response.headers.get("content-length") ?? "0");
if (Number.isFinite(declared) && declared > this.maxBytes) {
return { url: request.url, kind: request.kind, status: "too_large", sizeBytes: declared };
}
const contentType = normalizeContentType(response.headers.get("content-type"), request.kind);
const bytes = Buffer.from(await response.arrayBuffer());
if (bytes.byteLength > this.maxBytes) {
return { url: request.url, kind: request.kind, status: "too_large", sizeBytes: bytes.byteLength };
}
const stored: StoredObject & { deduplicated: boolean } = await this.storage.putContent(bytes, contentType);
return {
url: request.url, kind: request.kind, status: "stored",
storageKey: stored.key, hash: stored.hash, sizeBytes: stored.sizeBytes,
contentType: stored.contentType, deduplicated: stored.deduplicated,
};
} catch (error) {
const retryAfter = error instanceof IntakeHttpError ? intakeRetryAfterMs(error.headers, this.now()) : 0;
return {
url: request.url, kind: request.kind, status: "failed",
error: retryAfter > 0 ? `http ${error instanceof IntakeHttpError ? error.status : "network"} (retry-after ${Math.round(retryAfter / 1000)}s)` : error instanceof Error ? error.message.slice(0, 120) : "harvest failed",
};
}
}
}
function normalizeContentType(raw: string | null, kind: "image" | "video"): string {
const value = (raw ?? "").split(";")[0]?.trim().toLowerCase() ?? "";
if (value !== "") {
if (kind === "image" && IMAGE_TYPES.test(value)) return value;
if (kind === "video" && (value.startsWith("video/") || value === "application/octet-stream" || value.startsWith("audio/"))) return value;
if (value === "application/octet-stream") return value;
if (IMAGE_TYPES.test(value)) return value;
if (value.startsWith("video/") || value.startsWith("audio/")) return value;
}
return kind === "image" ? "image/jpeg" : "video/mp4";
}
/** The host class a kind's feed fetch draws on (re-exported for the service). */
export function feedHostClassFor(kind: IntakeKind): string {
return HOST_CLASS_BY_KIND[kind];
}