glass beadian
aa5e262f3a
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
100 lines
4.4 KiB
TypeScript
100 lines
4.4 KiB
TypeScript
/**
|
|
* Page-watch adapter — the changedetection.io pattern in ~100 lines
|
|
* (research §1.8): fetch the page through the egress budget, normalize the
|
|
* text (strip scripts/styles, collapse whitespace, strip rotating
|
|
* timestamps/CSRF nonces), hash, and emit an item only when the hash moves.
|
|
*
|
|
* Plain fetch only — no headless browser (per the task's "no headless"
|
|
* bound); a page that needs JS renders as its static DOM, which is the right
|
|
* default for press pages and gazettes.
|
|
*/
|
|
import { createHash } from "node:crypto";
|
|
import { fetchText, hostClassFor, type AdapterContext, type AdapterPollResult, type IntakeAdapter, type RawIntakeItem } from "./adapter.js";
|
|
import { KIND_WEB_WATCH } from "../intake-kinds.js";
|
|
import type { WebWatchParams } from "../intake-params.js";
|
|
|
|
/**
|
|
* Diff normalization: what changes on a page without changing its content —
|
|
* numbers (counts, timestamps), long hex/base64 tokens (CSRF, cache busters).
|
|
* Hashing the normalized text is what keeps phantom diffs out of the archive.
|
|
*/
|
|
export function normalizePageText(html: string, selector?: string): string {
|
|
let work = html
|
|
.replace(/<script[\s\S]*?<\/script>/gi, " ")
|
|
.replace(/<style[\s\S]*?<\/style>/gi, " ")
|
|
.replace(/<noscript[\s\S]*?<\/noscript>/gi, " ")
|
|
.replace(/<!--[\s\S]*?-->/g, " ")
|
|
.replace(/<[^>]+>/g, " ")
|
|
.replace(/ /gi, " ")
|
|
.replace(/&/gi, "&")
|
|
.replace(/</gi, "<")
|
|
.replace(/>/gi, ">")
|
|
.replace(/"/gi, '"')
|
|
.replace(/�?39;/g, "'");
|
|
if (selector !== undefined && selector.trim() !== "") {
|
|
// Cheap id/class containment: keep only lines containing the selector's
|
|
// leaf token when the selector looks like #id or .class (a full CSS
|
|
// engine is out of scope for phase 1; a wrong selector degrades to the
|
|
// whole page's diff, which is still correct — just noisier).
|
|
const leaf = selector.trim().split(/[ >]+/).at(-1) ?? "";
|
|
const match = /^([#.])([\w-]+)$/.exec(leaf);
|
|
if (match) {
|
|
// Extract the region around the marker token from the raw HTML instead.
|
|
const marker = match[1] === "#" ? `id="${match[2]}"` : `class="${match[2]}"`;
|
|
const at = html.toLowerCase().indexOf(marker.toLowerCase());
|
|
if (at >= 0) {
|
|
const from = html.lastIndexOf("<", at);
|
|
const to = html.indexOf("</body>", at);
|
|
work = normalizePageText(html.slice(from > 0 ? from : 0, to > 0 ? to : undefined));
|
|
}
|
|
}
|
|
}
|
|
return work
|
|
.replace(/\b[0-9]{2,}\b/g, " ") // counts, epochs, timestamps
|
|
.replace(/\b[0-9a-f]{16,}\b/gi, " ") // CSRF tokens, digests
|
|
.replace(/\s+/g, " ")
|
|
.trim();
|
|
}
|
|
|
|
export function pageHash(normalized: string): string {
|
|
return createHash("sha256").update(normalized).digest("hex");
|
|
}
|
|
|
|
export class WebWatchAdapter implements IntakeAdapter {
|
|
readonly kind = KIND_WEB_WATCH;
|
|
|
|
async poll(context: AdapterContext): Promise<AdapterPollResult> {
|
|
const params = context.params as WebWatchParams;
|
|
const { text, unchanged } = await fetchText(context.fetch, params.url, hostClassFor(KIND_WEB_WATCH), {
|
|
"cache-control": "no-cache",
|
|
});
|
|
if (unchanged) return { items: [], state: context.state, unchanged: true };
|
|
const normalized = normalizePageText(text, params.selector);
|
|
const hash = pageHash(normalized);
|
|
if (typeof context.state.contentHash === "string" && context.state.contentHash === hash) {
|
|
return { items: [], state: context.state, unchanged: true };
|
|
}
|
|
const now = new Date();
|
|
const items: RawIntakeItem[] = [];
|
|
if (typeof context.state.contentHash === "string") {
|
|
// First fetch seeds the baseline; only a CHANGED page (second fetch
|
|
// onward) emits — a new subscription never manufactures an item for a
|
|
// page that was already in that state.
|
|
items.push({
|
|
id: hash.slice(0, 24),
|
|
remoteId: hash.slice(0, 24),
|
|
remoteRef: `${params.url}#${hash.slice(0, 16)}`,
|
|
provider: "web",
|
|
authorHandle: new URL(params.url).hostname,
|
|
text: normalized.slice(0, 8_000),
|
|
mediaUrls: [],
|
|
createdAt: now,
|
|
metrics: {},
|
|
normalize: { contentKind: "text", url: params.url },
|
|
attribution: { url: params.url, provider: "web", changedAt: now.toISOString(), diffHash: hash },
|
|
media: [],
|
|
});
|
|
}
|
|
return { items, state: { ...context.state, contentHash: hash, lastChangedAt: now.toISOString() } };
|
|
}
|
|
}
|