Tactical Media Broadcast Hub — initial release
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
This commit is contained in:
commit
aa5e262f3a
880 changed files with 111042 additions and 0 deletions
217
packages/connector-sdk/src/http.ts
Normal file
217
packages/connector-sdk/src/http.ts
Normal file
|
|
@ -0,0 +1,217 @@
|
|||
/**
|
||||
* Rate-limited HTTP client for connector packages: per-host token bucket,
|
||||
* exponential backoff with jitter on 429/5xx, fetch-based.
|
||||
*
|
||||
* Retry classification: only idempotent methods (GET/HEAD/OPTIONS/PUT/DELETE)
|
||||
* are retried automatically. A non-idempotent request (POST/PATCH) that gets
|
||||
* 429/5xx or loses its transport is never resent here — the caller receives a
|
||||
* typed error recording whether delivery was definitely refused or unknown.
|
||||
* Read-only POST endpoints can opt back in with `{ retry: true }`; publishing
|
||||
* mutations can opt out entirely with `{ retry: false }`.
|
||||
*/
|
||||
import {
|
||||
ProviderRejectedError,
|
||||
ProviderServerError,
|
||||
ProviderTransportError,
|
||||
RateLimitError,
|
||||
} from "./errors.js";
|
||||
|
||||
interface Bucket {
|
||||
tokens: number;
|
||||
lastRefillMs: number;
|
||||
}
|
||||
|
||||
export interface HttpClientOptions {
|
||||
/** Requests per second allowed per host (default 1). */
|
||||
requestsPerSecond?: number;
|
||||
/** Max backoff attempt count (default 4). */
|
||||
maxRetries?: number;
|
||||
fetchImpl?: typeof fetch;
|
||||
sleep?: (ms: number) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface HttpRequestOptions {
|
||||
/**
|
||||
* Auto-retry on 429/5xx. Default: true for idempotent methods, false
|
||||
* otherwise. `true` on a POST asserts the endpoint is side-effect free.
|
||||
*/
|
||||
retry?: boolean;
|
||||
}
|
||||
|
||||
/** Nonsecret request description handed to hooks — never the full URL. */
|
||||
export interface HttpRequestEvent {
|
||||
provider: string;
|
||||
method: string;
|
||||
origin: string;
|
||||
/** 0 for the first send, then one per automatic retry. */
|
||||
attempt: number;
|
||||
/** Whether this request may have side effects on the provider. */
|
||||
mutating: boolean;
|
||||
}
|
||||
|
||||
export interface HttpResponseEvent extends HttpRequestEvent {
|
||||
status: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-execution hook, resolved for every request. `beforeRequest` runs after
|
||||
* the rate-limiter wait and immediately before each actual send; throwing
|
||||
* from it blocks the send (zero HTTP requests leave the process).
|
||||
*/
|
||||
export interface HttpRequestHook {
|
||||
aroundMutation?(event: HttpRequestEvent, send: () => Promise<Response>): Promise<Response>;
|
||||
beforeRequest?(event: HttpRequestEvent): void | Promise<void>;
|
||||
afterResponse?(event: HttpResponseEvent): void;
|
||||
afterTransportError?(event: HttpRequestEvent): void;
|
||||
}
|
||||
|
||||
let hookProvider: (() => HttpRequestHook | undefined) | undefined;
|
||||
|
||||
/**
|
||||
* Install the resolver for the current execution's hook (e.g. an
|
||||
* AsyncLocalStorage lookup in the publishing runtime). The resolver is
|
||||
* consulted per request, so concurrent executions never share a hook.
|
||||
* Pass undefined to uninstall.
|
||||
*/
|
||||
export function setHttpRequestHookProvider(provider: (() => HttpRequestHook | undefined) | undefined): void {
|
||||
hookProvider = provider;
|
||||
}
|
||||
|
||||
const IDEMPOTENT_METHODS = new Set(["GET", "HEAD", "OPTIONS", "PUT", "DELETE"]);
|
||||
const SAFE_METHODS = new Set(["GET", "HEAD", "OPTIONS"]);
|
||||
|
||||
export function isIdempotentMethod(method: string): boolean {
|
||||
return IDEMPOTENT_METHODS.has(method.toUpperCase());
|
||||
}
|
||||
|
||||
const defaultSleep = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
/** Retry-After header: delta-seconds or HTTP-date → ms. */
|
||||
function headerRetryAfterMs(response: Response): number | undefined {
|
||||
const raw = response.headers.get("retry-after");
|
||||
if (raw === null || raw.trim().length === 0) return undefined;
|
||||
const seconds = Number(raw);
|
||||
if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000);
|
||||
const at = Date.parse(raw);
|
||||
return Number.isFinite(at) ? Math.max(0, at - Date.now()) : undefined;
|
||||
}
|
||||
|
||||
/** `retry_after` (seconds) in a JSON error body, top level or under `parameters`. */
|
||||
async function bodyRetryAfterMs(response: Response): Promise<number | undefined> {
|
||||
try {
|
||||
const body = (await response.json()) as {
|
||||
retry_after?: unknown;
|
||||
parameters?: { retry_after?: unknown } | null;
|
||||
} | null;
|
||||
const value = body?.parameters?.retry_after ?? body?.retry_after;
|
||||
return typeof value === "number" && Number.isFinite(value) ? Math.max(0, value * 1000) : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export class RateLimitedHttpClient {
|
||||
private readonly buckets = new Map<string, Bucket>();
|
||||
private readonly rps: number;
|
||||
private readonly maxRetries: number;
|
||||
private readonly fetchImpl: typeof fetch;
|
||||
private readonly sleep: (ms: number) => Promise<void>;
|
||||
|
||||
constructor(opts: HttpClientOptions = {}) {
|
||||
this.rps = opts.requestsPerSecond ?? 1;
|
||||
this.maxRetries = opts.maxRetries ?? 4;
|
||||
this.fetchImpl = opts.fetchImpl ?? fetch;
|
||||
this.sleep = opts.sleep ?? defaultSleep;
|
||||
}
|
||||
|
||||
private async takeToken(host: string): Promise<void> {
|
||||
for (;;) {
|
||||
const now = Date.now();
|
||||
const bucket = this.buckets.get(host) ?? { tokens: 1, lastRefillMs: now };
|
||||
const elapsed = (now - bucket.lastRefillMs) / 1000;
|
||||
bucket.tokens = Math.min(1, bucket.tokens + elapsed * this.rps);
|
||||
bucket.lastRefillMs = now;
|
||||
if (bucket.tokens >= 1) {
|
||||
bucket.tokens -= 1;
|
||||
this.buckets.set(host, bucket);
|
||||
return;
|
||||
}
|
||||
this.buckets.set(host, bucket);
|
||||
const waitMs = Math.ceil(((1 - bucket.tokens) / this.rps) * 1000);
|
||||
await this.sleep(waitMs);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Never resolves with a 429/5xx: those are retried (when allowed) and
|
||||
* otherwise thrown as RateLimitError / ProviderServerError.
|
||||
*/
|
||||
async request(
|
||||
provider: string,
|
||||
url: string,
|
||||
init: RequestInit = {},
|
||||
options: HttpRequestOptions = {},
|
||||
): Promise<Response> {
|
||||
const parsed = new URL(url);
|
||||
const method = (init.method ?? "GET").toUpperCase();
|
||||
const retry = options.retry ?? isIdempotentMethod(method);
|
||||
// A POST opted into retries is asserted side-effect free.
|
||||
const mutating = !SAFE_METHODS.has(method) && !(options.retry === true && !isIdempotentMethod(method));
|
||||
let attempt = 0;
|
||||
for (;;) {
|
||||
await this.takeToken(parsed.host);
|
||||
const hook = hookProvider?.();
|
||||
const event: HttpRequestEvent = { provider, method, origin: parsed.origin, attempt, mutating };
|
||||
await hook?.beforeRequest?.(event);
|
||||
const send = async (): Promise<Response> => {
|
||||
try { return await this.fetchImpl(url, init); }
|
||||
catch (cause) {
|
||||
hook?.afterTransportError?.(event);
|
||||
if (retry) throw cause;
|
||||
throw new ProviderTransportError(provider, method, cause);
|
||||
}
|
||||
};
|
||||
const response = event.mutating && hook?.aroundMutation !== undefined
|
||||
? await hook.aroundMutation(event, send)
|
||||
: await send();
|
||||
hook?.afterResponse?.({ ...event, status: response.status });
|
||||
const failed = response.status === 429 || response.status >= 500;
|
||||
if (!failed) {
|
||||
return response;
|
||||
}
|
||||
const headerAfter = headerRetryAfterMs(response);
|
||||
if (!retry || attempt >= this.maxRetries) {
|
||||
if (response.status === 429) {
|
||||
throw new RateLimitError(provider, headerAfter ?? (await bodyRetryAfterMs(response)));
|
||||
}
|
||||
const delivery = SAFE_METHODS.has(method) ? "rejected" : "unknown";
|
||||
throw new ProviderServerError(provider, response.status, method, delivery, headerAfter);
|
||||
}
|
||||
const backoffMs = headerAfter ?? Math.min(30_000, 2 ** attempt * 500 + Math.random() * 250);
|
||||
await this.sleep(backoffMs);
|
||||
attempt += 1;
|
||||
}
|
||||
}
|
||||
|
||||
async json<T>(
|
||||
provider: string,
|
||||
url: string,
|
||||
init: RequestInit = {},
|
||||
options: HttpRequestOptions = {},
|
||||
): Promise<T> {
|
||||
const response = await this.request(
|
||||
provider,
|
||||
url,
|
||||
{ ...init, headers: { accept: "application/json", ...init.headers } },
|
||||
options,
|
||||
);
|
||||
if (!response.ok) {
|
||||
throw new ProviderRejectedError(
|
||||
`${provider}: HTTP ${response.status} for ${url}`,
|
||||
provider,
|
||||
response.status,
|
||||
);
|
||||
}
|
||||
return (await response.json()) as T;
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue