Tactical Media Broadcast Hub — initial release
Self-hosted multi-platform broadcast newsroom for activist collectives: publishing pipeline with group approval and DST-proof scheduling, TweetDeck-style column wall with hostile-content-safe rendering, live control room with Auto Director, bridged chat governance with polls and quorum, distributed sandboxed transcode workers, encrypted content-addressed archive, monthly analytics. MIT licensed. See README.md and docs/.
This commit is contained in:
commit
aa5e262f3a
880 changed files with 111042 additions and 0 deletions
23
packages/egress/src/budget.d.ts
vendored
Normal file
23
packages/egress/src/budget.d.ts
vendored
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
import type { Pool } from "pg";
|
||||
export declare const REFILL_INTERVAL_MS = 10000;
|
||||
export interface HostBudget {
|
||||
capacity: number;
|
||||
refillPerInterval: number;
|
||||
}
|
||||
export declare const DEFAULT_HOST_BUDGET: Readonly<HostBudget>;
|
||||
export interface HostBudgetOptions {
|
||||
pool: Pick<Pool, "connect">;
|
||||
hostClasses?: Readonly<Record<string, HostBudget>>;
|
||||
/** Test clock only. Production uses the DB clock sampled after acquiring the row lock. */
|
||||
now?: () => Date;
|
||||
}
|
||||
export declare function retryAfterDeadline(headers: Headers, now: number): number | undefined;
|
||||
export declare class HostBudgets {
|
||||
private readonly options;
|
||||
constructor(options: HostBudgetOptions);
|
||||
private configuration;
|
||||
private locked;
|
||||
consume(host: string, hostClass: string): Promise<void>;
|
||||
/** Header hints can only tighten a shared budget; parking is monotone under concurrent responses. */
|
||||
observe(host: string, hostClass: string, status: number, headers: Headers): Promise<void>;
|
||||
}
|
||||
99
packages/egress/src/budget.js
Normal file
99
packages/egress/src/budget.js
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
import { EgressError } from "./errors.js";
|
||||
export const REFILL_INTERVAL_MS = 10_000;
|
||||
export const DEFAULT_HOST_BUDGET = Object.freeze({ capacity: 3, refillPerInterval: 10 });
|
||||
function numericHeader(headers, name) {
|
||||
const raw = headers.get(name);
|
||||
if (raw === null || !/^\d+(?:\.\d+)?$/.test(raw.trim()))
|
||||
return undefined;
|
||||
const value = Number(raw);
|
||||
return Number.isFinite(value) ? value : undefined;
|
||||
}
|
||||
export function retryAfterDeadline(headers, now) {
|
||||
const value = headers.get("retry-after");
|
||||
if (value === null)
|
||||
return undefined;
|
||||
if (/^\d+(?:\.\d+)?$/.test(value.trim())) {
|
||||
const deadline = now + Number(value) * 1000;
|
||||
return Number.isFinite(deadline) && deadline <= 8.64e15 ? deadline : undefined;
|
||||
}
|
||||
const deadline = Date.parse(value);
|
||||
return Number.isFinite(deadline) ? Math.max(now, deadline) : undefined;
|
||||
}
|
||||
export class HostBudgets {
|
||||
options;
|
||||
constructor(options) {
|
||||
this.options = options;
|
||||
}
|
||||
configuration(hostClass) {
|
||||
const config = this.options.hostClasses?.[hostClass] ?? DEFAULT_HOST_BUDGET;
|
||||
if (!Number.isFinite(config.capacity) || config.capacity < 1 || !Number.isFinite(config.refillPerInterval) || config.refillPerInterval <= 0) {
|
||||
throw new EgressError("INVALID_CONFIGURATION", "Invalid host budget");
|
||||
}
|
||||
return config;
|
||||
}
|
||||
async locked(host, hostClass, action) {
|
||||
const config = this.configuration(hostClass);
|
||||
const client = await this.options.pool.connect().catch(() => { throw new EgressError("BUDGET_UNAVAILABLE", "Budget database is unavailable"); });
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
// Existing rows remain authoritative: another caller/class cannot reset or enlarge a bucket.
|
||||
await client.query(`INSERT INTO host_budgets(host, capacity, refill_per_interval, tokens, updated_at)
|
||||
VALUES ($1, $2, $3, $2, COALESCE($4::timestamptz, clock_timestamp())) ON CONFLICT (host) DO NOTHING`, [host, config.capacity, config.refillPerInterval, this.options.now?.() ?? null]);
|
||||
const row = (await client.query("SELECT capacity, refill_per_interval, tokens, updated_at FROM host_budgets WHERE host = $1 FOR UPDATE", [host])).rows[0];
|
||||
// Keep microseconds in SQL parameters. pg's Date parser truncates them and can make
|
||||
// a freshly inserted bucket appear parked if both queries finish in one millisecond.
|
||||
const timestamp = this.options.now?.() ?? (await client.query("SELECT clock_timestamp()::text AS now")).rows[0].now;
|
||||
const now = typeof timestamp === "string" ? new Date(timestamp) : timestamp;
|
||||
const result = await action(client, row, now, timestamp);
|
||||
await client.query("COMMIT");
|
||||
return result;
|
||||
}
|
||||
catch (error) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw error instanceof EgressError ? error : new EgressError("BUDGET_UNAVAILABLE", "Budget transaction failed");
|
||||
}
|
||||
finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
async consume(host, hostClass) {
|
||||
const retryAfterMs = await this.locked(host, hostClass, async (client, row, now, timestamp) => {
|
||||
const result = await client.query(`UPDATE host_budgets
|
||||
SET tokens = LEAST(capacity, tokens + GREATEST(0, EXTRACT(EPOCH FROM ($2::timestamptz - updated_at))) * refill_per_interval / 10) - 1,
|
||||
updated_at = $2
|
||||
WHERE host = $1 AND updated_at <= $2
|
||||
AND LEAST(capacity, tokens + GREATEST(0, EXTRACT(EPOCH FROM ($2::timestamptz - updated_at))) * refill_per_interval / 10) >= 1
|
||||
RETURNING tokens`, [host, timestamp]);
|
||||
if (result.rowCount)
|
||||
return undefined;
|
||||
const elapsed = Math.max(0, now.getTime() - row.updated_at.getTime());
|
||||
return Math.ceil(Math.max(0, row.updated_at.getTime() - now.getTime()) + Math.max(0, (1 - row.tokens) * REFILL_INTERVAL_MS / row.refill_per_interval - elapsed));
|
||||
});
|
||||
if (retryAfterMs !== undefined)
|
||||
throw new EgressError("BUDGET_EXHAUSTED", "Host budget is exhausted", retryAfterMs);
|
||||
}
|
||||
/** Header hints can only tighten a shared budget; parking is monotone under concurrent responses. */
|
||||
async observe(host, hostClass, status, headers) {
|
||||
const remaining = numericHeader(headers, "x-ratelimit-remaining");
|
||||
const limit = numericHeader(headers, "x-ratelimit-limit");
|
||||
if (status !== 429 && !headers.has("retry-after") && remaining === undefined && limit === undefined)
|
||||
return;
|
||||
await this.locked(host, hostClass, async (client, row, now, timestamp) => {
|
||||
const time = now.getTime();
|
||||
const reset = numericHeader(headers, "x-ratelimit-reset");
|
||||
const proposedReset = reset === undefined ? undefined : reset >= 1e12 ? reset : reset >= 1e9 ? reset * 1000 : time + reset * 1000;
|
||||
const resetAt = proposedReset !== undefined && proposedReset <= 8.64e15 ? proposedReset : undefined;
|
||||
const retryAt = retryAfterDeadline(headers, time);
|
||||
const exhausted = status === 429 || remaining === 0 || retryAt !== undefined;
|
||||
const capacity = Math.min(row.capacity, limit !== undefined && limit >= 1 ? limit : row.capacity);
|
||||
// When no window is supplied, conservatively interpret Limit per our fixed 10-second interval.
|
||||
const refill = limit !== undefined && limit >= 1 ? Math.min(row.refill_per_interval, limit * REFILL_INTERVAL_MS / Math.max(REFILL_INTERVAL_MS, (resetAt ?? time) - time)) : row.refill_per_interval;
|
||||
const deadline = exhausted ? Math.max(time, retryAt ?? 0, resetAt ?? 0, retryAt === undefined && resetAt === undefined ? time + REFILL_INTERVAL_MS : 0) : time;
|
||||
await client.query(`UPDATE host_budgets SET capacity = $2,
|
||||
tokens = CASE WHEN $6::boolean THEN 0 ELSE LEAST($2, $3,
|
||||
tokens + GREATEST(0, EXTRACT(EPOCH FROM ($7::timestamptz - updated_at))) * refill_per_interval / 10) END,
|
||||
updated_at = GREATEST(updated_at, $4::timestamptz), refill_per_interval = $5 WHERE host = $1`, [host, capacity, remaining ?? capacity, exhausted ? new Date(Math.max(row.updated_at.getTime(), deadline)) : timestamp, refill, exhausted, timestamp]);
|
||||
});
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=budget.js.map
|
||||
1
packages/egress/src/budget.js.map
Normal file
1
packages/egress/src/budget.js.map
Normal file
File diff suppressed because one or more lines are too long
71
packages/egress/src/budget.test.ts
Normal file
71
packages/egress/src/budget.test.ts
Normal file
|
|
@ -0,0 +1,71 @@
|
|||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { HostBudgets } from "./budget.js";
|
||||
import { testDatabase } from "./test-db.js";
|
||||
|
||||
describe("PostgreSQL host budgets", () => {
|
||||
const pool = testDatabase();
|
||||
afterEach(() => { vi.useRealTimers(); });
|
||||
function clock() {
|
||||
vi.useFakeTimers({ toFake: ["Date"] });
|
||||
vi.setSystemTime(new Date("2026-01-01T00:00:00Z"));
|
||||
return () => new Date();
|
||||
}
|
||||
it("parallel callers share one atomic capacity even on first initialization", async () => {
|
||||
const now = clock();
|
||||
const callers = Array.from({ length: 16 }, () => new HostBudgets({ pool, now }));
|
||||
const results = await Promise.allSettled(Array.from({ length: 80 }, (_, index) => callers[index % callers.length]!.consume("example.com:443", "monitor")));
|
||||
expect(results.filter((r) => r.status === "fulfilled")).toHaveLength(3);
|
||||
for (const result of results) if (result.status === "rejected") expect(result.reason).toMatchObject({ code: "BUDGET_EXHAUSTED", retryAfterMs: 1000 });
|
||||
expect((await pool.query("SELECT tokens FROM host_budgets")).rows[0].tokens).toBe(0);
|
||||
});
|
||||
it("refills fractional tokens, caps idle refill, and tolerates a backwards clock", async () => {
|
||||
const now = clock();
|
||||
const bucket = new HostBudgets({ pool, now });
|
||||
for (let i = 0; i < 3; i++) await bucket.consume("example.com:443", "monitor");
|
||||
vi.setSystemTime(Date.now() + 999);
|
||||
await expect(bucket.consume("example.com:443", "monitor")).rejects.toMatchObject({ code: "BUDGET_EXHAUSTED", retryAfterMs: 1 });
|
||||
vi.setSystemTime(Date.now() + 1);
|
||||
await bucket.consume("example.com:443", "monitor");
|
||||
vi.setSystemTime(Date.now() + 60_000);
|
||||
for (let i = 0; i < 3; i++) await bucket.consume("example.com:443", "monitor");
|
||||
await expect(bucket.consume("example.com:443", "monitor")).rejects.toMatchObject({ code: "BUDGET_EXHAUSTED" });
|
||||
vi.setSystemTime(Date.now() - 5000);
|
||||
await expect(bucket.consume("example.com:443", "monitor")).rejects.toMatchObject({ code: "BUDGET_EXHAUSTED", retryAfterMs: 6000 });
|
||||
});
|
||||
it.each(["seconds", "date"])("parks on Retry-After (%s) without a concurrent response shortening the deadline", async (kind) => {
|
||||
const now = clock();
|
||||
const bucket = new HostBudgets({ pool, now });
|
||||
const retry = kind === "seconds" ? "5" : new Date(Date.now() + 5000).toUTCString();
|
||||
await bucket.consume("example.com:443", "monitor");
|
||||
await bucket.observe("example.com:443", "monitor", 429, new Headers({ "Retry-After": retry }));
|
||||
await bucket.observe("example.com:443", "monitor", 429, new Headers({ "Retry-After": "1" }));
|
||||
await bucket.observe("example.com:443", "monitor", 200, new Headers({ "X-RateLimit-Remaining": "100" }));
|
||||
vi.setSystemTime(Date.now() + 5000);
|
||||
await expect(bucket.consume("example.com:443", "monitor")).rejects.toMatchObject({ retryAfterMs: 1000 });
|
||||
vi.setSystemTime(Date.now() + 1000);
|
||||
await bucket.consume("example.com:443", "monitor");
|
||||
});
|
||||
it("honors reset/remaining/limit and defaults missing 429 hints to ten seconds", async () => {
|
||||
const now = clock();
|
||||
const bucket = new HostBudgets({ pool, now });
|
||||
await bucket.consume("example.com:443", "monitor");
|
||||
await bucket.observe("example.com:443", "monitor", 200, new Headers({ "X-RateLimit-Remaining": "0", "X-RateLimit-Reset": String((Date.now() + 7000) / 1000), "X-RateLimit-Limit": "2" }));
|
||||
await expect(bucket.consume("example.com:443", "monitor")).rejects.toMatchObject({ retryAfterMs: 12000 });
|
||||
expect((await pool.query("SELECT capacity, tokens, refill_per_interval FROM host_budgets")).rows[0]).toMatchObject({ capacity: 2, tokens: 0, refill_per_interval: 2 });
|
||||
await bucket.observe("other.com:443", "monitor", 429, new Headers({ "Retry-After": "invalid" }));
|
||||
await expect(bucket.consume("other.com:443", "monitor")).rejects.toMatchObject({ retryAfterMs: 11000 });
|
||||
});
|
||||
it("uses DB time by default, isolates ports/hosts and prevents class-based resets", async () => {
|
||||
const bucket = new HostBudgets({ pool, hostClasses: { large: { capacity: 100, refillPerInterval: 100 } } });
|
||||
await bucket.consume("example.com:443", "monitor");
|
||||
await bucket.consume("example.com:443", "large");
|
||||
await bucket.consume("example.com:80", "large");
|
||||
const rows = (await pool.query("SELECT host, capacity FROM host_budgets ORDER BY host")).rows;
|
||||
expect(rows).toEqual([{ host: "example.com:443", capacity: 3 }, { host: "example.com:80", capacity: 100 }]);
|
||||
});
|
||||
it("never mistakes newly created buckets for parked buckets with the DB microsecond clock", async () => {
|
||||
const bucket = new HostBudgets({ pool });
|
||||
for (let i = 0; i < 40; i++) await bucket.consume(`fresh-${i}.example:443`, "monitor");
|
||||
expect((await pool.query("SELECT count(*)::int AS count FROM host_budgets WHERE tokens >= 2 AND tokens <= 3")).rows[0].count).toBe(40);
|
||||
});
|
||||
});
|
||||
106
packages/egress/src/budget.ts
Normal file
106
packages/egress/src/budget.ts
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
import type { Pool, PoolClient } from "pg";
|
||||
import { EgressError } from "./errors.js";
|
||||
|
||||
export const REFILL_INTERVAL_MS = 10_000;
|
||||
export interface HostBudget { capacity: number; refillPerInterval: number }
|
||||
export const DEFAULT_HOST_BUDGET: Readonly<HostBudget> = Object.freeze({ capacity: 3, refillPerInterval: 10 });
|
||||
export interface HostBudgetOptions {
|
||||
pool: Pick<Pool, "connect">;
|
||||
hostClasses?: Readonly<Record<string, HostBudget>>;
|
||||
/** Test clock only. Production uses the DB clock sampled after acquiring the row lock. */
|
||||
now?: () => Date;
|
||||
}
|
||||
interface BudgetRow { capacity: number; refill_per_interval: number; tokens: number; updated_at: Date }
|
||||
|
||||
function numericHeader(headers: Headers, name: string): number | undefined {
|
||||
const raw = headers.get(name);
|
||||
if (raw === null || !/^\d+(?:\.\d+)?$/.test(raw.trim())) return undefined;
|
||||
const value = Number(raw);
|
||||
return Number.isFinite(value) ? value : undefined;
|
||||
}
|
||||
|
||||
export function retryAfterDeadline(headers: Headers, now: number): number | undefined {
|
||||
const value = headers.get("retry-after");
|
||||
if (value === null) return undefined;
|
||||
if (/^\d+(?:\.\d+)?$/.test(value.trim())) {
|
||||
const deadline = now + Number(value) * 1000;
|
||||
return Number.isFinite(deadline) && deadline <= 8.64e15 ? deadline : undefined;
|
||||
}
|
||||
const deadline = Date.parse(value);
|
||||
return Number.isFinite(deadline) ? Math.max(now, deadline) : undefined;
|
||||
}
|
||||
|
||||
export class HostBudgets {
|
||||
constructor(private readonly options: HostBudgetOptions) {}
|
||||
private configuration(hostClass: string): HostBudget {
|
||||
const config = this.options.hostClasses?.[hostClass] ?? DEFAULT_HOST_BUDGET;
|
||||
if (!Number.isFinite(config.capacity) || config.capacity < 1 || !Number.isFinite(config.refillPerInterval) || config.refillPerInterval <= 0) {
|
||||
throw new EgressError("INVALID_CONFIGURATION", "Invalid host budget");
|
||||
}
|
||||
return config;
|
||||
}
|
||||
|
||||
private async locked<T>(host: string, hostClass: string, action: (client: PoolClient, row: BudgetRow, now: Date, timestamp: Date | string) => Promise<T>): Promise<T> {
|
||||
const config = this.configuration(hostClass);
|
||||
const client = await this.options.pool.connect().catch(() => { throw new EgressError("BUDGET_UNAVAILABLE", "Budget database is unavailable"); });
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
// Existing rows remain authoritative: another caller/class cannot reset or enlarge a bucket.
|
||||
await client.query(`INSERT INTO host_budgets(host, capacity, refill_per_interval, tokens, updated_at)
|
||||
VALUES ($1, $2, $3, $2, COALESCE($4::timestamptz, clock_timestamp())) ON CONFLICT (host) DO NOTHING`,
|
||||
[host, config.capacity, config.refillPerInterval, this.options.now?.() ?? null]);
|
||||
const row = (await client.query<BudgetRow>("SELECT capacity, refill_per_interval, tokens, updated_at FROM host_budgets WHERE host = $1 FOR UPDATE", [host])).rows[0]!;
|
||||
// Keep microseconds in SQL parameters. pg's Date parser truncates them and can make
|
||||
// a freshly inserted bucket appear parked if both queries finish in one millisecond.
|
||||
const timestamp = this.options.now?.() ?? (await client.query<{ now: string }>("SELECT clock_timestamp()::text AS now")).rows[0]!.now;
|
||||
const now = typeof timestamp === "string" ? new Date(timestamp) : timestamp;
|
||||
const result = await action(client, row, now, timestamp);
|
||||
await client.query("COMMIT");
|
||||
return result;
|
||||
} catch (error) {
|
||||
await client.query("ROLLBACK").catch(() => undefined);
|
||||
throw error instanceof EgressError ? error : new EgressError("BUDGET_UNAVAILABLE", "Budget transaction failed");
|
||||
} finally { client.release(); }
|
||||
}
|
||||
|
||||
async consume(host: string, hostClass: string): Promise<void> {
|
||||
const retryAfterMs = await this.locked(host, hostClass, async (client, row, now, timestamp) => {
|
||||
const result = await client.query(`UPDATE host_budgets
|
||||
SET tokens = LEAST(capacity, tokens + GREATEST(0, EXTRACT(EPOCH FROM ($2::timestamptz - updated_at))) * refill_per_interval / 10) - 1,
|
||||
updated_at = $2
|
||||
WHERE host = $1 AND updated_at <= $2
|
||||
AND LEAST(capacity, tokens + GREATEST(0, EXTRACT(EPOCH FROM ($2::timestamptz - updated_at))) * refill_per_interval / 10) >= 1
|
||||
RETURNING tokens`, [host, timestamp]);
|
||||
if (result.rowCount) return undefined;
|
||||
const elapsed = Math.max(0, now.getTime() - row.updated_at.getTime());
|
||||
return Math.ceil(Math.max(0, row.updated_at.getTime() - now.getTime()) + Math.max(0, (1 - row.tokens) * REFILL_INTERVAL_MS / row.refill_per_interval - elapsed));
|
||||
});
|
||||
if (retryAfterMs !== undefined) throw new EgressError("BUDGET_EXHAUSTED", "Host budget is exhausted", retryAfterMs);
|
||||
}
|
||||
|
||||
/** Header hints can only tighten a shared budget; parking is monotone under concurrent responses. */
|
||||
async observe(host: string, hostClass: string, status: number, headers: Headers): Promise<void> {
|
||||
const remaining = numericHeader(headers, "x-ratelimit-remaining");
|
||||
const limit = numericHeader(headers, "x-ratelimit-limit");
|
||||
if (status !== 429 && !headers.has("retry-after") && remaining === undefined && limit === undefined) return;
|
||||
await this.locked(host, hostClass, async (client, row, now, timestamp) => {
|
||||
const time = now.getTime();
|
||||
const reset = numericHeader(headers, "x-ratelimit-reset");
|
||||
const proposedReset = reset === undefined ? undefined : reset >= 1e12 ? reset : reset >= 1e9 ? reset * 1000 : time + reset * 1000;
|
||||
const resetAt = proposedReset !== undefined && proposedReset <= 8.64e15 ? proposedReset : undefined;
|
||||
const retryAt = retryAfterDeadline(headers, time);
|
||||
const exhausted = status === 429 || remaining === 0 || retryAt !== undefined;
|
||||
const capacity = Math.min(row.capacity, limit !== undefined && limit >= 1 ? limit : row.capacity);
|
||||
// When no window is supplied, conservatively interpret Limit per our fixed 10-second interval.
|
||||
const refill = limit !== undefined && limit >= 1 ? Math.min(row.refill_per_interval,
|
||||
limit * REFILL_INTERVAL_MS / Math.max(REFILL_INTERVAL_MS, (resetAt ?? time) - time)) : row.refill_per_interval;
|
||||
const deadline = exhausted ? Math.max(time, retryAt ?? 0, resetAt ?? 0,
|
||||
retryAt === undefined && resetAt === undefined ? time + REFILL_INTERVAL_MS : 0) : time;
|
||||
await client.query(`UPDATE host_budgets SET capacity = $2,
|
||||
tokens = CASE WHEN $6::boolean THEN 0 ELSE LEAST($2, $3,
|
||||
tokens + GREATEST(0, EXTRACT(EPOCH FROM ($7::timestamptz - updated_at))) * refill_per_interval / 10) END,
|
||||
updated_at = GREATEST(updated_at, $4::timestamptz), refill_per_interval = $5 WHERE host = $1`,
|
||||
[host, capacity, remaining ?? capacity, exhausted ? new Date(Math.max(row.updated_at.getTime(), deadline)) : timestamp, refill, exhausted, timestamp]);
|
||||
});
|
||||
}
|
||||
}
|
||||
20
packages/egress/src/client.d.ts
vendored
Normal file
20
packages/egress/src/client.d.ts
vendored
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
import { HostBudgets, type HostBudgetOptions } from "./budget.js";
|
||||
import { EgressPolicy, type EgressPolicyOptions } from "./policy.js";
|
||||
export interface EgressClientOptions extends HostBudgetOptions, EgressPolicyOptions {
|
||||
timeoutMs?: number;
|
||||
maxResponseBytes?: number;
|
||||
maxRequestBytes?: number;
|
||||
maxRedirects?: number;
|
||||
}
|
||||
export interface EgressRequestOptions {
|
||||
hostClass: string;
|
||||
/** true pins to the initial origin; a URL string explicitly pins to that origin. */
|
||||
pinnedOrigin?: boolean | string | URL;
|
||||
}
|
||||
export declare function createEgressClient(options: EgressClientOptions): {
|
||||
egressFetch: (raw: string | URL, init: RequestInit | undefined, context: EgressRequestOptions) => Promise<Response>;
|
||||
policy: EgressPolicy;
|
||||
budgets: HostBudgets;
|
||||
};
|
||||
/** Stateless convenience form. Reuse createEgressClient when pool/config are fixed. */
|
||||
export declare function egressFetch(url: string | URL, init: RequestInit | undefined, options: EgressClientOptions & EgressRequestOptions): Promise<Response>;
|
||||
142
packages/egress/src/client.js
Normal file
142
packages/egress/src/client.js
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
import { HostBudgets } from "./budget.js";
|
||||
import { EgressError } from "./errors.js";
|
||||
import { EgressPolicy } from "./policy.js";
|
||||
import { requestPinned } from "./transport.js";
|
||||
const redirectStatuses = new Set([301, 302, 303, 307, 308]);
|
||||
export function createEgressClient(options) {
|
||||
const policy = new EgressPolicy(options);
|
||||
const budgets = new HostBudgets(options);
|
||||
const timeoutMs = options.timeoutMs ?? 15_000;
|
||||
const maxResponseBytes = options.maxResponseBytes ?? 8 * 1024 * 1024;
|
||||
const maxRequestBytes = options.maxRequestBytes ?? 8 * 1024 * 1024;
|
||||
const maxRedirects = options.maxRedirects ?? 5;
|
||||
if (![timeoutMs, maxResponseBytes, maxRequestBytes].every((value) => Number.isSafeInteger(value) && value > 0) || !Number.isSafeInteger(maxRedirects) || maxRedirects < 0) {
|
||||
throw new EgressError("INVALID_CONFIGURATION", "Invalid request limits");
|
||||
}
|
||||
async function egressFetch(raw, init = {}, context) {
|
||||
const controller = new AbortController();
|
||||
const abort = () => controller.abort(new EgressError("ABORTED", "Request aborted"));
|
||||
init.signal?.addEventListener("abort", abort, { once: true });
|
||||
if (init.signal?.aborted)
|
||||
abort();
|
||||
const timer = setTimeout(() => controller.abort(new EgressError("TIMEOUT", "Egress deadline exceeded")), timeoutMs);
|
||||
const check = () => { if (controller.signal.aborted)
|
||||
throw controller.signal.reason; };
|
||||
let abortListener;
|
||||
const cancelled = new Promise((_resolve, reject) => {
|
||||
abortListener = () => reject(controller.signal.reason);
|
||||
controller.signal.addEventListener("abort", abortListener, { once: true });
|
||||
if (controller.signal.aborted)
|
||||
abortListener();
|
||||
});
|
||||
const run = async () => {
|
||||
check();
|
||||
let url = policy.validateUrl(raw);
|
||||
const pinnedOrigin = context.pinnedOrigin === true ? url.origin : context.pinnedOrigin ? policy.validateUrl(context.pinnedOrigin).origin : undefined;
|
||||
if (pinnedOrigin && url.origin !== pinnedOrigin)
|
||||
throw new EgressError("ORIGIN_DENIED", "Request is outside the pinned origin");
|
||||
// Request provides fetch method/header/body normalization; the transport owns connection options.
|
||||
let normalized;
|
||||
try {
|
||||
normalized = new Request(url, init);
|
||||
}
|
||||
catch {
|
||||
throw new EgressError("INVALID_REQUEST", "Invalid fetch request initialization");
|
||||
}
|
||||
let method = normalized.method;
|
||||
const headers = new Headers(normalized.headers);
|
||||
let body;
|
||||
if (normalized.body) {
|
||||
const reader = normalized.body.getReader();
|
||||
const cancelBody = () => { void reader.cancel().catch(() => undefined); };
|
||||
controller.signal.addEventListener("abort", cancelBody, { once: true });
|
||||
const chunks = [];
|
||||
let size = 0;
|
||||
try {
|
||||
for (;;) {
|
||||
check();
|
||||
const { value, done } = await reader.read();
|
||||
if (done)
|
||||
break;
|
||||
size += value.length;
|
||||
if (size > maxRequestBytes)
|
||||
throw new EgressError("REQUEST_TOO_LARGE", "Request exceeds size limit");
|
||||
chunks.push(Buffer.from(value));
|
||||
}
|
||||
body = Buffer.concat(chunks);
|
||||
}
|
||||
finally {
|
||||
controller.signal.removeEventListener("abort", cancelBody);
|
||||
void reader.cancel().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
const visited = new Set();
|
||||
for (let hop = 0;; hop++) {
|
||||
check();
|
||||
if (visited.has(url.href))
|
||||
throw new EgressError("REDIRECT_LOOP", "Redirect cycle detected");
|
||||
visited.add(url.href);
|
||||
const endpoint = await policy.validate(url);
|
||||
check();
|
||||
await budgets.consume(endpoint.hostKey, context.hostClass);
|
||||
check();
|
||||
const wire = await requestPinned(endpoint, method, headers, body, controller.signal, maxResponseBytes, (status, responseHeaders) => budgets.observe(endpoint.hostKey, context.hostClass, status, responseHeaders));
|
||||
check();
|
||||
const location = wire.headers.get("location");
|
||||
if (!redirectStatuses.has(wire.status) || !location || init.redirect === "manual") {
|
||||
const response = new Response(method === "HEAD" || [204, 205, 304].includes(wire.status) ? null : new Uint8Array(wire.body), { status: wire.status, statusText: wire.statusText, headers: wire.headers });
|
||||
Object.defineProperties(response, { url: { value: url.href }, redirected: { value: hop > 0 } });
|
||||
return response;
|
||||
}
|
||||
if (init.redirect === "error" || hop >= maxRedirects)
|
||||
throw new EgressError("REDIRECT_LOOP", "Redirect limit exceeded");
|
||||
if (/[\s\\\u0000-\u001f\u007f]/u.test(location))
|
||||
throw new EgressError("INVALID_URL", "Malformed redirect target");
|
||||
let target;
|
||||
try {
|
||||
target = policy.validateUrl(new URL(location, url));
|
||||
}
|
||||
catch (error) {
|
||||
if (error instanceof EgressError)
|
||||
throw error;
|
||||
throw new EgressError("INVALID_URL", "Invalid redirect target");
|
||||
}
|
||||
if (pinnedOrigin && target.origin !== pinnedOrigin)
|
||||
throw new EgressError("ORIGIN_DENIED", "Redirect is outside the pinned origin");
|
||||
if (target.origin !== url.origin) {
|
||||
for (const name of ["authorization", "proxy-authorization", "cookie", "cookie2"])
|
||||
headers.delete(name);
|
||||
// A body can contain secrets just as headers can; refuse to replay it off-origin.
|
||||
if (body && (wire.status === 307 || wire.status === 308 || method !== "POST" && wire.status !== 303)) {
|
||||
throw new EgressError("ORIGIN_DENIED", "Request body cannot be redirected off-origin");
|
||||
}
|
||||
}
|
||||
if (wire.status === 303 && method !== "HEAD" || (wire.status === 301 || wire.status === 302) && method === "POST") {
|
||||
method = "GET";
|
||||
body = undefined;
|
||||
for (const name of ["content-type", "content-length", "content-encoding", "content-language", "content-location"])
|
||||
headers.delete(name);
|
||||
}
|
||||
url = target;
|
||||
}
|
||||
};
|
||||
try {
|
||||
return await Promise.race([run(), cancelled]);
|
||||
}
|
||||
catch (error) {
|
||||
throw error instanceof EgressError ? error : new EgressError("NETWORK_ERROR", "Egress request failed");
|
||||
}
|
||||
finally {
|
||||
clearTimeout(timer);
|
||||
init.signal?.removeEventListener("abort", abort);
|
||||
if (abortListener)
|
||||
controller.signal.removeEventListener("abort", abortListener);
|
||||
}
|
||||
}
|
||||
return { egressFetch, policy, budgets };
|
||||
}
|
||||
/** Stateless convenience form. Reuse createEgressClient when pool/config are fixed. */
|
||||
export function egressFetch(url, init = {}, options) {
|
||||
return createEgressClient(options).egressFetch(url, init, options);
|
||||
}
|
||||
//# sourceMappingURL=client.js.map
|
||||
1
packages/egress/src/client.js.map
Normal file
1
packages/egress/src/client.js.map
Normal file
File diff suppressed because one or more lines are too long
150
packages/egress/src/client.test.ts
Normal file
150
packages/egress/src/client.test.ts
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
import { EventEmitter } from "node:events";
|
||||
import http, { type ClientRequest, type IncomingMessage, type RequestOptions } from "node:http";
|
||||
import https from "node:https";
|
||||
import { PassThrough } from "node:stream";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { createEgressClient } from "./client.js";
|
||||
import { testDatabase } from "./test-db.js";
|
||||
|
||||
interface Fixture { status?: number; headers?: Record<string, string>; body?: string; hang?: boolean }
|
||||
interface Call { url: URL; options: RequestOptions; pin?: string; body?: Buffer }
|
||||
function mockWire(fixtures: Fixture[]) {
|
||||
const calls: Call[] = [];
|
||||
const implementation = ((url: URL, options: RequestOptions, callback: (response: IncomingMessage) => void) => {
|
||||
const fixture = fixtures.shift();
|
||||
if (!fixture) throw new Error("Unexpected connection");
|
||||
const call: Call = { url, options };
|
||||
calls.push(call);
|
||||
const request = new EventEmitter() as ClientRequest;
|
||||
request.destroy = () => { request.emit("close"); return request; };
|
||||
request.end = ((body?: Buffer) => {
|
||||
call.body = body;
|
||||
const lookup = options.lookup as (host: string, options: object, cb: (error: Error | null, address: string) => void) => void;
|
||||
lookup(url.hostname, {}, (_error, address) => { call.pin = address; });
|
||||
if (!fixture.hang) queueMicrotask(() => {
|
||||
const stream = new PassThrough();
|
||||
const incoming = stream as unknown as IncomingMessage;
|
||||
incoming.statusCode = fixture.status ?? 200;
|
||||
incoming.statusMessage = "OK";
|
||||
incoming.rawHeaders = Object.entries(fixture.headers ?? {}).flat();
|
||||
callback(incoming);
|
||||
stream.end(fixture.body ?? "ok");
|
||||
});
|
||||
return request;
|
||||
}) as ClientRequest["end"];
|
||||
return request;
|
||||
}) as typeof http.request;
|
||||
vi.spyOn(http, "request").mockImplementation(implementation);
|
||||
vi.spyOn(https, "request").mockImplementation(implementation as typeof https.request);
|
||||
return calls;
|
||||
}
|
||||
|
||||
describe("validated, pinned, budgeted fetch", () => {
|
||||
const pool = testDatabase();
|
||||
afterEach(() => { vi.restoreAllMocks(); });
|
||||
const publicResolver = async () => [{ address: "8.8.8.8", family: 4 }];
|
||||
it("connects to the pinned answer once, retaining TLS hostname and overriding Host spoofing", async () => {
|
||||
const calls = mockWire([{ body: "pinned" }]);
|
||||
const resolver = vi.fn().mockResolvedValueOnce([{ address: "8.8.8.8", family: 4 }]).mockResolvedValueOnce([{ address: "127.0.0.1", family: 4 }]);
|
||||
const client = createEgressClient({ pool, resolver });
|
||||
const response = await client.egressFetch("https://example.com/asset", { headers: { Host: "internal.local" } }, { hostClass: "monitor" });
|
||||
expect(await response.text()).toBe("pinned");
|
||||
expect(response.url).toBe("https://example.com/asset");
|
||||
expect(resolver).toHaveBeenCalledTimes(1);
|
||||
expect(calls[0]!.pin).toBe("8.8.8.8");
|
||||
expect(calls[0]!.url.hostname).toBe("example.com");
|
||||
expect(calls[0]!.options.agent).toBe(false);
|
||||
expect(calls[0]!.options.headers).toMatchObject({ host: "example.com" });
|
||||
});
|
||||
it.each(["http://127.0.0.1/secret", "https://[::1]/secret", "https://internal.example/secret"])("denies redirect to %s before making the second connection", async (location) => {
|
||||
const calls = mockWire([{ status: 302, headers: { location } }]);
|
||||
const resolver = vi.fn(async (host: string) => [{ address: host === "internal.example" ? "10.0.0.1" : "8.8.8.8", family: 4 }]);
|
||||
const client = createEgressClient({ pool, resolver });
|
||||
await expect(client.egressFetch("https://example.com/start", {}, { hostClass: "monitor" })).rejects.toMatchObject({ code: "DENIED_PRIVATE_ADDRESS" });
|
||||
expect(calls).toHaveLength(1);
|
||||
});
|
||||
it("resolves and pins again at each redirect hop, detecting same-host DNS rebinding", async () => {
|
||||
const calls = mockWire([{ status: 302, headers: { location: "/next" } }]);
|
||||
const resolver = vi.fn().mockResolvedValueOnce([{ address: "8.8.8.8", family: 4 }]).mockResolvedValueOnce([{ address: "127.0.0.1", family: 4 }]);
|
||||
await expect(createEgressClient({ pool, resolver }).egressFetch("https://example.com/start", {}, { hostClass: "monitor" })).rejects.toMatchObject({ code: "DENIED_PRIVATE_ADDRESS" });
|
||||
expect(resolver).toHaveBeenCalledTimes(2);
|
||||
expect(calls).toHaveLength(1);
|
||||
});
|
||||
it.each([302, 307, 308])("pinned-origin rejects an off-origin %s without forwarding auth or body", async (status) => {
|
||||
const calls = mockWire([{ status, headers: { location: "https://sink.example/token" } }]);
|
||||
const resolver = vi.fn(publicResolver);
|
||||
await expect(createEgressClient({ pool, resolver }).egressFetch("https://example.com/token",
|
||||
{ method: "POST", body: "secret", headers: { authorization: "Bearer secret" } }, { hostClass: "monitor", pinnedOrigin: true })).rejects.toMatchObject({ code: "ORIGIN_DENIED" });
|
||||
expect(calls).toHaveLength(1);
|
||||
expect(resolver).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
it("strips sensitive headers on public cross-origin redirects and normalizes POST to GET", async () => {
|
||||
const calls = mockWire([{ status: 302, headers: { location: "//cdn.example/next" } }, { body: "done" }]);
|
||||
const response = await createEgressClient({ pool, resolver: publicResolver }).egressFetch("https://example.com/start",
|
||||
{ method: "POST", body: "form", headers: { authorization: "secret", cookie: "session=secret", "content-type": "text/plain" } }, { hostClass: "monitor" });
|
||||
expect(await response.text()).toBe("done");
|
||||
expect(response.redirected).toBe(true);
|
||||
expect(calls[1]!.options.method).toBe("GET");
|
||||
expect(calls[1]!.body).toBeUndefined();
|
||||
expect(calls[1]!.options.headers).not.toHaveProperty("authorization");
|
||||
expect(calls[1]!.options.headers).not.toHaveProperty("cookie");
|
||||
expect(calls[1]!.options.headers).not.toHaveProperty("content-type");
|
||||
});
|
||||
it("follows same-origin 307 without changing method/body", async () => {
|
||||
const calls = mockWire([{ status: 307, headers: { location: "/next" } }, {}]);
|
||||
await createEgressClient({ pool, resolver: publicResolver }).egressFetch("https://example.com/start", { method: "POST", body: "payload" }, { hostClass: "monitor", pinnedOrigin: true });
|
||||
expect(calls[1]!.options.method).toBe("POST");
|
||||
expect(calls[1]!.body?.toString()).toBe("payload");
|
||||
});
|
||||
it("bounds cycles and redirect depth; manual returns without following", async () => {
|
||||
let calls = mockWire([{ status: 302, headers: { location: "/start" } }]);
|
||||
await expect(createEgressClient({ pool, resolver: publicResolver }).egressFetch("https://example.com/start", {}, { hostClass: "monitor" })).rejects.toMatchObject({ code: "REDIRECT_LOOP" });
|
||||
expect(calls).toHaveLength(1);
|
||||
calls = mockWire([{ status: 302, headers: { location: "/next" } }]);
|
||||
await expect(createEgressClient({ pool, resolver: publicResolver, maxRedirects: 0 }).egressFetch("https://other.example/start", {}, { hostClass: "monitor" })).rejects.toMatchObject({ code: "REDIRECT_LOOP" });
|
||||
calls = mockWire([{ status: 302, headers: { location: "http://127.0.0.1/" } }]);
|
||||
const response = await createEgressClient({ pool, resolver: publicResolver }).egressFetch("https://manual.example/start", { redirect: "manual" }, { hostClass: "monitor" });
|
||||
expect(response.status).toBe(302);
|
||||
expect(calls).toHaveLength(1);
|
||||
});
|
||||
it("parks on 429 before another caller can connect", async () => {
|
||||
const calls = mockWire([{ status: 429, headers: { "retry-after": "10" } }]);
|
||||
const client = createEgressClient({ pool, resolver: publicResolver });
|
||||
expect((await client.egressFetch("https://example.com/start", {}, { hostClass: "monitor" })).status).toBe(429);
|
||||
await expect(client.egressFetch("https://example.com/next", {}, { hostClass: "monitor" })).rejects.toMatchObject({ code: "BUDGET_EXHAUSTED" });
|
||||
expect(calls).toHaveLength(1);
|
||||
});
|
||||
it.each<Record<string, string>>([{}, { "content-length": "100" }])("enforces response caps for streamed and declared lengths: %j", async (headers) => {
|
||||
mockWire([{ headers, body: "12345" }]);
|
||||
await expect(createEgressClient({ pool, resolver: publicResolver, maxResponseBytes: 4 }).egressFetch("https://example.com", {}, { hostClass: "monitor" })).rejects.toMatchObject({ code: "RESPONSE_TOO_LARGE" });
|
||||
});
|
||||
it("caps request bodies before acquiring a token or connecting", async () => {
|
||||
const calls = mockWire([]);
|
||||
await expect(createEgressClient({ pool, resolver: publicResolver, maxRequestBytes: 4 }).egressFetch("https://example.com", { method: "POST", body: "12345" }, { hostClass: "monitor" })).rejects.toMatchObject({ code: "REQUEST_TOO_LARGE" });
|
||||
expect(calls).toHaveLength(0);
|
||||
expect((await pool.query("SELECT * FROM host_budgets")).rows).toHaveLength(0);
|
||||
});
|
||||
it("returns HEAD metadata without applying a body cap to Content-Length", async () => {
|
||||
mockWire([{ headers: { "content-length": "100000", "content-encoding": "gzip" }, body: "" }]);
|
||||
const response = await createEgressClient({ pool, resolver: publicResolver, maxResponseBytes: 4 }).egressFetch("https://example.com", { method: "HEAD" }, { hostClass: "monitor" });
|
||||
expect(response.headers.get("content-length")).toBe("100000");
|
||||
expect(await response.text()).toBe("");
|
||||
});
|
||||
it("rejects whitespace in redirect targets instead of normalizing it away", async () => {
|
||||
const calls = mockWire([{ status: 302, headers: { location: "https://next.example/a b" } }]);
|
||||
await expect(createEgressClient({ pool, resolver: publicResolver }).egressFetch("https://example.com", {}, { hostClass: "monitor" })).rejects.toMatchObject({ code: "INVALID_URL" });
|
||||
expect(calls).toHaveLength(1);
|
||||
});
|
||||
it("times out a stalled DNS resolution and a stalled connection", async () => {
|
||||
const stalled = () => new Promise<never>(() => undefined);
|
||||
await expect(createEgressClient({ pool, resolver: stalled, timeoutMs: 30 }).egressFetch("https://example.com", {}, { hostClass: "monitor" })).rejects.toMatchObject({ code: "TIMEOUT" });
|
||||
mockWire([{ hang: true }]);
|
||||
await expect(createEgressClient({ pool, resolver: publicResolver, timeoutMs: 100 }).egressFetch("https://example.com", {}, { hostClass: "monitor" })).rejects.toMatchObject({ code: "TIMEOUT" });
|
||||
});
|
||||
it("honors an already-aborted signal without DNS, debit, or network", async () => {
|
||||
const controller = new AbortController(); controller.abort();
|
||||
const resolver = vi.fn(publicResolver);
|
||||
await expect(createEgressClient({ pool, resolver }).egressFetch("https://example.com", { signal: controller.signal }, { hostClass: "monitor" })).rejects.toMatchObject({ code: "ABORTED" });
|
||||
expect(resolver).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
129
packages/egress/src/client.ts
Normal file
129
packages/egress/src/client.ts
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
import { HostBudgets, type HostBudgetOptions } from "./budget.js";
|
||||
import { EgressError } from "./errors.js";
|
||||
import { EgressPolicy, type EgressPolicyOptions } from "./policy.js";
|
||||
import { requestPinned } from "./transport.js";
|
||||
|
||||
export interface EgressClientOptions extends HostBudgetOptions, EgressPolicyOptions {
|
||||
timeoutMs?: number;
|
||||
maxResponseBytes?: number;
|
||||
maxRequestBytes?: number;
|
||||
maxRedirects?: number;
|
||||
}
|
||||
export interface EgressRequestOptions {
|
||||
hostClass: string;
|
||||
/** true pins to the initial origin; a URL string explicitly pins to that origin. */
|
||||
pinnedOrigin?: boolean | string | URL;
|
||||
}
|
||||
const redirectStatuses = new Set([301, 302, 303, 307, 308]);
|
||||
|
||||
export function createEgressClient(options: EgressClientOptions) {
|
||||
const policy = new EgressPolicy(options);
|
||||
const budgets = new HostBudgets(options);
|
||||
const timeoutMs = options.timeoutMs ?? 15_000;
|
||||
const maxResponseBytes = options.maxResponseBytes ?? 8 * 1024 * 1024;
|
||||
const maxRequestBytes = options.maxRequestBytes ?? 8 * 1024 * 1024;
|
||||
const maxRedirects = options.maxRedirects ?? 5;
|
||||
if (![timeoutMs, maxResponseBytes, maxRequestBytes].every((value) => Number.isSafeInteger(value) && value > 0) || !Number.isSafeInteger(maxRedirects) || maxRedirects < 0) {
|
||||
throw new EgressError("INVALID_CONFIGURATION", "Invalid request limits");
|
||||
}
|
||||
|
||||
async function egressFetch(raw: string | URL, init: RequestInit = {}, context: EgressRequestOptions): Promise<Response> {
|
||||
const controller = new AbortController();
|
||||
const abort = () => controller.abort(new EgressError("ABORTED", "Request aborted"));
|
||||
init.signal?.addEventListener("abort", abort, { once: true });
|
||||
if (init.signal?.aborted) abort();
|
||||
const timer = setTimeout(() => controller.abort(new EgressError("TIMEOUT", "Egress deadline exceeded")), timeoutMs);
|
||||
const check = () => { if (controller.signal.aborted) throw controller.signal.reason; };
|
||||
let abortListener: (() => void) | undefined;
|
||||
const cancelled = new Promise<never>((_resolve, reject) => {
|
||||
abortListener = () => reject(controller.signal.reason);
|
||||
controller.signal.addEventListener("abort", abortListener, { once: true });
|
||||
if (controller.signal.aborted) abortListener();
|
||||
});
|
||||
const run = async () => {
|
||||
check();
|
||||
let url = policy.validateUrl(raw);
|
||||
const pinnedOrigin = context.pinnedOrigin === true ? url.origin : context.pinnedOrigin ? policy.validateUrl(context.pinnedOrigin).origin : undefined;
|
||||
if (pinnedOrigin && url.origin !== pinnedOrigin) throw new EgressError("ORIGIN_DENIED", "Request is outside the pinned origin");
|
||||
// Request provides fetch method/header/body normalization; the transport owns connection options.
|
||||
let normalized: Request;
|
||||
try { normalized = new Request(url, init); }
|
||||
catch { throw new EgressError("INVALID_REQUEST", "Invalid fetch request initialization"); }
|
||||
let method = normalized.method;
|
||||
const headers = new Headers(normalized.headers);
|
||||
let body: Buffer | undefined;
|
||||
if (normalized.body) {
|
||||
const reader = normalized.body.getReader();
|
||||
const cancelBody = () => { void reader.cancel().catch(() => undefined); };
|
||||
controller.signal.addEventListener("abort", cancelBody, { once: true });
|
||||
const chunks: Buffer[] = [];
|
||||
let size = 0;
|
||||
try {
|
||||
for (;;) {
|
||||
check();
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
size += value.length;
|
||||
if (size > maxRequestBytes) throw new EgressError("REQUEST_TOO_LARGE", "Request exceeds size limit");
|
||||
chunks.push(Buffer.from(value));
|
||||
}
|
||||
body = Buffer.concat(chunks);
|
||||
} finally {
|
||||
controller.signal.removeEventListener("abort", cancelBody);
|
||||
void reader.cancel().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
const visited = new Set<string>();
|
||||
for (let hop = 0; ; hop++) {
|
||||
check();
|
||||
if (visited.has(url.href)) throw new EgressError("REDIRECT_LOOP", "Redirect cycle detected");
|
||||
visited.add(url.href);
|
||||
const endpoint = await policy.validate(url);
|
||||
check();
|
||||
await budgets.consume(endpoint.hostKey, context.hostClass);
|
||||
check();
|
||||
const wire = await requestPinned(endpoint, method, headers, body, controller.signal, maxResponseBytes,
|
||||
(status, responseHeaders) => budgets.observe(endpoint.hostKey, context.hostClass, status, responseHeaders));
|
||||
check();
|
||||
const location = wire.headers.get("location");
|
||||
if (!redirectStatuses.has(wire.status) || !location || init.redirect === "manual") {
|
||||
const response = new Response(method === "HEAD" || [204, 205, 304].includes(wire.status) ? null : new Uint8Array(wire.body),
|
||||
{ status: wire.status, statusText: wire.statusText, headers: wire.headers });
|
||||
Object.defineProperties(response, { url: { value: url.href }, redirected: { value: hop > 0 } });
|
||||
return response;
|
||||
}
|
||||
if (init.redirect === "error" || hop >= maxRedirects) throw new EgressError("REDIRECT_LOOP", "Redirect limit exceeded");
|
||||
if (/[\s\\\u0000-\u001f\u007f]/u.test(location)) throw new EgressError("INVALID_URL", "Malformed redirect target");
|
||||
let target: URL;
|
||||
try { target = policy.validateUrl(new URL(location, url)); }
|
||||
catch (error) { if (error instanceof EgressError) throw error; throw new EgressError("INVALID_URL", "Invalid redirect target"); }
|
||||
if (pinnedOrigin && target.origin !== pinnedOrigin) throw new EgressError("ORIGIN_DENIED", "Redirect is outside the pinned origin");
|
||||
if (target.origin !== url.origin) {
|
||||
for (const name of ["authorization", "proxy-authorization", "cookie", "cookie2"]) headers.delete(name);
|
||||
// A body can contain secrets just as headers can; refuse to replay it off-origin.
|
||||
if (body && (wire.status === 307 || wire.status === 308 || method !== "POST" && wire.status !== 303)) {
|
||||
throw new EgressError("ORIGIN_DENIED", "Request body cannot be redirected off-origin");
|
||||
}
|
||||
}
|
||||
if (wire.status === 303 && method !== "HEAD" || (wire.status === 301 || wire.status === 302) && method === "POST") {
|
||||
method = "GET"; body = undefined;
|
||||
for (const name of ["content-type", "content-length", "content-encoding", "content-language", "content-location"]) headers.delete(name);
|
||||
}
|
||||
url = target;
|
||||
}
|
||||
};
|
||||
try { return await Promise.race([run(), cancelled]); }
|
||||
catch (error) { throw error instanceof EgressError ? error : new EgressError("NETWORK_ERROR", "Egress request failed"); }
|
||||
finally {
|
||||
clearTimeout(timer);
|
||||
init.signal?.removeEventListener("abort", abort);
|
||||
if (abortListener) controller.signal.removeEventListener("abort", abortListener);
|
||||
}
|
||||
}
|
||||
return { egressFetch, policy, budgets };
|
||||
}
|
||||
|
||||
/** Stateless convenience form. Reuse createEgressClient when pool/config are fixed. */
|
||||
export function egressFetch(url: string | URL, init: RequestInit = {}, options: EgressClientOptions & EgressRequestOptions): Promise<Response> {
|
||||
return createEgressClient(options).egressFetch(url, init, options);
|
||||
}
|
||||
7
packages/egress/src/errors.d.ts
vendored
Normal file
7
packages/egress/src/errors.d.ts
vendored
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
export type EgressErrorCode = "INVALID_URL" | "SCHEME_DENIED" | "PORT_DENIED" | "DENIED_PRIVATE_ADDRESS" | "DNS_FAILED" | "BUDGET_EXHAUSTED" | "REDIRECT_LOOP" | "ORIGIN_DENIED" | "TIMEOUT" | "ABORTED" | "RESPONSE_TOO_LARGE" | "REQUEST_TOO_LARGE" | "NETWORK_ERROR" | "INVALID_PLAYLIST" | "INVALID_CONFIGURATION" | "INVALID_REQUEST" | "BUDGET_UNAVAILABLE";
|
||||
/** Messages intentionally omit URLs, credentials and upstream error strings. */
|
||||
export declare class EgressError extends Error {
|
||||
readonly code: EgressErrorCode;
|
||||
readonly retryAfterMs?: number | undefined;
|
||||
constructor(code: EgressErrorCode, message: string, retryAfterMs?: number | undefined);
|
||||
}
|
||||
12
packages/egress/src/errors.js
Normal file
12
packages/egress/src/errors.js
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
/** Messages intentionally omit URLs, credentials and upstream error strings. */
|
||||
export class EgressError extends Error {
|
||||
code;
|
||||
retryAfterMs;
|
||||
constructor(code, message, retryAfterMs) {
|
||||
super(message);
|
||||
this.code = code;
|
||||
this.retryAfterMs = retryAfterMs;
|
||||
this.name = "EgressError";
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=errors.js.map
|
||||
1
packages/egress/src/errors.js.map
Normal file
1
packages/egress/src/errors.js.map
Normal file
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"errors.js","sourceRoot":"","sources":["errors.ts"],"names":[],"mappings":"AAMA,gFAAgF;AAChF,MAAM,OAAO,WAAY,SAAQ,KAAK;IACR;IAAwD;IAApF,YAA4B,IAAqB,EAAE,OAAe,EAAkB,YAAqB;QACvG,KAAK,CAAC,OAAO,CAAC,CAAC;QADW,SAAI,GAAJ,IAAI,CAAiB;QAAmC,iBAAY,GAAZ,YAAY,CAAS;QAEvG,IAAI,CAAC,IAAI,GAAG,aAAa,CAAC;IAC5B,CAAC;CACF"}
|
||||
13
packages/egress/src/errors.ts
Normal file
13
packages/egress/src/errors.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
export type EgressErrorCode =
|
||||
| "INVALID_URL" | "SCHEME_DENIED" | "PORT_DENIED" | "DENIED_PRIVATE_ADDRESS"
|
||||
| "DNS_FAILED" | "BUDGET_EXHAUSTED" | "REDIRECT_LOOP" | "ORIGIN_DENIED"
|
||||
| "TIMEOUT" | "ABORTED" | "RESPONSE_TOO_LARGE" | "REQUEST_TOO_LARGE"
|
||||
| "NETWORK_ERROR" | "INVALID_PLAYLIST" | "INVALID_CONFIGURATION" | "INVALID_REQUEST" | "BUDGET_UNAVAILABLE";
|
||||
|
||||
/** Messages intentionally omit URLs, credentials and upstream error strings. */
|
||||
export class EgressError extends Error {
|
||||
constructor(public readonly code: EgressErrorCode, message: string, public readonly retryAfterMs?: number) {
|
||||
super(message);
|
||||
this.name = "EgressError";
|
||||
}
|
||||
}
|
||||
5
packages/egress/src/hls.d.ts
vendored
Normal file
5
packages/egress/src/hls.d.ts
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
import { EgressPolicy, type ValidatedEndpoint } from "./policy.js";
|
||||
/** Validates master variants and URI-bearing tags (renditions, keys, maps, etc.) with DNS pinning. */
|
||||
export declare function validateMasterPlaylist(playlist: string, base: string | URL, policy?: EgressPolicy): Promise<ValidatedEndpoint[]>;
|
||||
/** Produces absolute, validated child URLs, preserving comments, attributes and newline style. */
|
||||
export declare function rewriteSegmentUris(playlist: string, base: string | URL, policy?: EgressPolicy): Promise<string>;
|
||||
124
packages/egress/src/hls.js
Normal file
124
packages/egress/src/hls.js
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
import { EgressError } from "./errors.js";
|
||||
import { EgressPolicy } from "./policy.js";
|
||||
/** Parse attribute boundaries rather than finding URI= inside other quoted values. */
|
||||
function attributeUris(attributes, offset) {
|
||||
const uris = [];
|
||||
if (!/^\s*[A-Z0-9-]+\s*=/.test(attributes))
|
||||
return uris;
|
||||
let cursor = 0;
|
||||
const seen = new Set();
|
||||
while (cursor < attributes.length) {
|
||||
const key = /^\s*([A-Z0-9-]+)\s*=\s*/.exec(attributes.slice(cursor));
|
||||
if (!key)
|
||||
throw new EgressError("INVALID_PLAYLIST", "Malformed HLS attributes");
|
||||
cursor += key[0].length;
|
||||
const quoted = attributes[cursor] === '"';
|
||||
const start = cursor + (quoted ? 1 : 0);
|
||||
const end = quoted ? attributes.indexOf('"', start) : attributes.indexOf(",", start);
|
||||
const valueEnd = end < 0 && !quoted ? attributes.length : end;
|
||||
if (valueEnd < 0)
|
||||
throw new EgressError("INVALID_PLAYLIST", "Unterminated HLS attribute");
|
||||
if (key[1] === "URI" || key[1] === "SERVER-URI") {
|
||||
if (!quoted || start === valueEnd || seen.has(key[1]))
|
||||
throw new EgressError("INVALID_PLAYLIST", "Invalid HLS URI attribute");
|
||||
seen.add(key[1]);
|
||||
uris.push({ start: offset + start, end: offset + valueEnd, value: attributes.slice(start, valueEnd) });
|
||||
}
|
||||
cursor = valueEnd + (quoted ? 1 : 0);
|
||||
while (/\s/.test(attributes[cursor] ?? "") && cursor < attributes.length)
|
||||
cursor++;
|
||||
if (cursor === attributes.length)
|
||||
break;
|
||||
if (attributes[cursor] !== "," || cursor + 1 === attributes.length)
|
||||
throw new EgressError("INVALID_PLAYLIST", "Invalid HLS attribute suffix");
|
||||
cursor++;
|
||||
}
|
||||
return uris;
|
||||
}
|
||||
function playlistUris(playlist) {
|
||||
if (!/^#EXTM3U(?:\r?\n|$)/.test(playlist) || playlist.includes("\u0000"))
|
||||
throw new EgressError("INVALID_PLAYLIST", "Missing HLS header");
|
||||
const uris = [];
|
||||
let offset = 0;
|
||||
let awaitingVariant = false;
|
||||
let variants = 0;
|
||||
let media = false;
|
||||
for (const line of playlist.split(/(?<=\n)/)) {
|
||||
const text = line.replace(/\r?\n$/, "").trim();
|
||||
if (text && !text.startsWith("#")) {
|
||||
const start = offset + line.indexOf(text);
|
||||
uris.push({ start, end: start + text.length, value: text });
|
||||
if (awaitingVariant) {
|
||||
variants++;
|
||||
awaitingVariant = false;
|
||||
}
|
||||
}
|
||||
else if (text.startsWith("#EXT")) {
|
||||
if (text.startsWith("#EXTINF:"))
|
||||
media = true;
|
||||
if (text.startsWith("#EXT-X-STREAM-INF:")) {
|
||||
if (awaitingVariant)
|
||||
throw new EgressError("INVALID_PLAYLIST", "Variant URI is missing");
|
||||
awaitingVariant = true;
|
||||
}
|
||||
const colon = line.indexOf(":");
|
||||
if (colon >= 0) {
|
||||
const attributes = line.slice(colon + 1).replace(/\r?\n$/, "");
|
||||
const attributesFound = attributeUris(attributes, offset + colon + 1);
|
||||
uris.push(...attributesFound);
|
||||
if (text.startsWith("#EXT-X-I-FRAME-STREAM-INF:")) {
|
||||
if (!attributesFound.length)
|
||||
throw new EgressError("INVALID_PLAYLIST", "I-frame URI is missing");
|
||||
variants++;
|
||||
}
|
||||
}
|
||||
}
|
||||
offset += line.length;
|
||||
}
|
||||
if (awaitingVariant)
|
||||
throw new EgressError("INVALID_PLAYLIST", "Variant URI is missing");
|
||||
return { uris, master: variants > 0 && !media };
|
||||
}
|
||||
async function resolveUris(uris, base, policy) {
|
||||
const baseUrl = (await policy.validate(base)).url;
|
||||
const endpoints = [];
|
||||
// Sequential validation bounds DNS work even for large playlists; shared URI pins within this parse.
|
||||
const cache = new Map();
|
||||
for (const occurrence of uris) {
|
||||
if (!occurrence.value || occurrence.value.includes("{$") || /[\s\\\u0000-\u001f\u007f]/u.test(occurrence.value))
|
||||
throw new EgressError("INVALID_PLAYLIST", "Invalid or unexpanded HLS URI");
|
||||
let absolute;
|
||||
try {
|
||||
absolute = new URL(occurrence.value, baseUrl);
|
||||
}
|
||||
catch {
|
||||
throw new EgressError("INVALID_PLAYLIST", "Invalid HLS URI");
|
||||
}
|
||||
let endpoint = cache.get(absolute.href);
|
||||
if (!endpoint) {
|
||||
endpoint = await policy.validate(absolute);
|
||||
cache.set(absolute.href, endpoint);
|
||||
}
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
return endpoints;
|
||||
}
|
||||
/** Validates master variants and URI-bearing tags (renditions, keys, maps, etc.) with DNS pinning. */
|
||||
export async function validateMasterPlaylist(playlist, base, policy = new EgressPolicy()) {
|
||||
const { uris, master } = playlistUris(playlist);
|
||||
if (!master)
|
||||
throw new EgressError("INVALID_PLAYLIST", "Expected an HLS master playlist");
|
||||
return resolveUris(uris, base, policy);
|
||||
}
|
||||
/** Produces absolute, validated child URLs, preserving comments, attributes and newline style. */
|
||||
export async function rewriteSegmentUris(playlist, base, policy = new EgressPolicy()) {
|
||||
const { uris } = playlistUris(playlist);
|
||||
const endpoints = await resolveUris(uris, base, policy);
|
||||
let rewritten = playlist;
|
||||
for (let index = uris.length - 1; index >= 0; index--) {
|
||||
const uri = uris[index];
|
||||
rewritten = rewritten.slice(0, uri.start) + endpoints[index].url.href + rewritten.slice(uri.end);
|
||||
}
|
||||
return rewritten;
|
||||
}
|
||||
//# sourceMappingURL=hls.js.map
|
||||
1
packages/egress/src/hls.js.map
Normal file
1
packages/egress/src/hls.js.map
Normal file
File diff suppressed because one or more lines are too long
41
packages/egress/src/hls.test.ts
Normal file
41
packages/egress/src/hls.test.ts
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
import { rewriteSegmentUris, validateMasterPlaylist } from "./hls.js";
|
||||
import { EgressPolicy } from "./policy.js";
|
||||
|
||||
const policy = new EgressPolicy({ resolver: async (host) => [{ address: host === "private.example" ? "10.1.2.3" : "8.8.8.8", family: 4 }] });
|
||||
const base = "https://media.example/live/master.m3u8?signature=abc";
|
||||
describe("HLS child policy and URI rewriting", () => {
|
||||
it("rewrites relative, root-relative, absolute and protocol-relative segments with CRLF preserved", async () => {
|
||||
const source = '#EXTM3U\r\n# comment\r\n#EXTINF:10,\r\nsegment.ts?token=1\r\n../other.ts\r\n/root.ts\r\nhttps://cdn.example/a.ts\r\n//cdn.example/b.ts\r\n';
|
||||
expect(await rewriteSegmentUris(source, base, policy)).toBe('#EXTM3U\r\n# comment\r\n#EXTINF:10,\r\nhttps://media.example/live/segment.ts?token=1\r\nhttps://media.example/other.ts\r\nhttps://media.example/root.ts\r\nhttps://cdn.example/a.ts\r\nhttps://cdn.example/b.ts\r\n');
|
||||
});
|
||||
it("rewrites variant, rendition, map, encryption-key, part and iframe URIs including commas", async () => {
|
||||
const source = '#EXTM3U\n#EXT-X-MEDIA:TYPE=AUDIO,URI="audio/list.m3u8",NAME="Audio"\n#EXT-X-STREAM-INF:BANDWIDTH=1234,CODECS="avc1,mp4a"\nvideo/list.m3u8\n#EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=12,URI="//cdn.example/iframe.m3u8"\n#EXT-X-MAP:URI="init.mp4"\n#EXT-X-KEY:METHOD=AES-128,URI="keys/key?k=1,2"\n#EXT-X-PART:DURATION=0.1,URI="part.m4s"\n';
|
||||
const endpoints = await validateMasterPlaylist(source, base, policy);
|
||||
expect(endpoints).toHaveLength(6);
|
||||
expect(endpoints.every((endpoint) => endpoint.pinnedAddress === "8.8.8.8")).toBe(true);
|
||||
const rewritten = await rewriteSegmentUris(source, base, policy);
|
||||
expect(rewritten).toContain('URI="https://media.example/live/keys/key?k=1,2"');
|
||||
expect(rewritten).toContain('CODECS="avc1,mp4a"\nhttps://media.example/live/video/list.m3u8');
|
||||
expect(rewritten).toContain('URI="https://cdn.example/iframe.m3u8"');
|
||||
});
|
||||
it.each(["http://127.0.0.1/segment", "https://[::1]/segment", "//private.example/segment", "file:///tmp/segment", "https://media.example:80/segment"])("rejects unsafe child %s", async (uri) => {
|
||||
await expect(rewriteSegmentUris(`#EXTM3U\n${uri}\n`, base, policy)).rejects.toThrow();
|
||||
await expect(validateMasterPlaylist(`#EXTM3U\n#EXT-X-STREAM-INF:BANDWIDTH=1\n${uri}\n`, base, policy)).rejects.toThrow();
|
||||
await expect(rewriteSegmentUris(`#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI="${uri}"\n`, base, policy)).rejects.toThrow();
|
||||
});
|
||||
it.each(["not HLS", "#EXTM3U\n#EXT-X-STREAM-INF:BANDWIDTH=1\n", '#EXTM3U\n#EXT-X-MAP:URI=unquoted\n', '#EXTM3U\n#EXT-X-MAP:URI="{$var}/init.mp4"\n', '#EXTM3U\n#EXT-X-MAP:URI="a",URI="b"\n'])("rejects malformed or unexpanded playlist %s", async (playlist) => {
|
||||
await expect(rewriteSegmentUris(playlist, base, policy)).rejects.toMatchObject({ code: "INVALID_PLAYLIST" });
|
||||
});
|
||||
it("requires a master and validates the base even when no children exist", async () => {
|
||||
await expect(validateMasterPlaylist("#EXTM3U\n#EXTINF:2,\nseg.ts\n", base, policy)).rejects.toMatchObject({ code: "INVALID_PLAYLIST" });
|
||||
await expect(rewriteSegmentUris("#EXTM3U\n", "https://private.example/master.m3u8", policy)).rejects.toMatchObject({ code: "DENIED_PRIVATE_ADDRESS" });
|
||||
});
|
||||
it("preserves URI-looking text inside quoted names and validates content steering URIs", async () => {
|
||||
const source = '#EXTM3U\n#EXT-X-MEDIA:NAME="name,URI=ordinary text",URI="audio.m3u8"\n#EXT-X-CONTENT-STEERING:SERVER-URI="../steering.json",PATHWAY-ID="main"\n';
|
||||
const rewritten = await rewriteSegmentUris(source, base, policy);
|
||||
expect(rewritten).toContain('NAME="name,URI=ordinary text",URI="https://media.example/live/audio.m3u8"');
|
||||
expect(rewritten).toContain('SERVER-URI="https://media.example/steering.json"');
|
||||
await expect(rewriteSegmentUris('#EXTM3U\n#EXT-X-CONTENT-STEERING:SERVER-URI="//private.example/steer"\n', base, policy)).rejects.toMatchObject({ code: "DENIED_PRIVATE_ADDRESS" });
|
||||
});
|
||||
});
|
||||
104
packages/egress/src/hls.ts
Normal file
104
packages/egress/src/hls.ts
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
import { EgressError } from "./errors.js";
|
||||
import { EgressPolicy, type ValidatedEndpoint } from "./policy.js";
|
||||
|
||||
interface UriOccurrence { start: number; end: number; value: string }
|
||||
|
||||
/** Parse attribute boundaries rather than finding URI= inside other quoted values. */
|
||||
function attributeUris(attributes: string, offset: number): UriOccurrence[] {
|
||||
const uris: UriOccurrence[] = [];
|
||||
if (!/^\s*[A-Z0-9-]+\s*=/.test(attributes)) return uris;
|
||||
let cursor = 0;
|
||||
const seen = new Set<string>();
|
||||
while (cursor < attributes.length) {
|
||||
const key = /^\s*([A-Z0-9-]+)\s*=\s*/.exec(attributes.slice(cursor));
|
||||
if (!key) throw new EgressError("INVALID_PLAYLIST", "Malformed HLS attributes");
|
||||
cursor += key[0].length;
|
||||
const quoted = attributes[cursor] === '"';
|
||||
const start = cursor + (quoted ? 1 : 0);
|
||||
const end = quoted ? attributes.indexOf('"', start) : attributes.indexOf(",", start);
|
||||
const valueEnd = end < 0 && !quoted ? attributes.length : end;
|
||||
if (valueEnd < 0) throw new EgressError("INVALID_PLAYLIST", "Unterminated HLS attribute");
|
||||
if (key[1] === "URI" || key[1] === "SERVER-URI") {
|
||||
if (!quoted || start === valueEnd || seen.has(key[1])) throw new EgressError("INVALID_PLAYLIST", "Invalid HLS URI attribute");
|
||||
seen.add(key[1]);
|
||||
uris.push({ start: offset + start, end: offset + valueEnd, value: attributes.slice(start, valueEnd) });
|
||||
}
|
||||
cursor = valueEnd + (quoted ? 1 : 0);
|
||||
while (/\s/.test(attributes[cursor] ?? "") && cursor < attributes.length) cursor++;
|
||||
if (cursor === attributes.length) break;
|
||||
if (attributes[cursor] !== "," || cursor + 1 === attributes.length) throw new EgressError("INVALID_PLAYLIST", "Invalid HLS attribute suffix");
|
||||
cursor++;
|
||||
}
|
||||
return uris;
|
||||
}
|
||||
|
||||
function playlistUris(playlist: string): { uris: UriOccurrence[]; master: boolean } {
|
||||
if (!/^#EXTM3U(?:\r?\n|$)/.test(playlist) || playlist.includes("\u0000")) throw new EgressError("INVALID_PLAYLIST", "Missing HLS header");
|
||||
const uris: UriOccurrence[] = [];
|
||||
let offset = 0;
|
||||
let awaitingVariant = false;
|
||||
let variants = 0;
|
||||
let media = false;
|
||||
for (const line of playlist.split(/(?<=\n)/)) {
|
||||
const text = line.replace(/\r?\n$/, "").trim();
|
||||
if (text && !text.startsWith("#")) {
|
||||
const start = offset + line.indexOf(text);
|
||||
uris.push({ start, end: start + text.length, value: text });
|
||||
if (awaitingVariant) { variants++; awaitingVariant = false; }
|
||||
} else if (text.startsWith("#EXT")) {
|
||||
if (text.startsWith("#EXTINF:")) media = true;
|
||||
if (text.startsWith("#EXT-X-STREAM-INF:")) {
|
||||
if (awaitingVariant) throw new EgressError("INVALID_PLAYLIST", "Variant URI is missing");
|
||||
awaitingVariant = true;
|
||||
}
|
||||
const colon = line.indexOf(":");
|
||||
if (colon >= 0) {
|
||||
const attributes = line.slice(colon + 1).replace(/\r?\n$/, "");
|
||||
const attributesFound = attributeUris(attributes, offset + colon + 1);
|
||||
uris.push(...attributesFound);
|
||||
if (text.startsWith("#EXT-X-I-FRAME-STREAM-INF:")) {
|
||||
if (!attributesFound.length) throw new EgressError("INVALID_PLAYLIST", "I-frame URI is missing");
|
||||
variants++;
|
||||
}
|
||||
}
|
||||
}
|
||||
offset += line.length;
|
||||
}
|
||||
if (awaitingVariant) throw new EgressError("INVALID_PLAYLIST", "Variant URI is missing");
|
||||
return { uris, master: variants > 0 && !media };
|
||||
}
|
||||
|
||||
async function resolveUris(uris: UriOccurrence[], base: string | URL, policy: EgressPolicy): Promise<ValidatedEndpoint[]> {
|
||||
const baseUrl = (await policy.validate(base)).url;
|
||||
const endpoints: ValidatedEndpoint[] = [];
|
||||
// Sequential validation bounds DNS work even for large playlists; shared URI pins within this parse.
|
||||
const cache = new Map<string, ValidatedEndpoint>();
|
||||
for (const occurrence of uris) {
|
||||
if (!occurrence.value || occurrence.value.includes("{$") || /[\s\\\u0000-\u001f\u007f]/u.test(occurrence.value)) throw new EgressError("INVALID_PLAYLIST", "Invalid or unexpanded HLS URI");
|
||||
let absolute: URL;
|
||||
try { absolute = new URL(occurrence.value, baseUrl); } catch { throw new EgressError("INVALID_PLAYLIST", "Invalid HLS URI"); }
|
||||
let endpoint = cache.get(absolute.href);
|
||||
if (!endpoint) { endpoint = await policy.validate(absolute); cache.set(absolute.href, endpoint); }
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
return endpoints;
|
||||
}
|
||||
|
||||
/** Validates master variants and URI-bearing tags (renditions, keys, maps, etc.) with DNS pinning. */
|
||||
export async function validateMasterPlaylist(playlist: string, base: string | URL, policy = new EgressPolicy()): Promise<ValidatedEndpoint[]> {
|
||||
const { uris, master } = playlistUris(playlist);
|
||||
if (!master) throw new EgressError("INVALID_PLAYLIST", "Expected an HLS master playlist");
|
||||
return resolveUris(uris, base, policy);
|
||||
}
|
||||
|
||||
/** Produces absolute, validated child URLs, preserving comments, attributes and newline style. */
|
||||
export async function rewriteSegmentUris(playlist: string, base: string | URL, policy = new EgressPolicy()): Promise<string> {
|
||||
const { uris } = playlistUris(playlist);
|
||||
const endpoints = await resolveUris(uris, base, policy);
|
||||
let rewritten = playlist;
|
||||
for (let index = uris.length - 1; index >= 0; index--) {
|
||||
const uri = uris[index]!;
|
||||
rewritten = rewritten.slice(0, uri.start) + endpoints[index]!.url.href + rewritten.slice(uri.end);
|
||||
}
|
||||
return rewritten;
|
||||
}
|
||||
5
packages/egress/src/index.d.ts
vendored
Normal file
5
packages/egress/src/index.d.ts
vendored
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
export { EgressError, type EgressErrorCode } from "./errors.js";
|
||||
export { EgressPolicy, assertPublicAddress, validateUrl, hostBudgetKey, type EgressPolicyOptions, type ValidatedEndpoint, type Resolver, type ResolvedAddress } from "./policy.js";
|
||||
export { HostBudgets, DEFAULT_HOST_BUDGET, REFILL_INTERVAL_MS, retryAfterDeadline, type HostBudget, type HostBudgetOptions } from "./budget.js";
|
||||
export { createEgressClient, egressFetch, type EgressClientOptions, type EgressRequestOptions } from "./client.js";
|
||||
export { validateMasterPlaylist, rewriteSegmentUris } from "./hls.js";
|
||||
6
packages/egress/src/index.js
Normal file
6
packages/egress/src/index.js
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
export { EgressError } from "./errors.js";
|
||||
export { EgressPolicy, assertPublicAddress, validateUrl, hostBudgetKey } from "./policy.js";
|
||||
export { HostBudgets, DEFAULT_HOST_BUDGET, REFILL_INTERVAL_MS, retryAfterDeadline } from "./budget.js";
|
||||
export { createEgressClient, egressFetch } from "./client.js";
|
||||
export { validateMasterPlaylist, rewriteSegmentUris } from "./hls.js";
|
||||
//# sourceMappingURL=index.js.map
|
||||
1
packages/egress/src/index.js.map
Normal file
1
packages/egress/src/index.js.map
Normal file
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"index.js","sourceRoot":"","sources":["index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAwB,MAAM,aAAa,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,WAAW,EAAE,aAAa,EAAyF,MAAM,aAAa,CAAC;AACnL,OAAO,EAAE,WAAW,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,kBAAkB,EAA2C,MAAM,aAAa,CAAC;AAChJ,OAAO,EAAE,kBAAkB,EAAE,WAAW,EAAuD,MAAM,aAAa,CAAC;AACnH,OAAO,EAAE,sBAAsB,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC"}
|
||||
5
packages/egress/src/index.ts
Normal file
5
packages/egress/src/index.ts
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
export { EgressError, type EgressErrorCode } from "./errors.js";
|
||||
export { EgressPolicy, assertPublicAddress, validateUrl, hostBudgetKey, type EgressPolicyOptions, type ValidatedEndpoint, type Resolver, type ResolvedAddress } from "./policy.js";
|
||||
export { HostBudgets, DEFAULT_HOST_BUDGET, REFILL_INTERVAL_MS, retryAfterDeadline, type HostBudget, type HostBudgetOptions } from "./budget.js";
|
||||
export { createEgressClient, egressFetch, type EgressClientOptions, type EgressRequestOptions } from "./client.js";
|
||||
export { validateMasterPlaylist, rewriteSegmentUris } from "./hls.js";
|
||||
26
packages/egress/src/policy.d.ts
vendored
Normal file
26
packages/egress/src/policy.d.ts
vendored
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
export interface ResolvedAddress {
|
||||
address: string;
|
||||
family: number;
|
||||
}
|
||||
export type Resolver = (hostname: string) => Promise<readonly ResolvedAddress[]>;
|
||||
export interface EgressPolicyOptions {
|
||||
/** Operator-controlled allowlists; defaults to HTTP 80 and HTTPS 443 only. */
|
||||
ports?: Partial<Record<"http:" | "https:", readonly number[]>>;
|
||||
resolver?: Resolver;
|
||||
}
|
||||
export interface ValidatedEndpoint {
|
||||
url: URL;
|
||||
pinnedAddress: string;
|
||||
family: 4 | 6;
|
||||
hostKey: string;
|
||||
}
|
||||
export declare function assertPublicAddress(address: string): void;
|
||||
export declare function hostBudgetKey(url: URL): string;
|
||||
/** Pure syntactic/literal policy check; no DNS or budget is consumed. */
|
||||
export declare function validateUrl(raw: string | URL, options?: EgressPolicyOptions): URL;
|
||||
export declare class EgressPolicy {
|
||||
private readonly options;
|
||||
constructor(options?: EgressPolicyOptions);
|
||||
validateUrl(raw: string | URL): URL;
|
||||
validate(raw: string | URL): Promise<ValidatedEndpoint>;
|
||||
}
|
||||
93
packages/egress/src/policy.js
Normal file
93
packages/egress/src/policy.js
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
import { lookup } from "node:dns/promises";
|
||||
import { BlockList, isIP } from "node:net";
|
||||
import { domainToASCII } from "node:url";
|
||||
import { EgressError } from "./errors.js";
|
||||
const blocked = new BlockList();
|
||||
for (const [subnet, prefix] of [
|
||||
["0.0.0.0", 8], ["10.0.0.0", 8], ["100.64.0.0", 10], ["127.0.0.0", 8],
|
||||
["169.254.0.0", 16], ["172.16.0.0", 12], ["192.0.0.0", 24], ["192.168.0.0", 16],
|
||||
["192.0.2.0", 24], ["198.18.0.0", 15], ["198.51.100.0", 24], ["203.0.113.0", 24],
|
||||
["224.0.0.0", 4], ["240.0.0.0", 4],
|
||||
])
|
||||
blocked.addSubnet(subnet, prefix, "ipv4");
|
||||
for (const [subnet, prefix] of [
|
||||
["::", 96], ["::1", 128], ["fc00::", 7], ["fe80::", 10], ["fec0::", 10],
|
||||
["ff00::", 8], ["2001::", 32], ["2001:db8::", 32], ["2002::", 16],
|
||||
["64:ff9b::", 96], ["64:ff9b:1::", 48],
|
||||
])
|
||||
blocked.addSubnet(subnet, prefix, "ipv6");
|
||||
// Keep this separate: Node also maps native IPv4 into this range internally.
|
||||
const mapped = new BlockList();
|
||||
mapped.addSubnet("::ffff:0:0", 96, "ipv6");
|
||||
export function assertPublicAddress(address) {
|
||||
const normalized = address.replace(/^\[|\]$/g, "").toLowerCase();
|
||||
const family = isIP(normalized);
|
||||
if (!family)
|
||||
throw new EgressError("DNS_FAILED", "Invalid resolved address");
|
||||
if (normalized.includes("%") || family === 6 && mapped.check(normalized, "ipv6") || blocked.check(normalized, family === 4 ? "ipv4" : "ipv6")) {
|
||||
throw new EgressError("DENIED_PRIVATE_ADDRESS", "Non-public addresses are denied");
|
||||
}
|
||||
}
|
||||
export function hostBudgetKey(url) {
|
||||
const host = url.hostname.replace(/^\[|\]$/g, "").replace(/\.$/, "").toLowerCase();
|
||||
const canonical = isIP(host) ? host : domainToASCII(host);
|
||||
return `${isIP(canonical) === 6 ? `[${canonical}]` : canonical}:${url.port || (url.protocol === "https:" ? "443" : "80")}`;
|
||||
}
|
||||
/** Pure syntactic/literal policy check; no DNS or budget is consumed. */
|
||||
export function validateUrl(raw, options = {}) {
|
||||
const text = String(raw);
|
||||
if (/[\s\\\u0000-\u001f\u007f]/u.test(text))
|
||||
throw new EgressError("INVALID_URL", "Malformed URL");
|
||||
let url;
|
||||
try {
|
||||
url = new URL(text);
|
||||
}
|
||||
catch {
|
||||
throw new EgressError("INVALID_URL", "Malformed URL");
|
||||
}
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:")
|
||||
throw new EgressError("SCHEME_DENIED", "Only HTTP and HTTPS are allowed");
|
||||
if (!url.hostname || url.username || url.password)
|
||||
throw new EgressError("INVALID_URL", "URL credentials are denied");
|
||||
const port = Number(url.port || (url.protocol === "https:" ? 443 : 80));
|
||||
if (!(options.ports?.[url.protocol] ?? (url.protocol === "https:" ? [443] : [80])).includes(port)) {
|
||||
throw new EgressError("PORT_DENIED", "Port is not allowed for this scheme");
|
||||
}
|
||||
const host = url.hostname.replace(/^\[|\]$/g, "").replace(/\.$/, "").toLowerCase();
|
||||
if (host === "localhost" || host.endsWith(".localhost"))
|
||||
throw new EgressError("DENIED_PRIVATE_ADDRESS", "Localhost is denied");
|
||||
if (isIP(host))
|
||||
assertPublicAddress(host);
|
||||
else
|
||||
url.hostname = domainToASCII(host);
|
||||
url.hash = "";
|
||||
return url;
|
||||
}
|
||||
export class EgressPolicy {
|
||||
options;
|
||||
constructor(options = {}) {
|
||||
this.options = options;
|
||||
}
|
||||
validateUrl(raw) { return validateUrl(raw, this.options); }
|
||||
async validate(raw) {
|
||||
const url = this.validateUrl(raw);
|
||||
const host = url.hostname.replace(/^\[|\]$/g, "");
|
||||
let addresses;
|
||||
try {
|
||||
addresses = isIP(host) ? [{ address: host, family: isIP(host) }] : await (this.options.resolver ?? ((name) => lookup(name, { all: true, verbatim: true })))(host);
|
||||
}
|
||||
catch (error) {
|
||||
if (error instanceof EgressError)
|
||||
throw error;
|
||||
throw new EgressError("DNS_FAILED", "Hostname resolution failed");
|
||||
}
|
||||
if (!addresses.length)
|
||||
throw new EgressError("DNS_FAILED", "No resolved addresses");
|
||||
// Mixed public/private answers are denied, rather than relying on answer order.
|
||||
for (const entry of addresses)
|
||||
assertPublicAddress(entry.address);
|
||||
const pinnedAddress = addresses[0].address.replace(/^\[|\]$/g, "").toLowerCase();
|
||||
return { url, pinnedAddress, family: isIP(pinnedAddress), hostKey: hostBudgetKey(url) };
|
||||
}
|
||||
}
|
||||
//# sourceMappingURL=policy.js.map
|
||||
1
packages/egress/src/policy.js.map
Normal file
1
packages/egress/src/policy.js.map
Normal file
File diff suppressed because one or more lines are too long
35
packages/egress/src/policy.test.ts
Normal file
35
packages/egress/src/policy.test.ts
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
import { describe, expect, it, vi } from "vitest";
|
||||
import { assertPublicAddress, EgressPolicy, hostBudgetKey, validateUrl } from "./policy.js";
|
||||
|
||||
describe("unified public-address policy", () => {
|
||||
it.each([
|
||||
"0.0.0.0", "0.1.2.3", "10.0.0.1", "100.64.0.1", "100.127.255.255", "127.0.0.1", "127.255.1.1",
|
||||
"169.254.169.254", "169.254.1.1", "172.16.0.1", "172.31.255.255", "192.0.0.1", "192.168.1.1",
|
||||
"192.0.2.1", "198.18.0.1", "198.19.255.255", "198.51.100.1", "203.0.113.1", "224.0.0.1", "239.1.2.3",
|
||||
"240.0.0.1", "255.255.255.255", "::", "::1", "[::1]", "fc00::1", "fdff::1", "fe80::1", "fec0::1", "ff02::1",
|
||||
"2001::1", "2001:db8::1", "2002:0808:0808::1", "64:ff9b::808:808", "64:ff9b:1::1",
|
||||
"::ffff:127.0.0.1", "::ffff:8.8.8.8", "0:0:0:0:0:ffff:0808:0808", "::8.8.8.8",
|
||||
])("denies %s", (address) => { expect(() => assertPublicAddress(address)).toThrowError(expect.objectContaining({ code: "DENIED_PRIVATE_ADDRESS" })); });
|
||||
it.each(["8.8.8.8", "1.1.1.1", "100.63.255.255", "100.128.0.1", "172.15.0.1", "172.32.0.1", "192.169.0.1", "2606:4700:4700::1111", "2001:4860:4860::8888"])("allows %s", (address) => { expect(() => assertPublicAddress(address)).not.toThrow(); });
|
||||
it.each(["http://2130706433", "http://0177.0.0.1", "http://0x7f000001", "http://127.1", "http://[::ffff:0808:0808]", "http://LOCALHOST.", "http://foo.localhost"])("normalizes then denies %s", (url) => { expect(() => validateUrl(url)).toThrowError(expect.objectContaining({ code: "DENIED_PRIVATE_ADDRESS" })); });
|
||||
it.each(["file:///etc/passwd", "ftp://example.com", "rtmp://example.com", "concat:foo"])("denies scheme %s", (url) => { expect(() => validateUrl(url)).toThrowError(expect.objectContaining({ code: "SCHEME_DENIED" })); });
|
||||
it.each(["https://user:secret@example.com", " https://example.com", "https://example.com\n", "https:\\example.com", "-https://example.com"])("denies malformed input %s", (url) => { expect(() => validateUrl(url)).toThrow(); });
|
||||
it("has per-scheme port allowlists and stable IDNA/effective-port keys", () => {
|
||||
expect(() => validateUrl("https://example.com:80")).toThrowError(expect.objectContaining({ code: "PORT_DENIED" }));
|
||||
expect(() => validateUrl("http://example.com:443")).toThrowError(expect.objectContaining({ code: "PORT_DENIED" }));
|
||||
expect(validateUrl("https://example.com:8443", { ports: { "https:": [443, 8443] } }).port).toBe("8443");
|
||||
expect(hostBudgetKey(validateUrl("https://BÜCHER.Example.:443/a"))).toBe("xn--bcher-kva.example:443");
|
||||
expect(hostBudgetKey(validateUrl("http://[2606:4700:4700::1111]"))).toBe("[2606:4700:4700::1111]:80");
|
||||
});
|
||||
it("pins one validated answer and rejects mixed or invalid DNS results", async () => {
|
||||
const resolver = vi.fn().mockResolvedValueOnce([{ address: "8.8.8.8", family: 4 }]).mockResolvedValueOnce([{ address: "127.0.0.1", family: 4 }]);
|
||||
const policy = new EgressPolicy({ resolver });
|
||||
expect((await policy.validate("https://example.com")).pinnedAddress).toBe("8.8.8.8");
|
||||
expect(resolver).toHaveBeenCalledTimes(1);
|
||||
await expect(policy.validate("https://example.com")).rejects.toMatchObject({ code: "DENIED_PRIVATE_ADDRESS" });
|
||||
await expect(new EgressPolicy({ resolver: async () => [{ address: "8.8.8.8", family: 4 }, { address: "10.0.0.1", family: 4 }] }).validate("https://example.com")).rejects.toMatchObject({ code: "DENIED_PRIVATE_ADDRESS" });
|
||||
for (const answer of [[], [{ address: "bogus", family: 4 }]]) {
|
||||
await expect(new EgressPolicy({ resolver: async () => answer }).validate("https://example.com")).rejects.toMatchObject({ code: "DNS_FAILED" });
|
||||
}
|
||||
});
|
||||
});
|
||||
91
packages/egress/src/policy.ts
Normal file
91
packages/egress/src/policy.ts
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
import { lookup } from "node:dns/promises";
|
||||
import { BlockList, isIP } from "node:net";
|
||||
import { domainToASCII } from "node:url";
|
||||
import { EgressError } from "./errors.js";
|
||||
|
||||
export interface ResolvedAddress { address: string; family: number }
|
||||
export type Resolver = (hostname: string) => Promise<readonly ResolvedAddress[]>;
|
||||
export interface EgressPolicyOptions {
|
||||
/** Operator-controlled allowlists; defaults to HTTP 80 and HTTPS 443 only. */
|
||||
ports?: Partial<Record<"http:" | "https:", readonly number[]>>;
|
||||
resolver?: Resolver;
|
||||
}
|
||||
export interface ValidatedEndpoint {
|
||||
url: URL;
|
||||
pinnedAddress: string;
|
||||
family: 4 | 6;
|
||||
hostKey: string;
|
||||
}
|
||||
|
||||
const blocked = new BlockList();
|
||||
for (const [subnet, prefix] of [
|
||||
["0.0.0.0", 8], ["10.0.0.0", 8], ["100.64.0.0", 10], ["127.0.0.0", 8],
|
||||
["169.254.0.0", 16], ["172.16.0.0", 12], ["192.0.0.0", 24], ["192.168.0.0", 16],
|
||||
["192.0.2.0", 24], ["198.18.0.0", 15], ["198.51.100.0", 24], ["203.0.113.0", 24],
|
||||
["224.0.0.0", 4], ["240.0.0.0", 4],
|
||||
] as const) blocked.addSubnet(subnet, prefix, "ipv4");
|
||||
for (const [subnet, prefix] of [
|
||||
["::", 96], ["::1", 128], ["fc00::", 7], ["fe80::", 10], ["fec0::", 10],
|
||||
["ff00::", 8], ["2001::", 32], ["2001:db8::", 32], ["2002::", 16],
|
||||
["64:ff9b::", 96], ["64:ff9b:1::", 48],
|
||||
] as const) blocked.addSubnet(subnet, prefix, "ipv6");
|
||||
// Keep this separate: Node also maps native IPv4 into this range internally.
|
||||
const mapped = new BlockList();
|
||||
mapped.addSubnet("::ffff:0:0", 96, "ipv6");
|
||||
|
||||
export function assertPublicAddress(address: string): void {
|
||||
const normalized = address.replace(/^\[|\]$/g, "").toLowerCase();
|
||||
const family = isIP(normalized);
|
||||
if (!family) throw new EgressError("DNS_FAILED", "Invalid resolved address");
|
||||
if (normalized.includes("%") || family === 6 && mapped.check(normalized, "ipv6") || blocked.check(normalized, family === 4 ? "ipv4" : "ipv6")) {
|
||||
throw new EgressError("DENIED_PRIVATE_ADDRESS", "Non-public addresses are denied");
|
||||
}
|
||||
}
|
||||
|
||||
export function hostBudgetKey(url: URL): string {
|
||||
const host = url.hostname.replace(/^\[|\]$/g, "").replace(/\.$/, "").toLowerCase();
|
||||
const canonical = isIP(host) ? host : domainToASCII(host);
|
||||
return `${isIP(canonical) === 6 ? `[${canonical}]` : canonical}:${url.port || (url.protocol === "https:" ? "443" : "80")}`;
|
||||
}
|
||||
|
||||
/** Pure syntactic/literal policy check; no DNS or budget is consumed. */
|
||||
export function validateUrl(raw: string | URL, options: EgressPolicyOptions = {}): URL {
|
||||
const text = String(raw);
|
||||
if (/[\s\\\u0000-\u001f\u007f]/u.test(text)) throw new EgressError("INVALID_URL", "Malformed URL");
|
||||
let url: URL;
|
||||
try { url = new URL(text); } catch { throw new EgressError("INVALID_URL", "Malformed URL"); }
|
||||
if (url.protocol !== "http:" && url.protocol !== "https:") throw new EgressError("SCHEME_DENIED", "Only HTTP and HTTPS are allowed");
|
||||
if (!url.hostname || url.username || url.password) throw new EgressError("INVALID_URL", "URL credentials are denied");
|
||||
const port = Number(url.port || (url.protocol === "https:" ? 443 : 80));
|
||||
if (!(options.ports?.[url.protocol] ?? (url.protocol === "https:" ? [443] : [80])).includes(port)) {
|
||||
throw new EgressError("PORT_DENIED", "Port is not allowed for this scheme");
|
||||
}
|
||||
const host = url.hostname.replace(/^\[|\]$/g, "").replace(/\.$/, "").toLowerCase();
|
||||
if (host === "localhost" || host.endsWith(".localhost")) throw new EgressError("DENIED_PRIVATE_ADDRESS", "Localhost is denied");
|
||||
if (isIP(host)) assertPublicAddress(host);
|
||||
else url.hostname = domainToASCII(host);
|
||||
url.hash = "";
|
||||
return url;
|
||||
}
|
||||
|
||||
export class EgressPolicy {
|
||||
constructor(private readonly options: EgressPolicyOptions = {}) {}
|
||||
validateUrl(raw: string | URL): URL { return validateUrl(raw, this.options); }
|
||||
async validate(raw: string | URL): Promise<ValidatedEndpoint> {
|
||||
const url = this.validateUrl(raw);
|
||||
const host = url.hostname.replace(/^\[|\]$/g, "");
|
||||
let addresses: readonly ResolvedAddress[];
|
||||
try {
|
||||
addresses = isIP(host) ? [{ address: host, family: isIP(host) }] : await
|
||||
(this.options.resolver ?? ((name) => lookup(name, { all: true, verbatim: true })))(host);
|
||||
} catch (error) {
|
||||
if (error instanceof EgressError) throw error;
|
||||
throw new EgressError("DNS_FAILED", "Hostname resolution failed");
|
||||
}
|
||||
if (!addresses.length) throw new EgressError("DNS_FAILED", "No resolved addresses");
|
||||
// Mixed public/private answers are denied, rather than relying on answer order.
|
||||
for (const entry of addresses) assertPublicAddress(entry.address);
|
||||
const pinnedAddress = addresses[0]!.address.replace(/^\[|\]$/g, "").toLowerCase();
|
||||
return { url, pinnedAddress, family: isIP(pinnedAddress) as 4 | 6, hostKey: hostBudgetKey(url) };
|
||||
}
|
||||
}
|
||||
28
packages/egress/src/test-db.ts
Normal file
28
packages/egress/src/test-db.ts
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
import { randomUUID } from "node:crypto";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { Pool } from "pg";
|
||||
import { afterAll, beforeAll, beforeEach } from "vitest";
|
||||
|
||||
/** Dedicated schema per suite; never runs application migrations or modifies application rows. */
|
||||
export function testDatabase(): Pool {
|
||||
const connectionString = process.env.EGRESS_TEST_DATABASE_URL ?? "postgres://superposter:superposter_dev@127.0.0.1:5433/superposter";
|
||||
const schema = `egress_test_${randomUUID().replaceAll("-", "")}`;
|
||||
const owner = new Pool({ connectionString });
|
||||
const pool = new Pool({ connectionString, options: `-c search_path=${schema}`, max: 24 });
|
||||
beforeAll(async () => {
|
||||
await owner.query(`CREATE SCHEMA ${schema}`);
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
await client.query(await readFile(new URL("../migrations/001_host_budgets.sql", import.meta.url), "utf8"));
|
||||
await client.query("COMMIT");
|
||||
} finally { client.release(); }
|
||||
});
|
||||
beforeEach(async () => { await pool.query("TRUNCATE host_budgets"); });
|
||||
afterAll(async () => {
|
||||
await pool.end();
|
||||
await owner.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`);
|
||||
await owner.end();
|
||||
});
|
||||
return pool;
|
||||
}
|
||||
9
packages/egress/src/transport.d.ts
vendored
Normal file
9
packages/egress/src/transport.d.ts
vendored
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
import type { ValidatedEndpoint } from "./policy.js";
|
||||
export interface WireResponse {
|
||||
status: number;
|
||||
statusText: string;
|
||||
headers: Headers;
|
||||
body: Buffer;
|
||||
}
|
||||
/** A fresh socket for every hop: no proxy, second DNS lookup, or unvalidated pooled connection. */
|
||||
export declare function requestPinned(endpoint: ValidatedEndpoint, method: string, headers: Headers, body: Buffer | undefined, signal: AbortSignal, maxResponseBytes: number, onHeaders: (status: number, headers: Headers) => Promise<void>): Promise<WireResponse>;
|
||||
86
packages/egress/src/transport.js
Normal file
86
packages/egress/src/transport.js
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
import http from "node:http";
|
||||
import https from "node:https";
|
||||
import { EgressError } from "./errors.js";
|
||||
/** A fresh socket for every hop: no proxy, second DNS lookup, or unvalidated pooled connection. */
|
||||
export function requestPinned(endpoint, method, headers, body, signal, maxResponseBytes, onHeaders) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const requestHeaders = Object.fromEntries(headers.entries());
|
||||
requestHeaders.host = endpoint.url.host;
|
||||
requestHeaders["accept-encoding"] = "identity";
|
||||
delete requestHeaders["content-length"];
|
||||
delete requestHeaders["transfer-encoding"];
|
||||
delete requestHeaders.connection;
|
||||
if (body)
|
||||
requestHeaders["content-length"] = String(body.length);
|
||||
let response;
|
||||
let settled = false;
|
||||
const fail = (error) => {
|
||||
if (settled)
|
||||
return;
|
||||
settled = true;
|
||||
reject(error);
|
||||
response?.destroy();
|
||||
request.destroy();
|
||||
};
|
||||
const request = (endpoint.url.protocol === "https:" ? https : http).request(endpoint.url, {
|
||||
method, headers: requestHeaders, agent: false, family: endpoint.family,
|
||||
// Keep URL hostname for TLS verification/SNI and Host, but return only the vetted IP.
|
||||
lookup: (_hostname, options, callback) => {
|
||||
if (options.all)
|
||||
callback(null, [{ address: endpoint.pinnedAddress, family: endpoint.family }]);
|
||||
else
|
||||
callback(null, endpoint.pinnedAddress, endpoint.family);
|
||||
},
|
||||
}, (incoming) => {
|
||||
response = incoming;
|
||||
const responseHeaders = new Headers();
|
||||
for (let i = 0; i < incoming.rawHeaders.length; i += 2)
|
||||
responseHeaders.append(incoming.rawHeaders[i], incoming.rawHeaders[i + 1]);
|
||||
incoming.on("error", () => fail(new EgressError("NETWORK_ERROR", "Response stream failed")));
|
||||
incoming.on("aborted", () => fail(new EgressError("NETWORK_ERROR", "Response stream aborted")));
|
||||
void (async () => {
|
||||
const status = incoming.statusCode ?? 502;
|
||||
await onHeaders(status, responseHeaders);
|
||||
if (settled)
|
||||
return;
|
||||
const hasBody = method !== "HEAD" && ![204, 205, 304].includes(status);
|
||||
const length = Number(responseHeaders.get("content-length"));
|
||||
if (hasBody && Number.isFinite(length) && length > maxResponseBytes) {
|
||||
fail(new EgressError("RESPONSE_TOO_LARGE", "Response exceeds size limit"));
|
||||
return;
|
||||
}
|
||||
// Identity encoding makes both the size cap and returned byte representation explicit.
|
||||
const encoding = responseHeaders.get("content-encoding");
|
||||
if (hasBody && encoding && encoding.toLowerCase() !== "identity") {
|
||||
fail(new EgressError("NETWORK_ERROR", "Upstream ignored identity encoding"));
|
||||
return;
|
||||
}
|
||||
const chunks = [];
|
||||
let size = 0;
|
||||
incoming.on("data", (chunk) => {
|
||||
size += chunk.length;
|
||||
if (size > maxResponseBytes)
|
||||
fail(new EgressError("RESPONSE_TOO_LARGE", "Response exceeds size limit"));
|
||||
else
|
||||
chunks.push(chunk);
|
||||
});
|
||||
incoming.on("end", () => {
|
||||
if (settled)
|
||||
return;
|
||||
settled = true;
|
||||
resolve({ status, statusText: incoming.statusMessage ?? "", headers: responseHeaders, body: Buffer.concat(chunks) });
|
||||
});
|
||||
})().catch((error) => fail(error instanceof EgressError ? error : new EgressError("NETWORK_ERROR", "Response processing failed")));
|
||||
});
|
||||
const aborted = () => fail(signal.reason instanceof EgressError ? signal.reason : new EgressError("ABORTED", "Request aborted"));
|
||||
signal.addEventListener("abort", aborted, { once: true });
|
||||
request.on("close", () => signal.removeEventListener("abort", aborted));
|
||||
request.on("error", () => fail(new EgressError("NETWORK_ERROR", "Connection failed")));
|
||||
if (signal.aborted) {
|
||||
aborted();
|
||||
return;
|
||||
}
|
||||
request.end(body);
|
||||
});
|
||||
}
|
||||
//# sourceMappingURL=transport.js.map
|
||||
1
packages/egress/src/transport.js.map
Normal file
1
packages/egress/src/transport.js.map
Normal file
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"transport.js","sourceRoot":"","sources":["transport.ts"],"names":[],"mappings":"AAAA,OAAO,IAA8B,MAAM,WAAW,CAAC;AACvD,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAK1C,mGAAmG;AACnG,MAAM,UAAU,aAAa,CAAC,QAA2B,EAAE,MAAc,EAAE,OAAgB,EAAE,IAAwB,EACnH,MAAmB,EAAE,gBAAwB,EAC7C,SAA8D;IAC9D,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,cAAc,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7D,cAAc,CAAC,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;QACxC,cAAc,CAAC,iBAAiB,CAAC,GAAG,UAAU,CAAC;QAC/C,OAAO,cAAc,CAAC,gBAAgB,CAAC,CAAC;QACxC,OAAO,cAAc,CAAC,mBAAmB,CAAC,CAAC;QAC3C,OAAO,cAAc,CAAC,UAAU,CAAC;QACjC,IAAI,IAAI;YAAE,cAAc,CAAC,gBAAgB,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACjE,IAAI,QAAqC,CAAC;QAC1C,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,MAAM,IAAI,GAAG,CAAC,KAAkB,EAAE,EAAE;YAClC,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,CAAC;YACd,QAAQ,EAAE,OAAO,EAAE,CAAC;YACpB,OAAO,CAAC,OAAO,EAAE,CAAC;QACpB,CAAC,CAAC;QACF,MAAM,OAAO,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,EAAE;YACxF,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM;YACtE,sFAAsF;YACtF,MAAM,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;gBACvC,IAAI,OAAO,CAAC,GAAG;oBAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,aAAa,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;;oBAC3F,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,aAAa,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC/D,CAAC;SACF,EAAE,CAAC,QAAQ,EAAE,EAAE;YACd,QAAQ,GAAG,QAAQ,CAAC;YACpB,MAAM,eAAe,GAAG,IAAI,OAAO,EAAE,CAAC;YACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC;gBAAE,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAE,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,CAAC;YACrI,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,WAAW,CAAC,eAAe,EAAE,wBAAwB,CAAC,CAAC,CAAC,CAAC;YAC7F,QAAQ,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,WAAW,CAAC,eAAe,EAAE,yBAAyB,CAAC,CAAC,CAAC,CAAC;YAChG,KAAK,CAAC,KAAK,IAAI,EAAE;gBACf,MAAM,MAAM,GAAG,QAAQ,CAAC,UAAU,IAAI,GAAG,CAAC;gBAC1C,MAAM,SAAS,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;gBACzC,IAAI,OAAO;oBAAE,OAAO;gBACpB,MAAM,OAAO,GAAG,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACvE,MAAM,MAAM,GAAG,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC;gBAC7D,IAAI,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,gBAAgB,EAAE,CAAC;oBACpE,IAAI,CAAC,IAAI,WAAW,CAAC,oBAAoB,EAAE,6BAA6B,CAAC,CAAC,CAAC;oBAAC,OAAO;gBACrF,CAAC;gBACD,uFAAuF;gBACvF,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;gBACzD,IAAI,OAAO,IAAI,QAAQ,IAAI,QAAQ,CAAC,WAAW,EAAE,KAAK,UAAU,EAAE,CAAC;oBACjE,IAAI,CAAC,IAAI,WAAW,CAAC,eAAe,EAAE,oCAAoC,CAAC,CAAC,CAAC;oBAAC,OAAO;gBACvF,CAAC;gBACD,MAAM,MAAM,GAAa,EAAE,CAAC;gBAC5B,IAAI,IAAI,GAAG,CAAC,CAAC;gBACb,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;oBACpC,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;oBACrB,IAAI,IAAI,GAAG,gBAAgB;wBAAE,IAAI,CAAC,IAAI,WAAW,CAAC,oBAAoB,EAAE,6BAA6B,CAAC,CAAC,CAAC;;wBACnG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC1B,CAAC,CAAC,CAAC;gBACH,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;oBACtB,IAAI,OAAO;wBAAE,OAAO;oBACpB,OAAO,GAAG,IAAI,CAAC;oBACf,OAAO,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,aAAa,IAAI,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBACvH,CAAC,CAAC,CAAC;YACL,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,YAAY,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,WAAW,CAAC,eAAe,EAAE,4BAA4B,CAAC,CAAC,CAAC,CAAC;QAC9I,CAAC,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,YAAY,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,WAAW,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC,CAAC;QACjI,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QACxE,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,WAAW,CAAC,eAAe,EAAE,mBAAmB,CAAC,CAAC,CAAC,CAAC;QACvF,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAAC,OAAO,EAAE,CAAC;YAAC,OAAO;QAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC,CAAC,CAAC;AACL,CAAC"}
|
||||
77
packages/egress/src/transport.ts
Normal file
77
packages/egress/src/transport.ts
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
import http, { type IncomingMessage } from "node:http";
|
||||
import https from "node:https";
|
||||
import { EgressError } from "./errors.js";
|
||||
import type { ValidatedEndpoint } from "./policy.js";
|
||||
|
||||
export interface WireResponse { status: number; statusText: string; headers: Headers; body: Buffer }
|
||||
|
||||
/** A fresh socket for every hop: no proxy, second DNS lookup, or unvalidated pooled connection. */
|
||||
export function requestPinned(endpoint: ValidatedEndpoint, method: string, headers: Headers, body: Buffer | undefined,
|
||||
signal: AbortSignal, maxResponseBytes: number,
|
||||
onHeaders: (status: number, headers: Headers) => Promise<void>): Promise<WireResponse> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const requestHeaders = Object.fromEntries(headers.entries());
|
||||
requestHeaders.host = endpoint.url.host;
|
||||
requestHeaders["accept-encoding"] = "identity";
|
||||
delete requestHeaders["content-length"];
|
||||
delete requestHeaders["transfer-encoding"];
|
||||
delete requestHeaders.connection;
|
||||
if (body) requestHeaders["content-length"] = String(body.length);
|
||||
let response: IncomingMessage | undefined;
|
||||
let settled = false;
|
||||
const fail = (error: EgressError) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
reject(error);
|
||||
response?.destroy();
|
||||
request.destroy();
|
||||
};
|
||||
const request = (endpoint.url.protocol === "https:" ? https : http).request(endpoint.url, {
|
||||
method, headers: requestHeaders, agent: false, family: endpoint.family,
|
||||
// Keep URL hostname for TLS verification/SNI and Host, but return only the vetted IP.
|
||||
lookup: (_hostname, options, callback) => {
|
||||
if (options.all) callback(null, [{ address: endpoint.pinnedAddress, family: endpoint.family }]);
|
||||
else callback(null, endpoint.pinnedAddress, endpoint.family);
|
||||
},
|
||||
}, (incoming) => {
|
||||
response = incoming;
|
||||
const responseHeaders = new Headers();
|
||||
for (let i = 0; i < incoming.rawHeaders.length; i += 2) responseHeaders.append(incoming.rawHeaders[i]!, incoming.rawHeaders[i + 1]!);
|
||||
incoming.on("error", () => fail(new EgressError("NETWORK_ERROR", "Response stream failed")));
|
||||
incoming.on("aborted", () => fail(new EgressError("NETWORK_ERROR", "Response stream aborted")));
|
||||
void (async () => {
|
||||
const status = incoming.statusCode ?? 502;
|
||||
await onHeaders(status, responseHeaders);
|
||||
if (settled) return;
|
||||
const hasBody = method !== "HEAD" && ![204, 205, 304].includes(status);
|
||||
const length = Number(responseHeaders.get("content-length"));
|
||||
if (hasBody && Number.isFinite(length) && length > maxResponseBytes) {
|
||||
fail(new EgressError("RESPONSE_TOO_LARGE", "Response exceeds size limit")); return;
|
||||
}
|
||||
// Identity encoding makes both the size cap and returned byte representation explicit.
|
||||
const encoding = responseHeaders.get("content-encoding");
|
||||
if (hasBody && encoding && encoding.toLowerCase() !== "identity") {
|
||||
fail(new EgressError("NETWORK_ERROR", "Upstream ignored identity encoding")); return;
|
||||
}
|
||||
const chunks: Buffer[] = [];
|
||||
let size = 0;
|
||||
incoming.on("data", (chunk: Buffer) => {
|
||||
size += chunk.length;
|
||||
if (size > maxResponseBytes) fail(new EgressError("RESPONSE_TOO_LARGE", "Response exceeds size limit"));
|
||||
else chunks.push(chunk);
|
||||
});
|
||||
incoming.on("end", () => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
resolve({ status, statusText: incoming.statusMessage ?? "", headers: responseHeaders, body: Buffer.concat(chunks) });
|
||||
});
|
||||
})().catch((error: unknown) => fail(error instanceof EgressError ? error : new EgressError("NETWORK_ERROR", "Response processing failed")));
|
||||
});
|
||||
const aborted = () => fail(signal.reason instanceof EgressError ? signal.reason : new EgressError("ABORTED", "Request aborted"));
|
||||
signal.addEventListener("abort", aborted, { once: true });
|
||||
request.on("close", () => signal.removeEventListener("abort", aborted));
|
||||
request.on("error", () => fail(new EgressError("NETWORK_ERROR", "Connection failed")));
|
||||
if (signal.aborted) { aborted(); return; }
|
||||
request.end(body);
|
||||
});
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue