fix: produce worker claim — SECURITY DEFINER fn fixes 08P01 poll crash + stale-lease reclaim (040)

The store's hand-built claim interpolated the lane predicate into the SQL
text while still passing a bind parameter — node-postgres sent 1 param for
a 0-param statement (08P01) on every poll, so the live produce worker
store-errored every ~2s and never claimed. Migration 040 replaces the
inline claim with produce_job_claim(p_lane, p_stale_after): SECURITY
DEFINER (the 035/oidc_refresh_lookup pattern — the poller is lane-global,
FORCE RLS hides every row without a tenant context), atomic FOR UPDATE
SKIP LOCKED with the queued→transcoding bump as the lease, and stale-
transcoding reclaim so a killed worker's rows return to the queue.
produce_job_workspace(p_job_id) discovers the owning tenant for the
follow-up writes, which stay RLS-bound as sp_app. pg-store rewired; store
errors now surface err.message instead of err.name; pg-store tests added.
38/38 producer tests green.
This commit is contained in:
glass beadian 2026-10-14 21:55:41 +00:00
parent d9e8cdf789
commit ee8d5c6e76
5 changed files with 512 additions and 91 deletions

View file

@ -193,4 +193,59 @@ test("scratch is removed after a job, success or failure", async () => {
assert.equal(leftovers.stdout.trim(), "");
});
test("a store error is reported with the driver's real message, not just its name", async () => {
const d = doubles();
// node-postgres errors carry name:"error" and the cause in the message — the
// production symptom was detail:"error" with the cause swallowed.
const boom = new Error("bind message supplies 1 parameters, but prepared statement \"\" requires 0");
boom.name = "error";
const broken: ProduceJobStore = { ...d.store, claim: async () => { throw boom; } };
// pollOnce is the direct path (the loop swallows into the same event), so the
// rejection itself proves the error escaped the store call.
await assert.rejects(() => runner(d, { jobs: broken }).pollOnce(), /bind message supplies/);
const looping = runner(d, { jobs: broken, pollIntervalMs: 5, scheduler: { setTimeout: (_ms, action) => { const t = setTimeout(action, 0); return { cancel: () => clearTimeout(t) }; } } });
looping.start();
await new Promise((resolve) => setTimeout(resolve, 30));
await looping.stop();
const event = d.events.find((e) => e.type === "store-error");
assert.ok(event !== undefined);
assert.match(event.detail ?? "", /bind message supplies 1 parameters/);
assert.notEqual(event.detail, "error");
});
test("a store error while failing a job is also reported with its real message", async () => {
const d = doubles();
d.setQueue([job("sha256/" + "0".repeat(64) + ".mp4", "compose", {})]); // the input object does not exist
const boom = new Error("connection terminated unexpectedly");
boom.name = "error";
const failingStore: ProduceJobStore = { ...d.store, fail: async () => { throw boom; } };
const r = runner(d, { jobs: failingStore });
await r.pollOnce(); // run() swallows the fail() error into an event
const event = d.events.find((e) => e.type === "store-error");
assert.ok(event !== undefined);
assert.match(event.detail ?? "", /connection terminated/);
});
test("the poll loop survives a store error and idles on the next tick", async () => {
const d = doubles();
let failures = 0;
const flaky: ProduceJobStore = {
...d.store,
claim: async (lane) => { if (failures++ < 2) throw new Error("connection refused"); return d.store.claim(lane); },
};
const r = new ProduceRunner({
jobs: flaky, storage: d.storage, fontFile: FONT, scratchRoot: SCRATCH, lane: "remote",
redactor: { redact: (s) => s }, onStderr: () => undefined, onEvent: (e) => d.events.push(e),
pollIntervalMs: 5, scheduler: { setTimeout: (_ms, action) => { const t = setTimeout(action, 0); return { cancel: () => clearTimeout(t) }; } },
});
r.start();
await new Promise((resolve) => setTimeout(resolve, 30));
await r.stop();
assert.equal(d.laneOf(), "remote");
const storeErrors = d.events.filter((e) => e.type === "store-error");
assert.ok(storeErrors.length >= 2);
for (const event of storeErrors) assert.match(event.detail ?? "", /connection refused/);
assert.ok(d.events.some((e) => e.type === "idle"));
});
test.after(() => { rmSync(SCRATCH, { recursive: true, force: true }); });