fix: produce worker claim — SECURITY DEFINER fn fixes 08P01 poll crash + stale-lease reclaim (040)
The store's hand-built claim interpolated the lane predicate into the SQL text while still passing a bind parameter — node-postgres sent 1 param for a 0-param statement (08P01) on every poll, so the live produce worker store-errored every ~2s and never claimed. Migration 040 replaces the inline claim with produce_job_claim(p_lane, p_stale_after): SECURITY DEFINER (the 035/oidc_refresh_lookup pattern — the poller is lane-global, FORCE RLS hides every row without a tenant context), atomic FOR UPDATE SKIP LOCKED with the queued→transcoding bump as the lease, and stale- transcoding reclaim so a killed worker's rows return to the queue. produce_job_workspace(p_job_id) discovers the owning tenant for the follow-up writes, which stay RLS-bound as sp_app. pg-store rewired; store errors now surface err.message instead of err.name; pg-store tests added. 38/38 producer tests green.
This commit is contained in:
parent
d9e8cdf789
commit
ee8d5c6e76
5 changed files with 512 additions and 91 deletions
|
|
@ -193,4 +193,59 @@ test("scratch is removed after a job, success or failure", async () => {
|
|||
assert.equal(leftovers.stdout.trim(), "");
|
||||
});
|
||||
|
||||
test("a store error is reported with the driver's real message, not just its name", async () => {
|
||||
const d = doubles();
|
||||
// node-postgres errors carry name:"error" and the cause in the message — the
|
||||
// production symptom was detail:"error" with the cause swallowed.
|
||||
const boom = new Error("bind message supplies 1 parameters, but prepared statement \"\" requires 0");
|
||||
boom.name = "error";
|
||||
const broken: ProduceJobStore = { ...d.store, claim: async () => { throw boom; } };
|
||||
// pollOnce is the direct path (the loop swallows into the same event), so the
|
||||
// rejection itself proves the error escaped the store call.
|
||||
await assert.rejects(() => runner(d, { jobs: broken }).pollOnce(), /bind message supplies/);
|
||||
const looping = runner(d, { jobs: broken, pollIntervalMs: 5, scheduler: { setTimeout: (_ms, action) => { const t = setTimeout(action, 0); return { cancel: () => clearTimeout(t) }; } } });
|
||||
looping.start();
|
||||
await new Promise((resolve) => setTimeout(resolve, 30));
|
||||
await looping.stop();
|
||||
const event = d.events.find((e) => e.type === "store-error");
|
||||
assert.ok(event !== undefined);
|
||||
assert.match(event.detail ?? "", /bind message supplies 1 parameters/);
|
||||
assert.notEqual(event.detail, "error");
|
||||
});
|
||||
|
||||
test("a store error while failing a job is also reported with its real message", async () => {
|
||||
const d = doubles();
|
||||
d.setQueue([job("sha256/" + "0".repeat(64) + ".mp4", "compose", {})]); // the input object does not exist
|
||||
const boom = new Error("connection terminated unexpectedly");
|
||||
boom.name = "error";
|
||||
const failingStore: ProduceJobStore = { ...d.store, fail: async () => { throw boom; } };
|
||||
const r = runner(d, { jobs: failingStore });
|
||||
await r.pollOnce(); // run() swallows the fail() error into an event
|
||||
const event = d.events.find((e) => e.type === "store-error");
|
||||
assert.ok(event !== undefined);
|
||||
assert.match(event.detail ?? "", /connection terminated/);
|
||||
});
|
||||
|
||||
test("the poll loop survives a store error and idles on the next tick", async () => {
|
||||
const d = doubles();
|
||||
let failures = 0;
|
||||
const flaky: ProduceJobStore = {
|
||||
...d.store,
|
||||
claim: async (lane) => { if (failures++ < 2) throw new Error("connection refused"); return d.store.claim(lane); },
|
||||
};
|
||||
const r = new ProduceRunner({
|
||||
jobs: flaky, storage: d.storage, fontFile: FONT, scratchRoot: SCRATCH, lane: "remote",
|
||||
redactor: { redact: (s) => s }, onStderr: () => undefined, onEvent: (e) => d.events.push(e),
|
||||
pollIntervalMs: 5, scheduler: { setTimeout: (_ms, action) => { const t = setTimeout(action, 0); return { cancel: () => clearTimeout(t) }; } },
|
||||
});
|
||||
r.start();
|
||||
await new Promise((resolve) => setTimeout(resolve, 30));
|
||||
await r.stop();
|
||||
assert.equal(d.laneOf(), "remote");
|
||||
const storeErrors = d.events.filter((e) => e.type === "store-error");
|
||||
assert.ok(storeErrors.length >= 2);
|
||||
for (const event of storeErrors) assert.match(event.detail ?? "", /connection refused/);
|
||||
assert.ok(d.events.some((e) => e.type === "idle"));
|
||||
});
|
||||
|
||||
test.after(() => { rmSync(SCRATCH, { recursive: true, force: true }); });
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue