From 5808e89dded43f9640ac73b8d597195b65307e5e Mon Sep 17 00:00:00 2001 From: glass beadian Date: Thu, 15 Oct 2026 21:27:09 +0000 Subject: [PATCH] fix: synth silence bed for audio-less produce sources Audio-less inputs no longer emit [0:a] maps that ffmpeg rejects; the worker branches to an anullsrc bed configured at the output preset's rate/layout. Producer tests 42/42. --- .../src/producer/command-builder.test.ts | 25 +++++++++++++ .../src/producer/command-builder.ts | 29 +++++++++++++-- .../src/producer/pg-store.test.ts | 27 ++++++++++++++ apps/media-worker/src/producer/pg-store.ts | 13 +++++-- apps/media-worker/src/producer/runner.test.ts | 37 +++++++++++++++++++ apps/media-worker/src/producer/runner.ts | 7 +++- 6 files changed, 130 insertions(+), 8 deletions(-) diff --git a/apps/media-worker/src/producer/command-builder.test.ts b/apps/media-worker/src/producer/command-builder.test.ts index a091018..c4b25bb 100644 --- a/apps/media-worker/src/producer/command-builder.test.ts +++ b/apps/media-worker/src/producer/command-builder.test.ts @@ -169,6 +169,31 @@ test("a source without audio mixes the voiceover over a synthetic silence bed", assert.ok(g.includes("duration=longest")); }); +test("an audio-less source never references [0:a]: the bed is a valid source-filter chain", () => { + // No voiceover: the synthetic bed alone is the output's audio. + const bare = graph({ input: { path: `${SCRATCH}/${HASH}.mp4`, durationSec: 2, audioBed: "none" } }); + const bareChains = bare.split(";"); + // anullsrc is a source filter, so aformat follows a COMMA — the missing comma + // produced `anullsrc=...aformat=...`, which ffmpeg rejects with "Option + // 'sample_rates' not found". + assert.equal(bareChains.at(-1), `anullsrc=r=${48000}:cl=stereo,aformat=sample_fmts=fltp:sample_rates=48000:channel_layouts=stereo[aout]`); + assert.ok(!bare.includes("[0:a]"), "an audio-less source must never be referenced as [0:a]"); + + // Ducking over silence: bedRaw gets the same comma-separated form. + const duck = graph({ input: { path: `${SCRATCH}/${HASH}.mp4`, durationSec: 2, audioBed: "none" }, + voiceover: { path: `${SCRATCH}/${VO_HASH}.m4a`, gainDb: -6, duck: true } }); + const duckChains = duck.split(";"); + const bedRaw = duckChains.find((c) => c.endsWith("[bedRaw]")); + assert.ok(bedRaw !== undefined, "ducking path must still produce [bedRaw]"); + assert.ok(bedRaw.startsWith(`anullsrc=r=${48000}:cl=stereo,aformat=`), "synthetic bed + aformat must be comma-separated"); + assert.ok(duck.includes("duration=longest"), "a synthetic bed mixes to longest, bounded by -t"); + assert.ok(!duck.includes("[0:a]")); + + // An audio-bearing source keeps the input-pad form (no comma after [0:a]). + const withAudio = graph({ input: { path: `${SCRATCH}/${HASH}.mp4`, durationSec: 2, audioBed: "source" } }); + assert.ok(withAudio.includes("[0:a]aformat=sample_fmts=fltp"), "the source bed stays an input pad chain"); +}); + test("a synthetic bed must be bounded by the source duration", () => { assert.throws(() => graph({ input: { path: `${SCRATCH}/${HASH}.mp4`, durationSec: null, audioBed: "none" }, diff --git a/apps/media-worker/src/producer/command-builder.ts b/apps/media-worker/src/producer/command-builder.ts index 81f3a64..60d8682 100644 --- a/apps/media-worker/src/producer/command-builder.ts +++ b/apps/media-worker/src/producer/command-builder.ts @@ -265,12 +265,33 @@ export interface ProducePlan { /** ffmpeg input indexes: source 0, then each logo, then the voiceover — argv order. */ export interface ProduceInputIndex { logos: number[]; voiceover: number | null } +/** `aformat` as a filter step (a real input's chain: comma-separated, after a comma). */ function audioFormat(): string { return `aformat=sample_fmts=fltp:sample_rates=${PRODUCE_PRESET.sampleRate}:channel_layouts=stereo`; } +/** + * The synthetic silence bed for an audio-less source. anullsrc is a source + * filter, so it cannot be followed by `aformat` — it is configured directly with + * the output preset's rate and stereo layout, and the encoder's `-ar`/channel + * handling downstream already guarantees the format. + */ +function silenceBed(): string { + return `anullsrc=r=${PRODUCE_PRESET.sampleRate}:cl=stereo`; +} + function gain(gainDb: number): string { return `volume=${gainDb}dB`; } +/** + * One bed chain: `[0:a]` is an input pad, so `aformat` follows it directly, but + * the synthetic bed is a source filter, so it needs the `,` separator before + * `aformat` (the missing comma is what made the audio-less path emit + * `anullsrc=...aformat=...`, which ffmpeg rejects outright). + */ +function bedChain(bed: string, label: string): string { + return `${bed}${bed.startsWith("[") ? "" : ","}${audioFormat()}[${label}]`; +} + /** * The `-filter_complex` string: video chain (scale → overlays in request order) and * the audio mix. `min(px,W-tw)`-style expressions are single-quoted because `min` @@ -312,10 +333,10 @@ export function buildProduceFilter(plan: ProducePlan, index: ProduceInputIndex): // on the bed. `duration=first` ends with the source bed; a synthetic bed is // unbounded, so it mixes to `longest` and only the source's `-t` can end the encode. const vo = plan.voiceover; - const bed = plan.input.audioBed === "source" ? "[0:a]" : `anullsrc=r=${PRODUCE_PRESET.sampleRate}:cl=stereo`; + const bed = plan.input.audioBed === "source" ? "[0:a]" : silenceBed(); if (vo === null) { - chains.push(`${bed}${audioFormat()}[aout]`); + chains.push(bedChain(bed, "aout")); return chains.join(";"); } const voIndex = index.voiceover ?? fail("voiceover input index required"); @@ -324,13 +345,13 @@ export function buildProduceFilter(plan: ProducePlan, index: ProduceInputIndex): if (vo.duck) { // asplit feeds the same voiceover to the compressor key and to the mix. chains.push(`${voIn}${audioFormat()},asplit=2[voMix][voKey]`); - chains.push(`${bed}${audioFormat()}[bedRaw]`); + chains.push(bedChain(bed, "bedRaw")); chains.push(`[bedRaw][voKey]sidechaincompress=threshold=0.03:ratio=8:attack=20:release=300:makeup=1:link=average[duck]`); if (vo.gainDb !== null && vo.gainDb !== 0) chains.push(`[voMix]${gain(vo.gainDb)}[voGain]`); const mixIn = vo.gainDb !== null && vo.gainDb !== 0 ? "voGain" : "voMix"; chains.push(`[duck][${mixIn}]amix=inputs=2:duration=${amixDuration}:dropout_transition=2:normalize=0[aout]`); } else { - chains.push(`${bed}${audioFormat()}[bedFmt]`); + chains.push(bedChain(bed, "bedFmt")); chains.push(`${voIn}${audioFormat()}${vo.gainDb === null || vo.gainDb === 0 ? "" : `,${gain(vo.gainDb)}`}[voGain]`); chains.push(`[bedFmt][voGain]amix=inputs=2:duration=${amixDuration}:dropout_transition=2:normalize=0[aout]`); } diff --git a/apps/media-worker/src/producer/pg-store.test.ts b/apps/media-worker/src/producer/pg-store.test.ts index c598c21..65ad864 100644 --- a/apps/media-worker/src/producer/pg-store.test.ts +++ b/apps/media-worker/src/producer/pg-store.test.ts @@ -140,6 +140,33 @@ test("claim goes through the definer function with typed lane + lease params", a assert.equal(job!.request.overlays[0]!.kind, "lower_third"); }); +test("a probed audio-less asset reports hasAudio:false so the runner uses a silent bed", async () => { + // The probe writes duration_ms and audio_codec together: audio_codec NULL with + // a duration means probed-and-silent (muted screen recording, GIF conversion). + // Before the fix this mapped to hasAudio:null → audioBed "source" → ffmpeg + // "Stream specifier ':a' matches no streams". + const pool = new FakePool(); + pool.expectClient({ rows: [{ ...assetRow(), audio_codec: null }] }); + pool.reply([claimRow()]); + + const job = await storeOf(pool).claim("local"); + + assert.ok(job !== null); + assert.equal(job.source.hasAudio, false, "probed + audio_codec NULL must be hasAudio:false"); + // The runner turns hasAudio===false into audioBed "none" (synthetic silence). +}); + +test("an unprobed asset (NULL duration AND NULL codec) stays hasAudio:null", async () => { + const pool = new FakePool(); + pool.expectClient({ rows: [{ ...assetRow(), audio_codec: null, duration_ms: null }] }); + pool.reply([claimRow()]); + + const job = await storeOf(pool).claim("local"); + + assert.ok(job !== null); + assert.equal(job.source.hasAudio, null, "unprobed is unknown, not audio-less — the source bed stays"); +}); + test("claim passes the lane through: a remote worker binds lane='remote'", async () => { const { pool } = healthyClaim(); await storeOf(pool).claim("remote"); diff --git a/apps/media-worker/src/producer/pg-store.ts b/apps/media-worker/src/producer/pg-store.ts index 7358ab3..ab414b2 100644 --- a/apps/media-worker/src/producer/pg-store.ts +++ b/apps/media-worker/src/producer/pg-store.ts @@ -47,6 +47,11 @@ function uuid(value: unknown, field: string): string { return text; } +/** True once 009's probe job has run: the probe writes duration + codecs together, so duration≠NULL means probed. */ +function probed(row: Record): boolean { + return row.duration_ms !== null && row.duration_ms !== undefined; +} + function nullableNumber(value: unknown): number | null { if (value === null || value === undefined) return null; const n = typeof value === "number" ? value : Number(value); @@ -63,9 +68,11 @@ function sourceFromRow(row: Record): ProduceSourceAsset { height: nullableNumber(row.height), durationMs: row.duration_ms === null || row.duration_ms === undefined ? null : Math.round(Number(row.duration_ms)), frameRate: nullableNumber(row.frame_rate), - // Probed "no audio track" (009 stores an empty codec) → a silent bed, so the - // output always has an audio stream. - hasAudio: row.audio_codec === null || row.audio_codec === undefined ? null : String(row.audio_codec) !== "", + // An unprobed asset has NULL codec AND NULL duration (009's probe fills all + // of them at once); a probed one always has a duration. So the probed + // audio-less case is duration≠NULL + audio_codec NULL/'' — which must become + // a silent bed, or the graph references [0:a] on a file with no audio track. + hasAudio: probed(row) ? String(row.audio_codec ?? "") !== "" : null, }; } diff --git a/apps/media-worker/src/producer/runner.test.ts b/apps/media-worker/src/producer/runner.test.ts index 112f585..ffbda83 100644 --- a/apps/media-worker/src/producer/runner.test.ts +++ b/apps/media-worker/src/producer/runner.test.ts @@ -43,6 +43,17 @@ function makeClip(path: string): void { function key(bytes: Buffer): string { return `sha256/${createHash("sha256").update(bytes).digest("hex")}.mp4`; } +/** + * The audio-less class from the bug report: video only, no lavfi sine, no audio + * track at all (muted screen recording / GIF conversion / synthetic test clip). + */ +function makeSilentClip(path: string): void { + const r = spawnSync("ffmpeg", ["-hide_banner", "-loglevel", "error", "-y", + "-f", "lavfi", "-i", "testsrc2=s=320x180:r=30:d=2", + "-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", "-movflags", "+faststart", path]); + if (r.status !== 0) throw new Error(`silent fixture encode failed: ${r.stderr?.toString() ?? ""}`); +} + function doubles() { const events: Array<{ type: string; jobId?: string; detail?: string }> = []; const jobs: Array<{ jobId: string; state: string; reason?: string; output?: { storageKey: string; producedAssetId: string } }> = []; @@ -132,6 +143,32 @@ test("compose job: encode, publish, derived asset, complete", { skip: !hasFfmpeg assert.ok(body !== undefined && body.subarray(4, 8).toString("latin1") === "ftyp"); }); +test("an audio-less source produces output WITH audio (synthetic bed), not a [0:a] crash", { skip: !hasFfmpeg() || !hasFont() ? "needs ffmpeg + a ttf font" : false }, async () => { + const d = doubles(); + const clip = join(SCRATCH, "source-silent.mp4"); + makeSilentClip(clip); + const bytes = await import("node:fs/promises").then((fs) => fs.readFile(clip)); + const inputKey = key(bytes); + d.objects.set(inputKey, bytes); + // hasAudio:false is what the pg store reports for a probed audio-less asset. + d.setQueue([job(inputKey, "compose", + { overlays: [{ kind: "lower_third", text: "Ada Lovelace", position: { x: 8, y: 85 } }], lane: "local" }, + { hasAudio: false, durationMs: 2000 })]); + + await runner(d).pollOnce(); + + assert.equal(d.jobs[0]?.state, "complete", d.jobs[0]?.reason); + assert.ok(d.jobs[0]?.output?.storageKey.startsWith("sha256/")); + // The produced mp4 must actually carry an audio stream — the whole point of the bed. + const body = d.objects.get(d.published()!.key); + assert.ok(body !== undefined && body.subarray(4, 8).toString("latin1") === "ftyp"); + const probe = spawnSync("ffprobe", ["-v", "error", "-print_format", "json", "-show_streams", "-i", `pipe:0`], + { input: body, encoding: "buffer" }); + const streams = JSON.parse(probe.stdout.toString("utf8")).streams as Array<{ codec_type: string }>; + assert.ok(streams.some((s) => s.codec_type === "audio"), "output must have an audio stream"); + assert.ok(streams.some((s) => s.codec_type === "video"), "output must keep its video stream"); +}); + test("a mix job with a voiceover runs the ducking path end to end", { skip: !hasFfmpeg() || !hasFont() ? "needs ffmpeg + a ttf font" : false }, async () => { const d = doubles(); const clip = join(SCRATCH, "source-mix.mp4"); diff --git a/apps/media-worker/src/producer/runner.ts b/apps/media-worker/src/producer/runner.ts index 4302f9e..2f0c4ba 100644 --- a/apps/media-worker/src/producer/runner.ts +++ b/apps/media-worker/src/producer/runner.ts @@ -171,7 +171,12 @@ export class ProduceRunner { }; } const durationSec = job.source.durationMs === null ? null : job.source.durationMs / 1000; - // A probed asset without an audio track gets a silent bed, so the output always has audio. + // An audio-less source (muted screen recording, GIF conversion, synthetic + // test clip) must not feed [0:a] into the graph — ffmpeg dies with "Stream + // specifier ':a' matches no streams". hasAudio===false means probed and + // audio-less; null means the probe never ran, where the source bed stays + // (the [0:a] reference then fails loudly rather than silently dropping audio + // from a file that may well have a track). const audioBed = job.source.hasAudio === false ? "none" : "source"; return { input: { path: source, durationSec, audioBed },