Merge pull request 'Feature: non-realtime exporting' (#1674) from BANanaD3V/strudel:feat/non-realtime-exporting into main

Reviewed-on: https://codeberg.org/uzu/strudel/pulls/1674

packages/codemirror/codemirror.mjs

@@ -293,9 +293,9 @@ export class StrudelMirror {
       console.warn('first frame could not be painted');
     }
   }
-  async evaluate() {
+  async evaluate(autostart = true) {
     this.flash();
-    await this.repl.evaluate(this.code);
+    await this.repl.evaluate(this.code, autostart);
   }
   async stop() {
     this.repl.scheduler.stop();

packages/superdough/audioContext.mjs

@@ -5,6 +5,11 @@ export const setDefaultAudioContext = () => {
   return audioContext;
 };
 
+export const setAudioContext = (context) => {
+  audioContext = context;
+  return audioContext;
+};
+
 export const getAudioContext = () => {
   if (!audioContext) {
     return setDefaultAudioContext();

packages/superdough/feedbackdelay.mjs

@@ -25,7 +25,7 @@ if (typeof DelayNode !== 'undefined') {
     }
   }
 
-  AudioContext.prototype.createFeedbackDelay = function (wet, time, feedback) {
+  BaseAudioContext.prototype.createFeedbackDelay = function (wet, time, feedback) {
     return new FeedbackDelayNode(this, wet, time, feedback);
   };
 }

packages/superdough/reverb.mjs

@@ -2,7 +2,7 @@ import reverbGen from './reverbGen.mjs';
 import { clamp } from './util.mjs';
 
 if (typeof AudioContext !== 'undefined') {
-  AudioContext.prototype.adjustLength = function (duration, buffer, speed = 1, offsetAmount = 0) {
+  BaseAudioContext.prototype.adjustLength = function (duration, buffer, speed = 1, offsetAmount = 0) {
     const sampleOffset = Math.floor(clamp(offsetAmount, 0, 1) * buffer.length);
     const newLength = buffer.sampleRate * duration;
     const newBuffer = this.createBuffer(buffer.numberOfChannels, buffer.length, buffer.sampleRate);
@@ -23,7 +23,7 @@ if (typeof AudioContext !== 'undefined') {
     return newBuffer;
   };
 
-  AudioContext.prototype.createReverb = function (duration, fade, lp, dim, ir, irspeed, irbegin) {
+  BaseAudioContext.prototype.createReverb = function (duration, fade, lp, dim, ir, irspeed, irbegin) {
     const convolver = this.createConvolver();
     convolver.generate = (d = 2, fade = 0.1, lp = 15000, dim = 1000, ir, irspeed, irbegin) => {
       convolver.duration = d;

packages/superdough/superdough.mjs

@@ -22,13 +22,14 @@ import {
 import { map } from 'nanostores';
 import { logger } from './logger.mjs';
 import { loadBuffer } from './sampler.mjs';
-import { getAudioContext } from './audioContext.mjs';
+import { getAudioContext, setAudioContext } from './audioContext.mjs';
 import { SuperdoughAudioController } from './superdoughoutput.mjs';
+import { resetSeenKeys } from './wavetable.mjs';
 
 export const DEFAULT_MAX_POLYPHONY = 128;
 const DEFAULT_AUDIO_DEVICE_NAME = 'System Standard';
 
-let maxPolyphony = DEFAULT_MAX_POLYPHONY;
+export let maxPolyphony = DEFAULT_MAX_POLYPHONY;
 
 /**
  * Set the max polyphony. If notes are ringing out via `release` then they will
@@ -45,7 +46,7 @@ export function setMaxPolyphony(polyphony) {
   maxPolyphony = parseInt(polyphony) ?? DEFAULT_MAX_POLYPHONY;
 }
 
-let multiChannelOrbits = false;
+export let multiChannelOrbits = false;
 export function setMultiChannelOrbits(bool) {
   multiChannelOrbits = bool == true;
 }
@@ -234,11 +235,13 @@ export function registerWorklet(url) {
 }
 
 let workletsLoading;
-function loadWorklets() {
+export function loadWorklets() {
   if (!workletsLoading) {
     const audioCtx = getAudioContext();
     const allWorkletURLs = externalWorklets.concat([workletsUrl]);
-    workletsLoading = Promise.all(allWorkletURLs.map((workletURL) => audioCtx.audioWorklet.addModule(workletURL)));
+    workletsLoading = Promise.all(allWorkletURLs.map((workletURL) => audioCtx.audioWorklet.addModule(workletURL))).then(
+      () => (workletsLoading = undefined),
+    );
   }
 
   return workletsLoading;
@@ -255,6 +258,7 @@ export async function initAudio(options = {}) {
 
   setMaxPolyphony(maxPolyphony);
   setMultiChannelOrbits(multiChannelOrbits);
+  resetSeenKeys();
   if (typeof window === 'undefined') {
     return;
   }
@@ -276,8 +280,9 @@ export async function initAudio(options = {}) {
       logger('[superdough] failed to set audio interface', 'warning');
     }
   }
-
+  if ((!audioCtx) instanceof OfflineAudioContext) {
-  await audioCtx.resume();
+    await audioCtx.resume();
+  }
   if (disableWorklets) {
     logger('[superdough]: AudioWorklets disabled with disableWorklets');
     return;
@@ -311,6 +316,12 @@ export function getSuperdoughAudioController() {
   }
   return controller;
 }
+
+export function setSuperdoughAudioController(newController) {
+  controller = newController;
+  return controller;
+}
+
 export function connectToDestination(input, channels) {
   const controller = getSuperdoughAudioController();
   controller.output.connectToDestination(input, channels);
@@ -348,7 +359,7 @@ export let analysers = {},
   analysersData = {};
 
 export function getAnalyserById(id, fftSize = 1024, smoothingTimeConstant = 0.5) {
-  if (!analysers[id]) {
+  if (!analysers[id] || analysers[id].audioContext != getAudioContext()) {
     // make sure this doesn't happen too often as it piles up garbage
     const analyserNode = getAudioContext().createAnalyser();
     analyserNode.fftSize = fftSize;
@@ -410,7 +421,6 @@ export const superdough = async (value, t, hapDuration, cps = 0.5, cycle = 0.5)
   // duration is passed as value too..
   value.duration = hapDuration;
   // calculate absolute time
-
   if (t < ac.currentTime) {
     console.warn(
       `[superdough]: cannot schedule sounds in the past (target: ${t.toFixed(2)}, now: ${ac.currentTime.toFixed(2)})`,
@@ -782,7 +792,7 @@ export const superdough = async (value, t, hapDuration, cps = 0.5, cycle = 0.5)
   }
 
   // analyser
-  if (analyze) {
+  if (analyze && !(ac instanceof OfflineAudioContext)) {
     const analyserNode = getAnalyserById(analyze, 2 ** (fft + 5));
     const analyserSend = effectSend(post, analyserNode, 1);
     audioNodes.push(analyserSend);

packages/superdough/vowel.mjs

@@ -75,7 +75,7 @@ if (typeof GainNode !== 'undefined') {
     }
   }
 
-  AudioContext.prototype.createVowelFilter = function (letter) {
+  BaseAudioContext.prototype.createVowelFilter = function (letter) {
     return new VowelNode(this, letter);
   };
 }

packages/superdough/wavetable.mjs

@@ -40,6 +40,11 @@ export const Warpmode = Object.freeze({
 });
 
 const seenKeys = new Set();
+
+export function resetSeenKeys() {
+  seenKeys.clear();
+}
+
 async function getPayload(url, label, frameLen = 2048) {
   const key = `${url},${frameLen}`;
   if (!seenKeys.has(key)) {

packages/webaudio/webaudio.mjs

@@ -5,10 +5,21 @@ This program is free software: you can redistribute it and/or modify it under th
 */
 
 import * as strudel from '@strudel/core';
-import { superdough, getAudioContext, setLogger, doughTrigger, registerWorklet } from 'superdough';
+import {
+  superdough,
+  getAudioContext,
+  setLogger,
+  doughTrigger,
+  registerWorklet,
+  setAudioContext,
+  initAudio,
+  setSuperdoughAudioController,
+  resetGlobalEffects,
+  errorLogger,
+} from 'superdough';
 import './supradough.mjs';
 import { workletUrl } from 'supradough';
-
+import { SuperdoughAudioController } from 'superdough/superdoughoutput.mjs';
 registerWorklet(workletUrl);
 
 const { Pattern, logger, repl } = strudel;
@@ -26,6 +37,71 @@ export const webaudioOutput = (hap, _deadline, hapDuration, cps, t) => {
   return superdough(hap2value(hap), t, hapDuration, cps, hap.whole?.begin.valueOf());
 };
 
+export async function renderPatternAudio(
+  pattern,
+  cps,
+  begin,
+  end,
+  sampleRate,
+  maxPolyphony,
+  multiChannelOrbits,
+  downloadName = undefined,
+) {
+  let audioContext = getAudioContext();
+  await audioContext.close();
+  audioContext = new OfflineAudioContext(2, ((end - begin) / cps) * sampleRate, sampleRate);
+  setAudioContext(audioContext);
+  setSuperdoughAudioController(new SuperdoughAudioController(audioContext));
+  await initAudio({
+    maxPolyphony,
+    multiChannelOrbits,
+  });
+  logger('[webaudio] preloading');
+
+  // Calling superdough(...) in ascending onset time order is important
+  // for controls that depend on the audio graph state like `cut`
+  let haps = pattern
+    .queryArc(begin, end, { _cps: cps })
+    .sort((a, b) => a.whole.begin.valueOf() - b.whole.begin.valueOf());
+  for (const hap of haps) {
+    if (hap.hasOnset()) {
+      try {
+        await superdough(
+          hap2value(hap),
+          (hap.whole.begin.valueOf() - begin) / cps,
+          hap.duration / cps,
+          cps,
+          (hap.whole?.begin.valueOf() - begin) / cps,
+        );
+      } catch (err) {
+        errorLogger(err, 'webaudio');
+      }
+    }
+  }
+  logger('[webaudio] start rendering');
+
+  return audioContext
+    .startRendering()
+    .then((renderedBuffer) => {
+      const wavBuffer = audioBufferToWav(renderedBuffer);
+      const blob = new Blob([wavBuffer], { type: 'audio/wav' });
+      const url = URL.createObjectURL(blob);
+      const a = document.createElement('a');
+      a.href = url;
+      downloadName = downloadName ? `${downloadName}.wav` : `${new Date().toISOString()}.wav`;
+      a.download = `${downloadName}`;
+      document.body.appendChild(a);
+      a.click();
+      document.body.removeChild(a);
+      URL.revokeObjectURL(url);
+    })
+    .finally(async () => {
+      setAudioContext(null);
+      setSuperdoughAudioController(null);
+      resetGlobalEffects();
+    });
+}
+
 export function webaudioRepl(options = {}) {
   options = {
     getTime: () => getAudioContext().currentTime,
@@ -38,3 +114,98 @@ export function webaudioRepl(options = {}) {
 Pattern.prototype.dough = function () {
   return this.onTrigger(doughTrigger, 1);
 };
+
+function audioBufferToWav(buffer, opt) {
+  opt = opt || {};
+
+  var numChannels = buffer.numberOfChannels;
+  var sampleRate = buffer.sampleRate;
+  var format = opt.float32 ? 3 : 1;
+  var bitDepth = format === 3 ? 32 : 16;
+
+  var result;
+  if (numChannels === 2) {
+    result = interleave(buffer.getChannelData(0), buffer.getChannelData(1));
+  } else {
+    result = buffer.getChannelData(0);
+  }
+
+  return encodeWAV(result, format, sampleRate, numChannels, bitDepth);
+}
+
+function encodeWAV(samples, format, sampleRate, numChannels, bitDepth) {
+  var bytesPerSample = bitDepth / 8;
+  var blockAlign = numChannels * bytesPerSample;
+
+  var buffer = new ArrayBuffer(44 + samples.length * bytesPerSample);
+  var view = new DataView(buffer);
+
+  /* RIFF identifier */
+  writeString(view, 0, 'RIFF');
+  /* RIFF chunk length */
+  view.setUint32(4, 36 + samples.length * bytesPerSample, true);
+  /* RIFF type */
+  writeString(view, 8, 'WAVE');
+  /* format chunk identifier */
+  writeString(view, 12, 'fmt ');
+  /* format chunk length */
+  view.setUint32(16, 16, true);
+  /* sample format (raw) */
+  view.setUint16(20, format, true);
+  /* channel count */
+  view.setUint16(22, numChannels, true);
+  /* sample rate */
+  view.setUint32(24, sampleRate, true);
+  /* byte rate (sample rate * block align) */
+  view.setUint32(28, sampleRate * blockAlign, true);
+  /* block align (channel count * bytes per sample) */
+  view.setUint16(32, blockAlign, true);
+  /* bits per sample */
+  view.setUint16(34, bitDepth, true);
+  /* data chunk identifier */
+  writeString(view, 36, 'data');
+  /* data chunk length */
+  view.setUint32(40, samples.length * bytesPerSample, true);
+  if (format === 1) {
+    // Raw PCM
+    floatTo16BitPCM(view, 44, samples);
+  } else {
+    writeFloat32(view, 44, samples);
+  }
+
+  return buffer;
+}
+
+function interleave(inputL, inputR) {
+  var length = inputL.length + inputR.length;
+  var result = new Float32Array(length);
+
+  var index = 0;
+  var inputIndex = 0;
+
+  while (index < length) {
+    result[index++] = inputL[inputIndex];
+    result[index++] = inputR[inputIndex];
+    inputIndex++;
+  }
+  return result;
+}
+
+function writeFloat32(output, offset, input) {
+  for (var i = 0; i < input.length; i++, offset += 4) {
+    output.setFloat32(offset, input[i], true);
+  }
+}
+
+function floatTo16BitPCM(output, offset, input) {
+  for (var i = 0; i < input.length; i++, offset += 2) {
+    var s = Math.max(-1, Math.min(1, input[i]));
+    output.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
+  }
+}
+
+function writeString(view, offset, string) {
+  for (var i = 0; i < string.length; i++) {
+    view.setUint8(offset + i, string.charCodeAt(i));
+  }
+}

website/src/repl/components/panel/ExportTab.jsx

@@ -0,0 +1,197 @@
+import PlayCircleIcon from '@heroicons/react/20/solid/PlayCircleIcon';
+import cx from '@src/cx.mjs';
+import NumberInput from '@src/repl/components/NumberInput';
+import { useEffect, useState } from 'react';
+import { Textbox } from '../textbox/Textbox';
+import { getAudioContext } from '@strudel/webaudio';
+import XMarkIcon from '@heroicons/react/24/outline/XMarkIcon';
+
+function Checkbox({ label, value, onChange, disabled = false }) {
+  return (
+    <label className={cx(disabled && 'opacity-50')}>
+      <input disabled={disabled} type="checkbox" checked={value} onChange={onChange} />
+      {' ' + label}
+    </label>
+  );
+}
+
+function FormItem({ label, children, disabled }) {
+  return (
+    <div className="grid gap-2 w-full">
+      <label className={cx(disabled && 'opacity-50')}>{label}</label>
+      {children}
+    </div>
+  );
+}
+
+export default function ExportTab(Props) {
+  const { handleExport } = Props;
+
+  const [downloadName, setDownloadName] = useState('');
+  const [startCycle, setStartCycle] = useState(0);
+  const [endCycle, setEndCycle] = useState(1);
+  const [sampleRate, setSampleRate] = useState(48000);
+  const [multiChannelOrbits, setMultiChannelOrbits] = useState(true);
+  const [maxPolyphony, setMaxPolyphony] = useState(1024);
+  const [exporting, setExporting] = useState(false);
+  const [progress, setProgress] = useState(0);
+  const [length, setLength] = useState(1);
+
+  const refreshProgress = () => {
+    const audioContext = getAudioContext();
+    if (audioContext instanceof OfflineAudioContext) {
+      setProgress(audioContext.currentTime);
+      setLength(audioContext.length / sampleRate);
+      setTimeout(refreshProgress, 100);
+    }
+  };
+
+  return (
+    <>
+      <div className="text-foreground w-full p-4 space-y-4">
+        <FormItem label="File name" disabled={exporting}>
+          <Textbox
+            onBlur={(e) => {
+              setDownloadName(e.target.value);
+            }}
+            onChange={(v) => {
+              setDownloadName(v);
+            }}
+            disabled={exporting}
+            placeholder="Leave empty to use current date"
+            className={cx('placeholder:opacity-50', exporting && 'opacity-50 border-opacity-50')}
+            value={downloadName ?? ''}
+          />
+        </FormItem>
+        <div className="flex flex-row gap-4 w-full">
+          <FormItem label="Start cycle" disabled={exporting}>
+            <Textbox
+              min={1}
+              max={Infinity}
+              onBlur={(e) => {
+                let v = parseInt(e.target.value);
+                v = isNaN(v) ? 0 : Math.max(0, v);
+                setStartCycle(v);
+              }}
+              onChange={(v) => {
+                v = parseInt(v);
+                setStartCycle(v);
+              }}
+              type="number"
+              placeholder=""
+              disabled={exporting}
+              className={cx(exporting && 'opacity-50 border-opacity-50', 'w-full')}
+              value={startCycle ?? ''}
+            />
+          </FormItem>
+          <FormItem label="End cycle" disabled={exporting}>
+            <Textbox
+              min={1}
+              max={Infinity}
+              onBlur={(e) => {
+                let v = parseInt(e.target.value);
+                v = isNaN(v) ? Math.max(startCycle + 1, parseInt(v)) : v;
+                setEndCycle(v);
+              }}
+              onChange={(v) => {
+                v = parseInt(v);
+                setEndCycle(v);
+              }}
+              type="number"
+              placeholder=""
+              disabled={exporting}
+              className={cx(exporting && 'opacity-50 border-opacity-50', 'w-full')}
+              value={endCycle ?? ''}
+            />
+          </FormItem>
+        </div>
+        <div className="flex flex-row gap-4">
+          <FormItem label="Sample rate" disabled={exporting}>
+            <Textbox
+              min={1}
+              max={Infinity}
+              onBlur={(e) => {
+                let v = parseInt(e.target.value);
+                v = isNaN(v) ? 1 : Math.max(1, v);
+                setSampleRate(v);
+              }}
+              onChange={(v) => {
+                v = parseInt(v);
+                setSampleRate(v);
+              }}
+              type="number"
+              placeholder=""
+              disabled={exporting}
+              className={cx(exporting && 'opacity-50 border-opacity-50')}
+              value={sampleRate ?? ''}
+            />
+          </FormItem>
+          <FormItem label="Maximum polyphony" disabled={exporting}>
+            <Textbox
+              min={1}
+              max={Infinity}
+              onBlur={(e) => {
+                let v = parseInt(e.target.value);
+                v = isNaN(v) ? Math.max(1, parseInt(v)) : v;
+                setMaxPolyphony(v);
+              }}
+              onChange={(v) => {
+                v = Math.max(1, parseInt(v));
+                setMaxPolyphony(v);
+              }}
+              type="number"
+              placeholder=""
+              disabled={exporting}
+              className={cx(exporting && 'opacity-50 border-opacity-50')}
+              value={maxPolyphony ?? ''}
+            />
+          </FormItem>
+        </div>
+        <div>
+          <Checkbox
+            label="Multi Channel Orbits"
+            onChange={(cbEvent) => {
+              const val = cbEvent.target.checked;
+              setMultiChannelOrbits(val);
+            }}
+            disabled={exporting}
+            value={multiChannelOrbits}
+          />
+        </div>
+        <button
+          className={cx('bg-background p-2 w-full rounded-md hover:opacity-75 relative', exporting && 'opacity-50')}
+          disabled={exporting}
+          onClick={async () => {
+            setExporting(true);
+            setTimeout(refreshProgress, 2000);
+            const modal = document.getElementById('exportProgressModal');
+            modal.showModal();
+            await handleExport(startCycle, endCycle, sampleRate, maxPolyphony, multiChannelOrbits, downloadName)
+              .then(() => {
+                const modal = document.getElementById('exportProgressModal');
+                modal.close();
+              })
+              .finally(() => {
+                setExporting(false);
+                setProgress(0);
+                setLength(1);
+              });
+          }}
+        >
+          <div
+            className="absolute top-0 left-0 right-0 bottom-0 backdrop-invert"
+            style={{
+              width: `${(exporting ? 1 : 0) + (progress / length) * 99}%`,
+            }}
+          />
+          <span className="text-foreground">{exporting ? 'Exporting...' : 'Export to WAV'}</span>
+        </button>
+      </div>
+      <dialog
+        closedby={exporting ? 'none' : 'closerequest'}
+        id="exportProgressModal"
+        className="text-md bg-background text-foreground rounded-lg backdrop:bg-background backdrop:opacity-25"
+      />
+    </>
+  );
+}

website/src/repl/components/panel/Panel.jsx

@@ -9,6 +9,7 @@ import { useLogger } from '../useLogger';
 import { WelcomeTab } from './WelcomeTab';
 import { PatternsTab } from './PatternsTab';
 import { ChevronLeftIcon, XMarkIcon } from '@heroicons/react/16/solid';
+import ExportTab from './ExportTab';
 
 const TAURI = typeof window !== 'undefined' && window.__TAURI__;
 
@@ -80,6 +81,7 @@ const tabNames = {
   patterns: 'patterns',
   sounds: 'sounds',
   reference: 'reference',
+  export: 'export',
   console: 'console',
   settings: 'settings',
 };
@@ -126,6 +128,8 @@ function PanelContent({ context, tab }) {
       return <SoundsTab />;
     case tabNames.reference:
       return <Reference />;
+    case tabNames.export:
+      return <ExportTab handleExport={context.handleExport} />;
     case tabNames.settings:
       return <SettingsTab started={context.started} />;
     case tabNames.files:

website/src/repl/useReplContext.jsx

@@ -9,11 +9,13 @@ import { getDrawContext } from '@strudel/draw';
 import { evaluate, transpiler } from '@strudel/transpiler';
 import {
   getAudioContextCurrentTime,
+  renderPatternAudio,
   webaudioOutput,
   resetGlobalEffects,
   resetLoadedSounds,
   initAudioOnFirstClick,
   resetDefaults,
+  initAudio,
 } from '@strudel/webaudio';
 import { setVersionDefaultsFrom } from './util.mjs';
 import { StrudelMirror, defaultSettings } from '@strudel/codemirror';
@@ -207,6 +209,30 @@ export function useReplContext() {
   const handleEvaluate = () => {
     editorRef.current.evaluate();
   };
+
+  const handleExport = async (begin, end, sampleRate, maxPolyphony, multiChannelOrbits, downloadName = undefined) => {
+    await editorRef.current.evaluate(false);
+    editorRef.current.repl.scheduler.stop();
+    await renderPatternAudio(
+      editorRef.current.repl.state.pattern,
+      editorRef.current.repl.scheduler.cps,
+      begin,
+      end,
+      sampleRate,
+      maxPolyphony,
+      multiChannelOrbits,
+      downloadName,
+    ).finally(async () => {
+      const { latestCode, maxPolyphony, audioDeviceName, multiChannelOrbits } = settingsMap.get();
+      await initAudio({
+        latestCode,
+        maxPolyphony,
+        audioDeviceName,
+        multiChannelOrbits,
+      });
+      editorRef.current.repl.scheduler.stop();
+    });
+  };
   const handleShuffle = async () => {
     const patternData = await getRandomTune();
     const code = patternData.code;
@@ -235,6 +261,7 @@ export function useReplContext() {
     handleShuffle,
     handleShare,
     handleEvaluate,
+    handleExport,
     init,
     error,
     editorRef,