Merge pull request 'Feat: MIDI Keyboard 🎹🐈' (#1828) from glossing/keyboard into main

Reviewed-on: https://codeberg.org/uzu/strudel/pulls/1828

packages/core/index.mjs

@@ -22,7 +22,7 @@ export * from './repl.mjs';
 export * from './signal.mjs';
 export * from './speak.mjs';
 export * from './state.mjs';
-export * from './time.mjs';
+export * from './schedulerState.mjs';
 export * from './timespan.mjs';
 export * from './ui.mjs';
 export * from './util.mjs';

packages/core/repl.mjs

@@ -2,7 +2,13 @@ import { NeoCyclist } from './neocyclist.mjs';
 import { Cyclist } from './cyclist.mjs';
 import { evaluate as _evaluate } from './evaluate.mjs';
 import { errorLogger, logger } from './logger.mjs';
-import { setTime } from './time.mjs';
+import {
+  setCpsFunc,
+  setIsStarted,
+  setPattern as exposeSchedulerPattern,
+  setTime,
+  setTriggerFunc,
+} from './schedulerState.mjs';
 import { evalScope } from './evaluate.mjs';
 import { register, Pattern, isPattern, silence, stack } from './pattern.mjs';
 import { reset_state } from './impure.mjs';
@@ -52,6 +58,7 @@ export function repl({
     getTime,
     onToggle: (started) => {
       updateState({ started });
+      setIsStarted(started);
       onToggle?.(started);
       if (!started) {
         reset_state();
@@ -65,6 +72,8 @@ export function repl({
   // NeoCyclist uses a shared worker to communicate between instances, which is not supported on mobile chrome
   const scheduler =
     sync && typeof SharedWorker != 'undefined' ? new NeoCyclist(schedulerOptions) : new Cyclist(schedulerOptions);
+  setTriggerFunc(schedulerOptions.onTrigger);
+  setCpsFunc(() => scheduler.cps);
   let pPatterns = {};
   let anonymousIndex = 0;
   let allTransform;
@@ -89,6 +98,7 @@ export function repl({
   const setPattern = async (pattern, autostart = true) => {
     pattern = editPattern?.(pattern) || pattern;
     await scheduler.setPattern(pattern, autostart);
+    exposeSchedulerPattern(pattern);
     return pattern;
   };
   setTime(() => scheduler.now()); // TODO: refactor?

packages/core/schedulerState.mjs

@@ -0,0 +1,53 @@
+/*
+schedulerState.mjs - Module to pipe out various parameters from the scheduler for global consumption
+Copyright (C) 2026 Strudel contributors - see <https://codeberg.org/uzu/strudel/src/branch/main/packages/core/schedulerState.mjs>
+This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+let time;
+let cpsFunc;
+let pattern;
+let triggerFunc;
+let isStarted;
+export function getTime() {
+  if (!time) {
+    throw new Error('no time set! use setTime to define a time source');
+  }
+  return time();
+}
+
+export function setTime(func) {
+  time = func;
+}
+
+export function setCpsFunc(func) {
+  cpsFunc = func;
+}
+
+export function getCps() {
+  return cpsFunc?.();
+}
+
+export function setPattern(pat) {
+  pattern = pat;
+}
+
+export function getPattern() {
+  return pattern;
+}
+
+export function setTriggerFunc(func) {
+  triggerFunc = func;
+}
+
+export function getTriggerFunc() {
+  return triggerFunc;
+}
+
+export function setIsStarted(val) {
+  isStarted = !!val;
+}
+
+export function getIsStarted() {
+  return isStarted;
+}

packages/core/time.mjs

@@ -1,11 +0,0 @@
-let time;
-export function getTime() {
-  if (!time) {
-    throw new Error('no time set! use setTime to define a time source');
-  }
-  return time();
-}
-
-export function setTime(func) {
-  time = func;
-}

packages/midi/midi.mjs

@@ -5,9 +5,23 @@ This program is free software: you can redistribute it and/or modify it under th
 */
 
 import * as _WebMidi from 'webmidi';
-import { Pattern, isPattern, logger, ref } from '@strudel/core';
+import {
+  Hap,
+  Pattern,
+  TimeSpan,
+  getCps,
+  getIsStarted,
+  getPattern,
+  getTime,
+  getTriggerFunc,
+  isPattern,
+  logger,
+  ref,
+  reify,
+} from '@strudel/core';
 import { noteToMidi, getControlName } from '@strudel/core';
 import { Note } from 'webmidi';
+import { getAudioContext } from '@strudel/webaudio';
 import { scheduleAtTime } from '../superdough/helpers.mjs';
 
 // if you use WebMidi from outside of this package, make sure to import that instance:
@@ -477,14 +491,41 @@ Pattern.prototype.midi = function (midiport, options = {}) {
   });
 };
 
-let listeners = {};
+/**
-const refs = {};
+ * Initialize a midi device
-const refsByChan = {};
+ */
+async function _initialize(input) {
+  if (isPattern(input)) {
+    throw new Error(
+      `[midi] Midi input cannot be a pattern. Make sure to pass device name with single quotes. Example: midin('${
+        WebMidi.outputs?.[0]?.name || 'IAC Driver Bus 1'
+      }')`,
+    );
+  }
+  const initial = await enableWebMidi(); // only returns on first init
+  const device = getDevice(input, WebMidi.inputs);
+  if (!device) {
+    throw new Error(
+      `[midi] Midi device "${input}" not found.. connected devices: ${getMidiDeviceNamesString(WebMidi.inputs)}`,
+    );
+  }
+  if (initial) {
+    const otherInputs = WebMidi.inputs.filter((o) => o.name !== device.name);
+    logger(
+      `[midi] Midi enabled! Using "${device.name}". ${
+        otherInputs?.length ? `Also available: ${getMidiDeviceNamesString(otherInputs)}` : ''
+      }`,
+    );
+  }
+  return device;
+}
 
 /**
  * MIDI input: Opens a MIDI input port to receive MIDI control change messages.
  *
  * The output is a function that accepts a midi cc value to query as well as (optionally) a midi channel
+ *
+ * @name midin
  * @param {string | number} input MIDI device name or index defaulting to 0
  * @returns {function(number, number=): Pattern} A function from (cc, channel?) to a pattern.
  *   When queried, the pattern will produces the most recently received midi value (normalized to 0 to 1)
@@ -498,29 +539,11 @@ const refsByChan = {};
  * note("c a f e").s("saw")
  *   .when(cc(0).gt(0), x => x.postgain(0))
  */
+let listeners = {};
+const refs = {};
+const refsByChan = {};
 export async function midin(input) {
-  if (isPattern(input)) {
+  const device = await _initialize(input);
-    throw new Error(
-      `midin: does not accept Pattern as input. Make sure to pass device name with single quotes. Example: midin('${
-        WebMidi.outputs?.[0]?.name || 'IAC Driver Bus 1'
-      }')`,
-    );
-  }
-  const initial = await enableWebMidi(); // only returns on first init
-  const device = getDevice(input, WebMidi.inputs);
-  if (!device) {
-    throw new Error(
-      `midiin: device "${input}" not found.. connected devices: ${getMidiDeviceNamesString(WebMidi.inputs)}`,
-    );
-  }
-  if (initial) {
-    const otherInputs = WebMidi.inputs.filter((o) => o.name !== device.name);
-    logger(
-      `Midi enabled! Using "${device.name}". ${
-        otherInputs?.length ? `Also available: ${getMidiDeviceNamesString(otherInputs)}` : ''
-      }`,
-    );
-  }
   refs[input] ??= {};
   refsByChan[input] ??= {};
   const cc = (cc, chan) => {
@@ -532,8 +555,7 @@ export async function midin(input) {
 
   listeners[input] && device.removeListener('midimessage', listeners[input]);
   listeners[input] = (e) => {
-    const ccNum = e.dataBytes[0];
+    const [ccNum, v] = e.dataBytes;
-    const v = e.dataBytes[1];
     const chan = e.message.channel;
     const scaled = v / 127;
     refsByChan[input][ccNum] ??= {};
@@ -543,3 +565,132 @@ export async function midin(input) {
   device.addListener('midimessage', listeners[input]);
   return cc;
 }
+
+/**
+ * MIDI keyboard: Opens a MIDI input port to receive MIDI keyboard messages.
+ *
+ * The note length is fixed as Superdough is not currently set up for undetermined
+ * note durations
+ *
+ * @name midikeys
+ * @param {string | number} input MIDI device name or index defaulting to 0
+ * @returns {function((number | Pattern)=): Pattern} A function that produces a pattern.
+ *   When queried, the pattern will produces the most recently played midi notes and velocities,
+ *   lasting for the specified duration
+ * @example
+ * const kb = await midikeys('Arturia KeyStep 32')
+ * kb().s("tri").lpf(80).lpe(6).lpd(0.1).room(2).delay(0.35)
+ * @example
+ * const kb = await midikeys('Arturia KeyStep 32')
+ * kb("0.5 1")
+ *   .s("saw")
+ *   .add(note(rand.mul(0.3)))
+ *   .lpf(1000).lpe(2).room(0.5)
+ */
+const kHaps = {};
+const kListeners = {};
+
+function _triggerKeyboard(input, cps, now, latencyCycles) {
+  const pattern = getPattern();
+  const trigger = getTriggerFunc();
+  if (!pattern || !trigger) {
+    return false;
+  }
+  const t = now + latencyCycles;
+  const eps = 1e-6;
+  const haps = pattern.queryArc(t - eps, t + eps, { _cps: cps });
+  // Only keep haps coming from `midikeys`
+  const kbHaps = haps.filter((hap) => hap.value?.midikey?.startsWith(`${input}_`));
+  const ctxNow = getAudioContext().currentTime;
+  if (!kbHaps.length) {
+    return false;
+  }
+  kbHaps.forEach((hap) => {
+    if (!hap.hasOnset()) {
+      return;
+    }
+    const t = ctxNow + (hap.whole.begin - now) / cps;
+    const duration = hap.duration / cps;
+    trigger(hap, t - ctxNow, duration, cps, t);
+  });
+
+  return true;
+}
+export async function midikeys(input) {
+  const device = await _initialize(input);
+  if (!kHaps[input]) {
+    kHaps[input] = [];
+  }
+  kListeners[input] && device.removeListener('midimessage', kListeners[input]);
+  kListeners[input] = (e) => {
+    const { dataBytes, message } = e;
+    const noteon = message.command === 9;
+    let noteoff = message.command === 8;
+    // Don't enqueue or trigger midi notes if scheduler is not started
+    const notStarted = !getIsStarted();
+    // Ignore non-note messages (e.g. CC, pitchbend, modwheel, etc.)
+    const notANote = !noteon && !noteoff;
+    if (notStarted || notANote) {
+      return;
+    }
+    const [note, velocity] = dataBytes;
+    noteoff ||= noteon && velocity === 0; // handle devices which may use velocity = 0 to signal noteoff
+    const key = `${input}_${note}`;
+    const cps = getCps() ?? 0.5;
+    const triggerAvailable = !!(getPattern() && getTriggerFunc());
+    const latencySeconds = triggerAvailable ? 0.01 : 0.06; // avoid missing notes due to cyclist / trigger latency
+    const now = getTime();
+    const t = now + latencySeconds * cps;
+    const span = new TimeSpan(t, t);
+    let value = { midikey: key };
+    if (noteoff) {
+      /* TODO: It's a big effort, but we could modify superdough to allow for situations where
+      we don't know the hap duration in advance. This would mean, for example, that if the hap
+      is flagged as such a special note-on event, we have all effects be persistent & all ADSR
+      envelopes stop at the S stage [and store references to them by `midikey`]
+      If this is implemented, then getting full keyboard functionality should be as simple
+      as sending the corresponding note-off event below and triggering `release` on each of those
+      referenced effects/envelopes
+      
+      value = { ...value, noteoff: true };
+  
+      If this is achieved, we can remove the noteLength parameter
+      */
+      return;
+    } else {
+      value = { ...value, note: Math.round(note), velocity: velocity / 127 };
+    }
+    kHaps[input].push(new Hap(span, span, value, {}));
+    if (!noteoff && triggerAvailable) {
+      // If we have access to a trigger function, we call it to immediately
+      // dispatch to the audio engine, rather than waiting for cyclist to catch these haps
+      const triggered = _triggerKeyboard(input, cps, now, latencySeconds * cps);
+      if (triggered) {
+        kHaps[input] = [];
+      }
+    }
+  };
+  device.addListener('midimessage', kListeners[input]);
+  const kb = (noteLength = 0.5) => {
+    const nlPat = reify(noteLength);
+    const query = (state) => {
+      const haps = kHaps[input].flatMap((hap) => {
+        const lenHaps = nlPat.query(state.setSpan(hap.wholeOrPart()));
+        return lenHaps.map((lenHap) => {
+          const nl = lenHap.value ?? 0.5;
+          const whole = new TimeSpan(hap.whole.begin, hap.whole.begin.add(nl));
+          const part = new TimeSpan(hap.part.begin, hap.part.begin.add(nl));
+          const context = hap.combineContext(lenHap);
+          return new Hap(whole, part, hap.value, context);
+        });
+      });
+      if (state.controls.cyclist) {
+        // Notes have been sent; clear them
+        kHaps[input] = [];
+      }
+      return haps;
+    };
+    return new Pattern(query);
+  };
+  return kb;
+}

test/__snapshots__/examples.test.mjs.snap

@@ -7241,6 +7241,10 @@ exports[`runs examples > example "midicmd" example index 0 1`] = `
 ]
 `;
 
+exports[`runs examples > example "midikeys" example index 0 1`] = `[]`;
+
+exports[`runs examples > example "midikeys" example index 1 1`] = `[]`;
+
 exports[`runs examples > example "midin" example index 0 1`] = `
 [
   "[ 0/1 → 1/4 | note:c cutoff:0 resonance:0 s:sawtooth ]",

test/runtime.mjs

@@ -131,6 +131,10 @@ const midin = () => {
   return (ccNum) => strudel.ref(() => 0); // returns ref with default value 0
 };
 
+const midikeys = async () => {
+  return () => strudel.silence;
+};
+
 const sysex = ([id, data]) => {};
 
 // TODO: refactor to evalScope
@@ -150,6 +154,7 @@ evalScope(
   */
   {
     midin,
+    midikeys,
     sysex,
     // gist,
     // euclid,

website/src/pages/learn/input-output.mdx

@@ -16,10 +16,14 @@ It is also possible to pattern other things with Strudel, such as software and h
 
 Strudel supports MIDI without any additional software (thanks to [webmidi](https://npmjs.com/package/webmidi)), just by adding methods to your pattern:
 
-## midiin(inputName?)
+## midin(inputName?)
 
 <JsDoc client:idle name="midin" h={0} />
 
+## midikeys(inputName?)
+
+<JsDoc client:idle name="midikeys" h={0} />
+
 ## midi(outputName?,options?)
 
 Either connect a midi device or use the IAC Driver (Mac) or Midi Through Port (Linux) for internal midi messages.