hey, after attending a FluCoMa workshop at Standford in 2023, i have spent some time writing a universal Prototype for the MLPRegressor, so you could plugin your abitrary SynthDef and navigate a 2D space for preset interpolation.
The Prototype is based on @elgiano awesome ProtoDef class. You additionally need the awesome NodeProxyGui2. The implementation works for single and array NamedControls and additionally has the possibility to use a MIDI Controller or an LFO to navigate the 2D space.
Currently the additional LFO to navigate the 2D space needs an additional Bus on top of the already existing Buffer which is necessary for the MLPRegressor and is a bit unfortunate imo.
Additional things i would like to implement and im not really sure how to do that are:
- instead of placing individual points on the 2D grid i would like to use a bunch of
Nodeproxypresets and organize them on a grid withFluidGrid - im not sure if the current LFO is the best way to navigate the 2D space. There are additional ways for surface scanning which could be interesting and i havent implemented them yet. But maybe instead of using an LFO it would be possible to create a language side attempt with different functions via
SkipJackto get rid of the additionalBus. Or maybe rethink the implementation of theMLPRegressorand its needs forBuffers, couldnt theMLPregressorbe implemented withBussesinstead? - currently every single param inside
~startMLPneeds its ownChangedUgen (e.g. the more params you have the more ugens you need). I was wondering ifFluidBufToKris the best option here.
After spending an unimaginable amount of hours on this, I really hope this could be a community effort 
Here is the Prototype:
(
ProtoDef(\makeMLP) { |synthDefName, initArgs=#[], excludeParams=#[], ignoreParams=#[], numChannels = 2|
~init = { |self|
self.getNodeProxy;
self.params = IdentityDictionary.new();
self.paramValsUni = IdentityDictionary.new();
self.dsXY = FluidDataSet(s);
self.dsParams = FluidDataSet(s);
self.bufXY = Buffer.alloc(s, 2);
self.bufParams = Buffer.alloc(s, self.getNumParamVals);
self.busXY = Bus.control(s, 2);
self.mlp = FluidMLPRegressor(
server: s,
hiddenLayers: [7],
activation: FluidMLPRegressor.sigmoid,
outputActivation: FluidMLPRegressor.sigmoid,
maxIter: 1000,
learnRate: 0.1,
batchSize: 1,
validation: 0,
);
self.window = Window.new(self.synthDefName, Rect(10, 50, 440, 990)).front;
self.window.layout = VLayout.new();
self.initFont;
self.window.view.children.do{ |c| c.font = self.font };
self.setUpDependencies;
self.makeParamSection;
};
~initFont = { |self|
var fontSize = 14;
self.font = Font.monospace(fontSize, bold: false, italic: false);
};
~getNodeProxy = { |self|
self.synthDef = try{ SynthDescLib.global[self.synthDefName].def } ?? {
Error("SynthDef '%' not found".format(self.synthDefName)).throw
};
self.nodeProxy = NodeProxy.audio(s, self.numChannels);
self.nodeProxy.prime(self.synthDef).set(*self.initArgs);
};
~nodeProxyGui = { |self|
NodeProxyGui2(self.nodeProxy, show: false)
.excludeParams_(self.excludeParams)
.ignoreParams_(self.ignoreParams)
};
~getNumParamVals = { |self|
var paramVals;
paramVals = self.nodeProxy
.getKeysValues(except: self.excludeParams ++ self.ignoreParams)
.flop[1];
paramVals.collect{ |val, i|
case
{ val.isNumber } { 1 }
{ val.isArray} { val.size };
}.sum;
};
~setUpDependencies = { |self|
var dataChangedFunc = { |obj ...args| self.dataChanged(*args) };
self.addDependant(dataChangedFunc);
self.window.onClose_{
self.stopLFO;
self.nodeProxy.stop;
self.nodeProxy.free;
self.unmapMidi;
self.removeDependant(dataChangedFunc);
self.bufXY.free;
self.bufParams.free;
self.busXY.free;
self.params.clear;
self.paramValsUni.clear;
self.stopMLP;
self.mlp.clear;
self.dsXY.clear;
self.dsParams.clear;
};
};
// notify change
~setData = { |self, values|
self.changed(\slider, *values);
self.changed(\buffer, *values);
};
// dataChanged is called when values change
~dataChanged = { |self, what ...args|
case
{ what == \slider } {
self.bufXY.setn(0, args);
}
{ what == \buffer } {
self.xySlider.setXY(*args);
}
{ what == \controller } {
var updateSliderFunc, index, value;
index = args[0];
value = args[1];
self.bufXY.set(index, value);
updateSliderFunc = {
switch(index,
0, { self.xySlider.x_(value) },
1, { self.xySlider.y_(value) }
);
};
{ updateSliderFunc.() }.defer;
};
};
~startLFO = { |self|
var limitUpdateRate = 0.01;
self.stopLFO;
self.lfoSynth = {
var reset = \reset.tr(0);
var rate = \rate.kr(0.1) * (1 - \freeze.kr(0));
var phase = Phasor.kr(reset, rate * ControlDur.ir);
var sig = SinOsc.kr(DC.kr(0), phase + [0.0, 0.25] * 2pi);
sig = (sig * \radius.kr(1)).linlin(-1, 1, 0, 1);
BufWr.kr(sig[0], self.bufXY, 0);
BufWr.kr(sig[1], self.bufXY, 1);
sig;
}.play(outbus: self.busXY);
SkipJack.new(
updateFunc: { self.changed(\buffer, *self.busXY.getnSynchronous(2)) },
dt: limitUpdateRate,
stopTest: { self.window.isClosed or: self.lfoSynth.isNil },
clock: AppClock
);
};
~stopLFO = { |self|
self.lfoSynth !? { self.lfoSynth.free; self.lfoSynth = nil; };
};
~resetLFO = { |self|
self.lfoSynth !? { self.lfoSynth.set(\reset, 1); };
};
~pauseLFO = { |self|
self.lfoSynth !? { self.lfoSynth.set(\freeze, 1); };
};
~resumeLFO = { |self|
self.lfoSynth !? { self.lfoSynth.set(\freeze, 0); };
};
~trainMLP = { |self, train|
self.trainTask = self.trainTask ?? {
Task {
loop {
var condition = CondVar.new;
var done = false;
self.mlp.fit(self.dsXY, self.dsParams) { |loss|
done = true;
condition.signalOne;
"> Training done. Loss: %".format(loss).postln;
};
condition.wait { done };
0.1.wait;
};
};
};
if (train) {
self.trainTask.play;
} {
self.trainTask.pause;
};
};
~startMLP = { |self|
self.stopMLP;
self.mlpSynth = {
var xyData, xyTrig, paramValUni, paramTrig;
// when data in xy buffer has changed, ...
xyData = FluidBufToKr.kr(self.bufXY);
xyTrig = Changed.kr(xyData.round(0.01)).sum;
// ... trigger MLP ...
self.mlp.kr(xyTrig, self.bufXY, self.bufParams);
// ... to get associated param value from parameter buffer ...
paramValUni = FluidBufToKr.kr(self.bufParams);
paramTrig = Changed.kr(paramValUni.round(0.01)).sum;
// ... and send OSC message to update NodeProxy!
SendReply.kr(paramTrig, "/paramsChanged", paramValUni);
Silent.ar;
}.play;
self.responder = OSCFunc({ |msg|
var paramValsUniOSC = msg.drop(3);
self.setParamVals(paramValsUniOSC);
}, "/paramsChanged", argTemplate:[self.mlpSynth.nodeID]);
};
~stopMLP = { |self|
self.mlpSynth !? { self.mlpSynth.free; self.mlpSynth = nil };
self.responder !? { self.responder.free; self.responder = nil };
};
~makeParamSection = { |self|
self.params.clear;
self.nodeProxy
.controlKeysValues(except: self.nodeProxy.internalKeys ++ self.excludeParams ++ self.ignoreParams)
.pairsDo{ |key, val|
var spec;
spec = case
{ val.isNumber } {
(self.nodeProxy.specs.at(key) ?? { Spec.specs.at(key) }).asSpec;
}
{ val.isArray } {
(self.nodeProxy.specs.at(key) ?? { Spec.specs.at(key) }).asSpec.dup(val.size);
};
self.params.put(key, spec);
};
if(self.paramLayout.notNil) { self.paramLayout.remove };
self.paramLayout = self.makeParamLayout;
self.window.layout.add(self.paramLayout);
self.window.layout.add(self.nodeProxyGui);
};
~getParamValsUni = { |self|
self.paramValsUni.clear;
self.params.sortedKeysValuesDo{ |key, spec|
var paramVal;
paramVal = self.nodeProxy.get(key);
case
{ paramVal.isNumber } {
var paramValUni;
paramValUni = spec.unmap(paramVal);
self.paramValsUni.put(key, paramValUni);
}
{ paramVal.isArray } {
var paramValsUni;
paramValsUni = paramVal.collect{ |val, n|
spec.wrapAt(n).unmap(val);
};
self.paramValsUni.put(key, paramValsUni);
};
};
self.paramValsUni;
};
~addPoint = { |self, id|
var paramValsUniSerialised;
self.getParamValsUni;
paramValsUniSerialised = self.paramValsUni.asSortedArray.flop[1].flat;
self.bufParams.loadCollection(paramValsUniSerialised, 0) {
self.dsParams.addPoint(id, self.bufParams);
};
self.dsXY.addPoint(id, self.bufXY);
};
~setParamVals = { |self, paramValsUniOSC|
var paramValsUni;
paramValsUni = paramValsUniOSC.reshapeLike(self.paramValsUni.asSortedArray.flop[1]);
self.params.sortedKeysValuesDo{ |key, spec, i|
var paramVal;
paramVal = self.nodeProxy.get(key);
case
{ paramVal.isNumber } {
var value = spec.map(paramValsUni[i]);
self.nodeProxy.set(key, value);
}
{ paramVal.isArray } {
paramVal.collect{ |val, n|
var value = spec.wrapAt(n).map(paramValsUni[i].wrapAt(n));
self.nodeProxy.seti(key, n, value);
};
};
};
};
~makeParamLayout = { |self|
var view, layout, buttons;
var graphView, pointView, plotDataSetXY;
var counter = 0;
view = View.new().layout_(VLayout.new());
plotDataSetXY = {
self.dsXY.dump { |v|
if (v["cols"] == 0) {
v = Dictionary["cols" -> 2, "data" -> Dictionary[]]
};
pointView.dict = v
};
};
pointView = FluidPlotter(standalone: false).pointSizeScale_(20/6);
plotDataSetXY.();
self.xySlider = Slider2D()
.background_(Color.white.alpha_(0))
.action_{ |obj|
self.changed(\slider, obj.x, obj.y);
};
graphView = StackLayout(self.xySlider, View.new().layout_(
VLayout(pointView).margins_(10)
)).mode_(\stackAll);
buttons = VLayout(
Button(bounds: 100@20).states_([["Add Point"]])
.action_{
var id = "point-%".format(counter);
self.addPoint(id);
counter = counter + 1;
plotDataSetXY.();
},
Button(bounds: 100@20).states_([["Save Data"]])
.action_{
FileDialog({ |folder|
self.dsXY.write(folder +/+ "xydata.json");
self.dsParams.write(folder +/+ "paramsdata.json");
}, {}, 2, 0, true);
},
Button(bounds: 100@20).states_([["Load Data"]])
.action_{
FileDialog({ |folder|
self.dsXY.read(folder +/+ "xydata.json");
self.dsParams.read(folder +/+ "paramsdata.json");
}, fileMode: 2, acceptMode: 0, stripResult: true);
plotDataSetXY.();
},
Button(bounds: 100@20).states_([
["Train"], ["Train", Color.white, Color.yellow(0.5)]])
.action_{ |obj|
self.trainMLP(obj.value > 0)
},
Button(bounds: 100@20).states_([["Save MLP"]])
.action_{
Dialog.savePanel({ |path|
if(PathName(path).extension != "json"){
path = "%.json".format(path);
};
self.mlp.write(path);
});
},
Button(bounds: 100@20).states_([["Load MLP"]])
.action_{
Dialog.openPanel({ |path|
self.mlp.read(path, action: {
{ plotDataSetXY.() }.defer
});
});
},
Button(bounds: 100@20).states_([["Start MLP"],
["Stop MLP", Color.white, Color.red(0.8)]])
.action_{ |obj|
if (obj.value > 0) { self.startMLP } { self.stopMLP };
}.value_(self.mlpSynth.notNil),
Button(bounds: 100@20).states_([["Clear"]])
.action_{|state|
self.mlp.clear;
self.dsXY.clear;
self.dsParams.clear;
plotDataSetXY.();
},
Button(bounds: 100@20).states_([["Start LFO"],
["Stop LFO", Color.white, Color.red(0.8)]])
.action_{ |obj|
if (obj.value > 0) { self.startLFO } { self.stopLFO };
}.value_(self.lfoSynth.notNil)
);
layout = HLayout([graphView, s: 1], buttons);
view.layout.add(layout);
view.children.do{ |c| c.font = self.font };
view;
};
~mapMidi = { |self, ccs|
self.midiResponders = self.midiResponders ?? { Order[] };
self.unmapMidi;
ccs.do { |cc, n|
self.midiResponders[cc] = MIDIFunc.cc({ |v, c|
var val = v / 127;
self.changed(\controller, n, val);
}, cc).fix
};
};
~unmapMidi = { |self|
self.midiResponders.do(_.free);
};
};
)
create a test SynthDef and use the Prototype for preset interpolation:
(
SynthDef(\test, {
var tFreq, trig, env, freqRange, freq, fmod, sig;
tFreq = \tFreq.kr(4, spec: ControlSpec(1, 12));
trig = Impulse.ar(tFreq);
env = Env.perc(\atk.kr(0.1, spec: ControlSpec(0.005, 1)), \rel.kr(1, spec: ControlSpec(0.05, 1))).ar(0, trig);
//freqRange = \freqRange.kr([440, 440], spec: ControlSpec(100, 8000));
//freq = exprand(freqRange[0], freqRange[1]);
freq = \freq.kr(440, spec: ControlSpec(20, 1000));
fmod = SinOsc.ar(\fmFreq.kr(3, spec: ControlSpec(1, 100)));
sig = SinOsc.ar(freq + (freq * fmod * \fmIndex.kr(2, spec: ControlSpec(0, 5))));
sig = sig * env;
sig = Pan2.ar(sig, \pan.kr(0, spec: ControlSpec(-1, 1)));
sig = sig * \amp.kr(-25, spec: ControlSpec(-35, -15, \lin, 1)).dbamp;
Out.ar(\out.kr(0), sig);
}).add;
)
(
~testMLP = Prototype(\makeMLP) {
~synthDefName = \test;
~initArgs = [
];
~excludeParams = [
];
~ignoreParams = [
\amp,
];
};
)
~testMLP.startLFO;
~testMLP.resetLFO;
~testMLP.pauseLFO;
~testMLP.resumeLFO;
~testMLP.lfoSynth.set(\radius, 0.8);
~testMLP.lfoSynth.set(\rate, 0.03);
~testMLP.stopLFO;
~testMLP.setData([0.5, 0.5]);
~testMLP.bufXY.getn(0, 2, { |a| a.postln });
~testMLP.mapMidi([0, 1]);
~testMLP.unmapMidi;