Here’s a little test I did a while ago of drawing onto a spectrogram with the mouse… it wasn’t fast enough for real time display so you only see what you’ve drawn after you’ve lifted the mouse button (play it by clicking to move the playhead and pressing space)
~winsize = 2048;
~hop = 0.15;
//~hop = 0.25;
~hop = (~hop * 2048 / 64).round * 64 / 2048;
~wintype = 1;
//~wintype = 0;
~brushHeight = 10;
~brushWidth = 10;
~inbuf.free;~inbuf = Buffer.read(s, Platform.resourceDir +/+ "sounds/a11wlk01.wav");
~buf.free;~buf = Buffer.alloc(s, ~inbuf.duration.calcPVRecSize(~winsize, ~hop, s.sampleRate));
(
// note the window type and overlaps... this is important for resynth parameters
SynthDef("pvrec", { |recbuf, inbuf, winsize = 2048, hop = 0.25, wintype = 1|
var in, chain, bufnum;
bufnum = LocalBuf.new(winsize, 1);
Line.kr(1, 1, BufDur.kr(inbuf), doneAction: 2);
in = PlayBuf.ar(1, inbuf, BufRateScale.kr(inbuf), loop: 0);
chain = FFT(bufnum, in, hop, wintype);
chain = PV_RecordBuf(chain, recbuf, 0, 1, 0, hop, wintype);
// no ouput
}).add;
)
a = Synth("pvrec", [recbuf: ~buf, inbuf: ~inbuf, winsize: ~winsize, hop: ~hop, wintype: ~wintype]);
(
SynthDef("pvplay", { |out, buf = 1, offset = 0, rate = 1, winsize = 2048, hop = 0.25, wintype = 1|
var in, chain, bufnum;
bufnum = LocalBuf.new(winsize);
chain = PV_PlayBuf(bufnum, buf, rate, offset, 0);
Out.ar(out, IFFT(chain, 1).dup * hop);
}).add;
)
b = Synth("pvplay", [out: 0, buf: ~buf, rate: 0.75, winsize: ~winsize, hop: ~hop, wintype: ~wintype]);
// stop the synth
b.free;
// load to ~data array
~buf.loadToFloatArray(action: { |data| ~data = data.postln })
// to zero it out
~data = ~data[0..2] ++ (~data.size - 3).collect { 0 }
~data[0] = ~winsize;
~data[1] = ~hop;
~data[2] = ~wintype;
(
var sampleRate = s.sampleRate;
var numFrames = ((~data.size - 3) / ~winsize);
var width = numFrames.asInteger;
var height = (~winsize / 2).asInteger;
var colorFunc = { |mag| Color.gray(1, mag.log10.lincurve(-2, 2, 0, 1, curve: 0)) };
var scaleFunc = { |pixel| pixel.lincurve(0, height, 0, height, curve: 3) }; // curve factor for Y scaleing (0 = linear, 3 =~ mel scale)
~image = Image.new(width, height);
~magAt = { |frame, index|
~data[(frame * ~winsize) + (index * 2) + 4];
};
~setMag = { |frame, index, mag, phase|
var i = (frame * ~winsize) + (index * 2) + 3;
~data[i + 1] = mag;
if (phase.notNil) {
~data[i] = phase;
};
};
~playrate = 0.1;
~synth = nil;
~playhead = 0;
if (~win.notNil) { ~win.close };
~win = Window("", Rect(0, 0, ~image.width, ~image.height)).front.background_(Color.white)
.view.keyDownAction_({ |v, char|
if (char == $ ) {
if (~synth.notNil) {
~synth.free;
~synth = nil;
~playrout.stop;
} {
~synth = Synth("pvplay", [
out: 0,
buf: ~buf,
offset: ~playhead / ~inbuf.duration * numFrames,
rate: ~playrate,
winsize: ~winsize,
hop: ~hop,
wintype: ~wintype
]);
//~playhead = 0;
~playrout = fork {
(~inbuf.duration / ~playrate * 30).do {
~playhead = ~playhead + (~playrate / 30);
defer { ~playheadview.refresh };
(1/30).wait;
};
};
}
};
});
~makeView = {
var mouseDownPoint;
var refreshImage = {
~image.setPixels(Int32Array.fill(width * height, { |i|
var index = scaleFunc.(height - (i / width).floor).asInteger;
var frame = i % width;
var mag = ~magAt.(frame, index);
Image.colorToPixel(colorFunc.(mag));
}), Rect(0, 0, ~image.width, ~image.height));
};
refreshImage.();
~view.remove;
~view = UserView(~win, ~win.bounds).drawFunc_({
Pen.use {
Pen.scale(~view.bounds.width / ~image.width, ~view.bounds.height / ~image.height);
Pen.drawImage(0@0, ~image)
}
}).resize_(5).background_(Color.black);
~playheadview = UserView(~win, ~win.bounds).drawFunc_({ |v|
Pen.use {
Pen.addRect(Rect(~playhead / ~inbuf.duration * v.bounds.width, 0, 2, v.bounds.height));
Pen.color = Color.red;
Pen.fill;
}
}).resize_(5)
.mouseDownAction_({ |v, x, y|
[x, y].postln;
mouseDownPoint = x@y;
})
.mouseMoveAction_({ |v, x, y|
/*
var frame = x / v.bounds.width * numFrames;
//var index = scaleFunc.((v.bounds.height - y) / v.bounds.height * height - 1).asInteger;
var index = ((y.postln / v.bounds.height).lincurve(0.8, 0.0, 0.0, 0.8, 3, nil).postln * (~winsize / 2)).asInteger;
*/
var index = scaleFunc.(height - y.linlin(0, v.bounds.height, 0, height)).asInteger;
var frame = x.linlin(0, v.bounds.width, 0, width).asInteger;
//var mag = ~magAt.(frame, index);
var halfBW = (~brushWidth / 2).asInteger;
var halfBH = (~brushHeight / 2).asInteger;
~brushWidth.do { |bwi|
~brushHeight.do { |bhi|
var thisFrame = frame - halfBW + bwi;
var thisIndex = index - halfBH + bhi;
if (thisFrame.isPositive and: thisIndex.isPositive) {
var distance = sqrt((thisFrame - frame).squared + (thisIndex - index).squared);
var mag = ~magAt.(thisFrame, thisIndex);
~setMag.(thisFrame, thisIndex, mag + distance.lincurve(0, 10, 1, 0, -5).postln, rrand(0, 2pi));
}
}
};
//refreshImage.();
})
.mouseUpAction_({ |v, x, y|
if (x@y == mouseDownPoint) {
~playhead = x / v.bounds.width * ~inbuf.duration;
};
~buf.loadCollection(~data, action: { "loaded".postln });
~makeView.();
});
};
~makeView.();
)
this generates the image from the PV buffer contents but it wouldn’t be too hard I think to do the reverse, generate the PV buffer from the image instead