Cepstral Cross Synthesis

Questions
1

Hi everyone!

I am trying to run this code:


Audio examples:
https://ccrma.stanford.edu/~jhsu/421b/

and I am getting no sound… What could the problem here ?

// load things
(
b = Buffer.readChannel(s,"pathto/some/audiofile.wav", channels: 0);
c = Bus.new('control', 0, 13);
d = Buffer.readChannel(s,"pathto/some/audiofile.wav", channels: 0);
~fftbufc = Buffer.alloc(s, 2048);
~fftbufm = Buffer.alloc(s, 2048);
~cepbufc = Buffer.alloc(s, 1024);
~cepbufm = Buffer.alloc(s, 1024);
~envc = Buffer.alloc(s, 2048);
~envm = Buffer.alloc(s, 2048);
)

(
SynthDef(\morphintime, {|out = 0, bufnum = 0, bufnum2 = 1|
	var in, in2, chain, chain2, chain3, cepsch, cepsch2, fftsize;
	fftsize = 2048;
	//bufnum = b.bufnum;
	//bufnum2 = d.bufnum;

	// 1. STFT of signal
	// 2. smooth spectral envelope
	// get cepstrum of modulating signal
	in = PlayBuf.ar(1, bufnum, BufRateScale.kr(bufnum), 1, 0, 1);
	chain = FFT(~fftbufm, in);
	cepsch = Cepstrum(~cepbufm, chain);
	// get cepstrum of carrier signal
	in2 = PlayBuf.ar(1, bufnum2, BufRateScale.kr(bufnum2), 1, 0, 1);
	chain2 = FFT(~fftbufc, in2);
	cepsch2 = Cepstrum(~cepbufc, chain2);

	// PV_BrickWall can act as a low-pass filter, or here, as a wol-pass lifter...
	// ...in practical terms, produces a smoothed version of the spectrum
	// get smooth version of modulator
	cepsch = PV_BrickWall(cepsch, -0.95);
	ICepstrum(cepsch, ~envm);
	// get smoothed version of carrier
	cepsch2 = PV_BrickWall(cepsch2, -0.95);
	ICepstrum(cepsch2, ~envc);

	// 3. divide spectrum of each carrier frame by smooth spectral envelope (to flatten)
	chain2 = chain2.pvcalc2(~envc, fftsize, {|mags, phases, mags2, phases2|
		[mags / mags2, phases - phases2]
	}, frombin: 0, tobin: 125, zeroothers: 0);

	// 4. multiply flattened spectral carrier frame with smooth spectral envelope of modulator
	chain2 = chain2.pvcalc2(~envm, fftsize, {|mags, phases, mags2, phases2|
		[mags * mags2, phases + phases2]
	}, frombin: 0, tobin: 125, zeroothers: 0);

	Out.ar( out, Pan2.ar(IFFT(chain2)) );

}).add;

)

x = Synth.new(\morphintime, [\bufnum, b, \bufnum2, d]);
x = Synth.new(\morphintime);

All the best,
Fellipe

2

I’ve found out where the bug is: it seems that Cepstrum only accepts local buffers and it doesn’t work if you allocate the buffer outside the Synthdef function. Does anyone know the reason for this? Legacy safety issues regarding the usage of PV UGens ? (the allocation outside the synthesis buffer works fine in UGens like PV_XFade…)

This is the example that works as expected. (Also replaced the .pvcalc2 for PV_UGens in order to get lot more efficiency)

// load things
(
~modulator = Buffer.read(s,Platform.resourceDir +/+ "sounds/a11wlk01.wav");
~carrier = Buffer.alloc(s, 2048);
~carrier.loadCollection((({ 0.3.bilinrand.sin.sum3rand}!1024) ++ ({ 0.05.sum3rand}!1024))); //some custom noise
~cepbufc = Buffer.alloc(s, 1024);
~cepbufm = Buffer.alloc(s, 1024);
~envc = Buffer.alloc(s, 2048);
~envm = Buffer.alloc(s, 2048);
)


(
SynthDef(\morphintime, {|out = 0, modBuf = 0, carBuf = 1|
var in, in2, chain, chain2, chain3, cepsch, cepsch2, fftsize;
fftsize = 2048;


// 1. STFT of signal
// 2. smooth spectral envelope
// get cepstrum of modulating signal
in = PlayBuf.ar(1, modBuf, BufRateScale.kr(modBuf), 1, 0, 1);
chain = FFT(LocalBuf(2048), in);
cepsch = Cepstrum(~cepbufm, chain);
// get cepstrum of carrier signal
in2 = PlayBuf.ar(1, carBuf, BufRateScale.kr(carBuf), 1, 0, 1);
//in2 = WhiteNoise.ar(0.1);
chain2 = FFT(LocalBuf(2048), in2);
cepsch2 = Cepstrum(~cepbufc, chain2);

// PV_BrickWall can act as a low-pass filter, or here, as a wol-pass lifter...
// ...in practical terms, produces a smoothed version of the spectrum
// get smooth version of modulator
cepsch = PV_BrickWall(cepsch, -0.92);
ICepstrum(cepsch, ~envm);
// get smoothed version of carrier
cepsch2 = PV_BrickWall(cepsch2, -0.92);
ICepstrum(cepsch2, ~envc);

// 3. divide spectrum of each carrier frame by smooth spectral envelope (to flatten)
chain2 = PV_MagDiv(chain2,~envc);

// 4. multiply flattened spectral carrier frame with smooth spectral envelope of modulator
chain2 = PV_MagMul(chain2, ~envm);

Out.ar( out, Pan2.ar(IFFT(chain2) *0.5) );

}).add;

)

x = Synth.new(\morphintime, [\modBuf, ~modulator, \carBuf, ~carrier  ]);

All the best,
Fellipe

3

The problem is here. ~envc must be a PV chain, but you’re supplying a buffer directly.

It might work to wrap it in a FFTTrigger here…?

hjh

4

Pretty impressive vocoder - I tried this with some orchestral music as carrier…

2 Likes
5

Hjh, can’t still figure out what is causing this, but I don’t think it’s the absence of PV chain because we I am using pvcalc2 with LocalBuf it is working fine:

// load things
(
~modulator = Buffer.read(s,Platform.resourceDir +/+ "sounds/a11wlk01.wav");
~carrier = Buffer.alloc(s, 2048);
~carrier.loadCollection((({ 0.3.bilinrand.sin.sum3rand}!1024) ++ ({ 0.05.sum3rand}!1024))); //some custom noise
~cepbufc = Buffer.alloc(s, 1024);
~cepbufm = Buffer.alloc(s, 1024);
~envc = Buffer.alloc(s, 2048);
~envm = Buffer.alloc(s, 2048);
)


(
SynthDef(\morphintime, {|out = 0, modBuf = 0, carBuf = 1|
	var in, in2, chain, chain2, chain3, cepsch, cepsch2, fftsize;
	fftsize = 2048;
	
	
	// 1. STFT of signal
	// 2. smooth spectral envelope
	// get cepstrum of modulating signal
	in = PlayBuf.ar(1, modBuf, BufRateScale.kr(modBuf), 1, 0, 1);
	chain = FFT(LocalBuf(2048), in);
	cepsch = Cepstrum(~cepbufm, chain);
	// get cepstrum of carrier signal
	in2 = PlayBuf.ar(1, carBuf, BufRateScale.kr(carBuf), 1, 0, 1);
	//in2 = WhiteNoise.ar(0.1);
	chain2 = FFT(LocalBuf(2048), in2);
	cepsch2 = Cepstrum(~cepbufc, chain2);
	
	// PV_BrickWall can act as a low-pass filter, or here, as a wol-pass lifter...
	// ...in practical terms, produces a smoothed version of the spectrum
	// get smooth version of modulator
	cepsch = PV_BrickWall(cepsch, -0.92);
	ICepstrum(cepsch, ~envm);
	// get smoothed version of carrier
	cepsch2 = PV_BrickWall(cepsch2, -0.92);
	ICepstrum(cepsch2, ~envc);
	
	// 3. divide spectrum of each carrier frame by smooth spectral envelope (to flatten)
	chain2 = chain2.pvcalc2(~envc, fftsize, {|mags, phases, mags2, phases2|
		[mags / mags2, phases - phases2]
	}, frombin: 0, tobin: 1023, zeroothers: 0);
	
	// 4. multiply flattened spectral carrier frame with smooth spectral envelope of modulator
	
	chain2 = chain2.pvcalc2(~envm, fftsize, {|mags, phases, mags2, phases2|
		[mags * mags2, phases + phases2]
	}, frombin: 0, tobin: 1023, zeroothers: 0);
	
	Out.ar( out, Pan2.ar(IFFT(chain2) *0.5) );
	
}).add;

)

x = Synth.new(\morphintime, [\modBuf, ~modulator, \carBuf, ~carrier  ]);

I can only guess that there might be some something regarding legacy safety issues of PV UGens…

6

hey, i was trying to use the code with In.ar for the modulating signal to send sources from other synth. Ive tried to send it to a bus, unfortunately you already have sound from the carrier without even playing the modulator synth. is there a workaround? thanks.

(
~grp_vocoders  = Group.new(s, \addToTail);
~bus_modulator = Bus.audio(s,1);
)

(
~cepbufc = Buffer.alloc(s, ~windowSize/2);
~cepbufm = Buffer.alloc(s, ~windowSize/2);
~envc = Buffer.alloc(s, ~windowSize);
~envm = Buffer.alloc(s, ~windowSize);

SynthDef(\vocoder2, {
	arg out=0, amp=0.5, in=0, pan=0, freq=220, atk=0.1, rel=1;

	var sig, input, in2, chain, chain2, cepsch, cepsch2, pos, posRate, env;

	env = EnvGen.ar(Env.perc(atk, rel), doneAction:2);
	input = In.ar(in, 2) * env;
	chain = FFT(LocalBuf(~windowSize), input);	
	cepsch = Cepstrum(~cepbufm, chain);

	// get cepstrum of carrier signal
	in2 = Saw.ar(freq);
	chain2 = FFT(LocalBuf(~windowSize), in2);
	cepsch2 = Cepstrum(~cepbufc, chain2);

	// get smooth version of modulator
	cepsch = PV_BrickWall(cepsch, -0.92);
	ICepstrum(cepsch, ~envm);

	// get smoothed version of carrier
	cepsch2 = PV_BrickWall(cepsch2, -0.92);
	ICepstrum(cepsch2, ~envc);

	// 3. divide spectrum of each carrier frame by smooth spectral envelope (to flatten)
	chain2 = PV_MagDiv(chain2, ~envc);

	// 4. multiply flattened spectral carrier frame with smooth spectral envelope of modulator
	chain2 = PV_MagMul(chain2, ~envm);

	Out.ar(out, Pan2.ar(IFFT(chain2) * 0.5));
}).add;
)

(
Pbindef(\vocoderfx,
	\instrument, \vocoder2,
	\scale, Scale.minor,
	\root, 1,
	\octave, Prand([[3,4,5,6]], inf),
	\degree, Prand([ [0,2,4,7] ], inf),
	\mtranspose, Pwhite(-0.02, 0.02, inf),
	\width, 0.01, //Pwhite(0.01, 0.5, inf),
	\in, ~bus_modulator,
	\atk, 0.5,
	\rel, 1,
	\amp, 0.125,
	\dur, Pseq([Pn(0.25,8)], inf),
	\group, ~grp_vocoders,
	\pan, Pwhite(-1.0, 1.0, inf).clump(4)
).quant_([4]).play;
)
7

As this UGen is showing some strange behavior, maybe it is a good idea to avoid allocating the buffer in the same scope as the SynthDef.add .

Moreover, maybe changing FFT size in FFT(LocalBuf(~windowSize) from an environment variable to a local variable or to an argument would also a good idea.

If nothing helps, than maybe you could post a code with the synths that you are using for people here helping with other details

8

hey thanks. even if I put all the Buffers in the SynthDef and declare an argument/variable for the fft size I get sound on the meter, barely audible but loud in volume, its more like a hum, only by playing the carrier Pbindef(\vocoderfx) and sending it to the bus ~bus_modulator
Because its doing its math on the carrier signal and sending it to the bus, even without having a modulating signal? when i then send any source to \out, ~bus_modulator the sound is really distorted and gets a weird amp envelope/gated effect.

9

ive low-pass liftered the modulator and high-pass liftered the carrier. its working quite nicely now.

// get smooth version of modulator
cepschM = PV_BrickWall(cepschM, \wipeM.kr(-0.92));
ICepstrum(cepschM, ~envM);

// get smoothed version of carrier
cepschC = PV_BrickWall(cepschC, \wipeC.kr(0.92));
ICepstrum(cepschC, ~envC);

full code:

(
~cepbufC = Buffer.alloc(s, ~windowSize/2);
~cepbufM = Buffer.alloc(s, ~windowSize/2);
~envC = Buffer.alloc(s, ~windowSize);
~envM = Buffer.alloc(s, ~windowSize);
)

(
SynthDef(\ceptral_cross_synthesis, {
	arg out=0, pan=0, recBuf=0, sndBuf=0, posLo=0.1, posHi=0.9,
	posRateE=0, posRateM=1, amp=1, clear=0, freq=150, numharm=25, freeze=1;

	var xModEnv = \gainEnv.kr(Env.newClear(8).asArray);
	var sig, pos, posRate;
	var chain, chainM, chainC, cepschM, cepschC, inC;
	var fftSize = 2048;

	posRate = 10 ** posRateE * posRateM;

	// phasor bounds must be between 0 and 1
	pos = Phasor.ar(0, posRate * SampleDur.ir, posLo, posHi);

	// get cepstrum of modulating signal
	chainM = FFT(LocalBuf(fftSize), PlayBuf.ar(1, sndBuf, BufRateScale.kr(sndBuf)), hop:0.5, wintype:0);
	chainM = PV_BinBufRd(chainM, recBuf, pos, 0, 2, 12, clear);

	chainM = PV_Freeze(chainM, freeze);
	cepschM = Cepstrum(~cepbufM, chainM);

	chainM = chainM.pvcollect(fftSize, {
		arg mag, phase;
		var pmod;
		pmod = LFNoise1.kr(rrand(0.5, 1.1)).range(-pi, pi);
		[mag, (pmod - phase) * freeze + phase]
	},
	frombin: 0, tobin: 250, zeroothers: 1);

	// get cepstrum of carrier signal
	inC = Blip.ar(freq, numharm);
	chainC = FFT(LocalBuf(fftSize), inC, hop:0.5, wintype:0);
	cepschC = Cepstrum(~cepbufC, chainC);

	// get smooth version of modulator
	cepschM = PV_BrickWall(cepschM, \wipeM.kr(-0.92));
	ICepstrum(cepschM, ~envM);

	// get smoothed version of carrier
	cepschC = PV_BrickWall(cepschC, \wipeC.kr(0.92));
	ICepstrum(cepschC, ~envC);

	// 3. divide spectrum of each carrier frame by smooth spectral envelope (to flatten)
	chainC = PV_MagDiv(chainC, ~envC);

	// 4. multiply flattened spectral carrier frame with smooth spectral envelope of modulator
	chainC = PV_MagMul(chainC, ~envM);

	chain = PV_XFade(chainC, chainM, SinOsc.ar(0.2));

	//Bandpassfilter
	chain = PV_BrickWall(chain, \lpf.kr(20000).linlin(20,20000,-1,0));
	chain = PV_BrickWall(chain, \hpf.kr(20).linlin(20,20000,0,1));

	sig = IFFT(chain) * 35.neg.dbamp;

	sig = Splay.ar(sig, level:amp);
	sig = sig + NHHall.ar(sig, 0.3);
	sig = Limiter.ar(sig);
	Out.ar(out, sig);
}).add;
)
2 Likes