diff --git a/src/Reveal.hpp b/src/Reveal.hpp index e1cf895ae9b22e68a632299ab53f50631ef0cbcf..a4937e57328b9c9a71fdb2abbe34db649377cb0f 100644 --- a/src/Reveal.hpp +++ b/src/Reveal.hpp @@ -76,7 +76,10 @@ class Reveal : public GroupModule { DEPTHTYPE, DEPTHID, OUTPUTCURS, PATW, PATPOSX, PATPOSY, PATNBX, PATNBY, PATBRIGHT, PATTERNTEX, PATRANGE, PATOFFSET, - POSTFILTER, RENDERTEX + POSTFILTER, RENDERTEX, + + AUDIOTEX, AUDIORATE, MAXAUDIOVALUE, AUDIOBUFSIZE, AUDIOMIXSIZE, + PROCESSAUDIO }; enum GEOM_ID { diff --git a/src/audio/AudioManager.cpp b/src/audio/AudioManager.cpp index 16c294169c63ae5bff9f3e96e678f944a4ce9ee6..38500423fc315bcbb9235d51e39f8fa41c11faee 100644 --- a/src/audio/AudioManager.cpp +++ b/src/audio/AudioManager.cpp @@ -57,7 +57,7 @@ static int rtaudio_callback(void *outbuf, void *inbuf, unsigned int nFrames, dou int audioIndex = 0; //check if there is audio value to play - if(ringbuffer_read_space(data->audioBuf) > 2*(nFrames + data->mixSize)){//!=0){ + if(ringbuffer_read_space(data->audioBuf) > 2*(nFrames + data->mixSize)){ //read the audio values ringbuffer_read(data->audioBuf, data->audio, nFrames + data->mixSize); @@ -80,6 +80,10 @@ static int rtaudio_callback(void *outbuf, void *inbuf, unsigned int nFrames, dou //save the value for the next crossfading memcpy(&data->mix[0], data->audio + nFrames, data->mixSize*sizeof(float)); } + else { + memset(buf, 0, nFrames*data->nChannel*sizeof(float)); + cout<<"AudioManager : Could not read buffer"<<endl; + } return 0; } @@ -89,10 +93,13 @@ AudioManager::AudioManager(){ param = new RtAudio::StreamParameters(); param->deviceId = audio->getDefaultOutputDevice(); param->nChannels = 2; + options = new RtAudio::StreamOptions(); + options->streamName = "Rivill"; - audio->openStream(param, NULL, RTAUDIO_FLOAT32, 44100, &buf, rtaudio_callback, &data); + audio->openStream(param, NULL, RTAUDIO_FLOAT32, 44100, &buf, + rtaudio_callback, &data, options); - data.mixSize = 100; + data.mixSize = 1000; data.nRate = audio->getDeviceInfo(param->deviceId).preferredSampleRate; data.nChannel = param->nChannels; data.ratioSampleFreq = data.nRate/440.0; @@ -102,7 +109,7 @@ AudioManager::AudioManager(){ data.audio = new float[buf + data.mixSize]; data.mix = new float[data.mixSize]; - memset(&data.mix[0], 0, data.mixSize*4); + memset(&data.mix[0], 0, data.mixSize*sizeof(float)); audioValue = new float[buf + data.mixSize]; m_thread = new std::thread(audioThreadFunction, this); @@ -141,6 +148,9 @@ void AudioManager::changeBuf(float* outputBuf, float maxSinValue){ ringbuffer_write(data.audioBuf, constAudioValue, buf + data.mixSize); } + else { + cout<<"AudioManager : Could not write buffer"<<endl; + } } diff --git a/src/audio/AudioManager.hpp b/src/audio/AudioManager.hpp index f67d62a5c41b7ea10ec74dc470a6e0dc7e7c5e58..00a5790383d65d35646719a9a58548fba2a99feb 100644 --- a/src/audio/AudioManager.hpp +++ b/src/audio/AudioManager.hpp @@ -80,6 +80,7 @@ class AudioManager { AudioManager(); float * audioValue; //array to save the audio value before writing const float *constAudioValue; //array to write the audio value in the ringbuffer + RtAudio::StreamOptions *options; }; diff --git a/src/modules/DepthCamModule.cpp b/src/modules/DepthCamModule.cpp index ec9eb294f8eeed7b05f878524333c59e8eeb70cd..f41f8bde190d8300c0dcf1856125ed2a9e441849 100644 --- a/src/modules/DepthCamModule.cpp +++ b/src/modules/DepthCamModule.cpp @@ -417,10 +417,19 @@ void DepthCamModule::draw(const int& contextID, glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); if(m_open) { - glTexImage2D(GL_TEXTURE_2D, 0, GL_R16, - m_width, m_height, 0, - GL_RED, GL_UNSIGNED_SHORT, - (unsigned char*)m_depthFrame.getData()); + if(!m_init) { + glTexImage2D(GL_TEXTURE_2D, 0, GL_R16, + m_width, m_height, 0, + GL_RED, GL_UNSIGNED_SHORT, + (unsigned char*)m_depthFrame.getData()); + m_init=true; + } + else { + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, + m_width, m_height, + GL_RED, GL_UNSIGNED_SHORT, + (unsigned char*)m_depthFrame.getData()); + } } m_camTexUpMap[contextID]=false; } @@ -705,6 +714,7 @@ void DepthCamModule::openDevice(const std::string& devStr, bool calib) { m_colorActivated=false; m_open=true; + m_init=false; m_deviceID=devStr; cout<<"Opened OpenNI device "<<m_deviceID <<" of "<<m_width<<"x"<<m_height diff --git a/src/modules/DepthCamModule.hpp b/src/modules/DepthCamModule.hpp index 9fcae7e5d2e51e307acb169dc925f13a27e2aa45..706df9ea0d1724eb854904cfdf01609783daf9d1 100644 --- a/src/modules/DepthCamModule.hpp +++ b/src/modules/DepthCamModule.hpp @@ -224,6 +224,7 @@ class DepthCamModule : public GroupModule, public DepthModule { private: std::string m_deviceID; bool m_open; + bool m_init; std::vector<std::string> m_modes; std::map<std::string, DepthMode> m_depthModes; diff --git a/src/modules/ProjectorModule.cpp b/src/modules/ProjectorModule.cpp index 3b0bbecc041accbe9c51f555b651e576542ac260..610155cd906250c3e17d316e4f6bd515547c4a82 100644 --- a/src/modules/ProjectorModule.cpp +++ b/src/modules/ProjectorModule.cpp @@ -40,6 +40,7 @@ https://github.com/kylemcdonald/ofxCv #include "RevealedModule.hpp" #include "shaders.h.in" +#include "../audio/AudioManager.hpp" using namespace std; using namespace glm; @@ -55,6 +56,7 @@ ProjectorModule::ProjectorModule(): Module() { m_posX=0; m_posY=0; m_processOutput=true; + m_processAudio=true; m_active=true; m_postFilter=0; m_transparentBackground=false; @@ -128,6 +130,10 @@ ProjectorModule::ProjectorModule(): Module() { Attribute::BOOL_ATTRIBUTE, transparentWindowCallback, this, 1, Attribute::LOCAL)); + addAttribute(new Attribute("output_audio", + Attribute::BOOL_ATTRIBUTE, + outputAudioCallback, this, 1, + Attribute::LOCAL)); //initialize attributes vector<string> labels; @@ -183,6 +189,18 @@ ProjectorModule::ProjectorModule(): Module() { m_patternPosRange = Size(m_width, m_height); m_patternPosOffset = Size(0,0); + + //audio + m_nbTracks=10; + m_maxAudioValue = (std::numeric_limits<int>::max())/(m_width * m_height); + m_audioMixSize = AudioManager::getInstance()->getMixSize(); + m_audioBufSize = AudioManager::getInstance()->getBufferSize() + + m_audioMixSize; + m_imgBufSize = m_audioBufSize+3; + m_audioBuffer = new float[m_audioBufSize]; + memset(&m_audioBuffer[0], 0, m_audioBufSize*sizeof(float)); + + Reveal::getInstance()->addContextHandler(this); m_initialized=false; @@ -312,16 +330,35 @@ void ProjectorModule::draw() { glUniform1i(m_uniforms[Reveal::RENDERPROG][Reveal::SELECTTEX], 1); glBindTexture(GL_TEXTURE_2D, m_selectTex); #ifdef GL43 - glActiveTexture(GL_TEXTURE7); - glUniform1i(m_uniforms[Reveal::RENDERPROG][Reveal::OUTPUTTEX], 7); - glBindImageTexture(7, m_outputTex, 0, GL_FALSE, 0, + //glActiveTexture(GL_TEXTURE7); + glUniform1i(m_uniforms[Reveal::RENDERPROG][Reveal::OUTPUTTEX], 0); + glBindImageTexture(0, m_outputTex, 0, GL_FALSE, 0, GL_READ_WRITE, GL_R32UI); + + glUniform1i(m_uniforms[Reveal::RENDERPROG][Reveal::OUTSHAPESIZE], m_outShapeSize); glUniform1i(m_uniforms[Reveal::RENDERPROG][Reveal::OUTDEPTHSIZE], m_outDepthSize); glUniform1i(m_uniforms[Reveal::RENDERPROG][Reveal::OUTNBDEPTH], m_outNbDepth); + + glUniform1i(m_uniforms[Reveal::RENDERPROG][Reveal::PROCESSAUDIO], + m_processAudio); + glUniform1i(m_uniforms[Reveal::RENDERPROG][Reveal::AUDIOTEX], 1); + glBindImageTexture(1, m_audioTex, 0, GL_FALSE, 0, + GL_READ_WRITE, GL_R32I); + glUniform1i(m_uniforms[Reveal::RENDERPROG][Reveal::AUDIORATE], + AudioManager::getInstance()->getRate()); + glUniform1f(m_uniforms[Reveal::RENDERPROG][Reveal::MAXAUDIOVALUE], + m_maxAudioValue); + glUniform1i(m_uniforms[Reveal::RENDERPROG][Reveal::AUDIOBUFSIZE], + m_audioBufSize); + glUniform1i(m_uniforms[Reveal::RENDERPROG][Reveal::AUDIOMIXSIZE], + m_audioMixSize); + + + #endif for(auto& geom : sceneGeoms) { geom->draw(m_contextHandlerID, @@ -361,6 +398,9 @@ void ProjectorModule::draw() { if(m_processOutput) { processOutput(); } + if(m_processAudio) { + processAudio(); + } #endif } @@ -523,6 +563,33 @@ void ProjectorModule::initGL() { glTexImage2D(GL_TEXTURE_2D, 0, GL_R32UI, m_outputImgSize, m_outputImgSize, 0, GL_RED_INTEGER, GL_UNSIGNED_INT, m_outputImgInit); + + //Audio texture + m_audioImg = new int[m_imgBufSize*m_nbTracks]; + memset(&m_audioImg[0], 0, m_imgBufSize*m_nbTracks*sizeof(int)); + m_audioImgInit = new int[2*m_nbTracks]; + memset(&m_audioImgInit[0], 0, 2*m_nbTracks*sizeof(int)); + glGenTextures(1, &m_audioTex); + glBindTexture(GL_TEXTURE_2D, m_audioTex); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glTexImage2D(GL_TEXTURE_2D, 0, GL_R32I, + m_imgBufSize, m_nbTracks, 0, + GL_RED_INTEGER, GL_INT, m_audioImg); + + m_uniforms[Reveal::RENDERPROG][Reveal::PROCESSAUDIO] + = glGetUniformLocation(m_programs[Reveal::RENDERPROG], "processAudio"); + m_uniforms[Reveal::RENDERPROG][Reveal::AUDIOTEX] + = glGetUniformLocation(m_programs[Reveal::RENDERPROG], "audioTex"); + m_uniforms[Reveal::RENDERPROG][Reveal::AUDIOMIXSIZE] + = glGetUniformLocation(m_programs[Reveal::RENDERPROG], "audioMixSize"); + m_uniforms[Reveal::RENDERPROG][Reveal::AUDIOBUFSIZE] + = glGetUniformLocation(m_programs[Reveal::RENDERPROG], "audioBufSize"); + m_uniforms[Reveal::RENDERPROG][Reveal::MAXAUDIOVALUE] + = glGetUniformLocation(m_programs[Reveal::RENDERPROG], "maxAudioValue"); + m_uniforms[Reveal::RENDERPROG][Reveal::AUDIORATE] + = glGetUniformLocation(m_programs[Reveal::RENDERPROG], "audioRate"); + m_uniforms[Reveal::RENDERPROG][Reveal::OUTPUTTEX] = glGetUniformLocation(m_programs[Reveal::RENDERPROG], "outputTex"); m_uniforms[Reveal::RENDERPROG][Reveal::REVSIZE] @@ -850,6 +917,10 @@ void ProjectorModule::outputRevealed(bool output) { m_processOutput=output; } +void ProjectorModule::outputAudio(bool audio) { + m_processAudio=audio; +} + void ProjectorModule::postFilter(const int& filt) { m_postFilter=filt; } @@ -870,9 +941,9 @@ void ProjectorModule::processOutput() { if(revModules.size()>0 || camModules.size()>0) { //retrieve output image glBindTexture(GL_TEXTURE_2D, m_outputTex); - glGetTexImage(GL_TEXTURE_2D, 0, GL_RGBA, - GL_UNSIGNED_BYTE, - m_outputImg); + glGetTexImage(GL_TEXTURE_2D, 0, GL_RED_INTEGER, + GL_INT, + m_outputImg); } @@ -925,10 +996,67 @@ void ProjectorModule::processOutput() { glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, + m_outputImgSize, m_outputImgSize, + GL_RED_INTEGER, GL_UNSIGNED_INT, m_outputImgInit); + /* glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, m_outputImgSize, m_outputImgSize, 0, GL_RGBA, GL_UNSIGNED_BYTE, m_outputImgInit); + */ + + } +} + +void ProjectorModule::processAudio() { + glBindTexture(GL_TEXTURE_2D, m_audioTex); + glGetTexImage(GL_TEXTURE_2D, 0, GL_RED_INTEGER, + GL_INT, + m_audioImg); + + //FIXME mix in a compute shader + + float totalGains = 0; + for(int i = 0; i<m_nbTracks; i++) { + //check if the note has to be played this frame + float trackGain = m_audioImg[m_imgBufSize * i]; + if(trackGain>0) { + totalGains += trackGain; + //compute the new phase for the note i + //m_phase[i] = (m_phase[i] + bufSize - 2 - mixSize) % m_outputAudio[ (bufSize*i) + 1]; + //convert the audio value from int to float and save it in outputAudio + for(int j=0; j<m_audioBufSize; j++){ + m_audioBuffer[j] += trackGain * m_audioImg[(m_imgBufSize*i)+3+j] + / m_maxAudioValue; + } + cout<<"read phase "<<i<<" "<<m_audioImg[m_imgBufSize*i+1]<<endl; + cout<<"write phase "<<i<<" "<<m_audioImg[m_imgBufSize*i+2]<<endl; + cout<<"nb frags = "<<m_audioImg[m_imgBufSize*i]<<endl; + m_audioImgInit[2*i+1] = m_audioImg[m_imgBufSize*i+2]; + } } + + + float gain = 0.8; + if(totalGains > 0){ + for(int i=0; i<m_audioBufSize; i++) { + m_audioBuffer[i] = gain*m_audioBuffer[i]/totalGains; + } + AudioManager::getInstance()->changeBuf(m_audioBuffer, m_maxAudioValue); + memset(&m_audioBuffer[0], 0, (m_audioBufSize)*sizeof(float)); + } + else{ + memset(&m_audioBuffer[0], 0, (m_audioBufSize)*sizeof(float)); + AudioManager::getInstance()->changeBuf(m_audioBuffer, m_maxAudioValue); + } + + //reset the first column of audio values + glBindTexture(GL_TEXTURE_2D, m_audioTex); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, 2, m_nbTracks, + GL_RED_INTEGER, GL_INT, &m_audioImgInit[0]); + } void ProjectorModule::updateWindowSize(const int& w, const int& h) { diff --git a/src/modules/ProjectorModule.hpp b/src/modules/ProjectorModule.hpp index 3cbdf06447ec37235a3f72872b0bd667145e1fe9..3c3e2a26ad4b9b30d9d8fabe4d32e8793d3eab53 100644 --- a/src/modules/ProjectorModule.hpp +++ b/src/modules/ProjectorModule.hpp @@ -1,6 +1,6 @@ /*************************************************************************** * ProjectorModule.hpp - * Part of Reveal + * Part of Rivill * 2015- Florent Berthaut * hitmuri.net ****************************************************************************/ @@ -111,6 +111,11 @@ class ProjectorModule : public Module, public ContextHandler { static_cast<ProjectorModule*>(mod)->outputRevealed(vals[0]); } void outputRevealed(bool output); + static void outputAudioCallback(Module* mod, + const std::vector<bool>& vals) { + static_cast<ProjectorModule*>(mod)->outputAudio(vals[0]); + } + void outputAudio(bool audio); static void mirroredCallback(Module* mod, const std::vector<bool>& vals) { static_cast<ProjectorModule*>(mod)->setMirrored(vals[0]); @@ -153,6 +158,7 @@ class ProjectorModule : public Module, public ContextHandler { void refreshDepthCamList(const std::vector<DepthCamModule*>); void processOutput(); + void processAudio(); private: void initGL(); @@ -183,6 +189,7 @@ class ProjectorModule : public Module, public ContextHandler { float m_posX, m_posY; int m_rttWidth, m_rttHeight; bool m_processOutput; + bool m_processAudio; float m_postFilter; bool m_transparentBackground; @@ -220,6 +227,20 @@ class ProjectorModule : public Module, public ContextHandler { int m_outHistoSize; int m_outDepthSize; int m_outNbDepth; + + //audio output + bool m_audioActive; + GLuint m_audioTex; + int* m_audioImg; + int* m_audioImgInit; + int m_nbTracks; + int m_audioBufSize; + int m_audioMixSize; + int m_imgBufSize; + float* m_audioBuffer; + float m_maxAudioValue; + + }; diff --git a/src/shaders/render43.fs b/src/shaders/render43.fs index f1eb72e53b6e2032b39685ac2b1af2dc2c4b1645..2127e7609cea35dfa4b481576ebd06dc948f4a64 100644 --- a/src/shaders/render43.fs +++ b/src/shaders/render43.fs @@ -1,4 +1,5 @@ #version 430 +#define M_PI 3.1415926535897932384626433832795 const int BOX=0; const int SPHERE=1; @@ -18,15 +19,15 @@ layout(binding=3) uniform sampler2D surfaceTex; layout(binding=4) uniform sampler3D reactTex; layout(binding=5) uniform sampler3D surfDistTex; layout(binding=6) uniform sampler2D gradientTex; + layout(r32ui, binding=0) uniform coherent uimage2D outputTex; -layout(r32i, binding=1) uniform coherent iimage2D outputAudioTex; -layout(r32i, binding=2) uniform coherent iimage2D phases; +layout(r32i, binding=1) uniform coherent iimage2D audioTex; -uniform int mixSize; -uniform float maxSinValue; -uniform int nRate; -uniform float phase; -uniform int bufSize; +uniform int audioMixSize; +uniform int audioBufSize; +uniform float maxAudioValue; +uniform int audioRate; +uniform int processAudio; uniform float viewportWidth; uniform float viewportHeight; @@ -107,17 +108,14 @@ float random (vec3 st){ out vec4 color; -void additiveSynth(vec4 color){ - int freq; - int i = 2; - float note = 20 + color.x * 50;//color.x * 127; +void additiveSynth(vec4 color) { + float freq=0.0; + //float note = 20 + color.x * 50;//color.x * 127; + float note = 20.0; + int ind=0; + int octave = int(note/12); float modNote = int(note) % 12; - int ratioSampleFreq; - float ratioFreqSample; - int initPhase; - int fragPhase; - int ind; if(modNote >= 8.5){ note = octave * 12 + 10; @@ -136,47 +134,43 @@ void additiveSynth(vec4 color){ ind = 4 * octave; } - /*int n = int(color.x * 3); - int note = 0; - if(n == 0){ - note = 40; - } - else if(n == 1){ - note = 44; - } - else if(n == 2){ - note = 47; - } - else{ - note = 50; - } - ind = n;*/ - freq = int(440 * pow(2, (note-69)/12)); + freq = float(440 * pow(2, (note-69)/12)); - ratioSampleFreq = nRate/freq; - ratioFreqSample = 1.0/ratioSampleFreq; - fragPhase = imageLoad(phases, ivec2(ind, 0)).x; + //retrieve phase stored from previous frame + float phase = float(imageLoad(audioTex, ivec2(1, ind)).x) + / maxAudioValue; + float samplesPerPeriod = audioRate/freq; + float phaseStep = 1.0/samplesPerPeriod; + for(int i=0; i<audioBufSize; i++) { + //compute value + float s = sin(phase * 2.0 * M_PI * freq); - float a = sin(fragPhase *(3.14f) * freq/nRate); + //write starting from 3rd column + imageStore(audioTex, ivec2(i+3, ind), ivec4(int(s*maxAudioValue))); + phase=mod(phase + phaseStep, 1.0); + } + /* + float a = 2.0*sin(phase*freq/float(audioRate)); float s[2]; - s[0] = 1.0f; s[1] = 0.0f; - imageStore(outputAudioTex, ivec2(0, ind), ivec4(42)); - - while (i < bufSize) { - imageStore(outputAudioTex, ivec2(i, ind), ivec4(int(s[1]*maxSinValue))); - + int i = 2; + while (i < audioBufSize) { + imageStore(audioTex, ivec2(i, ind), ivec4(int(s[1]*maxAudioValue))); s[0] = s[0] - a*s[1]; s[1] = s[1] + a*s[0]; - i += 1; - } + }*/ - imageStore(outputAudioTex, ivec2(1, ind), ivec4(ratioSampleFreq)); + //increase phase by (audioBufSize-audioMixSize) steps + imageStore(audioTex, ivec2(2, ind), + ivec4(int(phase*maxAudioValue))); + + //count this fragment + imageAtomicAdd(audioTex, ivec2(0, ind), 1); } void main(void) { @@ -392,19 +386,6 @@ void main(void) { } break; } } - /*imageAtomicAdd(outputAudioTex, ivec2(0,0), 1); - int i = 1; - float ratioSampleFreq = nRate/700.0; - float ratioFreqSample = 1.0/ratioSampleFreq; - int fragPhase = int(phase); - while (i < bufSize) { - int val = int(cos(fragPhase * ratioFreqSample * (2 * 3.14))*maxSinValue); - imageAtomicAdd(outputAudioTex, ivec2(i,0), val); - //imageAtomicAdd(outputAudioTex, ivec2(i+1,0),val); - fragPhase += 1; - fragPhase = fragPhase % int(ratioSampleFreq); - i += 1; - }*/ //texture coordinates vec3 texCoords = vec3(1-ratioWithinBox.x, 1-ratioWithinBox.y, ratioWithinBox.z); @@ -544,7 +525,9 @@ void main(void) { if((insideVisible>0 && rendered==1) || (surface>0 && rendered==2)) { color = vec4(finalColor, 1.0); - additiveSynth(color); + if(processAudio>0) { + additiveSynth(color); + } } else { color = vec4(0.0, 0.0, 0.0, 1.0);