From 598c0c4ae7137718a65afed676fa715203329aec Mon Sep 17 00:00:00 2001 From: scrawl Date: Tue, 29 Jul 2014 00:26:26 +0200 Subject: [PATCH] Implement mouth animation for NPCs based on say sound (Fixes #642) --- apps/openmw/CMakeLists.txt | 2 +- apps/openmw/mwbase/soundmanager.hpp | 5 ++ apps/openmw/mwrender/npcanimation.cpp | 47 +++++++++++-- apps/openmw/mwrender/npcanimation.hpp | 11 ++- apps/openmw/mwsound/loudness.cpp | 53 ++++++++++++++ apps/openmw/mwsound/loudness.hpp | 20 ++++++ apps/openmw/mwsound/openal_output.cpp | 94 +++++++++++++++---------- apps/openmw/mwsound/openal_output.hpp | 4 +- apps/openmw/mwsound/sound.cpp | 23 ++++++ apps/openmw/mwsound/sound.hpp | 10 +++ apps/openmw/mwsound/sound_output.hpp | 2 +- apps/openmw/mwsound/soundmanagerimp.cpp | 17 ++++- apps/openmw/mwsound/soundmanagerimp.hpp | 5 ++ 13 files changed, 245 insertions(+), 48 deletions(-) create mode 100644 apps/openmw/mwsound/loudness.cpp create mode 100644 apps/openmw/mwsound/loudness.hpp create mode 100644 apps/openmw/mwsound/sound.cpp diff --git a/apps/openmw/CMakeLists.txt b/apps/openmw/CMakeLists.txt index 0dda1131be..25cf48b09e 100644 --- a/apps/openmw/CMakeLists.txt +++ b/apps/openmw/CMakeLists.txt @@ -48,7 +48,7 @@ add_openmw_dir (mwscript ) add_openmw_dir (mwsound - soundmanagerimp openal_output ffmpeg_decoder sound + soundmanagerimp openal_output ffmpeg_decoder sound sound_decoder sound_output loudness ) add_openmw_dir (mwworld diff --git a/apps/openmw/mwbase/soundmanager.hpp b/apps/openmw/mwbase/soundmanager.hpp index 15739730ba..a02a463dde 100644 --- a/apps/openmw/mwbase/soundmanager.hpp +++ b/apps/openmw/mwbase/soundmanager.hpp @@ -101,6 +101,11 @@ namespace MWBase virtual void stopSay(const MWWorld::Ptr &reference=MWWorld::Ptr()) = 0; ///< Stop an actor speaking + virtual float getSaySoundLoudness(const MWWorld::Ptr& reference) const = 0; + ///< Check the currently playing say sound for this actor + /// and get an average loudness value (scale [0,1]) at the current time position. + /// If the actor is not saying anything, returns 0. + virtual SoundPtr playTrack(const MWSound::DecoderPtr& decoder, PlayType type) = 0; ///< Play a 2D audio track, using a custom decoder diff --git a/apps/openmw/mwrender/npcanimation.cpp b/apps/openmw/mwrender/npcanimation.cpp index be2b262fc2..597d0c2dfa 100644 --- a/apps/openmw/mwrender/npcanimation.cpp +++ b/apps/openmw/mwrender/npcanimation.cpp @@ -66,15 +66,40 @@ std::string getVampireHead(const std::string& race, bool female) namespace MWRender { +HeadAnimationTime::HeadAnimationTime(MWWorld::Ptr reference) + : mReference(reference), mTalkStart(0), mTalkStop(0), mBlinkStart(0), mBlinkStop(0) +{ +} + float HeadAnimationTime::getValue() const { - // TODO use time from text keys (Talk Start/Stop, Blink Start/Stop) // TODO: Handle eye blinking if (MWBase::Environment::get().getSoundManager()->sayDone(mReference)) - return 0; + return mBlinkStop; else - // TODO: Use the loudness of the currently playing sound - return 1; + return mTalkStart + + (mTalkStop - mTalkStart) * + std::min(1.f, MWBase::Environment::get().getSoundManager()->getSaySoundLoudness(mReference)*4); // Rescale a bit (most voices are not very loud) +} + +void HeadAnimationTime::setTalkStart(float value) +{ + mTalkStart = value; +} + +void HeadAnimationTime::setTalkStop(float value) +{ + mTalkStop = value; +} + +void HeadAnimationTime::setBlinkStart(float value) +{ + mBlinkStart = value; +} + +void HeadAnimationTime::setBlinkStop(float value) +{ + mBlinkStop = value; } static NpcAnimation::PartBoneMap createPartListMap() @@ -620,7 +645,21 @@ bool NpcAnimation::addOrReplaceIndividualPart(ESM::PartReferenceType type, int g ctrl->setSource(mNullAnimationTimePtr); if (type == ESM::PRT_Head) + { ctrl->setSource(mHeadAnimationTime); + const NifOgre::TextKeyMap& keys = mObjectParts[type]->mTextKeys; + for (NifOgre::TextKeyMap::const_iterator it = keys.begin(); it != keys.end(); ++it) + { + if (Misc::StringUtils::ciEqual(it->second, "talk: start")) + mHeadAnimationTime->setTalkStart(it->first); + if (Misc::StringUtils::ciEqual(it->second, "talk: stop")) + mHeadAnimationTime->setTalkStop(it->first); + if (Misc::StringUtils::ciEqual(it->second, "blink: start")) + mHeadAnimationTime->setBlinkStart(it->first); + if (Misc::StringUtils::ciEqual(it->second, "blink: stop")) + mHeadAnimationTime->setBlinkStop(it->first); + } + } else if (type == ESM::PRT_Weapon) ctrl->setSource(mWeaponAnimationTime); } diff --git a/apps/openmw/mwrender/npcanimation.hpp b/apps/openmw/mwrender/npcanimation.hpp index 8ec46facda..057f67e2f3 100644 --- a/apps/openmw/mwrender/npcanimation.hpp +++ b/apps/openmw/mwrender/npcanimation.hpp @@ -19,8 +19,17 @@ class HeadAnimationTime : public Ogre::ControllerValue { private: MWWorld::Ptr mReference; + float mTalkStart; + float mTalkStop; + float mBlinkStart; + float mBlinkStop; public: - HeadAnimationTime(MWWorld::Ptr reference) : mReference(reference) {} + HeadAnimationTime(MWWorld::Ptr reference); + + void setTalkStart(float value); + void setTalkStop(float value); + void setBlinkStart(float value); + void setBlinkStop(float value); virtual Ogre::Real getValue() const; virtual void setValue(Ogre::Real value) diff --git a/apps/openmw/mwsound/loudness.cpp b/apps/openmw/mwsound/loudness.cpp new file mode 100644 index 0000000000..88c706a91f --- /dev/null +++ b/apps/openmw/mwsound/loudness.cpp @@ -0,0 +1,53 @@ +#include "loudness.hpp" + +#include "soundmanagerimp.hpp" + +namespace MWSound +{ + + void analyzeLoudness(const std::vector &data, int sampleRate, ChannelConfig chans, + SampleType type, std::vector &out, float valuesPerSecond) + { + int samplesPerSegment = sampleRate / valuesPerSecond; + int numSamples = bytesToFrames(data.size(), chans, type); + int advance = framesToBytes(1, chans, type); + + out.reserve(numSamples/samplesPerSegment); + + int segment=0; + int sample=0; + while (segment < numSamples/samplesPerSegment) + { + float sum=0; + int samplesAdded = 0; + while (sample < numSamples && sample < (segment+1)*samplesPerSegment) + { + // get sample on a scale from -1 to 1 + float value = 0; + if (type == SampleType_UInt8) + value = data[sample*advance]/128.f; + else if (type == SampleType_Int16) + { + value = *reinterpret_cast(&data[sample*advance]); + value /= float(std::numeric_limits().max()); + } + else if (type == SampleType_Float32) + { + value = *reinterpret_cast(&data[sample*advance]); + value /= std::numeric_limits().max(); + } + + sum += value*value; + ++samplesAdded; + ++sample; + } + + float rms = 0; // root mean square + if (samplesAdded > 0) + rms = std::sqrt(sum / samplesAdded); + out.push_back(rms); + ++segment; + } + } + +} diff --git a/apps/openmw/mwsound/loudness.hpp b/apps/openmw/mwsound/loudness.hpp new file mode 100644 index 0000000000..df727bd0b2 --- /dev/null +++ b/apps/openmw/mwsound/loudness.hpp @@ -0,0 +1,20 @@ +#include "sound_decoder.hpp" + +namespace MWSound +{ + +/** + * Analyzes the energy (closely related to loudness) of a sound buffer. + * The buffer will be divided into segments according to \a valuesPerSecond, + * and for each segment a loudness value in the range of [0,1] will be computed. + * @param data the sound buffer to analyze, containing raw samples + * @param sampleRate the sample rate of the sound buffer + * @param chans channel layout of the buffer + * @param type sample type of the buffer + * @param out Will contain the output loudness values. + * @param valuesPerSecond How many loudness values per second of audio to compute. + */ +void analyzeLoudness (const std::vector& data, int sampleRate, ChannelConfig chans, SampleType type, + std::vector& out, float valuesPerSecond); + +} diff --git a/apps/openmw/mwsound/openal_output.cpp b/apps/openmw/mwsound/openal_output.cpp index b245b92414..b20bd4f7e9 100644 --- a/apps/openmw/mwsound/openal_output.cpp +++ b/apps/openmw/mwsound/openal_output.cpp @@ -11,11 +11,16 @@ #include "sound_decoder.hpp" #include "sound.hpp" #include "soundmanagerimp.hpp" +#include "loudness.hpp" #ifndef ALC_ALL_DEVICES_SPECIFIER #define ALC_ALL_DEVICES_SPECIFIER 0x1013 #endif +namespace +{ + const int loudnessFPS = 20; // loudness values per second of audio +} namespace MWSound { @@ -750,7 +755,7 @@ void OpenAL_Output::deinit() } -ALuint OpenAL_Output::getBuffer(const std::string &fname) +ALuint OpenAL_Output::getBuffer(const std::string &fname, std::vector* loudnessBuffer) { ALuint buf = 0; @@ -765,11 +770,12 @@ ALuint OpenAL_Output::getBuffer(const std::string &fname) if(iter != mUnusedBuffers.end()) mUnusedBuffers.erase(iter); } - - return buf; } throwALerror(); + if (buf != 0 && loudnessBuffer == NULL) + return buf; + std::vector data; ChannelConfig chans; SampleType type; @@ -795,42 +801,50 @@ ALuint OpenAL_Output::getBuffer(const std::string &fname) decoder->readAll(data); decoder->close(); - alGenBuffers(1, &buf); - throwALerror(); - - alBufferData(buf, format, &data[0], data.size(), srate); - mBufferCache[fname] = buf; - mBufferRefs[buf] = 1; - - ALint bufsize = 0; - alGetBufferi(buf, AL_SIZE, &bufsize); - mBufferCacheMemSize += bufsize; - - // NOTE: Max buffer cache: 15MB - while(mBufferCacheMemSize > 15*1024*1024) + if (loudnessBuffer != NULL) { - if(mUnusedBuffers.empty()) + analyzeLoudness(data, srate, chans, type, *loudnessBuffer, loudnessFPS); + } + + if (buf == 0) + { + alGenBuffers(1, &buf); + throwALerror(); + + alBufferData(buf, format, &data[0], data.size(), srate); + mBufferCache[fname] = buf; + mBufferRefs[buf] = 1; + + ALint bufsize = 0; + alGetBufferi(buf, AL_SIZE, &bufsize); + mBufferCacheMemSize += bufsize; + + // NOTE: Max buffer cache: 15MB + while(mBufferCacheMemSize > 15*1024*1024) { - std::cout <<"No more unused buffers to clear!"<< std::endl; - break; + if(mUnusedBuffers.empty()) + { + std::cout <<"No more unused buffers to clear!"<< std::endl; + break; + } + + ALuint oldbuf = mUnusedBuffers.front(); + mUnusedBuffers.pop_front(); + + NameMap::iterator nameiter = mBufferCache.begin(); + while(nameiter != mBufferCache.end()) + { + if(nameiter->second == oldbuf) + mBufferCache.erase(nameiter++); + else + ++nameiter; + } + + bufsize = 0; + alGetBufferi(oldbuf, AL_SIZE, &bufsize); + alDeleteBuffers(1, &oldbuf); + mBufferCacheMemSize -= bufsize; } - - ALuint oldbuf = mUnusedBuffers.front(); - mUnusedBuffers.pop_front(); - - NameMap::iterator nameiter = mBufferCache.begin(); - while(nameiter != mBufferCache.end()) - { - if(nameiter->second == oldbuf) - mBufferCache.erase(nameiter++); - else - ++nameiter; - } - - bufsize = 0; - alGetBufferi(oldbuf, AL_SIZE, &bufsize); - alDeleteBuffers(1, &oldbuf); - mBufferCacheMemSize -= bufsize; } return buf; } @@ -883,7 +897,7 @@ MWBase::SoundPtr OpenAL_Output::playSound(const std::string &fname, float vol, f } MWBase::SoundPtr OpenAL_Output::playSound3D(const std::string &fname, const Ogre::Vector3 &pos, float vol, float basevol, float pitch, - float min, float max, int flags, float offset) + float min, float max, int flags, float offset, bool extractLoudness) { boost::shared_ptr sound; ALuint src=0, buf=0; @@ -895,8 +909,12 @@ MWBase::SoundPtr OpenAL_Output::playSound3D(const std::string &fname, const Ogre try { - buf = getBuffer(fname); + std::vector loudnessVector; + + buf = getBuffer(fname, extractLoudness ? &loudnessVector : NULL); + sound.reset(new OpenAL_Sound3D(*this, src, buf, pos, vol, basevol, pitch, min, max, flags)); + sound->setLoudnessVector(loudnessVector, loudnessFPS); } catch(std::exception&) { diff --git a/apps/openmw/mwsound/openal_output.hpp b/apps/openmw/mwsound/openal_output.hpp index 31edf73599..f96c588cf3 100644 --- a/apps/openmw/mwsound/openal_output.hpp +++ b/apps/openmw/mwsound/openal_output.hpp @@ -36,7 +36,7 @@ namespace MWSound typedef std::vector SoundVec; SoundVec mActiveSounds; - ALuint getBuffer(const std::string &fname); + ALuint getBuffer(const std::string &fname, std::vector* loudnessBuffer=NULL); void bufferFinished(ALuint buffer); Environment mLastEnvironment; @@ -49,7 +49,7 @@ namespace MWSound virtual MWBase::SoundPtr playSound(const std::string &fname, float vol, float basevol, float pitch, int flags, float offset); /// @param offset Value from [0,1] meaning from which fraction the sound the playback starts. virtual MWBase::SoundPtr playSound3D(const std::string &fname, const Ogre::Vector3 &pos, - float vol, float basevol, float pitch, float min, float max, int flags, float offset); + float vol, float basevol, float pitch, float min, float max, int flags, float offset, bool extractLoudness=false); virtual MWBase::SoundPtr streamSound(DecoderPtr decoder, float volume, float pitch, int flags); virtual void updateListener(const Ogre::Vector3 &pos, const Ogre::Vector3 &atdir, const Ogre::Vector3 &updir, Environment env); diff --git a/apps/openmw/mwsound/sound.cpp b/apps/openmw/mwsound/sound.cpp new file mode 100644 index 0000000000..b3105a82c2 --- /dev/null +++ b/apps/openmw/mwsound/sound.cpp @@ -0,0 +1,23 @@ +#include "sound.hpp" + +namespace MWSound +{ + + float Sound::getCurrentLoudness() + { + if (mLoudnessVector.empty()) + return 0.f; + int index = getTimeOffset() * mLoudnessFPS; + + index = std::max(0, std::min(index, int(mLoudnessVector.size()-1))); + + return mLoudnessVector[index]; + } + + void Sound::setLoudnessVector(const std::vector &loudnessVector, float loudnessFPS) + { + mLoudnessVector = loudnessVector; + mLoudnessFPS = loudnessFPS; + } + +} diff --git a/apps/openmw/mwsound/sound.hpp b/apps/openmw/mwsound/sound.hpp index 670002a30f..1b5c001966 100644 --- a/apps/openmw/mwsound/sound.hpp +++ b/apps/openmw/mwsound/sound.hpp @@ -24,6 +24,9 @@ namespace MWSound int mFlags; float mFadeOutTime; + std::vector mLoudnessVector; + float mLoudnessFPS; + public: virtual void stop() = 0; virtual bool isPlaying() = 0; @@ -31,6 +34,12 @@ namespace MWSound void setPosition(const Ogre::Vector3 &pos) { mPos = pos; } void setVolume(float volume) { mVolume = volume; } void setFadeout(float duration) { mFadeOutTime=duration; } + void setLoudnessVector(const std::vector& loudnessVector, float loudnessFPS); + + /// Get loudness at the current time position on a [0,1] scale. + /// Requires that loudnessVector was filled in by the user. + float getCurrentLoudness(); + MWBase::SoundManager::PlayType getPlayType() const { return (MWBase::SoundManager::PlayType)(mFlags&MWBase::SoundManager::Play_TypeMask); } @@ -44,6 +53,7 @@ namespace MWSound , mMaxDistance(maxdist) , mFlags(flags) , mFadeOutTime(0) + , mLoudnessFPS(20) { } virtual ~Sound() { } diff --git a/apps/openmw/mwsound/sound_output.hpp b/apps/openmw/mwsound/sound_output.hpp index 91e25db521..a9a999a5cb 100644 --- a/apps/openmw/mwsound/sound_output.hpp +++ b/apps/openmw/mwsound/sound_output.hpp @@ -28,7 +28,7 @@ namespace MWSound virtual MWBase::SoundPtr playSound(const std::string &fname, float vol, float basevol, float pitch, int flags, float offset) = 0; /// @param offset Value from [0,1] meaning from which fraction the sound the playback starts. virtual MWBase::SoundPtr playSound3D(const std::string &fname, const Ogre::Vector3 &pos, - float vol, float basevol, float pitch, float min, float max, int flags, float offset) = 0; + float vol, float basevol, float pitch, float min, float max, int flags, float offset, bool extractLoudness=false) = 0; virtual MWBase::SoundPtr streamSound(DecoderPtr decoder, float volume, float pitch, int flags) = 0; virtual void updateListener(const Ogre::Vector3 &pos, const Ogre::Vector3 &atdir, const Ogre::Vector3 &updir, Environment env) = 0; diff --git a/apps/openmw/mwsound/soundmanagerimp.cpp b/apps/openmw/mwsound/soundmanagerimp.cpp index ba7b4f3bab..fced81f7be 100644 --- a/apps/openmw/mwsound/soundmanagerimp.cpp +++ b/apps/openmw/mwsound/soundmanagerimp.cpp @@ -256,7 +256,7 @@ namespace MWSound const Ogre::Vector3 objpos(pos.pos); MWBase::SoundPtr sound = mOutput->playSound3D(filePath, objpos, 1.0f, basevol, 1.0f, - 20.0f, 1500.0f, Play_Normal|Play_TypeVoice, 0); + 20.0f, 1500.0f, Play_Normal|Play_TypeVoice, 0, true); mActiveSounds[sound] = std::make_pair(ptr, std::string("_say_sound")); } catch(std::exception &e) @@ -265,6 +265,21 @@ namespace MWSound } } + float SoundManager::getSaySoundLoudness(const MWWorld::Ptr &ptr) const + { + SoundMap::const_iterator snditer = mActiveSounds.begin(); + while(snditer != mActiveSounds.end()) + { + if(snditer->second.first == ptr && snditer->second.second == "_say_sound") + break; + ++snditer; + } + if (snditer == mActiveSounds.end()) + return 0.f; + + return snditer->first->getCurrentLoudness(); + } + void SoundManager::say(const std::string& filename) { if(!mOutput->isInitialized()) diff --git a/apps/openmw/mwsound/soundmanagerimp.hpp b/apps/openmw/mwsound/soundmanagerimp.hpp index 380cfe2552..250cb0d51c 100644 --- a/apps/openmw/mwsound/soundmanagerimp.hpp +++ b/apps/openmw/mwsound/soundmanagerimp.hpp @@ -110,6 +110,11 @@ namespace MWSound virtual void stopSay(const MWWorld::Ptr &reference=MWWorld::Ptr()); ///< Stop an actor speaking + virtual float getSaySoundLoudness(const MWWorld::Ptr& reference) const; + ///< Check the currently playing say sound for this actor + /// and get an average loudness value (scale [0,1]) at the current time position. + /// If the actor is not saying anything, returns 0. + virtual MWBase::SoundPtr playTrack(const DecoderPtr& decoder, PlayType type); ///< Play a 2D audio track, using a custom decoder