mirror/OpenMW
1
0
Fork 0
mirror of https://gitlab.com/OpenMW/openmw.git synced 2025-01-05 15:55:45 +00:00
Code Issues Projects Releases Wiki Activity

Implement mouth animation for NPCs based on say sound (Fixes #642)

Browse Source
This commit is contained in:
scrawl 2014-07-29 00:26:26 +02:00
parent 7dfb624ee2
commit 598c0c4ae7
13 changed files with 245 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/openmw/CMakeLists.txt
View File

@ -48,7 +48,7 @@ add_openmw_dir (mwscript
)
add_openmw_dir (mwsound
soundmanagerimp openal_output ffmpeg_decoder sound
soundmanagerimp openal_output ffmpeg_decoder sound sound_decoder sound_output loudness
)
add_openmw_dir (mwworld

5
apps/openmw/mwbase/soundmanager.hpp
View File

@ -101,6 +101,11 @@ namespace MWBase
virtual void stopSay(const MWWorld::Ptr &reference=MWWorld::Ptr()) = 0;
///< Stop an actor speaking
virtual float getSaySoundLoudness(const MWWorld::Ptr& reference) const = 0;
///< Check the currently playing say sound for this actor
/// and get an average loudness value (scale [0,1]) at the current time position.
/// If the actor is not saying anything, returns 0.
virtual SoundPtr playTrack(const MWSound::DecoderPtr& decoder, PlayType type) = 0;
///< Play a 2D audio track, using a custom decoder

47
apps/openmw/mwrender/npcanimation.cpp
View File

@ -66,15 +66,40 @@ std::string getVampireHead(const std::string& race, bool female)
namespace MWRender
{
HeadAnimationTime::HeadAnimationTime(MWWorld::Ptr reference)
: mReference(reference), mTalkStart(0), mTalkStop(0), mBlinkStart(0), mBlinkStop(0)
{
}
float HeadAnimationTime::getValue() const
{
// TODO use time from text keys (Talk Start/Stop, Blink Start/Stop)
// TODO: Handle eye blinking
if (MWBase::Environment::get().getSoundManager()->sayDone(mReference))
return 0;
return mBlinkStop;
else
// TODO: Use the loudness of the currently playing sound
return 1;
return mTalkStart +
(mTalkStop - mTalkStart) *
std::min(1.f, MWBase::Environment::get().getSoundManager()->getSaySoundLoudness(mReference)*4); // Rescale a bit (most voices are not very loud)
}
void HeadAnimationTime::setTalkStart(float value)
{
mTalkStart = value;
}
void HeadAnimationTime::setTalkStop(float value)
{
mTalkStop = value;
}
void HeadAnimationTime::setBlinkStart(float value)
{
mBlinkStart = value;
}
void HeadAnimationTime::setBlinkStop(float value)
{
mBlinkStop = value;
}
static NpcAnimation::PartBoneMap createPartListMap()
@ -620,7 +645,21 @@ bool NpcAnimation::addOrReplaceIndividualPart(ESM::PartReferenceType type, int g
ctrl->setSource(mNullAnimationTimePtr);
if (type == ESM::PRT_Head)
{
ctrl->setSource(mHeadAnimationTime);
const NifOgre::TextKeyMap& keys = mObjectParts[type]->mTextKeys;
for (NifOgre::TextKeyMap::const_iterator it = keys.begin(); it != keys.end(); ++it)
{
if (Misc::StringUtils::ciEqual(it->second, "talk: start"))
mHeadAnimationTime->setTalkStart(it->first);
if (Misc::StringUtils::ciEqual(it->second, "talk: stop"))
mHeadAnimationTime->setTalkStop(it->first);
if (Misc::StringUtils::ciEqual(it->second, "blink: start"))
mHeadAnimationTime->setBlinkStart(it->first);
if (Misc::StringUtils::ciEqual(it->second, "blink: stop"))
mHeadAnimationTime->setBlinkStop(it->first);
}
}
else if (type == ESM::PRT_Weapon)
ctrl->setSource(mWeaponAnimationTime);
}

11
apps/openmw/mwrender/npcanimation.hpp
View File

@ -19,8 +19,17 @@ class HeadAnimationTime : public Ogre::ControllerValue<Ogre::Real>
{
private:
MWWorld::Ptr mReference;
float mTalkStart;
float mTalkStop;
float mBlinkStart;
float mBlinkStop;
public:
HeadAnimationTime(MWWorld::Ptr reference) : mReference(reference) {}
HeadAnimationTime(MWWorld::Ptr reference);
void setTalkStart(float value);
void setTalkStop(float value);
void setBlinkStart(float value);
void setBlinkStop(float value);
virtual Ogre::Real getValue() const;
virtual void setValue(Ogre::Real value)

53
apps/openmw/mwsound/loudness.cpp Normal file
View File

@ -0,0 +1,53 @@
#include "loudness.hpp"
#include "soundmanagerimp.hpp"
namespace MWSound
{
void analyzeLoudness(const std::vector<char> &data, int sampleRate, ChannelConfig chans,
SampleType type, std::vector<float> &out, float valuesPerSecond)
{
int samplesPerSegment = sampleRate / valuesPerSecond;
int numSamples = bytesToFrames(data.size(), chans, type);
int advance = framesToBytes(1, chans, type);
out.reserve(numSamples/samplesPerSegment);
int segment=0;
int sample=0;
while (segment < numSamples/samplesPerSegment)
{
float sum=0;
int samplesAdded = 0;
while (sample < numSamples && sample < (segment+1)*samplesPerSegment)
{
// get sample on a scale from -1 to 1
float value = 0;
if (type == SampleType_UInt8)
value = data[sample*advance]/128.f;
else if (type == SampleType_Int16)
{
value = *reinterpret_cast<const Ogre::int16*>(&data[sample*advance]);
value /= float(std::numeric_limits<Ogre::uint16>().max());
}
else if (type == SampleType_Float32)
{
value = *reinterpret_cast<const float*>(&data[sample*advance]);
value /= std::numeric_limits<float>().max();
}
sum += value*value;
++samplesAdded;
++sample;
}
float rms = 0; // root mean square
if (samplesAdded > 0)
rms = std::sqrt(sum / samplesAdded);
out.push_back(rms);
++segment;
}
}
}

20
apps/openmw/mwsound/loudness.hpp Normal file
View File

@ -0,0 +1,20 @@
#include "sound_decoder.hpp"
namespace MWSound
{
/**
* Analyzes the energy (closely related to loudness) of a sound buffer.
* The buffer will be divided into segments according to \a valuesPerSecond,
* and for each segment a loudness value in the range of [0,1] will be computed.
* @param data the sound buffer to analyze, containing raw samples
* @param sampleRate the sample rate of the sound buffer
* @param chans channel layout of the buffer
* @param type sample type of the buffer
* @param out Will contain the output loudness values.
* @param valuesPerSecond How many loudness values per second of audio to compute.
*/
void analyzeLoudness (const std::vector<char>& data, int sampleRate, ChannelConfig chans, SampleType type,
std::vector<float>& out, float valuesPerSecond);
}

94
apps/openmw/mwsound/openal_output.cpp
View File

@ -11,11 +11,16 @@
#include "sound_decoder.hpp"
#include "sound.hpp"
#include "soundmanagerimp.hpp"
#include "loudness.hpp"
#ifndef ALC_ALL_DEVICES_SPECIFIER
#define ALC_ALL_DEVICES_SPECIFIER 0x1013
#endif
namespace
{
const int loudnessFPS = 20; // loudness values per second of audio
}
namespace MWSound
{
@ -750,7 +755,7 @@ void OpenAL_Output::deinit()
}
ALuint OpenAL_Output::getBuffer(const std::string &fname)
ALuint OpenAL_Output::getBuffer(const std::string &fname, std::vector<float>* loudnessBuffer)
{
ALuint buf = 0;
@ -765,11 +770,12 @@ ALuint OpenAL_Output::getBuffer(const std::string &fname)
if(iter != mUnusedBuffers.end())
mUnusedBuffers.erase(iter);
}
return buf;
}
throwALerror();
if (buf != 0 && loudnessBuffer == NULL)
return buf;
std::vector<char> data;
ChannelConfig chans;
SampleType type;
@ -795,42 +801,50 @@ ALuint OpenAL_Output::getBuffer(const std::string &fname)
decoder->readAll(data);
decoder->close();
alGenBuffers(1, &buf);
throwALerror();
alBufferData(buf, format, &data[0], data.size(), srate);
mBufferCache[fname] = buf;
mBufferRefs[buf] = 1;
ALint bufsize = 0;
alGetBufferi(buf, AL_SIZE, &bufsize);
mBufferCacheMemSize += bufsize;
// NOTE: Max buffer cache: 15MB
while(mBufferCacheMemSize > 15*1024*1024)
if (loudnessBuffer != NULL)
{
if(mUnusedBuffers.empty())
analyzeLoudness(data, srate, chans, type, *loudnessBuffer, loudnessFPS);
}
if (buf == 0)
{
alGenBuffers(1, &buf);
throwALerror();
alBufferData(buf, format, &data[0], data.size(), srate);
mBufferCache[fname] = buf;
mBufferRefs[buf] = 1;
ALint bufsize = 0;
alGetBufferi(buf, AL_SIZE, &bufsize);
mBufferCacheMemSize += bufsize;
// NOTE: Max buffer cache: 15MB
while(mBufferCacheMemSize > 15*1024*1024)
{
std::cout <<"No more unused buffers to clear!"<< std::endl;
break;
if(mUnusedBuffers.empty())
{
std::cout <<"No more unused buffers to clear!"<< std::endl;
break;
}
ALuint oldbuf = mUnusedBuffers.front();
mUnusedBuffers.pop_front();
NameMap::iterator nameiter = mBufferCache.begin();
while(nameiter != mBufferCache.end())
{
if(nameiter->second == oldbuf)
mBufferCache.erase(nameiter++);
else
++nameiter;
}
bufsize = 0;
alGetBufferi(oldbuf, AL_SIZE, &bufsize);
alDeleteBuffers(1, &oldbuf);
mBufferCacheMemSize -= bufsize;
}
ALuint oldbuf = mUnusedBuffers.front();
mUnusedBuffers.pop_front();
NameMap::iterator nameiter = mBufferCache.begin();
while(nameiter != mBufferCache.end())
{
if(nameiter->second == oldbuf)
mBufferCache.erase(nameiter++);
else
++nameiter;
}
bufsize = 0;
alGetBufferi(oldbuf, AL_SIZE, &bufsize);
alDeleteBuffers(1, &oldbuf);
mBufferCacheMemSize -= bufsize;
}
return buf;
}
@ -883,7 +897,7 @@ MWBase::SoundPtr OpenAL_Output::playSound(const std::string &fname, float vol, f
}
MWBase::SoundPtr OpenAL_Output::playSound3D(const std::string &fname, const Ogre::Vector3 &pos, float vol, float basevol, float pitch,
float min, float max, int flags, float offset)
float min, float max, int flags, float offset, bool extractLoudness)
{
boost::shared_ptr<OpenAL_Sound> sound;
ALuint src=0, buf=0;
@ -895,8 +909,12 @@ MWBase::SoundPtr OpenAL_Output::playSound3D(const std::string &fname, const Ogre
try
{
buf = getBuffer(fname);
std::vector<float> loudnessVector;
buf = getBuffer(fname, extractLoudness ? &loudnessVector : NULL);
sound.reset(new OpenAL_Sound3D(*this, src, buf, pos, vol, basevol, pitch, min, max, flags));
sound->setLoudnessVector(loudnessVector, loudnessFPS);
}
catch(std::exception&)
{

4
apps/openmw/mwsound/openal_output.hpp
View File

@ -36,7 +36,7 @@ namespace MWSound
typedef std::vector<Sound*> SoundVec;
SoundVec mActiveSounds;
ALuint getBuffer(const std::string &fname);
ALuint getBuffer(const std::string &fname, std::vector<float>* loudnessBuffer=NULL);
void bufferFinished(ALuint buffer);
Environment mLastEnvironment;
@ -49,7 +49,7 @@ namespace MWSound
virtual MWBase::SoundPtr playSound(const std::string &fname, float vol, float basevol, float pitch, int flags, float offset);
/// @param offset Value from [0,1] meaning from which fraction the sound the playback starts.
virtual MWBase::SoundPtr playSound3D(const std::string &fname, const Ogre::Vector3 &pos,
float vol, float basevol, float pitch, float min, float max, int flags, float offset);
float vol, float basevol, float pitch, float min, float max, int flags, float offset, bool extractLoudness=false);
virtual MWBase::SoundPtr streamSound(DecoderPtr decoder, float volume, float pitch, int flags);
virtual void updateListener(const Ogre::Vector3 &pos, const Ogre::Vector3 &atdir, const Ogre::Vector3 &updir, Environment env);

23
apps/openmw/mwsound/sound.cpp Normal file
View File

@ -0,0 +1,23 @@
#include "sound.hpp"
namespace MWSound
{
float Sound::getCurrentLoudness()
{
if (mLoudnessVector.empty())
return 0.f;
int index = getTimeOffset() * mLoudnessFPS;
index = std::max(0, std::min(index, int(mLoudnessVector.size()-1)));
return mLoudnessVector[index];
}
void Sound::setLoudnessVector(const std::vector<float> &loudnessVector, float loudnessFPS)
{
mLoudnessVector = loudnessVector;
mLoudnessFPS = loudnessFPS;
}
}

10
apps/openmw/mwsound/sound.hpp
View File

@ -24,6 +24,9 @@ namespace MWSound
int mFlags;
float mFadeOutTime;
std::vector<float> mLoudnessVector;
float mLoudnessFPS;
public:
virtual void stop() = 0;
virtual bool isPlaying() = 0;
@ -31,6 +34,12 @@ namespace MWSound
void setPosition(const Ogre::Vector3 &pos) { mPos = pos; }
void setVolume(float volume) { mVolume = volume; }
void setFadeout(float duration) { mFadeOutTime=duration; }
void setLoudnessVector(const std::vector<float>& loudnessVector, float loudnessFPS);
/// Get loudness at the current time position on a [0,1] scale.
/// Requires that loudnessVector was filled in by the user.
float getCurrentLoudness();
MWBase::SoundManager::PlayType getPlayType() const
{ return (MWBase::SoundManager::PlayType)(mFlags&MWBase::SoundManager::Play_TypeMask); }
@ -44,6 +53,7 @@ namespace MWSound
, mMaxDistance(maxdist)
, mFlags(flags)
, mFadeOutTime(0)
, mLoudnessFPS(20)
{ }
virtual ~Sound() { }

2
apps/openmw/mwsound/sound_output.hpp
View File

@ -28,7 +28,7 @@ namespace MWSound
virtual MWBase::SoundPtr playSound(const std::string &fname, float vol, float basevol, float pitch, int flags, float offset) = 0;
/// @param offset Value from [0,1] meaning from which fraction the sound the playback starts.
virtual MWBase::SoundPtr playSound3D(const std::string &fname, const Ogre::Vector3 &pos,
float vol, float basevol, float pitch, float min, float max, int flags, float offset) = 0;
float vol, float basevol, float pitch, float min, float max, int flags, float offset, bool extractLoudness=false) = 0;
virtual MWBase::SoundPtr streamSound(DecoderPtr decoder, float volume, float pitch, int flags) = 0;
virtual void updateListener(const Ogre::Vector3 &pos, const Ogre::Vector3 &atdir, const Ogre::Vector3 &updir, Environment env) = 0;

17
apps/openmw/mwsound/soundmanagerimp.cpp
View File

@ -256,7 +256,7 @@ namespace MWSound
const Ogre::Vector3 objpos(pos.pos);
MWBase::SoundPtr sound = mOutput->playSound3D(filePath, objpos, 1.0f, basevol, 1.0f,
20.0f, 1500.0f, Play_Normal|Play_TypeVoice, 0);
20.0f, 1500.0f, Play_Normal|Play_TypeVoice, 0, true);
mActiveSounds[sound] = std::make_pair(ptr, std::string("_say_sound"));
}
catch(std::exception &e)
@ -265,6 +265,21 @@ namespace MWSound
}
}
float SoundManager::getSaySoundLoudness(const MWWorld::Ptr &ptr) const
{
SoundMap::const_iterator snditer = mActiveSounds.begin();
while(snditer != mActiveSounds.end())
{
if(snditer->second.first == ptr && snditer->second.second == "_say_sound")
break;
++snditer;
}
if (snditer == mActiveSounds.end())
return 0.f;
return snditer->first->getCurrentLoudness();
}
void SoundManager::say(const std::string& filename)
{
if(!mOutput->isInitialized())

5
apps/openmw/mwsound/soundmanagerimp.hpp
View File

@ -110,6 +110,11 @@ namespace MWSound
virtual void stopSay(const MWWorld::Ptr &reference=MWWorld::Ptr());
///< Stop an actor speaking
virtual float getSaySoundLoudness(const MWWorld::Ptr& reference) const;
///< Check the currently playing say sound for this actor
/// and get an average loudness value (scale [0,1]) at the current time position.
/// If the actor is not saying anything, returns 0.
virtual MWBase::SoundPtr playTrack(const DecoderPtr& decoder, PlayType type);
///< Play a 2D audio track, using a custom decoder