1
0
mirror of https://gitlab.com/OpenMW/openmw.git synced 2025-03-03 04:13:24 +00:00

Limit the number of simultaneously open not actively used content files

Use LRU cache for ESMReaders. When cache capacity is reached close least
recently used ESMReader. Remember the file name if a reader was open. Once the
reader requested again open the file if there is stored name for it. Put
released ESMReader to the back of the free items list. Close ESMReader's from
the front of the free items list.

Cached item can be used only by one client at the same time. If the same item is
requested twice exception is thrown. This should never happen in practice. If
this happens need to fix the client logic.

It's allowed to go over the capacity limit when requesting different readers.
Ideally this should never happen but there will be system error anyway
signalizing about too many open files. Need to fix client logic in this case.

All places that were using a vector of ESMReaders now using the cache. Cache is
local for each use case and there is no need for a thread safety.
This commit is contained in:
elsid 2022-06-01 22:53:18 +02:00
parent e78d36ff50
commit 3affe9913f
No known key found for this signature in database
GPG Key ID: B845CB9FEE18AB40
30 changed files with 420 additions and 169 deletions

View File

@ -15,6 +15,7 @@
#include <components/version/version.hpp>
#include <components/vfs/manager.hpp>
#include <components/vfs/registerarchives.hpp>
#include <components/esm3/readerscache.hpp>
#include <boost/program_options.hpp>
@ -156,7 +157,7 @@ namespace
Settings::Manager settings;
settings.load(config);
std::vector<ESM::ESMReader> readers(contentFiles.size());
ESM::ReadersCache readers;
EsmLoader::Query query;
query.mLoadActivators = true;
query.mLoadCells = true;

View File

@ -20,6 +20,7 @@
#include <components/version/version.hpp>
#include <components/vfs/manager.hpp>
#include <components/vfs/registerarchives.hpp>
#include <components/esm3/readerscache.hpp>
#include <osg/Vec3f>
@ -174,7 +175,7 @@ namespace NavMeshTool
DetourNavigator::NavMeshDb db(dbPath, maxDbFileSize);
std::vector<ESM::ESMReader> readers(contentFiles.size());
ESM::ReadersCache readers;
EsmLoader::Query query;
query.mLoadActivators = true;
query.mLoadCells = true;

View File

@ -20,6 +20,7 @@
#include <components/vfs/manager.hpp>
#include <components/debug/debugging.hpp>
#include <components/navmeshtool/protocol.hpp>
#include <components/esm3/readerscache.hpp>
#include <LinearMath/btVector3.h>
@ -68,17 +69,17 @@ namespace NavMeshTool
}
std::vector<CellRef> loadCellRefs(const ESM::Cell& cell, const EsmLoader::EsmData& esmData,
std::vector<ESM::ESMReader>& readers)
ESM::ReadersCache& readers)
{
std::vector<EsmLoader::Record<CellRef>> cellRefs;
for (std::size_t i = 0; i < cell.mContextList.size(); i++)
{
ESM::ESMReader& reader = readers[static_cast<std::size_t>(cell.mContextList[i].index)];
cell.restore(reader, static_cast<int>(i));
ESM::ReadersCache::BusyItem reader = readers.get(static_cast<std::size_t>(cell.mContextList[i].index));
cell.restore(*reader, static_cast<int>(i));
ESM::CellRef cellRef;
bool deleted = false;
while (ESM::Cell::getNextRef(reader, cellRef, deleted))
while (ESM::Cell::getNextRef(*reader, cellRef, deleted))
{
Misc::StringUtils::lowerCaseInPlace(cellRef.mRefID);
const ESM::RecNameInts type = getType(esmData, cellRef.mRefID);
@ -101,7 +102,7 @@ namespace NavMeshTool
template <class F>
void forEachObject(const ESM::Cell& cell, const EsmLoader::EsmData& esmData, const VFS::Manager& vfs,
Resource::BulletShapeManager& bulletShapeManager, std::vector<ESM::ESMReader>& readers,
Resource::BulletShapeManager& bulletShapeManager, ESM::ReadersCache& readers,
F&& f)
{
std::vector<CellRef> cellRefs = loadCellRefs(cell, esmData, readers);
@ -233,7 +234,7 @@ namespace NavMeshTool
mAabb.m_max = btVector3(0, 0, 0);
}
WorldspaceData gatherWorldspaceData(const DetourNavigator::Settings& settings, std::vector<ESM::ESMReader>& readers,
WorldspaceData gatherWorldspaceData(const DetourNavigator::Settings& settings, ESM::ReadersCache& readers,
const VFS::Manager& vfs, Resource::BulletShapeManager& bulletShapeManager, const EsmLoader::EsmData& esmData,
bool processInteriorCells, bool writeBinaryLog)
{

View File

@ -18,6 +18,7 @@
namespace ESM
{
class ESMReader;
class ReadersCache;
}
namespace VFS
@ -89,7 +90,7 @@ namespace NavMeshTool
std::vector<std::vector<float>> mHeightfields;
};
WorldspaceData gatherWorldspaceData(const DetourNavigator::Settings& settings, std::vector<ESM::ESMReader>& readers,
WorldspaceData gatherWorldspaceData(const DetourNavigator::Settings& settings, ESM::ReadersCache& readers,
const VFS::Manager& vfs, Resource::BulletShapeManager& bulletShapeManager, const EsmLoader::EsmData& esmData,
bool processInteriorCells, bool writeBinaryLog);
}

View File

@ -158,8 +158,6 @@ namespace MWBase
virtual const MWWorld::ESMStore& getStore() const = 0;
virtual std::vector<ESM::ESMReader>& getEsmReader() = 0;
virtual MWWorld::LocalScripts& getLocalScripts() = 0;
virtual bool hasCellChanged() const = 0;

View File

@ -12,6 +12,7 @@
#include <components/sceneutil/nodecallback.hpp>
#include <components/terrain/quadtreenode.hpp>
#include <components/shader/shadermanager.hpp>
#include <components/esm3/readerscache.hpp>
#include "../mwworld/groundcoverstore.hpp"
@ -176,7 +177,7 @@ namespace MWRender
osg::Vec2f minBound = (center - osg::Vec2f(size/2.f, size/2.f));
osg::Vec2f maxBound = (center + osg::Vec2f(size/2.f, size/2.f));
DensityCalculator calculator(mDensity);
std::vector<ESM::ESMReader> esm;
ESM::ReadersCache readers;
osg::Vec2i startCell = osg::Vec2i(std::floor(center.x() - size/2.f), std::floor(center.y() - size/2.f));
for (int cellX = startCell.x(); cellX < startCell.x() + size; ++cellX)
{
@ -190,14 +191,13 @@ namespace MWRender
std::map<ESM::RefNum, ESM::CellRef> refs;
for (size_t i=0; i<cell.mContextList.size(); ++i)
{
unsigned int index = cell.mContextList[i].index;
if (esm.size() <= index)
esm.resize(index+1);
cell.restore(esm[index], i);
const std::size_t index = static_cast<std::size_t>(cell.mContextList[i].index);
const ESM::ReadersCache::BusyItem reader = readers.get(index);
cell.restore(*reader, i);
ESM::CellRef ref;
ref.mRefNum.unset();
bool deleted = false;
while(cell.getNextRef(esm[index], ref, deleted))
while (cell.getNextRef(*reader, ref, deleted))
{
if (!deleted && refs.find(ref.mRefNum) == refs.end() && !calculator.isInstanceEnabled()) deleted = true;
if (!deleted && !isInChunkBorders(ref, minBound, maxBound)) deleted = true;

View File

@ -17,6 +17,7 @@
#include <components/sceneutil/clone.hpp>
#include <components/sceneutil/util.hpp>
#include <components/vfs/manager.hpp>
#include <components/esm3/readerscache.hpp>
#include <osgParticle/ParticleProcessor>
#include <osgParticle/ParticleSystemUpdater>
@ -412,7 +413,7 @@ namespace MWRender
osg::Vec3f relativeViewPoint = viewPoint - worldCenter;
std::map<ESM::RefNum, ESM::CellRef> refs;
std::vector<ESM::ESMReader> esm;
ESM::ReadersCache readers;
const MWWorld::ESMStore& store = MWBase::Environment::get().getWorld()->getStore();
for (int cellX = startCell.x(); cellX < startCell.x() + size; ++cellX)
@ -425,17 +426,16 @@ namespace MWRender
{
try
{
unsigned int index = cell->mContextList[i].index;
if (esm.size()<=index)
esm.resize(index+1);
cell->restore(esm[index], i);
const std::size_t index = static_cast<std::size_t>(cell->mContextList[i].index);
const ESM::ReadersCache::BusyItem reader = readers.get(index);
cell->restore(*reader, i);
ESM::CellRef ref;
ref.mRefNum.unset();
ESM::MovedCellRef cMRef;
cMRef.mRefNum.mIndex = 0;
bool deleted = false;
bool moved = false;
while (ESM::Cell::getNextRef(esm[index], ref, deleted, cMRef, moved, ESM::Cell::GetNextRefMode::LoadOnlyNotMoved))
while (ESM::Cell::getNextRef(*reader, ref, deleted, cMRef, moved, ESM::Cell::GetNextRefMode::LoadOnlyNotMoved))
{
if (moved)
continue;

View File

@ -68,9 +68,7 @@ MWWorld::CellStore *MWWorld::Cells::getCellStore (const ESM::Cell *cell)
std::map<std::string, CellStore>::iterator result = mInteriors.find (lowerName);
if (result==mInteriors.end())
{
result = mInteriors.insert (std::make_pair (lowerName, CellStore (cell, mStore, mReader))).first;
}
result = mInteriors.emplace(std::move(lowerName), CellStore(cell, mStore, mReaders)).first;
return &result->second;
}
@ -80,11 +78,8 @@ MWWorld::CellStore *MWWorld::Cells::getCellStore (const ESM::Cell *cell)
mExteriors.find (std::make_pair (cell->getGridX(), cell->getGridY()));
if (result==mExteriors.end())
{
result = mExteriors.insert (std::make_pair (
std::make_pair (cell->getGridX(), cell->getGridY()), CellStore (cell, mStore, mReader))).first;
}
result = mExteriors.emplace(std::make_pair(cell->getGridX(), cell->getGridY()),
CellStore(cell, mStore, mReaders)).first;
return &result->second;
}
@ -130,9 +125,10 @@ void MWWorld::Cells::writeCell (ESM::ESMWriter& writer, CellStore& cell) const
writer.endRecord (ESM::REC_CSTA);
}
MWWorld::Cells::Cells (const MWWorld::ESMStore& store, std::vector<ESM::ESMReader>& reader)
: mStore (store), mReader (reader),
mIdCacheIndex (0)
MWWorld::Cells::Cells (const MWWorld::ESMStore& store, ESM::ReadersCache& readers)
: mStore(store)
, mReaders(readers)
, mIdCacheIndex(0)
{
int cacheSize = std::clamp(Settings::Manager::getInt("pointers cache size", "Cells"), 40, 1000);
mIdCache = IdCache(cacheSize, std::pair<std::string, CellStore *> ("", (CellStore*)nullptr));
@ -165,8 +161,7 @@ MWWorld::CellStore *MWWorld::Cells::getExterior (int x, int y)
cell = MWBase::Environment::get().getWorld()->createRecord (record);
}
result = mExteriors.insert (std::make_pair (
std::make_pair (x, y), CellStore (cell, mStore, mReader))).first;
result = mExteriors.emplace(std::make_pair(x, y), CellStore(cell, mStore, mReaders)).first;
}
if (result->second.getState()!=CellStore::State_Loaded)
@ -186,7 +181,7 @@ MWWorld::CellStore *MWWorld::Cells::getInterior (const std::string& name)
{
const ESM::Cell *cell = mStore.get<ESM::Cell>().find(lowerName);
result = mInteriors.insert (std::make_pair (lowerName, CellStore (cell, mStore, mReader))).first;
result = mInteriors.emplace(std::move(lowerName), CellStore(cell, mStore, mReaders)).first;
}
if (result->second.getState()!=CellStore::State_Loaded)

View File

@ -11,6 +11,7 @@ namespace ESM
{
class ESMReader;
class ESMWriter;
class ReadersCache;
struct CellId;
struct Cell;
struct RefNum;
@ -30,7 +31,7 @@ namespace MWWorld
{
typedef std::vector<std::pair<std::string, CellStore *> > IdCache;
const MWWorld::ESMStore& mStore;
std::vector<ESM::ESMReader>& mReader;
ESM::ReadersCache& mReaders;
mutable std::map<std::string, CellStore> mInteriors;
mutable std::map<std::pair<int, int>, CellStore> mExteriors;
IdCache mIdCache;
@ -51,7 +52,7 @@ namespace MWWorld
void clear();
Cells (const MWWorld::ESMStore& store, std::vector<ESM::ESMReader>& reader);
explicit Cells(const MWWorld::ESMStore& store, ESM::ReadersCache& reader);
CellStore *getExterior (int x, int y);

View File

@ -17,6 +17,7 @@
#include <components/esm3/fogstate.hpp>
#include <components/esm3/creaturelevliststate.hpp>
#include <components/esm3/doorstate.hpp>
#include <components/esm3/readerscache.hpp>
#include "../mwbase/environment.hpp"
#include "../mwbase/luamanager.hpp"
@ -387,8 +388,14 @@ namespace MWWorld
return false;
}
CellStore::CellStore (const ESM::Cell *cell, const MWWorld::ESMStore& esmStore, std::vector<ESM::ESMReader>& readerList)
: mStore(esmStore), mReader(readerList), mCell (cell), mState (State_Unloaded), mHasState (false), mLastRespawn(0,0), mRechargingItemsUpToDate(false)
CellStore::CellStore(const ESM::Cell* cell, const MWWorld::ESMStore& esmStore, ESM::ReadersCache& readers)
: mStore(esmStore)
, mReaders(readers)
, mCell(cell)
, mState(State_Unloaded)
, mHasState(false)
, mLastRespawn(0, 0)
, mRechargingItemsUpToDate(false)
{
mWaterLevel = cell->mWater;
}
@ -545,8 +552,6 @@ namespace MWWorld
void CellStore::listRefs()
{
std::vector<ESM::ESMReader>& esm = mReader;
assert (mCell);
if (mCell->mContextList.empty())
@ -558,8 +563,9 @@ namespace MWWorld
try
{
// Reopen the ESM reader and seek to the right position.
int index = mCell->mContextList[i].index;
mCell->restore (esm[index], i);
const std::size_t index = static_cast<std::size_t>(mCell->mContextList[i].index);
const ESM::ReadersCache::BusyItem reader = mReaders.get(index);
mCell->restore(*reader, i);
ESM::CellRef ref;
@ -568,7 +574,7 @@ namespace MWWorld
cMRef.mRefNum.mIndex = 0;
bool deleted = false;
bool moved = false;
while (ESM::Cell::getNextRef(esm[index], ref, deleted, cMRef, moved, ESM::Cell::GetNextRefMode::LoadOnlyNotMoved))
while (ESM::Cell::getNextRef(*reader, ref, deleted, cMRef, moved, ESM::Cell::GetNextRefMode::LoadOnlyNotMoved))
{
if (deleted || moved)
continue;
@ -602,8 +608,6 @@ namespace MWWorld
void CellStore::loadRefs()
{
std::vector<ESM::ESMReader>& esm = mReader;
assert (mCell);
if (mCell->mContextList.empty())
@ -617,8 +621,9 @@ namespace MWWorld
try
{
// Reopen the ESM reader and seek to the right position.
int index = mCell->mContextList[i].index;
mCell->restore (esm[index], i);
const std::size_t index = static_cast<std::size_t>(mCell->mContextList[i].index);
const ESM::ReadersCache::BusyItem reader = mReaders.get(index);
mCell->restore(*reader, i);
ESM::CellRef ref;
ref.mRefNum.unset();
@ -628,7 +633,7 @@ namespace MWWorld
cMRef.mRefNum.mIndex = 0;
bool deleted = false;
bool moved = false;
while (ESM::Cell::getNextRef(esm[index], ref, deleted, cMRef, moved, ESM::Cell::GetNextRefMode::LoadOnlyNotMoved))
while (ESM::Cell::getNextRef(*reader, ref, deleted, cMRef, moved, ESM::Cell::GetNextRefMode::LoadOnlyNotMoved))
{
if (moved)
continue;

View File

@ -38,6 +38,7 @@
namespace ESM
{
class ReadersCache;
struct Cell;
struct CellState;
struct CellId;
@ -61,7 +62,7 @@ namespace MWWorld
private:
const MWWorld::ESMStore& mStore;
std::vector<ESM::ESMReader>& mReader;
ESM::ReadersCache& mReaders;
// Even though fog actually belongs to the player and not cells,
// it makes sense to store it here since we need it once for each cell.
@ -211,9 +212,7 @@ namespace MWWorld
}
/// @param readerList The readers to use for loading of the cell on-demand.
CellStore (const ESM::Cell *cell_,
const MWWorld::ESMStore& store,
std::vector<ESM::ESMReader>& readerList);
CellStore(const ESM::Cell* cell, const MWWorld::ESMStore& store, ESM::ReadersCache& readers);
const ESM::Cell *getCell() const;

View File

@ -2,13 +2,13 @@
#include "esmstore.hpp"
#include <components/esm3/esmreader.hpp>
#include <components/esm3/readerscache.hpp>
namespace MWWorld
{
EsmLoader::EsmLoader(MWWorld::ESMStore& store, std::vector<ESM::ESMReader>& readers,
ToUTF8::Utf8Encoder* encoder)
: mEsm(readers)
EsmLoader::EsmLoader(MWWorld::ESMStore& store, ESM::ReadersCache& readers, ToUTF8::Utf8Encoder* encoder)
: mReaders(readers)
, mStore(store)
, mEncoder(encoder)
, mDialogue(nullptr) // A content file containing INFO records without a DIAL record appends them to the previous file's dialogue
@ -17,13 +17,25 @@ EsmLoader::EsmLoader(MWWorld::ESMStore& store, std::vector<ESM::ESMReader>& read
void EsmLoader::load(const boost::filesystem::path& filepath, int& index, Loading::Listener* listener)
{
ESM::ESMReader lEsm;
lEsm.setEncoder(mEncoder);
lEsm.setIndex(index);
lEsm.open(filepath.string());
lEsm.resolveParentFileIndices(mEsm);
mEsm[index] = std::move(lEsm);
mStore.load(mEsm[index], listener, mDialogue);
const ESM::ReadersCache::BusyItem reader = mReaders.get(static_cast<std::size_t>(index));
reader->setEncoder(mEncoder);
reader->setIndex(index);
reader->open(filepath.string());
reader->resolveParentFileIndices(mReaders);
assert(reader->getGameFiles().size() == reader->getParentFileIndices().size());
for (std::size_t i = 0, n = reader->getParentFileIndices().size(); i < n; ++i)
if (i == static_cast<std::size_t>(reader->getIndex()))
throw std::runtime_error("File " + reader->getName() + " asks for parent file "
+ reader->getGameFiles()[i].name
+ ", but it is not available or has been loaded in the wrong order. "
"Please run the launcher to fix this issue.");
mStore.load(*reader, listener, mDialogue);
if (!mMasterFileFormat.has_value() && Misc::StringUtils::ciEndsWith(reader->getName(), ".esm") && !Misc::StringUtils::ciEndsWith(reader->getName(), ".omwgame"))
mMasterFileFormat = reader->getFormat();
}
} /* namespace MWWorld */

View File

@ -1,7 +1,7 @@
#ifndef ESMLOADER_HPP
#define ESMLOADER_HPP
#include <vector>
#include <optional>
#include "contentloader.hpp"
@ -12,7 +12,7 @@ namespace ToUTF8
namespace ESM
{
class ESMReader;
class ReadersCache;
struct Dialogue;
}
@ -23,16 +23,18 @@ class ESMStore;
struct EsmLoader : public ContentLoader
{
EsmLoader(MWWorld::ESMStore& store, std::vector<ESM::ESMReader>& readers,
ToUTF8::Utf8Encoder* encoder);
explicit EsmLoader(MWWorld::ESMStore& store, ESM::ReadersCache& readers, ToUTF8::Utf8Encoder* encoder);
std::optional<int> getMasterFileFormat() const { return mMasterFileFormat; }
void load(const boost::filesystem::path& filepath, int& index, Loading::Listener* listener) override;
private:
std::vector<ESM::ESMReader>& mEsm;
ESM::ReadersCache& mReaders;
MWWorld::ESMStore& mStore;
ToUTF8::Utf8Encoder* mEncoder;
ESM::Dialogue* mDialogue;
std::optional<int> mMasterFileFormat;
};
} /* namespace MWWorld */

View File

@ -9,6 +9,7 @@
#include <components/loadinglistener/loadinglistener.hpp>
#include <components/lua/configuration.hpp>
#include <components/misc/algorithm.hpp>
#include <components/esm3/readerscache.hpp>
#include "../mwmechanics/spelllist.hpp"
@ -24,19 +25,18 @@ namespace
constexpr std::size_t deletedRefID = std::numeric_limits<std::size_t>::max();
void readRefs(const ESM::Cell& cell, std::vector<Ref>& refs, std::vector<std::string>& refIDs, std::vector<ESM::ESMReader>& readers)
void readRefs(const ESM::Cell& cell, std::vector<Ref>& refs, std::vector<std::string>& refIDs, ESM::ReadersCache& readers)
{
// TODO: we have many similar copies of this code.
for (size_t i = 0; i < cell.mContextList.size(); i++)
{
size_t index = cell.mContextList[i].index;
if (readers.size() <= index)
readers.resize(index + 1);
cell.restore(readers[index], i);
const std::size_t index = static_cast<std::size_t>(cell.mContextList[i].index);
const ESM::ReadersCache::BusyItem reader = readers.get(index);
cell.restore(*reader, i);
ESM::CellRef ref;
ref.mRefNum.unset();
bool deleted = false;
while(cell.getNextRef(readers[index], ref, deleted))
while (cell.getNextRef(*reader, ref, deleted))
{
if(deleted)
refs.emplace_back(ref.mRefNum, deletedRefID);
@ -241,7 +241,7 @@ ESM::LuaScriptsCfg ESMStore::getLuaScriptsCfg() const
return cfg;
}
void ESMStore::setUp(bool validateRecords)
void ESMStore::setUp()
{
mIds.clear();
@ -267,15 +267,15 @@ void ESMStore::setUp(bool validateRecords)
mMagicEffects.setUp();
mAttributes.setUp();
mDialogs.setUp();
if (validateRecords)
{
validate();
countAllCellRefs();
}
}
void ESMStore::countAllCellRefs()
void ESMStore::validateRecords(ESM::ReadersCache& readers)
{
validate();
countAllCellRefs(readers);
}
void ESMStore::countAllCellRefs(ESM::ReadersCache& readers)
{
// TODO: We currently need to read entire files here again.
// We should consider consolidating or deferring this reading.
@ -283,7 +283,6 @@ void ESMStore::countAllCellRefs()
return;
std::vector<Ref> refs;
std::vector<std::string> refIDs;
std::vector<ESM::ESMReader> readers;
for(auto it = mCells.intBegin(); it != mCells.intEnd(); ++it)
readRefs(*it, refs, refIDs, readers);
for(auto it = mCells.extBegin(); it != mCells.extEnd(); ++it)

View File

@ -20,6 +20,11 @@ namespace MWMechanics
class SpellList;
}
namespace ESM
{
class ReadersCache;
}
namespace MWWorld
{
class ESMStore
@ -90,7 +95,7 @@ namespace MWWorld
/// Validate entries in store after setup
void validate();
void countAllCellRefs();
void countAllCellRefs(ESM::ReadersCache& readers);
template<class T>
void removeMissingObjects(Store<T>& store);
@ -266,7 +271,8 @@ namespace MWWorld
// This method must be called once, after loading all master/plugin files. This can only be done
// from the outside, so it must be public.
void setUp(bool validateRecords = false);
void setUp();
void validateRecords(ESM::ReadersCache& readers);
int countSavedGameRecords() const;

View File

@ -2,6 +2,7 @@
#include <components/esmloader/load.hpp>
#include <components/misc/stringops.hpp>
#include <components/esm3/readerscache.hpp>
namespace MWWorld
{
@ -11,7 +12,7 @@ namespace MWWorld
query.mLoadStatics = true;
query.mLoadCells = true;
std::vector<ESM::ESMReader> readers(groundcoverFiles.size());
ESM::ReadersCache readers;
const ::EsmLoader::EsmData content = ::EsmLoader::loadEsmData(query, groundcoverFiles, fileCollections, readers, encoder);
for (const ESM::Static& stat : statics)

View File

@ -145,8 +145,8 @@ namespace MWWorld
ToUTF8::Utf8Encoder* encoder, int activationDistanceOverride,
const std::string& startCell, const std::string& startupScript,
const std::string& resourcePath, const std::string& userDataPath)
: mResourceSystem(resourceSystem), mLocalScripts (mStore),
mCells (mStore, mEsm), mSky (true),
: mResourceSystem(resourceSystem), mLocalScripts(mStore),
mCells(mStore, mReaders), mSky(true),
mGodMode(false), mScriptsEnabled(true), mDiscardMovements(true), mContentFiles (contentFiles),
mUserDataPath(userDataPath),
mDefaultHalfExtents(Settings::Manager::getVector3("default actor pathfind half extents", "Game")),
@ -156,30 +156,20 @@ namespace MWWorld
mLevitationEnabled(true), mGoToJail(false), mDaysInPrison(0),
mPlayerTraveling(false), mPlayerInJail(false), mSpellPreloadTimer(0.f)
{
mEsm.resize(contentFiles.size());
Loading::Listener* listener = MWBase::Environment::get().getWindowManager()->getLoadingScreen();
listener->loadingOn();
loadContentFiles(fileCollections, contentFiles, mStore, mEsm, encoder, listener);
loadContentFiles(fileCollections, contentFiles, encoder, listener);
loadGroundcoverFiles(fileCollections, groundcoverFiles, encoder);
listener->loadingOff();
// Find main game file
for (const ESM::ESMReader& reader : mEsm)
{
if (!Misc::StringUtils::ciEndsWith(reader.getName(), ".esm") && !Misc::StringUtils::ciEndsWith(reader.getName(), ".omwgame"))
continue;
if (reader.getFormat() == 0)
ensureNeededRecords(); // and insert records that may not be present in all versions of MW.
break;
}
mCurrentDate = std::make_unique<DateTimeManager>();
fillGlobalVariables();
mStore.setUp(true);
mStore.setUp();
mStore.validateRecords(mReaders);
mStore.movePlayerRecord();
mSwimHeightScale = mStore.get<ESM::GameSetting>().find("fSwimHeightScale")->mValue.getFloat();
@ -417,23 +407,6 @@ namespace MWWorld
}
}
void World::validateMasterFiles(const std::vector<ESM::ESMReader>& readers)
{
for (const auto& esm : readers)
{
assert(esm.getGameFiles().size() == esm.getParentFileIndices().size());
for (unsigned int i=0; i<esm.getParentFileIndices().size(); ++i)
{
if (!esm.isValidParentFileIndex(i))
{
std::string fstring = "File " + esm.getName() + " asks for parent file " + esm.getGameFiles()[i].name
+ ", but it is not available or has been loaded in the wrong order. Please run the launcher to fix this issue.";
throw std::runtime_error(fstring);
}
}
}
}
void World::ensureNeededRecords()
{
std::map<std::string, ESM::Variant> gmst;
@ -641,11 +614,6 @@ namespace MWWorld
return mStore;
}
std::vector<ESM::ESMReader>& World::getEsmReader()
{
return mEsm;
}
LocalScripts& World::getLocalScripts()
{
return mLocalScripts;
@ -2939,11 +2907,11 @@ namespace MWWorld
return mScriptsEnabled;
}
void World::loadContentFiles(const Files::Collections& fileCollections, const std::vector<std::string>& content, ESMStore& store, std::vector<ESM::ESMReader>& readers, ToUTF8::Utf8Encoder* encoder, Loading::Listener* listener)
void World::loadContentFiles(const Files::Collections& fileCollections, const std::vector<std::string>& content,
ToUTF8::Utf8Encoder* encoder, Loading::Listener* listener)
{
GameContentLoader gameContentLoader;
EsmLoader esmLoader(store, readers, encoder);
validateMasterFiles(readers);
EsmLoader esmLoader(mStore, mReaders, encoder);
gameContentLoader.addLoader(".esm", esmLoader);
gameContentLoader.addLoader(".esp", esmLoader);
@ -2951,7 +2919,7 @@ namespace MWWorld
gameContentLoader.addLoader(".omwaddon", esmLoader);
gameContentLoader.addLoader(".project", esmLoader);
OMWScriptsLoader omwScriptsLoader(store);
OMWScriptsLoader omwScriptsLoader(mStore);
gameContentLoader.addLoader(".omwscripts", omwScriptsLoader);
int idx = 0;
@ -2970,6 +2938,9 @@ namespace MWWorld
}
idx++;
}
if (const auto v = esmLoader.getMasterFileFormat(); v.has_value() && *v == 0)
ensureNeededRecords(); // Insert records that may not be present in all versions of master files.
}
void World::loadGroundcoverFiles(const Files::Collections& fileCollections, const std::vector<std::string>& groundcoverFiles, ToUTF8::Utf8Encoder* encoder)

View File

@ -5,6 +5,7 @@
#include <components/settings/settings.hpp>
#include <components/misc/rng.hpp>
#include <components/esm3/readerscache.hpp>
#include "../mwbase/world.hpp"
@ -81,7 +82,7 @@ namespace MWWorld
private:
Resource::ResourceSystem* mResourceSystem;
std::vector<ESM::ESMReader> mEsm;
ESM::ReadersCache mReaders;
MWWorld::ESMStore mStore;
GroundcoverStore mGroundcoverStore;
LocalScripts mLocalScripts;
@ -162,13 +163,13 @@ namespace MWWorld
void updateNavigatorObject(const MWPhysics::Object& object);
void ensureNeededRecords();
void validateMasterFiles(const std::vector<ESM::ESMReader>& readers);
void fillGlobalVariables();
void updateSkyDate();
void loadContentFiles(const Files::Collections& fileCollections, const std::vector<std::string>& content, ESMStore& store, std::vector<ESM::ESMReader>& readers, ToUTF8::Utf8Encoder* encoder, Loading::Listener* listener);
void loadContentFiles(const Files::Collections& fileCollections, const std::vector<std::string>& content,
ToUTF8::Utf8Encoder* encoder, Loading::Listener* listener);
void loadGroundcoverFiles(const Files::Collections& fileCollections, const std::vector<std::string>& groundcoverFiles, ToUTF8::Utf8Encoder* encoder);
@ -241,8 +242,6 @@ namespace MWWorld
const MWWorld::ESMStore& getStore() const override;
std::vector<ESM::ESMReader>& getEsmReader() override;
LocalScripts& getLocalScripts() override;
bool hasCellChanged() const override;

View File

@ -81,6 +81,8 @@ if (GTEST_FOUND AND GMOCK_FOUND)
fx/lexer.cpp
fx/technique.cpp
esm3/readerscache.cpp
)
source_group(apps\\openmw_test_suite FILES openmw_test_suite.cpp ${UNITTEST_SRC_FILES})

View File

@ -0,0 +1,91 @@
#include <components/esm3/readerscache.hpp>
#include <components/files/collections.hpp>
#include <components/files/multidircollection.hpp>
#include <gtest/gtest.h>
#ifndef OPENMW_DATA_DIR
#error "OPENMW_DATA_DIR is not defined"
#endif
namespace
{
using namespace testing;
using namespace ESM;
TEST(ESM3ReadersCache, onAttemptToRequestTheSameReaderTwiceShouldThrowException)
{
ReadersCache readers(1);
const ReadersCache::BusyItem reader = readers.get(0);
EXPECT_THROW(readers.get(0), std::logic_error);
}
TEST(ESM3ReadersCache, shouldAllowToHaveBusyItemsMoreThanCapacity)
{
ReadersCache readers(1);
const ReadersCache::BusyItem reader0 = readers.get(0);
const ReadersCache::BusyItem reader1 = readers.get(1);
}
TEST(ESM3ReadersCache, shouldKeepClosedReleasedClosedItem)
{
ReadersCache readers(1);
readers.get(0);
const ReadersCache::BusyItem reader = readers.get(0);
EXPECT_FALSE(reader->isOpen());
}
struct ESM3ReadersCacheWithContentFile : Test
{
static constexpr std::size_t sInitialOffset = 324;
static constexpr std::size_t sSkip = 100;
const Files::PathContainer mDataDirs {{std::string(OPENMW_DATA_DIR)}};
const Files::Collections mFileCollections {mDataDirs, true};
const std::string mContentFile = "template.omwgame";
const std::string mContentFilePath = mFileCollections.getCollection(".omwgame").getPath(mContentFile).string();
};
TEST_F(ESM3ReadersCacheWithContentFile, shouldKeepOpenReleasedOpenReader)
{
ReadersCache readers(1);
{
const ReadersCache::BusyItem reader = readers.get(0);
reader->open(mContentFilePath);
ASSERT_TRUE(reader->isOpen());
ASSERT_EQ(reader->getFileOffset(), sInitialOffset);
ASSERT_GT(reader->getFileSize(), sInitialOffset + sSkip);
reader->skip(sSkip);
ASSERT_EQ(reader->getFileOffset(), sInitialOffset + sSkip);
}
{
const ReadersCache::BusyItem reader = readers.get(0);
EXPECT_TRUE(reader->isOpen());
EXPECT_EQ(reader->getName(), mContentFilePath);
EXPECT_EQ(reader->getFileOffset(), sInitialOffset + sSkip);
}
}
TEST_F(ESM3ReadersCacheWithContentFile, shouldCloseFreeReaderWhenReachingCapacityLimit)
{
ReadersCache readers(1);
{
const ReadersCache::BusyItem reader = readers.get(0);
reader->open(mContentFilePath);
ASSERT_TRUE(reader->isOpen());
ASSERT_EQ(reader->getFileOffset(), sInitialOffset);
ASSERT_GT(reader->getFileSize(), sInitialOffset + sSkip);
reader->skip(sSkip);
ASSERT_EQ(reader->getFileOffset(), sInitialOffset + sSkip);
}
{
const ReadersCache::BusyItem reader = readers.get(1);
reader->open(mContentFilePath);
ASSERT_TRUE(reader->isOpen());
}
{
const ReadersCache::BusyItem reader = readers.get(0);
EXPECT_TRUE(reader->isOpen());
EXPECT_EQ(reader->getFileOffset(), sInitialOffset);
}
}
}

View File

@ -10,6 +10,7 @@
#include <components/files/collections.hpp>
#include <components/files/multidircollection.hpp>
#include <components/to_utf8/to_utf8.hpp>
#include <components/esm3/readerscache.hpp>
#include <gtest/gtest.h>
@ -39,7 +40,7 @@ namespace
query.mLoadGameSettings = true;
query.mLoadLands = true;
query.mLoadStatics = true;
std::vector<ESM::ESMReader> readers(mContentFiles.size());
ESM::ReadersCache readers;
ToUTF8::Utf8Encoder* const encoder = nullptr;
const EsmData esmData = loadEsmData(query, mContentFiles, mFileCollections, readers, encoder);
EXPECT_EQ(esmData.mActivators.size(), 0);
@ -61,7 +62,7 @@ namespace
query.mLoadGameSettings = true;
query.mLoadLands = true;
query.mLoadStatics = true;
std::vector<ESM::ESMReader> readers(mContentFiles.size());
ESM::ReadersCache readers;
ToUTF8::Utf8Encoder* const encoder = nullptr;
const EsmData esmData = loadEsmData(query, mContentFiles, mFileCollections, readers, encoder);
EXPECT_EQ(esmData.mActivators.size(), 0);
@ -83,7 +84,7 @@ namespace
query.mLoadGameSettings = false;
query.mLoadLands = true;
query.mLoadStatics = true;
std::vector<ESM::ESMReader> readers(mContentFiles.size());
ESM::ReadersCache readers;
ToUTF8::Utf8Encoder* const encoder = nullptr;
const EsmData esmData = loadEsmData(query, mContentFiles, mFileCollections, readers, encoder);
EXPECT_EQ(esmData.mActivators.size(), 0);
@ -98,7 +99,7 @@ namespace
TEST_F(EsmLoaderTest, shouldIgnoreAllWithDefaultQuery)
{
const Query query;
std::vector<ESM::ESMReader> readers(mContentFiles.size());
ESM::ReadersCache readers;
ToUTF8::Utf8Encoder* const encoder = nullptr;
const EsmData esmData = loadEsmData(query, mContentFiles, mFileCollections, readers, encoder);
EXPECT_EQ(esmData.mActivators.size(), 0);
@ -121,7 +122,7 @@ namespace
query.mLoadLands = true;
query.mLoadStatics = true;
const std::vector<std::string> contentFiles {{"script.omwscripts"}};
std::vector<ESM::ESMReader> readers(contentFiles.size());
ESM::ReadersCache readers;
ToUTF8::Utf8Encoder* const encoder = nullptr;
const EsmData esmData = loadEsmData(query, contentFiles, mFileCollections, readers, encoder);
EXPECT_EQ(esmData.mActivators.size(), 0);

View File

@ -95,7 +95,7 @@ add_component_dir (esm3
savedgame journalentry queststate locals globalscript player objectstate cellid cellstate globalmap
inventorystate containerstate npcstate creaturestate dialoguestate statstate npcstats creaturestats
weatherstate quickkeys fogstate spellstate activespells creaturelevliststate doorstate projectilestate debugprofile
aisequence magiceffects custommarkerstate stolenitems transport animationstate controlsstate mappings
aisequence magiceffects custommarkerstate stolenitems transport animationstate controlsstate mappings readerscache
)
add_component_dir (esm3terrain

View File

@ -1,5 +1,7 @@
#include "esmreader.hpp"
#include "readerscache.hpp"
#include <components/misc/stringops.hpp>
#include <components/files/openfile.hpp>
@ -61,21 +63,22 @@ void ESMReader::clearCtx()
mCtx.subName.clear();
}
void ESMReader::resolveParentFileIndices(const std::vector<ESMReader>& allPlugins)
void ESMReader::resolveParentFileIndices(ReadersCache& readers)
{
mCtx.parentFileIndices.clear();
const std::vector<Header::MasterData> &masters = getGameFiles();
for (size_t j = 0; j < masters.size(); j++) {
const Header::MasterData &mast = masters[j];
std::string fname = mast.name;
for (const Header::MasterData &mast : getGameFiles())
{
const std::string& fname = mast.name;
int index = getIndex();
for (int i = 0; i < getIndex(); i++) {
const ESMReader& reader = allPlugins.at(i);
if (reader.getFileSize() == 0)
for (int i = 0; i < getIndex(); i++)
{
const ESM::ReadersCache::BusyItem reader = readers.get(static_cast<std::size_t>(i));
if (reader->getFileSize() == 0)
continue; // Content file in non-ESM format
const std::string& candidate = reader.getName();
const std::string& candidate = reader->getName();
std::string fnamecandidate = std::filesystem::path(candidate).filename().string();
if (Misc::StringUtils::ciEqual(fname, fnamecandidate)) {
if (Misc::StringUtils::ciEqual(fname, fnamecandidate))
{
index = i;
break;
}

View File

@ -16,6 +16,8 @@
namespace ESM
{
class ReadersCache;
class ESMReader
{
public:
@ -39,6 +41,7 @@ public:
const NAME &retSubName() const { return mCtx.subName; }
uint32_t getSubSize() const { return mCtx.leftSub; }
const std::string& getName() const { return mCtx.filename; };
bool isOpen() const { return mEsm != nullptr; }
/*************************************************************************
*
@ -85,9 +88,8 @@ public:
// all files/readers used by the engine. This is required for correct adjustRefNum() results
// as required for handling moved, deleted and edited CellRefs.
/// @note Does not validate.
void resolveParentFileIndices(const std::vector<ESMReader>& files);
void resolveParentFileIndices(ReadersCache& readers);
const std::vector<int>& getParentFileIndices() const { return mCtx.parentFileIndices; }
bool isValidParentFileIndex(int i) const { return i != getIndex(); }
/*************************************************************************
*

View File

@ -0,0 +1,87 @@
#include "readerscache.hpp"
#include <stdexcept>
namespace ESM
{
ReadersCache::BusyItem::BusyItem(ReadersCache& owner, std::list<Item>::iterator item) noexcept
: mOwner(owner)
, mItem(item)
{}
ReadersCache::BusyItem::~BusyItem() noexcept
{
mOwner.releaseItem(mItem);
}
ReadersCache::ReadersCache(std::size_t capacity)
: mCapacity(capacity)
{}
ReadersCache::BusyItem ReadersCache::get(std::size_t index)
{
const auto indexIt = mIndex.find(index);
std::list<Item>::iterator it;
if (indexIt == mIndex.end())
{
closeExtraReaders();
it = mBusyItems.emplace(mBusyItems.end());
mIndex.emplace(index, it);
}
else
{
switch (indexIt->second->mState)
{
case State::Busy:
throw std::logic_error("ESMReader at index " + std::to_string(index) + " is busy");
case State::Free:
it = indexIt->second;
mBusyItems.splice(mBusyItems.end(), mFreeItems, it);
break;
case State::Closed:
closeExtraReaders();
it = indexIt->second;
if (it->mName.has_value())
{
it->mReader.open(*it->mName);
it->mName.reset();
}
mBusyItems.splice(mBusyItems.end(), mClosedItems, it);
break;
}
it->mState = State::Busy;
}
return BusyItem(*this, it);
}
void ReadersCache::closeExtraReaders()
{
while (!mFreeItems.empty() && mBusyItems.size() + mFreeItems.size() + 1 > mCapacity)
{
const auto it = mFreeItems.begin();
if (it->mReader.isOpen())
{
it->mName = it->mReader.getName();
it->mReader.close();
}
mClosedItems.splice(mClosedItems.end(), mFreeItems, it);
it->mState = State::Closed;
}
}
void ReadersCache::releaseItem(std::list<Item>::iterator it) noexcept
{
assert(it->mState == State::Busy);
if (it->mReader.isOpen())
{
mFreeItems.splice(mFreeItems.end(), mBusyItems, it);
it->mState = State::Free;
}
else
{
mClosedItems.splice(mClosedItems.end(), mBusyItems, it);
it->mState = State::Closed;
}
}
}

View File

@ -0,0 +1,71 @@
#ifndef OPENMW_COMPONENTS_ESM3_READERSCACHE_H
#define OPENMW_COMPONENTS_ESM3_READERSCACHE_H
#include "esmreader.hpp"
#include <cstddef>
#include <list>
#include <map>
#include <optional>
#include <string>
namespace ESM
{
class ReadersCache
{
private:
enum class State
{
Busy,
Free,
Closed,
};
struct Item
{
State mState = State::Busy;
ESMReader mReader;
std::optional<std::string> mName;
Item() = default;
};
public:
class BusyItem
{
public:
explicit BusyItem(ReadersCache& owner, std::list<Item>::iterator item) noexcept;
BusyItem(const BusyItem& other) = delete;
~BusyItem() noexcept;
BusyItem& operator=(const BusyItem& other) = delete;
ESMReader& operator*() const noexcept { return mItem->mReader; }
ESMReader* operator->() const noexcept { return &mItem->mReader; }
private:
ReadersCache& mOwner;
std::list<Item>::iterator mItem;
};
explicit ReadersCache(std::size_t capacity = 100);
BusyItem get(std::size_t index);
private:
const std::size_t mCapacity;
std::map<std::size_t, std::list<Item>::iterator> mIndex;
std::list<Item> mBusyItems;
std::list<Item> mFreeItems;
std::list<Item> mClosedItems;
inline void closeExtraReaders();
inline void releaseItem(std::list<Item>::iterator it) noexcept;
};
}
#endif

View File

@ -18,6 +18,7 @@
#include <components/files/multidircollection.hpp>
#include <components/misc/resourcehelpers.hpp>
#include <components/misc/stringops.hpp>
#include <components/esm3/readerscache.hpp>
#include <algorithm>
#include <filesystem>
@ -207,7 +208,7 @@ namespace EsmLoader
}
ShallowContent shallowLoad(const Query& query, const std::vector<std::string>& contentFiles,
const Files::Collections& fileCollections, std::vector<ESM::ESMReader>& readers,
const Files::Collections& fileCollections, ESM::ReadersCache& readers,
ToUTF8::Utf8Encoder* encoder)
{
ShallowContent result;
@ -233,14 +234,14 @@ namespace EsmLoader
const Files::MultiDirCollection& collection = fileCollections.getCollection(extension);
ESM::ESMReader& reader = readers[i];
reader.setEncoder(encoder);
reader.setIndex(static_cast<int>(i));
reader.open(collection.getPath(file).string());
const ESM::ReadersCache::BusyItem reader = readers.get(i);
reader->setEncoder(encoder);
reader->setIndex(static_cast<int>(i));
reader->open(collection.getPath(file).string());
if (query.mLoadCells)
reader.resolveParentFileIndices(readers);
reader->resolveParentFileIndices(readers);
loadEsm(query, readers[i], result);
loadEsm(query, *reader, result);
}
return result;
@ -289,7 +290,7 @@ namespace EsmLoader
}
EsmData loadEsmData(const Query& query, const std::vector<std::string>& contentFiles,
const Files::Collections& fileCollections, std::vector<ESM::ESMReader>& readers, ToUTF8::Utf8Encoder* encoder)
const Files::Collections& fileCollections, ESM::ReadersCache& readers, ToUTF8::Utf8Encoder* encoder)
{
Log(Debug::Info) << "Loading ESM data...";

View File

@ -32,7 +32,7 @@ namespace EsmLoader
};
EsmData loadEsmData(const Query& query, const std::vector<std::string>& contentFiles,
const Files::Collections& fileCollections, std::vector<ESM::ESMReader>& readers,
const Files::Collections& fileCollections, ESM::ReadersCache& readers,
ToUTF8::Utf8Encoder* encoder);
}

View File

@ -13,6 +13,7 @@
#include <components/resource/bulletshapemanager.hpp>
#include <components/settings/settings.hpp>
#include <components/vfs/manager.hpp>
#include <components/esm3/readerscache.hpp>
#include <osg/ref_ptr>
@ -50,17 +51,17 @@ namespace Resource
}
std::vector<CellRef> loadCellRefs(const ESM::Cell& cell, const EsmLoader::EsmData& esmData,
std::vector<ESM::ESMReader>& readers)
ESM::ReadersCache& readers)
{
std::vector<EsmLoader::Record<CellRef>> cellRefs;
for (std::size_t i = 0; i < cell.mContextList.size(); i++)
{
ESM::ESMReader& reader = readers[static_cast<std::size_t>(cell.mContextList[i].index)];
cell.restore(reader, static_cast<int>(i));
const ESM::ReadersCache::BusyItem reader = readers.get(static_cast<std::size_t>(cell.mContextList[i].index));
cell.restore(*reader, static_cast<int>(i));
ESM::CellRef cellRef;
bool deleted = false;
while (ESM::Cell::getNextRef(reader, cellRef, deleted))
while (ESM::Cell::getNextRef(*reader, cellRef, deleted))
{
Misc::StringUtils::lowerCaseInPlace(cellRef.mRefID);
const ESM::RecNameInts type = getType(esmData, cellRef.mRefID);
@ -83,7 +84,7 @@ namespace Resource
template <class F>
void forEachObject(const ESM::Cell& cell, const EsmLoader::EsmData& esmData, const VFS::Manager& vfs,
Resource::BulletShapeManager& bulletShapeManager, std::vector<ESM::ESMReader>& readers,
Resource::BulletShapeManager& bulletShapeManager, ESM::ReadersCache& readers,
F&& f)
{
std::vector<CellRef> cellRefs = loadCellRefs(cell, esmData, readers);
@ -130,7 +131,7 @@ namespace Resource
}
}
void forEachBulletObject(std::vector<ESM::ESMReader>& readers, const VFS::Manager& vfs,
void forEachBulletObject(ESM::ReadersCache& readers, const VFS::Manager& vfs,
Resource::BulletShapeManager& bulletShapeManager, const EsmLoader::EsmData& esmData,
std::function<void (const ESM::Cell& cell, const BulletObject& object)> callback)
{

View File

@ -12,7 +12,7 @@
namespace ESM
{
class ESMReader;
class ReadersCache;
struct Cell;
}
@ -40,7 +40,7 @@ namespace Resource
float mScale;
};
void forEachBulletObject(std::vector<ESM::ESMReader>& readers, const VFS::Manager& vfs,
void forEachBulletObject(ESM::ReadersCache& readers, const VFS::Manager& vfs,
Resource::BulletShapeManager& bulletShapeManager, const EsmLoader::EsmData& esmData,
std::function<void (const ESM::Cell&, const BulletObject& object)> callback);
}