mirror of
https://gitlab.com/OpenMW/openmw.git
synced 2025-01-28 14:53:58 +00:00
3affe9913f
Use LRU cache for ESMReaders. When cache capacity is reached close least recently used ESMReader. Remember the file name if a reader was open. Once the reader requested again open the file if there is stored name for it. Put released ESMReader to the back of the free items list. Close ESMReader's from the front of the free items list. Cached item can be used only by one client at the same time. If the same item is requested twice exception is thrown. This should never happen in practice. If this happens need to fix the client logic. It's allowed to go over the capacity limit when requesting different readers. Ideally this should never happen but there will be system error anyway signalizing about too many open files. Need to fix client logic in this case. All places that were using a vector of ESMReaders now using the cache. Cache is local for each use case and there is no need for a thread safety.
359 lines
13 KiB
C++
359 lines
13 KiB
C++
#include "load.hpp"
|
|
#include "esmdata.hpp"
|
|
#include "lessbyid.hpp"
|
|
#include "record.hpp"
|
|
|
|
#include <components/debug/debuglog.hpp>
|
|
#include <components/esm3/cellref.hpp>
|
|
#include <components/esm/defs.hpp>
|
|
#include <components/esm3/esmreader.hpp>
|
|
#include <components/esm3/loadacti.hpp>
|
|
#include <components/esm3/loadcell.hpp>
|
|
#include <components/esm3/loadcont.hpp>
|
|
#include <components/esm3/loaddoor.hpp>
|
|
#include <components/esm3/loadgmst.hpp>
|
|
#include <components/esm3/loadland.hpp>
|
|
#include <components/esm3/loadstat.hpp>
|
|
#include <components/files/collections.hpp>
|
|
#include <components/files/multidircollection.hpp>
|
|
#include <components/misc/resourcehelpers.hpp>
|
|
#include <components/misc/stringops.hpp>
|
|
#include <components/esm3/readerscache.hpp>
|
|
|
|
#include <algorithm>
|
|
#include <filesystem>
|
|
#include <cstddef>
|
|
#include <map>
|
|
#include <set>
|
|
#include <string>
|
|
#include <string_view>
|
|
#include <type_traits>
|
|
#include <utility>
|
|
#include <vector>
|
|
#include <sstream>
|
|
|
|
namespace EsmLoader
|
|
{
|
|
namespace
|
|
{
|
|
struct GetKey
|
|
{
|
|
template <class T>
|
|
decltype(auto) operator()(const T& v) const
|
|
{
|
|
return (v.mId);
|
|
}
|
|
|
|
const ESM::CellId& operator()(const ESM::Cell& v) const
|
|
{
|
|
return v.mCellId;
|
|
}
|
|
|
|
std::pair<int, int> operator()(const ESM::Land& v) const
|
|
{
|
|
return std::pair(v.mX, v.mY);
|
|
}
|
|
|
|
template <class T>
|
|
decltype(auto) operator()(const Record<T>& v) const
|
|
{
|
|
return (*this)(v.mValue);
|
|
}
|
|
};
|
|
|
|
struct CellRecords
|
|
{
|
|
Records<ESM::Cell> mValues;
|
|
std::map<std::string, std::size_t> mByName;
|
|
std::map<std::pair<int, int>, std::size_t> mByPosition;
|
|
};
|
|
|
|
template <class T, class = std::void_t<>>
|
|
struct HasId : std::false_type {};
|
|
|
|
template <class T>
|
|
struct HasId<T, std::void_t<decltype(T::mId)>> : std::true_type {};
|
|
|
|
template <class T>
|
|
constexpr bool hasId = HasId<T>::value;
|
|
|
|
template <class T>
|
|
auto loadRecord(ESM::ESMReader& reader, Records<T>& records)
|
|
-> std::enable_if_t<hasId<T>>
|
|
{
|
|
T record;
|
|
bool deleted = false;
|
|
record.load(reader, deleted);
|
|
Misc::StringUtils::lowerCaseInPlace(record.mId);
|
|
if (Misc::ResourceHelpers::isHiddenMarker(record.mId))
|
|
return;
|
|
records.emplace_back(deleted, std::move(record));
|
|
}
|
|
|
|
template <class T>
|
|
auto loadRecord(ESM::ESMReader& reader, Records<T>& records)
|
|
-> std::enable_if_t<!hasId<T>>
|
|
{
|
|
T record;
|
|
bool deleted = false;
|
|
record.load(reader, deleted);
|
|
records.emplace_back(deleted, std::move(record));
|
|
}
|
|
|
|
void loadRecord(ESM::ESMReader& reader, CellRecords& records)
|
|
{
|
|
ESM::Cell record;
|
|
bool deleted = false;
|
|
record.loadNameAndData(reader, deleted);
|
|
Misc::StringUtils::lowerCaseInPlace(record.mName);
|
|
|
|
if ((record.mData.mFlags & ESM::Cell::Interior) != 0)
|
|
{
|
|
const auto it = records.mByName.find(record.mName);
|
|
if (it == records.mByName.end())
|
|
{
|
|
record.loadCell(reader, true);
|
|
records.mByName.emplace_hint(it, record.mName, records.mValues.size());
|
|
records.mValues.emplace_back(deleted, std::move(record));
|
|
}
|
|
else
|
|
{
|
|
Record<ESM::Cell>& old = records.mValues[it->second];
|
|
old.mValue.mData = record.mData;
|
|
old.mValue.loadCell(reader, true);
|
|
}
|
|
}
|
|
else
|
|
{
|
|
const std::pair<int, int> position(record.mData.mX, record.mData.mY);
|
|
const auto it = records.mByPosition.find(position);
|
|
if (it == records.mByPosition.end())
|
|
{
|
|
record.loadCell(reader, true);
|
|
records.mByPosition.emplace_hint(it, position, records.mValues.size());
|
|
records.mValues.emplace_back(deleted, std::move(record));
|
|
}
|
|
else
|
|
{
|
|
Record<ESM::Cell>& old = records.mValues[it->second];
|
|
old.mValue.mData = record.mData;
|
|
old.mValue.loadCell(reader, true);
|
|
}
|
|
}
|
|
}
|
|
|
|
struct ShallowContent
|
|
{
|
|
Records<ESM::Activator> mActivators;
|
|
CellRecords mCells;
|
|
Records<ESM::Container> mContainers;
|
|
Records<ESM::Door> mDoors;
|
|
Records<ESM::GameSetting> mGameSettings;
|
|
Records<ESM::Land> mLands;
|
|
Records<ESM::Static> mStatics;
|
|
};
|
|
|
|
void loadRecord(const Query& query, const ESM::NAME& name, ESM::ESMReader& reader, ShallowContent& content)
|
|
{
|
|
switch (name.toInt())
|
|
{
|
|
case ESM::REC_ACTI:
|
|
if (query.mLoadActivators)
|
|
return loadRecord(reader, content.mActivators);
|
|
break;
|
|
case ESM::REC_CELL:
|
|
if (query.mLoadCells)
|
|
return loadRecord(reader, content.mCells);
|
|
break;
|
|
case ESM::REC_CONT:
|
|
if (query.mLoadContainers)
|
|
return loadRecord(reader, content.mContainers);
|
|
break;
|
|
case ESM::REC_DOOR:
|
|
if (query.mLoadDoors)
|
|
return loadRecord(reader, content.mDoors);
|
|
break;
|
|
case ESM::REC_GMST:
|
|
if (query.mLoadGameSettings)
|
|
return loadRecord(reader, content.mGameSettings);
|
|
break;
|
|
case ESM::REC_LAND:
|
|
if (query.mLoadLands)
|
|
return loadRecord(reader, content.mLands);
|
|
break;
|
|
case ESM::REC_STAT:
|
|
if (query.mLoadStatics)
|
|
return loadRecord(reader, content.mStatics);
|
|
break;
|
|
}
|
|
|
|
reader.skipRecord();
|
|
}
|
|
|
|
void loadEsm(const Query& query, ESM::ESMReader& reader, ShallowContent& content)
|
|
{
|
|
Log(Debug::Info) << "Loading ESM file " << reader.getName();
|
|
|
|
while (reader.hasMoreRecs())
|
|
{
|
|
const ESM::NAME recName = reader.getRecName();
|
|
reader.getRecHeader();
|
|
if (reader.getRecordFlags() & ESM::FLAG_Ignored)
|
|
{
|
|
reader.skipRecord();
|
|
continue;
|
|
}
|
|
loadRecord(query, recName, reader, content);
|
|
}
|
|
}
|
|
|
|
ShallowContent shallowLoad(const Query& query, const std::vector<std::string>& contentFiles,
|
|
const Files::Collections& fileCollections, ESM::ReadersCache& readers,
|
|
ToUTF8::Utf8Encoder* encoder)
|
|
{
|
|
ShallowContent result;
|
|
|
|
const std::set<std::string> supportedFormats {
|
|
".esm",
|
|
".esp",
|
|
".omwgame",
|
|
".omwaddon",
|
|
".project",
|
|
};
|
|
|
|
for (std::size_t i = 0; i < contentFiles.size(); ++i)
|
|
{
|
|
const std::string &file = contentFiles[i];
|
|
const std::string extension = Misc::StringUtils::lowerCase(std::filesystem::path(file).extension().string());
|
|
|
|
if (supportedFormats.find(extension) == supportedFormats.end())
|
|
{
|
|
Log(Debug::Warning) << "Skipping unsupported content file: " << file;
|
|
continue;
|
|
}
|
|
|
|
const Files::MultiDirCollection& collection = fileCollections.getCollection(extension);
|
|
|
|
const ESM::ReadersCache::BusyItem reader = readers.get(i);
|
|
reader->setEncoder(encoder);
|
|
reader->setIndex(static_cast<int>(i));
|
|
reader->open(collection.getPath(file).string());
|
|
if (query.mLoadCells)
|
|
reader->resolveParentFileIndices(readers);
|
|
|
|
loadEsm(query, *reader, result);
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
struct WithType
|
|
{
|
|
ESM::RecNameInts mType;
|
|
|
|
template <class T>
|
|
RefIdWithType operator()(const T& v) const { return {v.mId, mType}; }
|
|
};
|
|
|
|
template <class T>
|
|
void addRefIdsTypes(const std::vector<T>& values, std::vector<RefIdWithType>& refIdsTypes)
|
|
{
|
|
std::transform(values.begin(), values.end(), std::back_inserter(refIdsTypes),
|
|
WithType {static_cast<ESM::RecNameInts>(T::sRecordId)});
|
|
}
|
|
|
|
void addRefIdsTypes(EsmData& content)
|
|
{
|
|
content.mRefIdTypes.reserve(
|
|
content.mActivators.size()
|
|
+ content.mContainers.size()
|
|
+ content.mDoors.size()
|
|
+ content.mStatics.size()
|
|
);
|
|
|
|
addRefIdsTypes(content.mActivators, content.mRefIdTypes);
|
|
addRefIdsTypes(content.mContainers, content.mRefIdTypes);
|
|
addRefIdsTypes(content.mDoors, content.mRefIdTypes);
|
|
addRefIdsTypes(content.mStatics, content.mRefIdTypes);
|
|
|
|
std::sort(content.mRefIdTypes.begin(), content.mRefIdTypes.end(), LessById {});
|
|
}
|
|
|
|
std::vector<ESM::Cell> prepareCellRecords(Records<ESM::Cell>& records)
|
|
{
|
|
std::vector<ESM::Cell> result;
|
|
for (Record<ESM::Cell>& v : records)
|
|
if (!v.mDeleted)
|
|
result.emplace_back(std::move(v.mValue));
|
|
return result;
|
|
}
|
|
}
|
|
|
|
EsmData loadEsmData(const Query& query, const std::vector<std::string>& contentFiles,
|
|
const Files::Collections& fileCollections, ESM::ReadersCache& readers, ToUTF8::Utf8Encoder* encoder)
|
|
{
|
|
Log(Debug::Info) << "Loading ESM data...";
|
|
|
|
ShallowContent content = shallowLoad(query, contentFiles, fileCollections, readers, encoder);
|
|
|
|
std::ostringstream loaded;
|
|
|
|
if (query.mLoadActivators)
|
|
loaded << ' ' << content.mActivators.size() << " activators,";
|
|
if (query.mLoadCells)
|
|
loaded << ' ' << content.mCells.mValues.size() << " cells,";
|
|
if (query.mLoadContainers)
|
|
loaded << ' ' << content.mContainers.size() << " containers,";
|
|
if (query.mLoadDoors)
|
|
loaded << ' ' << content.mDoors.size() << " doors,";
|
|
if (query.mLoadGameSettings)
|
|
loaded << ' ' << content.mGameSettings.size() << " game settings,";
|
|
if (query.mLoadLands)
|
|
loaded << ' ' << content.mLands.size() << " lands,";
|
|
if (query.mLoadStatics)
|
|
loaded << ' ' << content.mStatics.size() << " statics,";
|
|
|
|
Log(Debug::Info) << "Loaded" << loaded.str();
|
|
|
|
EsmData result;
|
|
|
|
if (query.mLoadActivators)
|
|
result.mActivators = prepareRecords(content.mActivators, GetKey {});
|
|
if (query.mLoadCells)
|
|
result.mCells = prepareCellRecords(content.mCells.mValues);
|
|
if (query.mLoadContainers)
|
|
result.mContainers = prepareRecords(content.mContainers, GetKey {});
|
|
if (query.mLoadDoors)
|
|
result.mDoors = prepareRecords(content.mDoors, GetKey {});
|
|
if (query.mLoadGameSettings)
|
|
result.mGameSettings = prepareRecords(content.mGameSettings, GetKey {});
|
|
if (query.mLoadLands)
|
|
result.mLands = prepareRecords(content.mLands, GetKey {});
|
|
if (query.mLoadStatics)
|
|
result.mStatics = prepareRecords(content.mStatics, GetKey {});
|
|
|
|
addRefIdsTypes(result);
|
|
|
|
std::ostringstream prepared;
|
|
|
|
if (query.mLoadActivators)
|
|
prepared << ' ' << result.mActivators.size() << " unique activators,";
|
|
if (query.mLoadCells)
|
|
prepared << ' ' << result.mCells.size() << " unique cells,";
|
|
if (query.mLoadContainers)
|
|
prepared << ' ' << result.mContainers.size() << " unique containers,";
|
|
if (query.mLoadDoors)
|
|
prepared << ' ' << result.mDoors.size() << " unique doors,";
|
|
if (query.mLoadGameSettings)
|
|
prepared << ' ' << result.mGameSettings.size() << " unique game settings,";
|
|
if (query.mLoadLands)
|
|
prepared << ' ' << result.mLands.size() << " unique lands,";
|
|
if (query.mLoadStatics)
|
|
prepared << ' ' << result.mStatics.size() << " unique statics,";
|
|
|
|
Log(Debug::Info) << "Prepared" << prepared.str();
|
|
|
|
return result;
|
|
}
|
|
}
|