mirror/OpenMW
1
0
Fork 0
mirror of https://gitlab.com/OpenMW/openmw.git synced 2025-02-26 06:40:22 +00:00
Code Issues Projects Releases Wiki Activity
OpenMW/components/esmloader/load.cpp

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

370 lines
13 KiB
C++
Raw Normal View History

Add ESM data loader A component to load ESM content files with limited support for record types and selection which of them to load. Supported record types are: ACTI, CELL, CONT, DOOR, GMST, LAND, STAT.
2021-10-16 19:07:55 +02:00
#include "load.hpp"
#include "esmdata.hpp"
#include "lessbyid.hpp"
#include "record.hpp"
#include <components/debug/debuglog.hpp>
move most of the files from esm to esm3, keep common code in esm; this is make space for a future with esm4 esm typo esm typo
2022-01-22 15:58:41 +01:00
#include <components/esm3/cellref.hpp>
Add ESM data loader A component to load ESM content files with limited support for record types and selection which of them to load. Supported record types are: ACTI, CELL, CONT, DOOR, GMST, LAND, STAT.
2021-10-16 19:07:55 +02:00
#include <components/esm/defs.hpp>
move most of the files from esm to esm3, keep common code in esm; this is make space for a future with esm4 esm typo esm typo
2022-01-22 15:58:41 +01:00
#include <components/esm3/esmreader.hpp>
#include <components/esm3/loadacti.hpp>
#include <components/esm3/loadcell.hpp>
#include <components/esm3/loadcont.hpp>
#include <components/esm3/loaddoor.hpp>
#include <components/esm3/loadgmst.hpp>
#include <components/esm3/loadland.hpp>
#include <components/esm3/loadstat.hpp>
Add ESM data loader A component to load ESM content files with limited support for record types and selection which of them to load. Supported record types are: ACTI, CELL, CONT, DOOR, GMST, LAND, STAT.
2021-10-16 19:07:55 +02:00
#include <components/files/collections.hpp>
#include <components/files/multidircollection.hpp>
#include <components/misc/resourcehelpers.hpp>
Split components/misc/stringops.hpp into multiple headers Replace all ciEqual overloads with one having std::string_view as argument.
2022-08-03 00:00:54 +02:00
#include <components/misc/strings/lower.hpp>
Limit the number of simultaneously open not actively used content files Use LRU cache for ESMReaders. When cache capacity is reached close least recently used ESMReader. Remember the file name if a reader was open. Once the reader requested again open the file if there is stored name for it. Put released ESMReader to the back of the free items list. Close ESMReader's from the front of the free items list. Cached item can be used only by one client at the same time. If the same item is requested twice exception is thrown. This should never happen in practice. If this happens need to fix the client logic. It's allowed to go over the capacity limit when requesting different readers. Ideally this should never happen but there will be system error anyway signalizing about too many open files. Need to fix client logic in this case. All places that were using a vector of ESMReaders now using the cache. Cache is local for each use case and there is no need for a thread safety.
2022-06-01 22:53:18 +02:00
#include <components/esm3/readerscache.hpp>
Show groundcover loading progress
2022-06-03 18:59:08 +02:00
#include <components/loadinglistener/loadinglistener.hpp>
Add ESM data loader A component to load ESM content files with limited support for record types and selection which of them to load. Supported record types are: ACTI, CELL, CONT, DOOR, GMST, LAND, STAT.
2021-10-16 19:07:55 +02:00
#include <algorithm>
Remove even more of boost::filesystem
2022-05-25 18:29:02 +00:00
#include <filesystem>
Add ESM data loader A component to load ESM content files with limited support for record types and selection which of them to load. Supported record types are: ACTI, CELL, CONT, DOOR, GMST, LAND, STAT.
2021-10-16 19:07:55 +02:00
#include <cstddef>
#include <map>
Load only supported content formats by EsmLoader
2021-11-21 16:16:15 +01:00
#include <set>
Add ESM data loader A component to load ESM content files with limited support for record types and selection which of them to load. Supported record types are: ACTI, CELL, CONT, DOOR, GMST, LAND, STAT.
2021-10-16 19:07:55 +02:00
#include <string>
#include <string_view>
#include <type_traits>
#include <utility>
#include <vector>
Remove redundant include from esm3/esmreader.hpp
2022-04-10 18:33:42 +02:00
#include <sstream>
Add ESM data loader A component to load ESM content files with limited support for record types and selection which of them to load. Supported record types are: ACTI, CELL, CONT, DOOR, GMST, LAND, STAT.
2021-10-16 19:07:55 +02:00
namespace EsmLoader
{
namespace
{
struct GetKey
{
template <class T>
decltype(auto) operator()(const T& v) const
{
return (v.mId);
}
const ESM::CellId& operator()(const ESM::Cell& v) const
{
return v.mCellId;
}
std::pair<int, int> operator()(const ESM::Land& v) const
{
return std::pair(v.mX, v.mY);
}
template <class T>
decltype(auto) operator()(const Record<T>& v) const
{
return (*this)(v.mValue);
}
};
struct CellRecords
{
Records<ESM::Cell> mValues;
std::map<std::string, std::size_t> mByName;
std::map<std::pair<int, int>, std::size_t> mByPosition;
};
template <class T, class = std::void_t<>>
struct HasId : std::false_type {};
template <class T>
struct HasId<T, std::void_t<decltype(T::mId)>> : std::true_type {};
template <class T>
constexpr bool hasId = HasId<T>::value;
template <class T>
auto loadRecord(ESM::ESMReader& reader, Records<T>& records)
-> std::enable_if_t<hasId<T>>
{
T record;
bool deleted = false;
record.load(reader, deleted);
Misc::StringUtils::lowerCaseInPlace(record.mId);
if (Misc::ResourceHelpers::isHiddenMarker(record.mId))
return;
records.emplace_back(deleted, std::move(record));
}
template <class T>
auto loadRecord(ESM::ESMReader& reader, Records<T>& records)
-> std::enable_if_t<!hasId<T>>
{
T record;
bool deleted = false;
record.load(reader, deleted);
records.emplace_back(deleted, std::move(record));
}
void loadRecord(ESM::ESMReader& reader, CellRecords& records)
{
ESM::Cell record;
bool deleted = false;
record.loadNameAndData(reader, deleted);
Misc::StringUtils::lowerCaseInPlace(record.mName);
if ((record.mData.mFlags & ESM::Cell::Interior) != 0)
{
const auto it = records.mByName.find(record.mName);
if (it == records.mByName.end())
{
record.loadCell(reader, true);
records.mByName.emplace_hint(it, record.mName, records.mValues.size());
records.mValues.emplace_back(deleted, std::move(record));
}
else
{
Record<ESM::Cell>& old = records.mValues[it->second];
old.mValue.mData = record.mData;
old.mValue.loadCell(reader, true);
}
}
else
{
const std::pair<int, int> position(record.mData.mX, record.mData.mY);
const auto it = records.mByPosition.find(position);
if (it == records.mByPosition.end())
{
record.loadCell(reader, true);
records.mByPosition.emplace_hint(it, position, records.mValues.size());
records.mValues.emplace_back(deleted, std::move(record));
}
else
{
Record<ESM::Cell>& old = records.mValues[it->second];
old.mValue.mData = record.mData;
old.mValue.loadCell(reader, true);
}
}
}
struct ShallowContent
{
Records<ESM::Activator> mActivators;
CellRecords mCells;
Records<ESM::Container> mContainers;
Records<ESM::Door> mDoors;
Records<ESM::GameSetting> mGameSettings;
Records<ESM::Land> mLands;
Records<ESM::Static> mStatics;
};
void loadRecord(const Query& query, const ESM::NAME& name, ESM::ESMReader& reader, ShallowContent& content)
{
switch (name.toInt())
{
case ESM::REC_ACTI:
if (query.mLoadActivators)
return loadRecord(reader, content.mActivators);
break;
case ESM::REC_CELL:
if (query.mLoadCells)
return loadRecord(reader, content.mCells);
break;
case ESM::REC_CONT:
if (query.mLoadContainers)
return loadRecord(reader, content.mContainers);
break;
case ESM::REC_DOOR:
if (query.mLoadDoors)
return loadRecord(reader, content.mDoors);
break;
case ESM::REC_GMST:
if (query.mLoadGameSettings)
return loadRecord(reader, content.mGameSettings);
break;
case ESM::REC_LAND:
if (query.mLoadLands)
return loadRecord(reader, content.mLands);
break;
case ESM::REC_STAT:
if (query.mLoadStatics)
return loadRecord(reader, content.mStatics);
break;
}
reader.skipRecord();
}
Show groundcover loading progress
2022-06-03 18:59:08 +02:00
void loadEsm(const Query& query, ESM::ESMReader& reader, ShallowContent& content, Loading::Listener* listener)
Add ESM data loader A component to load ESM content files with limited support for record types and selection which of them to load. Supported record types are: ACTI, CELL, CONT, DOOR, GMST, LAND, STAT.
2021-10-16 19:07:55 +02:00
{
Log(Debug::Info) << "Loading ESM file " << reader.getName();
while (reader.hasMoreRecs())
{
const ESM::NAME recName = reader.getRecName();
reader.getRecHeader();
Respect ignored flag in navmeshtool and groundcover
2022-04-17 09:23:09 +02:00
if (reader.getRecordFlags() & ESM::FLAG_Ignored)
{
reader.skipRecord();
continue;
}
Add ESM data loader A component to load ESM content files with limited support for record types and selection which of them to load. Supported record types are: ACTI, CELL, CONT, DOOR, GMST, LAND, STAT.
2021-10-16 19:07:55 +02:00
loadRecord(query, recName, reader, content);
Show groundcover loading progress
2022-06-03 18:59:08 +02:00
if (listener != nullptr)
listener->setProgress(fileProgress * reader.getFileOffset() / reader.getFileSize());
Add ESM data loader A component to load ESM content files with limited support for record types and selection which of them to load. Supported record types are: ACTI, CELL, CONT, DOOR, GMST, LAND, STAT.
2021-10-16 19:07:55 +02:00
}
}
ShallowContent shallowLoad(const Query& query, const std::vector<std::string>& contentFiles,
Limit the number of simultaneously open not actively used content files Use LRU cache for ESMReaders. When cache capacity is reached close least recently used ESMReader. Remember the file name if a reader was open. Once the reader requested again open the file if there is stored name for it. Put released ESMReader to the back of the free items list. Close ESMReader's from the front of the free items list. Cached item can be used only by one client at the same time. If the same item is requested twice exception is thrown. This should never happen in practice. If this happens need to fix the client logic. It's allowed to go over the capacity limit when requesting different readers. Ideally this should never happen but there will be system error anyway signalizing about too many open files. Need to fix client logic in this case. All places that were using a vector of ESMReaders now using the cache. Cache is local for each use case and there is no need for a thread safety.
2022-06-01 22:53:18 +02:00
const Files::Collections& fileCollections, ESM::ReadersCache& readers,
Show groundcover loading progress
2022-06-03 18:59:08 +02:00
ToUTF8::Utf8Encoder* encoder, Loading::Listener* listener)
Add ESM data loader A component to load ESM content files with limited support for record types and selection which of them to load. Supported record types are: ACTI, CELL, CONT, DOOR, GMST, LAND, STAT.
2021-10-16 19:07:55 +02:00
{
ShallowContent result;
Load only supported content formats by EsmLoader
2021-11-21 16:16:15 +01:00
const std::set<std::string> supportedFormats {
".esm",
".esp",
".omwgame",
".omwaddon",
".project",
};
Add ESM data loader A component to load ESM content files with limited support for record types and selection which of them to load. Supported record types are: ACTI, CELL, CONT, DOOR, GMST, LAND, STAT.
2021-10-16 19:07:55 +02:00
for (std::size_t i = 0; i < contentFiles.size(); ++i)
{
const std::string &file = contentFiles[i];
Make sure all paths are passed as std::filesystem::path instead of std::string where possible.
2022-06-19 13:28:33 +02:00
const std::string extension = Misc::StringUtils::lowerCase(std::filesystem::path(file).extension().string()); //TODO(Project579): let's hope unicode characters are never used in these extensions on windows or this will break
Load only supported content formats by EsmLoader
2021-11-21 16:16:15 +01:00
if (supportedFormats.find(extension) == supportedFormats.end())
{
Log(Debug::Warning) << "Skipping unsupported content file: " << file;
continue;
}
Show groundcover loading progress
2022-06-03 18:59:08 +02:00
if (listener != nullptr)
{
listener->setLabel(file);
listener->setProgressRange(fileProgress);
}
Load only supported content formats by EsmLoader
2021-11-21 16:16:15 +01:00
const Files::MultiDirCollection& collection = fileCollections.getCollection(extension);
Add ESM data loader A component to load ESM content files with limited support for record types and selection which of them to load. Supported record types are: ACTI, CELL, CONT, DOOR, GMST, LAND, STAT.
2021-10-16 19:07:55 +02:00
Limit the number of simultaneously open not actively used content files Use LRU cache for ESMReaders. When cache capacity is reached close least recently used ESMReader. Remember the file name if a reader was open. Once the reader requested again open the file if there is stored name for it. Put released ESMReader to the back of the free items list. Close ESMReader's from the front of the free items list. Cached item can be used only by one client at the same time. If the same item is requested twice exception is thrown. This should never happen in practice. If this happens need to fix the client logic. It's allowed to go over the capacity limit when requesting different readers. Ideally this should never happen but there will be system error anyway signalizing about too many open files. Need to fix client logic in this case. All places that were using a vector of ESMReaders now using the cache. Cache is local for each use case and there is no need for a thread safety.
2022-06-01 22:53:18 +02:00
const ESM::ReadersCache::BusyItem reader = readers.get(i);
reader->setEncoder(encoder);
reader->setIndex(static_cast<int>(i));
Make sure all paths are passed as std::filesystem::path instead of std::string where possible.
2022-06-19 13:28:33 +02:00
reader->open(collection.getPath(file));
Use a separate storage for groundcover data
2021-10-29 16:47:17 +04:00
if (query.mLoadCells)
Limit the number of simultaneously open not actively used content files Use LRU cache for ESMReaders. When cache capacity is reached close least recently used ESMReader. Remember the file name if a reader was open. Once the reader requested again open the file if there is stored name for it. Put released ESMReader to the back of the free items list. Close ESMReader's from the front of the free items list. Cached item can be used only by one client at the same time. If the same item is requested twice exception is thrown. This should never happen in practice. If this happens need to fix the client logic. It's allowed to go over the capacity limit when requesting different readers. Ideally this should never happen but there will be system error anyway signalizing about too many open files. Need to fix client logic in this case. All places that were using a vector of ESMReaders now using the cache. Cache is local for each use case and there is no need for a thread safety.
2022-06-01 22:53:18 +02:00
reader->resolveParentFileIndices(readers);
Add ESM data loader A component to load ESM content files with limited support for record types and selection which of them to load. Supported record types are: ACTI, CELL, CONT, DOOR, GMST, LAND, STAT.
2021-10-16 19:07:55 +02:00
Show groundcover loading progress
2022-06-03 18:59:08 +02:00
loadEsm(query, *reader, result, listener);
Add ESM data loader A component to load ESM content files with limited support for record types and selection which of them to load. Supported record types are: ACTI, CELL, CONT, DOOR, GMST, LAND, STAT.
2021-10-16 19:07:55 +02:00
}
return result;
}
struct WithType
{
ESM::RecNameInts mType;
template <class T>
RefIdWithType operator()(const T& v) const { return {v.mId, mType}; }
};
template <class T>
void addRefIdsTypes(const std::vector<T>& values, std::vector<RefIdWithType>& refIdsTypes)
{
std::transform(values.begin(), values.end(), std::back_inserter(refIdsTypes),
WithType {static_cast<ESM::RecNameInts>(T::sRecordId)});
}
void addRefIdsTypes(EsmData& content)
{
content.mRefIdTypes.reserve(
content.mActivators.size()
+ content.mContainers.size()
+ content.mDoors.size()
+ content.mStatics.size()
);
addRefIdsTypes(content.mActivators, content.mRefIdTypes);
addRefIdsTypes(content.mContainers, content.mRefIdTypes);
addRefIdsTypes(content.mDoors, content.mRefIdTypes);
addRefIdsTypes(content.mStatics, content.mRefIdTypes);
std::sort(content.mRefIdTypes.begin(), content.mRefIdTypes.end(), LessById {});
}
std::vector<ESM::Cell> prepareCellRecords(Records<ESM::Cell>& records)
{
std::vector<ESM::Cell> result;
for (Record<ESM::Cell>& v : records)
if (!v.mDeleted)
result.emplace_back(std::move(v.mValue));
return result;
}
}
EsmData loadEsmData(const Query& query, const std::vector<std::string>& contentFiles,
Show groundcover loading progress
2022-06-03 18:59:08 +02:00
const Files::Collections& fileCollections, ESM::ReadersCache& readers, ToUTF8::Utf8Encoder* encoder,
Loading::Listener* listener)
Add ESM data loader A component to load ESM content files with limited support for record types and selection which of them to load. Supported record types are: ACTI, CELL, CONT, DOOR, GMST, LAND, STAT.
2021-10-16 19:07:55 +02:00
{
Log(Debug::Info) << "Loading ESM data...";
Show groundcover loading progress
2022-06-03 18:59:08 +02:00
ShallowContent content = shallowLoad(query, contentFiles, fileCollections, readers, encoder, listener);
Add ESM data loader A component to load ESM content files with limited support for record types and selection which of them to load. Supported record types are: ACTI, CELL, CONT, DOOR, GMST, LAND, STAT.
2021-10-16 19:07:55 +02:00
std::ostringstream loaded;
if (query.mLoadActivators)
loaded << ' ' << content.mActivators.size() << " activators,";
if (query.mLoadCells)
loaded << ' ' << content.mCells.mValues.size() << " cells,";
if (query.mLoadContainers)
loaded << ' ' << content.mContainers.size() << " containers,";
if (query.mLoadDoors)
loaded << ' ' << content.mDoors.size() << " doors,";
if (query.mLoadGameSettings)
loaded << ' ' << content.mGameSettings.size() << " game settings,";
if (query.mLoadLands)
loaded << ' ' << content.mLands.size() << " lands,";
if (query.mLoadStatics)
loaded << ' ' << content.mStatics.size() << " statics,";
Log(Debug::Info) << "Loaded" << loaded.str();
EsmData result;
if (query.mLoadActivators)
result.mActivators = prepareRecords(content.mActivators, GetKey {});
if (query.mLoadCells)
result.mCells = prepareCellRecords(content.mCells.mValues);
if (query.mLoadContainers)
result.mContainers = prepareRecords(content.mContainers, GetKey {});
if (query.mLoadDoors)
result.mDoors = prepareRecords(content.mDoors, GetKey {});
if (query.mLoadGameSettings)
result.mGameSettings = prepareRecords(content.mGameSettings, GetKey {});
if (query.mLoadLands)
result.mLands = prepareRecords(content.mLands, GetKey {});
if (query.mLoadStatics)
result.mStatics = prepareRecords(content.mStatics, GetKey {});
addRefIdsTypes(result);
std::ostringstream prepared;
if (query.mLoadActivators)
prepared << ' ' << result.mActivators.size() << " unique activators,";
if (query.mLoadCells)
prepared << ' ' << result.mCells.size() << " unique cells,";
if (query.mLoadContainers)
prepared << ' ' << result.mContainers.size() << " unique containers,";
if (query.mLoadDoors)
prepared << ' ' << result.mDoors.size() << " unique doors,";
if (query.mLoadGameSettings)
prepared << ' ' << result.mGameSettings.size() << " unique game settings,";
if (query.mLoadLands)
prepared << ' ' << result.mLands.size() << " unique lands,";
if (query.mLoadStatics)
prepared << ' ' << result.mStatics.size() << " unique statics,";
Log(Debug::Info) << "Prepared" << prepared.str();
return result;
}
}
Reference in New Issue Copy Permalink