1
0
mirror of https://gitlab.com/OpenMW/openmw.git synced 2025-01-06 00:55:50 +00:00
OpenMW/components/esm3/esmreader.cpp
elsid 3affe9913f
Limit the number of simultaneously open not actively used content files
Use LRU cache for ESMReaders. When cache capacity is reached close least
recently used ESMReader. Remember the file name if a reader was open. Once the
reader requested again open the file if there is stored name for it. Put
released ESMReader to the back of the free items list. Close ESMReader's from
the front of the free items list.

Cached item can be used only by one client at the same time. If the same item is
requested twice exception is thrown. This should never happen in practice. If
this happens need to fix the client logic.

It's allowed to go over the capacity limit when requesting different readers.
Ideally this should never happen but there will be system error anyway
signalizing about too many open files. Need to fix client logic in this case.

All places that were using a vector of ESMReaders now using the cache. Cache is
local for each use case and there is no need for a thread safety.
2022-06-03 01:29:03 +02:00

374 lines
8.8 KiB
C++

#include "esmreader.hpp"
#include "readerscache.hpp"
#include <components/misc/stringops.hpp>
#include <components/files/openfile.hpp>
#include <stdexcept>
#include <sstream>
#include <filesystem>
#include <fstream>
namespace ESM
{
using namespace Misc;
ESM_Context ESMReader::getContext()
{
// Update the file position before returning
mCtx.filePos = mEsm->tellg();
return mCtx;
}
ESMReader::ESMReader()
: mRecordFlags(0)
, mBuffer(50*1024)
, mEncoder(nullptr)
, mFileSize(0)
{
clearCtx();
mCtx.index = 0;
}
void ESMReader::restoreContext(const ESM_Context &rc)
{
// Reopen the file if necessary
if (mCtx.filename != rc.filename)
openRaw(rc.filename);
// Copy the data
mCtx = rc;
// Make sure we seek to the right place
mEsm->seekg(mCtx.filePos);
}
void ESMReader::close()
{
mEsm.reset();
clearCtx();
mHeader.blank();
}
void ESMReader::clearCtx()
{
mCtx.filename.clear();
mCtx.leftFile = 0;
mCtx.leftRec = 0;
mCtx.leftSub = 0;
mCtx.subCached = false;
mCtx.recName.clear();
mCtx.subName.clear();
}
void ESMReader::resolveParentFileIndices(ReadersCache& readers)
{
mCtx.parentFileIndices.clear();
for (const Header::MasterData &mast : getGameFiles())
{
const std::string& fname = mast.name;
int index = getIndex();
for (int i = 0; i < getIndex(); i++)
{
const ESM::ReadersCache::BusyItem reader = readers.get(static_cast<std::size_t>(i));
if (reader->getFileSize() == 0)
continue; // Content file in non-ESM format
const std::string& candidate = reader->getName();
std::string fnamecandidate = std::filesystem::path(candidate).filename().string();
if (Misc::StringUtils::ciEqual(fname, fnamecandidate))
{
index = i;
break;
}
}
mCtx.parentFileIndices.push_back(index);
}
}
void ESMReader::openRaw(std::unique_ptr<std::istream>&& stream, std::string_view name)
{
close();
mEsm = std::move(stream);
mCtx.filename = name;
mEsm->seekg(0, mEsm->end);
mCtx.leftFile = mFileSize = mEsm->tellg();
mEsm->seekg(0, mEsm->beg);
}
void ESMReader::openRaw(std::string_view filename)
{
openRaw(Files::openBinaryInputFileStream(std::string(filename)), filename);
}
void ESMReader::open(std::unique_ptr<std::istream>&& stream, const std::string &name)
{
openRaw(std::move(stream), name);
if (getRecName() != "TES3")
fail("Not a valid Morrowind file");
getRecHeader();
mHeader.load (*this);
}
void ESMReader::open(const std::string &file)
{
open(Files::openBinaryInputFileStream(file), file);
}
std::string ESMReader::getHNOString(NAME name)
{
if (isNextSub(name))
return getHString();
return "";
}
void ESMReader::skipHNOString(NAME name)
{
if (isNextSub(name))
skipHString();
}
std::string ESMReader::getHNString(NAME name)
{
getSubNameIs(name);
return getHString();
}
std::string ESMReader::getHString()
{
getSubHeader();
// Hack to make MultiMark.esp load. Zero-length strings do not
// occur in any of the official mods, but MultiMark makes use of
// them. For some reason, they break the rules, and contain a byte
// (value 0) even if the header says there is no data. If
// Morrowind accepts it, so should we.
if (mCtx.leftSub == 0 && hasMoreSubs() && !mEsm->peek())
{
// Skip the following zero byte
mCtx.leftRec--;
char c;
getT(c);
return std::string();
}
return getString(mCtx.leftSub);
}
void ESMReader::skipHString()
{
getSubHeader();
// Hack to make MultiMark.esp load. Zero-length strings do not
// occur in any of the official mods, but MultiMark makes use of
// them. For some reason, they break the rules, and contain a byte
// (value 0) even if the header says there is no data. If
// Morrowind accepts it, so should we.
if (mCtx.leftSub == 0 && hasMoreSubs() && !mEsm->peek())
{
// Skip the following zero byte
mCtx.leftRec--;
skipT<char>();
return;
}
skip(mCtx.leftSub);
}
void ESMReader::getHExact(void*p, int size)
{
getSubHeader();
if (size != static_cast<int> (mCtx.leftSub))
reportSubSizeMismatch(size, mCtx.leftSub);
getExact(p, size);
}
// Read the given number of bytes from a named subrecord
void ESMReader::getHNExact(void*p, int size, NAME name)
{
getSubNameIs(name);
getHExact(p, size);
}
// Get the next subrecord name and check if it matches the parameter
void ESMReader::getSubNameIs(NAME name)
{
getSubName();
if (mCtx.subName != name)
fail("Expected subrecord " + name.toString() + " but got " + mCtx.subName.toString());
}
bool ESMReader::isNextSub(NAME name)
{
if (!hasMoreSubs())
return false;
getSubName();
// If the name didn't match, then mark the it as 'cached' so it's
// available for the next call to getSubName.
mCtx.subCached = (mCtx.subName != name);
// If subCached is false, then subName == name.
return !mCtx.subCached;
}
bool ESMReader::peekNextSub(NAME name)
{
if (!hasMoreSubs())
return false;
getSubName();
mCtx.subCached = true;
return mCtx.subName == name;
}
// Read subrecord name. This gets called a LOT, so I've optimized it
// slightly.
void ESMReader::getSubName()
{
// If the name has already been read, do nothing
if (mCtx.subCached)
{
mCtx.subCached = false;
return;
}
// reading the subrecord data anyway.
const std::size_t subNameSize = decltype(mCtx.subName)::sCapacity;
getExact(mCtx.subName.mData, static_cast<int>(subNameSize));
mCtx.leftRec -= static_cast<std::uint32_t>(subNameSize);
}
void ESMReader::skipHSub()
{
getSubHeader();
skip(mCtx.leftSub);
}
void ESMReader::skipHSubSize(int size)
{
skipHSub();
if (static_cast<int> (mCtx.leftSub) != size)
reportSubSizeMismatch(mCtx.leftSub, size);
}
void ESMReader::skipHSubUntil(NAME name)
{
while (hasMoreSubs() && !isNextSub(name))
{
mCtx.subCached = false;
skipHSub();
}
if (hasMoreSubs())
mCtx.subCached = true;
}
void ESMReader::getSubHeader()
{
if (mCtx.leftRec < sizeof(mCtx.leftSub))
fail("End of record while reading sub-record header");
// Get subrecord size
getT(mCtx.leftSub);
mCtx.leftRec -= sizeof(mCtx.leftSub);
// Adjust number of record bytes left
if (mCtx.leftRec < mCtx.leftSub)
fail("Record size is larger than rest of file");
mCtx.leftRec -= mCtx.leftSub;
}
NAME ESMReader::getRecName()
{
if (!hasMoreRecs())
fail("No more records, getRecName() failed");
getName(mCtx.recName);
mCtx.leftFile -= decltype(mCtx.recName)::sCapacity;
// Make sure we don't carry over any old cached subrecord
// names. This can happen in some cases when we skip parts of a
// record.
mCtx.subCached = false;
return mCtx.recName;
}
void ESMReader::skipRecord()
{
skip(mCtx.leftRec);
mCtx.leftRec = 0;
mCtx.subCached = false;
}
void ESMReader::getRecHeader(uint32_t &flags)
{
// General error checking
if (mCtx.leftFile < 3 * sizeof(uint32_t))
fail("End of file while reading record header");
if (mCtx.leftRec)
fail("Previous record contains unread bytes");
getUint(mCtx.leftRec);
getUint(flags);// This header entry is always zero
getUint(flags);
mCtx.leftFile -= 3 * sizeof(uint32_t);
// Check that sizes add up
if (mCtx.leftFile < mCtx.leftRec)
reportSubSizeMismatch(mCtx.leftFile, mCtx.leftRec);
// Adjust number of bytes mCtx.left in file
mCtx.leftFile -= mCtx.leftRec;
}
/*************************************************************************
*
* Lowest level data reading and misc methods
*
*************************************************************************/
std::string ESMReader::getString(int size)
{
size_t s = size;
if (mBuffer.size() <= s)
// Add some extra padding to reduce the chance of having to resize
// again later.
mBuffer.resize(3*s);
// And make sure the string is zero terminated
mBuffer[s] = 0;
// read ESM data
char *ptr = mBuffer.data();
getExact(ptr, size);
size = static_cast<int>(strnlen(ptr, size));
// Convert to UTF8 and return
if (mEncoder)
return std::string(mEncoder->getUtf8(std::string_view(ptr, size)));
return std::string (ptr, size);
}
[[noreturn]] void ESMReader::fail(const std::string &msg)
{
std::stringstream ss;
ss << "ESM Error: " << msg;
ss << "\n File: " << mCtx.filename;
ss << "\n Record: " << mCtx.recName.toStringView();
ss << "\n Subrecord: " << mCtx.subName.toStringView();
if (mEsm.get())
ss << "\n Offset: 0x" << std::hex << mEsm->tellg();
throw std::runtime_error(ss.str());
}
}