mirror/OpenMW
1
0
Fork 0
mirror of https://gitlab.com/OpenMW/openmw.git synced 2025-02-22 12:39:59 +00:00
Code Issues Projects Releases Wiki Activity

Initial commit to load ESM4

Browse Source 
Some data is actually loaded and store in ESM Store
Any new ESM4 will go through the same code path and be automatically sent to the right store
This commit is contained in:
florent.teppe 2022-12-28 20:48:25 +01:00
parent 80e2cd79ec
commit c721a6cafa
14 changed files with 221 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
apps/openmw/mwworld/esmloader.cpp
View File

@ -1,9 +1,13 @@
#include "esmloader.hpp"
#include "esmstore.hpp"
#include <components/esm/format.hpp>
#include <components/esm3/esmreader.hpp>
#include <components/esm3/readerscache.hpp>
#include <components/esm4/reader.hpp>
#include <components/files/conversion.hpp>
#include <components/files/openfile.hpp>
#include <fstream>
namespace MWWorld
{
@ -21,28 +25,50 @@ namespace MWWorld
void EsmLoader::load(const std::filesystem::path& filepath, int& index, Loading::Listener* listener)
{
const ESM::ReadersCache::BusyItem reader = mReaders.get(static_cast<std::size_t>(index));
reader->setEncoder(mEncoder);
reader->setIndex(index);
reader->open(filepath);
reader->resolveParentFileIndices(mReaders);
auto stream = Files::openBinaryInputFileStream(filepath);
if (!stream->is_open())
{
throw std::runtime_error(std::string("File Failed to open file: ") + std::strerror(errno) + "\n");
return;
}
const ESM::Format format = ESM::readFormat(*stream);
stream->seekg(0);
assert(reader->getGameFiles().size() == reader->getParentFileIndices().size());
for (std::size_t i = 0, n = reader->getParentFileIndices().size(); i < n; ++i)
if (i == static_cast<std::size_t>(reader->getIndex()))
throw std::runtime_error("File " + Files::pathToUnicodeString(reader->getName()) + " asks for parent file "
switch (format)
{
case ESM::Format::Tes3:
{
const ESM::ReadersCache::BusyItem reader = mReaders.get(static_cast<std::size_t>(index));
reader->setEncoder(mEncoder);
reader->setIndex(index);
reader->open(filepath);
reader->resolveParentFileIndices(mReaders);
assert(reader->getGameFiles().size() == reader->getParentFileIndices().size());
for (std::size_t i = 0, n = reader->getParentFileIndices().size(); i < n; ++i)
if (i == static_cast<std::size_t>(reader->getIndex()))
throw std::runtime_error("File " + Files::pathToUnicodeString(reader->getName()) + " asks for parent file "
+ reader->getGameFiles()[i].name
+ ", but it is not available or has been loaded in the wrong order. "
"Please run the launcher to fix this issue.");
mESMVersions[index] = reader->getVer();
mStore.load(*reader, listener, mDialogue);
mESMVersions[index] = reader->getVer();
mStore.load(*reader, listener, mDialogue);
if (!mMasterFileFormat.has_value()
&& (Misc::StringUtils::ciEndsWith(reader->getName().u8string(), u8".esm")
|| Misc::StringUtils::ciEndsWith(reader->getName().u8string(), u8".omwgame")))
mMasterFileFormat = reader->getFormat();
if (!mMasterFileFormat.has_value()
&& (Misc::StringUtils::ciEndsWith(reader->getName().u8string(), u8".esm")
|| Misc::StringUtils::ciEndsWith(reader->getName().u8string(), u8".omwgame")))
mMasterFileFormat = reader->getFormat();
break;
}
case ESM::Format::Tes4:
{
ESM4::Reader readerESM4(std::move(stream), filepath);
readerESM4.setEncoder(mEncoder->getStatelessEncoder());
mStore.loadESM4(readerESM4, listener, mDialogue);
}
}
}
} /* namespace MWWorld */

93
apps/openmw/mwworld/esmstore.cpp
View File

@ -13,6 +13,11 @@
#include <components/lua/configuration.hpp>
#include <components/misc/algorithm.hpp>
#include <components/esm4/loadcell.hpp>
#include <components/esm4/loadrefr.hpp>
#include <components/esm4/loadstat.hpp>
#include <components/esm4/reader.hpp>
#include <components/esm4/common.hpp>
#include <components/esmloader/load.hpp>
@ -180,6 +185,84 @@ namespace MWWorld
}
}
}
template <typename T>
static void typedReadRecordESM4(ESM4::Reader& reader, ESMStore& stores, Store<T>& store, int& found)
{
auto recordType = static_cast<ESM4::RecordTypes>(reader.hdr().record.typeId);
ESM::RecNameInts esm4RecName = static_cast<ESM::RecNameInts>(ESM::esm4Recname(recordType));
if constexpr (std::is_convertible_v<Store<T>*, DynamicStore*>)
{
if constexpr (ESM::isESM4Rec(T::sRecordId))
{
if (T::sRecordId == esm4RecName)
{
reader.getRecordData();
T value;
value.load(reader);
store.insertStatic(value);
found++;
}
}
}
}
static void readRecord(ESM4::Reader& reader, ESMStore& store)
{
int found = 0;
std::apply([&reader, &store, &found](
auto&... x) { (ESMStoreImp::typedReadRecordESM4(reader, store, x, found), ...); },
store.mStoreImp->mStores);
assert(found <= 1);
if (found == 0) // unhandled record
reader.skipRecordData();
}
static bool readItem(ESM4::Reader& reader, ESMStore& store)
{
if (!reader.getRecordHeader() || !reader.hasMoreRecs())
return false;
const ESM4::RecordHeader& header = reader.hdr();
if (header.record.typeId == ESM4::REC_GRUP)
return readGroup(reader, store);
readRecord(reader, store);
return true;
}
static bool readGroup(ESM4::Reader& reader, ESMStore& store)
{
const ESM4::RecordHeader& header = reader.hdr();
switch (static_cast<ESM4::GroupType>(header.group.type))
{
case ESM4::Grp_RecordType:
case ESM4::Grp_InteriorCell:
case ESM4::Grp_InteriorSubCell:
case ESM4::Grp_ExteriorCell:
case ESM4::Grp_ExteriorSubCell:
reader.enterGroup();
return readItem(reader, store);
case ESM4::Grp_WorldChild:
case ESM4::Grp_CellChild:
case ESM4::Grp_TopicChild:
case ESM4::Grp_CellPersistentChild:
case ESM4::Grp_CellTemporaryChild:
case ESM4::Grp_CellVisibleDistChild:
reader.adjustGRUPFormId();
reader.enterGroup();
if (!reader.hasMoreRecs())
return false;
return readItem(reader, store);
}
reader.skipGroup();
return true;
}
};
int ESMStore::find(const ESM::RefId& id) const
@ -338,6 +421,16 @@ namespace MWWorld
}
}
void ESMStore::loadESM4(ESM4::Reader& reader, Loading::Listener* listener, ESM::Dialogue*& dialogue)
{
while (reader.hasMoreRecs())
{
reader.exitGroupCheck();
if (!ESMStoreImp::readItem(reader, *this))
break;
}
}
void ESMStore::setIdType(const ESM::RefId& id, ESM::RecNameInts type)
{
mStoreImp->mIds[id] = type;

13
apps/openmw/mwworld/esmstore.hpp
View File

@ -24,6 +24,14 @@ namespace MWMechanics
class SpellList;
}
namespace ESM4
{
class Reader;
struct Static;
struct Cell;
struct Reference;
}
namespace ESM
{
class ReadersCache;
@ -95,7 +103,9 @@ namespace MWWorld
Store<ESM::MagicEffect>, Store<ESM::Skill>,
// Special entry which is hardcoded and not loaded from an ESM
Store<ESM::Attribute>>;
Store<ESM::Attribute>,
Store<ESM4::Static>, Store<ESM4::Cell>, Store<ESM4::Reference>>;
template <typename T>
static constexpr std::size_t getTypeIndex()
@ -162,6 +172,7 @@ namespace MWWorld
void validateDynamic();
void load(ESM::ESMReader& esm, Loading::Listener* listener, ESM::Dialogue*& dialogue);
void loadESM4(ESM4::Reader& esm, Loading::Listener* listener, ESM::Dialogue*& dialogue);
template <class T>
const Store<T>& get() const

31
apps/openmw/mwworld/store.cpp
View File

@ -13,6 +13,10 @@
#include <sstream>
#include <stdexcept>
#include <components/esm4/loadcell.hpp>
#include <components/esm4/loadrefr.hpp>
#include <components/esm4/loadstat.hpp>
namespace
{
// TODO: Switch to C++23 to get a working version of std::unordered_map::erase
@ -161,7 +165,10 @@ namespace MWWorld
if (ptr == nullptr)
{
std::stringstream msg;
msg << T::getRecordType() << " '" << id << "' not found";
if constexpr (!ESM::isESM4Rec(T::sRecordId))
{
msg << T::getRecordType() << " '" << id << "' not found";
}
throw std::runtime_error(msg.str());
}
return ptr;
@ -171,8 +178,10 @@ namespace MWWorld
{
T record;
bool isDeleted = false;
record.load(esm, isDeleted);
if constexpr (!ESM::isESM4Rec(T::sRecordId))
{
record.load(esm, isDeleted);
}
std::pair<typename Static::iterator, bool> inserted = mStatic.insert_or_assign(record.mId, record);
if (inserted.second)
@ -293,7 +302,11 @@ namespace MWWorld
for (typename Dynamic::const_iterator iter(mDynamic.begin()); iter != mDynamic.end(); ++iter)
{
writer.startRecord(T::sRecordId);
iter->second.save(writer);
if constexpr (!ESM::isESM4Rec(T::sRecordId))
{
iter->second.save(writer);
}
writer.endRecord(T::sRecordId);
}
}
@ -302,8 +315,10 @@ namespace MWWorld
{
T record;
bool isDeleted = false;
record.load(reader, isDeleted);
if constexpr (!ESM::isESM4Rec(T::sRecordId))
{
record.load(reader, isDeleted);
}
insert(record, overrideOnly);
return RecordId(record.mId, isDeleted);
@ -1196,3 +1211,7 @@ template class MWWorld::TypedDynamicStore<ESM::Spell>;
template class MWWorld::TypedDynamicStore<ESM::StartScript>;
template class MWWorld::TypedDynamicStore<ESM::Static>;
template class MWWorld::TypedDynamicStore<ESM::Weapon>;
template class MWWorld::TypedDynamicStore<ESM4::Static>;
template class MWWorld::TypedDynamicStore<ESM4::Reference>;
template class MWWorld::TypedDynamicStore<ESM4::Cell>;

5
components/esm/defs.hpp
View File

@ -333,6 +333,11 @@ namespace ESM
REC_MSET4 = esm4Recname(ESM4::REC_MSET) // Media Set
};
constexpr bool isESM4Rec(RecNameInts RecName)
{
return RecName & sEsm4RecnameFlag;
}
/// Common subrecords
enum SubRecNameInts
{

5
components/esm/refid.cpp
View File

@ -34,6 +34,11 @@ namespace ESM
return newRefId;
}
RefId RefId::formIdRefId(const ESM4::FormId id)
{
return ESM::RefId::stringRefId(ESM4::formIdToString(id));
}
bool RefId::operator==(std::string_view rhs) const
{
return Misc::StringUtils::ciEqual(mId, rhs);

2
components/esm/refid.hpp
View File

@ -1,6 +1,7 @@
#ifndef OPENMW_COMPONENTS_ESM_REFID_HPP
#define OPENMW_COMPONENTS_ESM_REFID_HPP
#include <compare>
#include <components/esm4/formid.hpp>
#include <functional>
#include <iosfwd>
#include <string>
@ -26,6 +27,7 @@ namespace ESM
// RefIds that are as string in the code. For serialization, and display. Using explicit conversions make it
// very clear where in the code we need to convert from string to RefId and Vice versa.
static RefId stringRefId(std::string_view id);
static RefId formIdRefId(const ESM4::FormId id);
const std::string& getRefIdString() const { return mId; }
private:

10
components/esm4/loadcell.cpp
View File

@ -37,7 +37,8 @@
#include <iostream> // FIXME: debug only
#include "reader.hpp"
//#include "writer.hpp"
#include <components/esm/refid.hpp>
// #include "writer.hpp"
// TODO: Try loading only EDID and XCLC (along with mFormId, mFlags and mParent)
//
@ -48,8 +49,9 @@
// longer/shorter/same as loading the subrecords.
void ESM4::Cell::load(ESM4::Reader& reader)
{
mFormId = reader.hdr().record.id;
reader.adjustFormId(mFormId);
auto formId = reader.hdr().record.id;
reader.adjustFormId(formId);
mId = ESM::RefId::formIdRefId(formId);
mFlags = reader.hdr().record.flags;
mParent = reader.currWorld();
@ -71,7 +73,7 @@ void ESM4::Cell::load(ESM4::Reader& reader)
// WARN: we need to call setCurrCell (and maybe setCurrCellGrid?) again before loading
// cell child groups if we are loading them after restoring the context
// (may be easier to update the context before saving?)
reader.setCurrCell(mFormId); // save for LAND (and other children) to access later
reader.setCurrCell(formId); // save for LAND (and other children) to access later
std::uint32_t esmVer = reader.esmVersion();
bool isFONV = esmVer == ESM::VER_132 || esmVer == ESM::VER_133 || esmVer == ESM::VER_134;

6
components/esm4/loadcell.hpp
View File

@ -33,6 +33,8 @@
#include "formid.hpp"
#include "lighting.hpp"
#include <components/esm/defs.hpp>
#include <components/esm/refid.hpp>
namespace ESM4
{
@ -61,7 +63,7 @@ namespace ESM4
{
FormId mParent; // world formId (for grouping cells), from the loading sequence
FormId mFormId; // from the header
ESM::RefId mId; // from the header
std::uint32_t mFlags; // from the header, see enum type RecordFlag for details
std::string mEditorId;
@ -95,6 +97,8 @@ namespace ESM4
// void save(ESM4::Writer& writer) const;
void blank();
static constexpr ESM::RecNameInts sRecordId = ESM::REC_CELL4;
};
}

13
components/esm4/loadrefr.cpp
View File

@ -30,14 +30,15 @@
#include <stdexcept>
#include "reader.hpp"
//#include "writer.hpp"
// #include "writer.hpp"
void ESM4::Reference::load(ESM4::Reader& reader)
{
mFormId = reader.hdr().record.id;
reader.adjustFormId(mFormId);
auto formId = reader.hdr().record.id;
reader.adjustFormId(formId);
mId = ESM::RefId::formIdRefId(formId);
mFlags = reader.hdr().record.flags;
mParent = reader.currCell(); // NOTE: only for persistent refs?
mParent = ESM::RefId::formIdRefId(reader.currCell()); // NOTE: only for persistent refs?
// TODO: Let the engine apply this? Saved games?
// mInitiallyDisabled = ((mFlags & ESM4::Rec_Disabled) != 0) ? true : false;
@ -60,7 +61,9 @@ void ESM4::Reference::load(ESM4::Reader& reader)
break;
case ESM4::SUB_NAME:
{
reader.getFormId(mBaseObj);
FormId BaseId;
reader.getFormId(BaseId);
mBaseObj = ESM::RefId::formIdRefId(BaseId);
#if 0
if (mFlags & ESM4::Rec_Disabled)
std::cout << "REFR disable at start " << formIdToString(mFormId) <<

12
components/esm4/loadrefr.hpp
View File

@ -30,6 +30,8 @@
#include <cstdint>
#include "reference.hpp" // FormId, Placement, EnableParent
#include <components/esm/defs.hpp>
#include <components/esm/refid.hpp>
namespace ESM4
{
@ -71,15 +73,15 @@ namespace ESM4
struct Reference
{
FormId mParent; // cell FormId (currently persistent refs only), from the loading sequence
// NOTE: for exterior cells it will be the dummy cell FormId
ESM::RefId mParent; // cell FormId (currently persistent refs only), from the loading sequence
// NOTE: for exterior cells it will be the dummy cell FormId
FormId mFormId; // from the header
ESM::RefId mId; // from the header
std::uint32_t mFlags; // from the header, see enum type RecordFlag for details
std::string mEditorId;
std::string mFullName;
FormId mBaseObj;
ESM::RefId mBaseObj;
Placement mPlacement;
float mScale = 1.0f;
@ -110,6 +112,8 @@ namespace ESM4
// void save(ESM4::Writer& writer) const;
void blank();
static constexpr ESM::RecNameInts sRecordId = ESM::REC_REFR4;
};
}

7
components/esm4/loadstat.cpp
View File

@ -30,12 +30,13 @@
#include <stdexcept>
#include "reader.hpp"
//#include "writer.hpp"
// #include "writer.hpp"
void ESM4::Static::load(ESM4::Reader& reader)
{
mFormId = reader.hdr().record.id;
reader.adjustFormId(mFormId);
FormId formId = reader.hdr().record.id;
reader.adjustFormId(formId);
mId = ESM::RefId::formIdRefId(formId);
mFlags = reader.hdr().record.flags;
while (reader.getSubRecordHeader())

6
components/esm4/loadstat.hpp
View File

@ -32,6 +32,8 @@
#include <vector>
#include "formid.hpp"
#include <components/esm/defs.hpp>
#include <components/esm/refid.hpp>
namespace ESM4
{
@ -40,7 +42,7 @@ namespace ESM4
struct Static
{
FormId mFormId; // from the header
ESM::RefId mId; // from the header
std::uint32_t mFlags; // from the header, see enum type RecordFlag for details
std::string mEditorId;
@ -53,6 +55,8 @@ namespace ESM4
// void save(ESM4::Writer& writer) const;
// void blank();
static constexpr ESM::RecNameInts sRecordId = ESM::REC_STAT4;
};
}

2
components/to_utf8/to_utf8.hpp
View File

@ -68,6 +68,8 @@ namespace ToUTF8
/// ASCII-only string. Otherwise returns a view to the input.
std::string_view getLegacyEnc(std::string_view input);
const StatelessUtf8Encoder* getStatelessEncoder() const { return &mImpl; }
private:
std::string mBuffer;
StatelessUtf8Encoder mImpl;