mirror/OpenMW
1
0
Fork 0
mirror of https://gitlab.com/OpenMW/openmw.git synced 2025-03-24 13:43:43 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'Load_ESM4' into 'master'

Browse Source 
Loading ESM4 data and storing them in the ESMStore

See merge request OpenMW/openmw!2557
This commit is contained in:
psi29a 2023-01-07 22:17:17 +00:00
commit 83718878b2
19 changed files with 521 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

320
apps/esmtool/tes4.cpp
View File

@ -8,6 +8,7 @@
#include <components/esm/esmcommon.hpp>
#include <components/esm4/reader.hpp>
#include <components/esm4/readerutils.hpp>
#include <components/esm4/records.hpp>
#include <components/to_utf8/to_utf8.hpp>
@ -69,6 +70,19 @@ namespace EsmTool
template <class T>
constexpr bool hasFormId = HasFormId<T>::value;
template <class T, class = std::void_t<>>
struct HasRefId : std::false_type
{
};
template <class T>
struct HasRefId<T, std::void_t<decltype(T::mId)>> : std::true_type
{
};
template <class T>
constexpr bool hasRefId = HasRefId<T>::value;
template <class T, class = std::void_t<>>
struct HasFlags : std::false_type
{
@ -169,6 +183,8 @@ namespace EsmTool
std::cout << "\n Record: " << ESM::NAME(reader.hdr().record.typeId).toStringView();
if constexpr (hasFormId<T>)
std::cout << "\n FormId: " << value.mFormId;
if constexpr (hasRefId<T>)
std::cout << "\n FormId: " << value.mId;
if constexpr (hasFlags<T>)
std::cout << "\n Record flags: " << recordFlags(value.mFlags);
if constexpr (hasEditorId<T>)
@ -182,60 +198,77 @@ namespace EsmTool
std::cout << '\n';
}
void readRecord(const Params& params, ESM4::Reader& reader)
bool readRecord(const Params& params, ESM4::Reader& reader)
{
switch (static_cast<ESM4::RecordTypes>(reader.hdr().record.typeId))
{
case ESM4::REC_AACT:
break;
case ESM4::REC_ACHR:
return readTypedRecord<ESM4::ActorCharacter>(params, reader);
readTypedRecord<ESM4::ActorCharacter>(params, reader);
return true;
case ESM4::REC_ACRE:
return readTypedRecord<ESM4::ActorCreature>(params, reader);
readTypedRecord<ESM4::ActorCreature>(params, reader);
return true;
case ESM4::REC_ACTI:
return readTypedRecord<ESM4::Activator>(params, reader);
readTypedRecord<ESM4::Activator>(params, reader);
return true;
case ESM4::REC_ADDN:
break;
case ESM4::REC_ALCH:
return readTypedRecord<ESM4::Potion>(params, reader);
readTypedRecord<ESM4::Potion>(params, reader);
return true;
case ESM4::REC_ALOC:
return readTypedRecord<ESM4::MediaLocationController>(params, reader);
readTypedRecord<ESM4::MediaLocationController>(params, reader);
return true;
case ESM4::REC_AMMO:
return readTypedRecord<ESM4::Ammunition>(params, reader);
readTypedRecord<ESM4::Ammunition>(params, reader);
return true;
case ESM4::REC_ANIO:
return readTypedRecord<ESM4::AnimObject>(params, reader);
readTypedRecord<ESM4::AnimObject>(params, reader);
return true;
case ESM4::REC_APPA:
return readTypedRecord<ESM4::Apparatus>(params, reader);
readTypedRecord<ESM4::Apparatus>(params, reader);
return true;
case ESM4::REC_ARMA:
return readTypedRecord<ESM4::ArmorAddon>(params, reader);
readTypedRecord<ESM4::ArmorAddon>(params, reader);
return true;
case ESM4::REC_ARMO:
return readTypedRecord<ESM4::Armor>(params, reader);
readTypedRecord<ESM4::Armor>(params, reader);
return true;
case ESM4::REC_ARTO:
break;
case ESM4::REC_ASPC:
return readTypedRecord<ESM4::AcousticSpace>(params, reader);
readTypedRecord<ESM4::AcousticSpace>(params, reader);
return true;
case ESM4::REC_ASTP:
break;
case ESM4::REC_AVIF:
break;
case ESM4::REC_BOOK:
return readTypedRecord<ESM4::Book>(params, reader);
readTypedRecord<ESM4::Book>(params, reader);
return true;
case ESM4::REC_BPTD:
return readTypedRecord<ESM4::BodyPartData>(params, reader);
readTypedRecord<ESM4::BodyPartData>(params, reader);
return true;
case ESM4::REC_CAMS:
break;
case ESM4::REC_CCRD:
break;
case ESM4::REC_CELL:
return readTypedRecord<ESM4::Cell>(params, reader);
readTypedRecord<ESM4::Cell>(params, reader);
return true;
case ESM4::REC_CLAS:
return readTypedRecord<ESM4::Class>(params, reader);
readTypedRecord<ESM4::Class>(params, reader);
return true;
case ESM4::REC_CLFM:
return readTypedRecord<ESM4::Colour>(params, reader);
readTypedRecord<ESM4::Colour>(params, reader);
return true;
case ESM4::REC_CLMT:
break;
case ESM4::REC_CLOT:
return readTypedRecord<ESM4::Clothing>(params, reader);
readTypedRecord<ESM4::Clothing>(params, reader);
return true;
case ESM4::REC_CMNY:
break;
case ESM4::REC_COBJ:
@ -243,25 +276,30 @@ namespace EsmTool
case ESM4::REC_COLL:
break;
case ESM4::REC_CONT:
return readTypedRecord<ESM4::Container>(params, reader);
readTypedRecord<ESM4::Container>(params, reader);
return true;
case ESM4::REC_CPTH:
break;
case ESM4::REC_CREA:
return readTypedRecord<ESM4::Creature>(params, reader);
readTypedRecord<ESM4::Creature>(params, reader);
return true;
case ESM4::REC_CSTY:
break;
case ESM4::REC_DEBR:
break;
case ESM4::REC_DIAL:
return readTypedRecord<ESM4::Dialogue>(params, reader);
readTypedRecord<ESM4::Dialogue>(params, reader);
return true;
case ESM4::REC_DLBR:
break;
case ESM4::REC_DLVW:
break;
case ESM4::REC_DOBJ:
return readTypedRecord<ESM4::DefaultObj>(params, reader);
readTypedRecord<ESM4::DefaultObj>(params, reader);
return true;
case ESM4::REC_DOOR:
return readTypedRecord<ESM4::Door>(params, reader);
readTypedRecord<ESM4::Door>(params, reader);
return true;
case ESM4::REC_DUAL:
break;
case ESM4::REC_ECZN:
@ -275,81 +313,103 @@ namespace EsmTool
case ESM4::REC_EXPL:
break;
case ESM4::REC_EYES:
return readTypedRecord<ESM4::Eyes>(params, reader);
readTypedRecord<ESM4::Eyes>(params, reader);
return true;
case ESM4::REC_FACT:
break;
case ESM4::REC_FLOR:
return readTypedRecord<ESM4::Flora>(params, reader);
readTypedRecord<ESM4::Flora>(params, reader);
return true;
case ESM4::REC_FLST:
return readTypedRecord<ESM4::FormIdList>(params, reader);
readTypedRecord<ESM4::FormIdList>(params, reader);
return true;
case ESM4::REC_FSTP:
break;
case ESM4::REC_FSTS:
break;
case ESM4::REC_FURN:
return readTypedRecord<ESM4::Furniture>(params, reader);
readTypedRecord<ESM4::Furniture>(params, reader);
return true;
case ESM4::REC_GLOB:
return readTypedRecord<ESM4::GlobalVariable>(params, reader);
readTypedRecord<ESM4::GlobalVariable>(params, reader);
return true;
case ESM4::REC_GMST:
break;
case ESM4::REC_GRAS:
return readTypedRecord<ESM4::Grass>(params, reader);
readTypedRecord<ESM4::Grass>(params, reader);
return true;
case ESM4::REC_GRUP:
break;
case ESM4::REC_HAIR:
return readTypedRecord<ESM4::Hair>(params, reader);
readTypedRecord<ESM4::Hair>(params, reader);
return true;
case ESM4::REC_HAZD:
break;
case ESM4::REC_HDPT:
return readTypedRecord<ESM4::HeadPart>(params, reader);
readTypedRecord<ESM4::HeadPart>(params, reader);
return true;
case ESM4::REC_IDLE:
// FIXME: ESM4::IdleAnimation::load does not work with Oblivion.esm
// return readTypedRecord<ESM4::IdleAnimation>(params, reader);
// readTypedRecord<ESM4::IdleAnimation>(params, reader);
return true;
break;
case ESM4::REC_IDLM:
return readTypedRecord<ESM4::IdleMarker>(params, reader);
readTypedRecord<ESM4::IdleMarker>(params, reader);
return true;
case ESM4::REC_IMAD:
break;
case ESM4::REC_IMGS:
break;
case ESM4::REC_IMOD:
return readTypedRecord<ESM4::ItemMod>(params, reader);
readTypedRecord<ESM4::ItemMod>(params, reader);
return true;
case ESM4::REC_INFO:
return readTypedRecord<ESM4::DialogInfo>(params, reader);
readTypedRecord<ESM4::DialogInfo>(params, reader);
return true;
case ESM4::REC_INGR:
return readTypedRecord<ESM4::Ingredient>(params, reader);
readTypedRecord<ESM4::Ingredient>(params, reader);
return true;
case ESM4::REC_IPCT:
break;
case ESM4::REC_IPDS:
break;
case ESM4::REC_KEYM:
return readTypedRecord<ESM4::Key>(params, reader);
readTypedRecord<ESM4::Key>(params, reader);
return true;
case ESM4::REC_KYWD:
break;
case ESM4::REC_LAND:
return readTypedRecord<ESM4::Land>(params, reader);
readTypedRecord<ESM4::Land>(params, reader);
return true;
case ESM4::REC_LCRT:
break;
case ESM4::REC_LCTN:
break;
case ESM4::REC_LGTM:
return readTypedRecord<ESM4::LightingTemplate>(params, reader);
readTypedRecord<ESM4::LightingTemplate>(params, reader);
return true;
case ESM4::REC_LIGH:
return readTypedRecord<ESM4::Light>(params, reader);
readTypedRecord<ESM4::Light>(params, reader);
return true;
case ESM4::REC_LSCR:
break;
case ESM4::REC_LTEX:
return readTypedRecord<ESM4::LandTexture>(params, reader);
readTypedRecord<ESM4::LandTexture>(params, reader);
return true;
case ESM4::REC_LVLC:
return readTypedRecord<ESM4::LevelledCreature>(params, reader);
readTypedRecord<ESM4::LevelledCreature>(params, reader);
return true;
case ESM4::REC_LVLI:
return readTypedRecord<ESM4::LevelledItem>(params, reader);
readTypedRecord<ESM4::LevelledItem>(params, reader);
return true;
case ESM4::REC_LVLN:
return readTypedRecord<ESM4::LevelledNpc>(params, reader);
readTypedRecord<ESM4::LevelledNpc>(params, reader);
return true;
case ESM4::REC_LVSP:
break;
case ESM4::REC_MATO:
return readTypedRecord<ESM4::Material>(params, reader);
readTypedRecord<ESM4::Material>(params, reader);
return true;
case ESM4::REC_MATT:
break;
case ESM4::REC_MESG:
@ -357,49 +417,66 @@ namespace EsmTool
case ESM4::REC_MGEF:
break;
case ESM4::REC_MISC:
return readTypedRecord<ESM4::MiscItem>(params, reader);
readTypedRecord<ESM4::MiscItem>(params, reader);
return true;
case ESM4::REC_MOVT:
break;
case ESM4::REC_MSET:
return readTypedRecord<ESM4::MediaSet>(params, reader);
readTypedRecord<ESM4::MediaSet>(params, reader);
return true;
case ESM4::REC_MSTT:
return readTypedRecord<ESM4::MovableStatic>(params, reader);
readTypedRecord<ESM4::MovableStatic>(params, reader);
return true;
case ESM4::REC_MUSC:
return readTypedRecord<ESM4::Music>(params, reader);
readTypedRecord<ESM4::Music>(params, reader);
return true;
case ESM4::REC_MUST:
break;
case ESM4::REC_NAVI:
return readTypedRecord<ESM4::Navigation>(params, reader);
readTypedRecord<ESM4::Navigation>(params, reader);
return true;
case ESM4::REC_NAVM:
return readTypedRecord<ESM4::NavMesh>(params, reader);
readTypedRecord<ESM4::NavMesh>(params, reader);
return true;
case ESM4::REC_NOTE:
return readTypedRecord<ESM4::Note>(params, reader);
readTypedRecord<ESM4::Note>(params, reader);
return true;
case ESM4::REC_NPC_:
return readTypedRecord<ESM4::Npc>(params, reader);
readTypedRecord<ESM4::Npc>(params, reader);
return true;
case ESM4::REC_OTFT:
return readTypedRecord<ESM4::Outfit>(params, reader);
readTypedRecord<ESM4::Outfit>(params, reader);
return true;
case ESM4::REC_PACK:
return readTypedRecord<ESM4::AIPackage>(params, reader);
readTypedRecord<ESM4::AIPackage>(params, reader);
return true;
case ESM4::REC_PERK:
break;
case ESM4::REC_PGRD:
return readTypedRecord<ESM4::Pathgrid>(params, reader);
readTypedRecord<ESM4::Pathgrid>(params, reader);
return true;
case ESM4::REC_PGRE:
return readTypedRecord<ESM4::PlacedGrenade>(params, reader);
readTypedRecord<ESM4::PlacedGrenade>(params, reader);
return true;
case ESM4::REC_PHZD:
break;
case ESM4::REC_PROJ:
break;
case ESM4::REC_PWAT:
return readTypedRecord<ESM4::PlaceableWater>(params, reader);
readTypedRecord<ESM4::PlaceableWater>(params, reader);
return true;
case ESM4::REC_QUST:
return readTypedRecord<ESM4::Quest>(params, reader);
readTypedRecord<ESM4::Quest>(params, reader);
return true;
case ESM4::REC_RACE:
return readTypedRecord<ESM4::Race>(params, reader);
readTypedRecord<ESM4::Race>(params, reader);
return true;
case ESM4::REC_REFR:
return readTypedRecord<ESM4::Reference>(params, reader);
readTypedRecord<ESM4::Reference>(params, reader);
return true;
case ESM4::REC_REGN:
return readTypedRecord<ESM4::Region>(params, reader);
readTypedRecord<ESM4::Region>(params, reader);
return true;
case ESM4::REC_RELA:
break;
case ESM4::REC_REVB:
@ -407,23 +484,30 @@ namespace EsmTool
case ESM4::REC_RFCT:
break;
case ESM4::REC_ROAD:
return readTypedRecord<ESM4::Road>(params, reader);
readTypedRecord<ESM4::Road>(params, reader);
return true;
case ESM4::REC_SBSP:
return readTypedRecord<ESM4::SubSpace>(params, reader);
readTypedRecord<ESM4::SubSpace>(params, reader);
return true;
case ESM4::REC_SCEN:
break;
case ESM4::REC_SCOL:
return readTypedRecord<ESM4::StaticCollection>(params, reader);
readTypedRecord<ESM4::StaticCollection>(params, reader);
return true;
case ESM4::REC_SCPT:
return readTypedRecord<ESM4::Script>(params, reader);
readTypedRecord<ESM4::Script>(params, reader);
return true;
case ESM4::REC_SCRL:
return readTypedRecord<ESM4::Scroll>(params, reader);
readTypedRecord<ESM4::Scroll>(params, reader);
return true;
case ESM4::REC_SGST:
return readTypedRecord<ESM4::SigilStone>(params, reader);
readTypedRecord<ESM4::SigilStone>(params, reader);
return true;
case ESM4::REC_SHOU:
break;
case ESM4::REC_SLGM:
return readTypedRecord<ESM4::SoulGem>(params, reader);
readTypedRecord<ESM4::SoulGem>(params, reader);
return true;
case ESM4::REC_SMBN:
break;
case ESM4::REC_SMEN:
@ -433,97 +517,56 @@ namespace EsmTool
case ESM4::REC_SNCT:
break;
case ESM4::REC_SNDR:
return readTypedRecord<ESM4::SoundReference>(params, reader);
readTypedRecord<ESM4::SoundReference>(params, reader);
return true;
case ESM4::REC_SOPM:
break;
case ESM4::REC_SOUN:
return readTypedRecord<ESM4::Sound>(params, reader);
readTypedRecord<ESM4::Sound>(params, reader);
return true;
case ESM4::REC_SPEL:
break;
case ESM4::REC_SPGD:
break;
case ESM4::REC_STAT:
return readTypedRecord<ESM4::Static>(params, reader);
readTypedRecord<ESM4::Static>(params, reader);
return true;
case ESM4::REC_TACT:
return readTypedRecord<ESM4::TalkingActivator>(params, reader);
readTypedRecord<ESM4::TalkingActivator>(params, reader);
return true;
case ESM4::REC_TERM:
return readTypedRecord<ESM4::Terminal>(params, reader);
readTypedRecord<ESM4::Terminal>(params, reader);
return true;
case ESM4::REC_TES4:
return readTypedRecord<ESM4::Header>(params, reader);
readTypedRecord<ESM4::Header>(params, reader);
return true;
case ESM4::REC_TREE:
return readTypedRecord<ESM4::Tree>(params, reader);
readTypedRecord<ESM4::Tree>(params, reader);
return true;
case ESM4::REC_TXST:
return readTypedRecord<ESM4::TextureSet>(params, reader);
readTypedRecord<ESM4::TextureSet>(params, reader);
return true;
case ESM4::REC_VTYP:
break;
case ESM4::REC_WATR:
break;
case ESM4::REC_WEAP:
return readTypedRecord<ESM4::Weapon>(params, reader);
readTypedRecord<ESM4::Weapon>(params, reader);
return true;
case ESM4::REC_WOOP:
break;
case ESM4::REC_WRLD:
return readTypedRecord<ESM4::World>(params, reader);
readTypedRecord<ESM4::World>(params, reader);
return true;
case ESM4::REC_WTHR:
break;
}
if (!params.mQuite)
std::cout << "\n Unsupported record: " << ESM::NAME(reader.hdr().record.typeId).toStringView() << '\n';
reader.skipRecordData();
return false;
}
bool readItem(const Params& params, ESM4::Reader& reader);
bool readGroup(const Params& params, ESM4::Reader& reader)
{
const ESM4::RecordHeader& header = reader.hdr();
if (!params.mQuite)
std::cout << "\nGroup: " << toString(static_cast<ESM4::GroupType>(header.group.type)) << " "
<< ESM::NAME(header.group.typeId).toStringView() << '\n';
switch (static_cast<ESM4::GroupType>(header.group.type))
{
case ESM4::Grp_RecordType:
case ESM4::Grp_InteriorCell:
case ESM4::Grp_InteriorSubCell:
case ESM4::Grp_ExteriorCell:
case ESM4::Grp_ExteriorSubCell:
reader.enterGroup();
return readItem(params, reader);
case ESM4::Grp_WorldChild:
case ESM4::Grp_CellChild:
case ESM4::Grp_TopicChild:
case ESM4::Grp_CellPersistentChild:
case ESM4::Grp_CellTemporaryChild:
case ESM4::Grp_CellVisibleDistChild:
reader.adjustGRUPFormId();
reader.enterGroup();
if (!reader.hasMoreRecs())
return false;
return readItem(params, reader);
}
reader.skipGroup();
return true;
}
bool readItem(const Params& params, ESM4::Reader& reader)
{
if (!reader.getRecordHeader() || !reader.hasMoreRecs())
return false;
const ESM4::RecordHeader& header = reader.hdr();
if (header.record.typeId == ESM4::REC_GRUP)
return readGroup(params, reader);
readRecord(params, reader);
return true;
}
}
int loadTes4(const Arguments& info, std::unique_ptr<std::ifstream>&& stream)
@ -551,12 +594,15 @@ namespace EsmTool
}
}
while (reader.hasMoreRecs())
{
reader.exitGroupCheck();
if (!readItem(params, reader))
break;
}
auto visitorRec = [&params](ESM4::Reader& reader) { return readRecord(params, reader); };
auto visitorGroup = [&params](ESM4::Reader& reader) {
if (params.mQuite)
return;
auto groupType = static_cast<ESM4::GroupType>(reader.hdr().group.type);
std::cout << "\nGroup: " << toString(groupType) << " "
<< ESM::NAME(reader.hdr().group.typeId).toStringView() << '\n';
};
ESM4::ReaderUtils::readAll(reader, visitorRec, visitorGroup);
}
catch (const std::exception& e)
{

54
apps/openmw/mwworld/esmloader.cpp
View File

@ -1,9 +1,14 @@
#include "esmloader.hpp"
#include "esmstore.hpp"
#include <fstream>
#include <components/esm/format.hpp>
#include <components/esm3/esmreader.hpp>
#include <components/esm3/readerscache.hpp>
#include <components/esm4/reader.hpp>
#include <components/files/conversion.hpp>
#include <components/files/openfile.hpp>
namespace MWWorld
{
@ -21,28 +26,47 @@ namespace MWWorld
void EsmLoader::load(const std::filesystem::path& filepath, int& index, Loading::Listener* listener)
{
const ESM::ReadersCache::BusyItem reader = mReaders.get(static_cast<std::size_t>(index));
reader->setEncoder(mEncoder);
reader->setIndex(index);
reader->open(filepath);
reader->resolveParentFileIndices(mReaders);
auto stream = Files::openBinaryInputFileStream(filepath);
const ESM::Format format = ESM::readFormat(*stream);
stream->seekg(0);
assert(reader->getGameFiles().size() == reader->getParentFileIndices().size());
for (std::size_t i = 0, n = reader->getParentFileIndices().size(); i < n; ++i)
if (i == static_cast<std::size_t>(reader->getIndex()))
throw std::runtime_error("File " + Files::pathToUnicodeString(reader->getName()) + " asks for parent file "
switch (format)
{
case ESM::Format::Tes3:
{
const ESM::ReadersCache::BusyItem reader = mReaders.get(static_cast<std::size_t>(index));
reader->setEncoder(mEncoder);
reader->setIndex(index);
reader->open(filepath);
reader->resolveParentFileIndices(mReaders);
assert(reader->getGameFiles().size() == reader->getParentFileIndices().size());
for (std::size_t i = 0, n = reader->getParentFileIndices().size(); i < n; ++i)
if (i == static_cast<std::size_t>(reader->getIndex()))
throw std::runtime_error("File " + Files::pathToUnicodeString(reader->getName()) + " asks for parent file "
+ reader->getGameFiles()[i].name
+ ", but it is not available or has been loaded in the wrong order. "
"Please run the launcher to fix this issue.");
mESMVersions[index] = reader->getVer();
mStore.load(*reader, listener, mDialogue);
mESMVersions[index] = reader->getVer();
mStore.load(*reader, listener, mDialogue);
if (!mMasterFileFormat.has_value()
&& (Misc::StringUtils::ciEndsWith(reader->getName().u8string(), u8".esm")
|| Misc::StringUtils::ciEndsWith(reader->getName().u8string(), u8".omwgame")))
mMasterFileFormat = reader->getFormat();
if (!mMasterFileFormat.has_value()
&& (Misc::StringUtils::ciEndsWith(reader->getName().u8string(), u8".esm")
|| Misc::StringUtils::ciEndsWith(reader->getName().u8string(), u8".omwgame")))
mMasterFileFormat = reader->getFormat();
break;
}
case ESM::Format::Tes4:
{
ESM4::Reader readerESM4(std::move(stream), filepath);
auto statelessEncoder = mEncoder->getStatelessEncoder();
readerESM4.setEncoder(&statelessEncoder);
mStore.loadESM4(readerESM4);
break;
}
}
}
} /* namespace MWWorld */

40
apps/openmw/mwworld/esmstore.cpp
View File

@ -14,6 +14,11 @@
#include <components/misc/algorithm.hpp>
#include <components/esm4/common.hpp>
#include <components/esm4/loadcell.hpp>
#include <components/esm4/loadrefr.hpp>
#include <components/esm4/loadstat.hpp>
#include <components/esm4/reader.hpp>
#include <components/esm4/readerutils.hpp>
#include <components/esmloader/load.hpp>
#include "../mwmechanics/spelllist.hpp"
@ -180,6 +185,35 @@ namespace MWWorld
}
}
}
template <typename T>
static bool typedReadRecordESM4(ESM4::Reader& reader, Store<T>& store)
{
auto recordType = static_cast<ESM4::RecordTypes>(reader.hdr().record.typeId);
ESM::RecNameInts esm4RecName = static_cast<ESM::RecNameInts>(ESM::esm4Recname(recordType));
if constexpr (std::is_convertible_v<Store<T>*, DynamicStore*> && HasRecordId<T>::value)
{
if constexpr (ESM::isESM4Rec(T::sRecordId))
{
if (T::sRecordId == esm4RecName)
{
reader.getRecordData();
T value;
value.load(reader);
store.insertStatic(value);
return true;
}
}
}
return false;
}
static bool readRecord(ESM4::Reader& reader, ESMStore& store)
{
return std::apply([&reader](auto&... x) { return (ESMStoreImp::typedReadRecordESM4(reader, x) || ...); },
store.mStoreImp->mStores);
}
};
int ESMStore::find(const ESM::RefId& id) const
@ -338,6 +372,12 @@ namespace MWWorld
}
}
void ESMStore::loadESM4(ESM4::Reader& reader)
{
auto visitorRec = [this](ESM4::Reader& reader) { return ESMStoreImp::readRecord(reader, *this); };
ESM4::ReaderUtils::readAll(reader, visitorRec, [](ESM4::Reader&) {});
}
void ESMStore::setIdType(const ESM::RefId& id, ESM::RecNameInts type)
{
mStoreImp->mIds[id] = type;

26
apps/openmw/mwworld/esmstore.hpp
View File

@ -24,6 +24,14 @@ namespace MWMechanics
class SpellList;
}
namespace ESM4
{
class Reader;
struct Static;
struct Cell;
struct Reference;
}
namespace ESM
{
class ReadersCache;
@ -78,7 +86,7 @@ namespace MWWorld
class ESMStore
{
friend struct ESMStoreImp; // This allows StoreImp to extend esmstore without beeing included everywhere
public:
using StoreTuple = std::tuple<Store<ESM::Activator>, Store<ESM::Potion>, Store<ESM::Apparatus>,
Store<ESM::Armor>, Store<ESM::BodyPart>, Store<ESM::Book>, Store<ESM::BirthSign>, Store<ESM::Class>,
Store<ESM::Clothing>, Store<ESM::Container>, Store<ESM::Creature>, Store<ESM::Dialogue>, Store<ESM::Door>,
@ -95,8 +103,11 @@ namespace MWWorld
Store<ESM::MagicEffect>, Store<ESM::Skill>,
// Special entry which is hardcoded and not loaded from an ESM
Store<ESM::Attribute>>;
Store<ESM::Attribute>,
Store<ESM4::Static>, Store<ESM4::Cell>, Store<ESM4::Reference>>;
private:
template <typename T>
static constexpr std::size_t getTypeIndex()
{
@ -162,6 +173,7 @@ namespace MWWorld
void validateDynamic();
void load(ESM::ESMReader& esm, Loading::Listener* listener, ESM::Dialogue*& dialogue);
void loadESM4(ESM4::Reader& esm);
template <class T>
const Store<T>& get() const
@ -252,6 +264,16 @@ namespace MWWorld
template <>
const ESM::NPC* ESMStore::insert<ESM::NPC>(const ESM::NPC& npc);
template <class T, class = std::void_t<>>
struct HasRecordId : std::false_type
{
};
template <class T>
struct HasRecordId<T, std::void_t<decltype(T::sRecordId)>> : std::true_type
{
};
}
#endif

68
apps/openmw/mwworld/store.cpp
View File

@ -1,18 +1,19 @@
#include "store.hpp"
#include <components/debug/debuglog.hpp>
#include <components/esm/records.hpp>
#include <components/esm3/esmreader.hpp>
#include <components/esm3/esmwriter.hpp>
#include <components/loadinglistener/loadinglistener.hpp>
#include <components/misc/rng.hpp>
#include <iterator>
#include <sstream>
#include <stdexcept>
#include <components/debug/debuglog.hpp>
#include <components/esm/records.hpp>
#include <components/esm3/esmreader.hpp>
#include <components/esm3/esmwriter.hpp>
#include <components/esm4/loadcell.hpp>
#include <components/esm4/loadrefr.hpp>
#include <components/esm4/loadstat.hpp>
#include <components/loadinglistener/loadinglistener.hpp>
#include <components/misc/rng.hpp>
namespace
{
// TODO: Switch to C++23 to get a working version of std::unordered_map::erase
@ -161,7 +162,15 @@ namespace MWWorld
if (ptr == nullptr)
{
std::stringstream msg;
msg << T::getRecordType() << " '" << id << "' not found";
if constexpr (!ESM::isESM4Rec(T::sRecordId))
{
msg << T::getRecordType();
}
else
{
msg << "ESM::REC_" << getRecNameString(T::sRecordId).toStringView();
}
msg << " '" << id << "' not found";
throw std::runtime_error(msg.str());
}
return ptr;
@ -171,8 +180,10 @@ namespace MWWorld
{
T record;
bool isDeleted = false;
record.load(esm, isDeleted);
if constexpr (!ESM::isESM4Rec(T::sRecordId))
{
record.load(esm, isDeleted);
}
std::pair<typename Static::iterator, bool> inserted = mStatic.insert_or_assign(record.mId, record);
if (inserted.second)
@ -292,9 +303,12 @@ namespace MWWorld
{
for (typename Dynamic::const_iterator iter(mDynamic.begin()); iter != mDynamic.end(); ++iter)
{
writer.startRecord(T::sRecordId);
iter->second.save(writer);
writer.endRecord(T::sRecordId);
if constexpr (!ESM::isESM4Rec(T::sRecordId))
{
writer.startRecord(T::sRecordId);
iter->second.save(writer);
writer.endRecord(T::sRecordId);
}
}
}
template <typename T>
@ -302,8 +316,10 @@ namespace MWWorld
{
T record;
bool isDeleted = false;
record.load(reader, isDeleted);
if constexpr (!ESM::isESM4Rec(T::sRecordId))
{
record.load(reader, isDeleted);
}
insert(record, overrideOnly);
return RecordId(record.mId, isDeleted);
@ -1152,6 +1168,20 @@ namespace MWWorld
return mKeywordSearch;
}
ESM::FixedString<6> getRecNameString(ESM::RecNameInts recName)
{
ESM::FixedString<6> name;
name.assign("");
ESM::NAME fourCCName(recName & ~ESM::sEsm4RecnameFlag);
for (int i = 0; i < 4; i++)
name.mData[i] = fourCCName.mData[i];
if (ESM::isESM4Rec(recName))
{
name.mData[4] = '4';
}
return name;
}
}
template class MWWorld::TypedDynamicStore<ESM::Activator>;
@ -1196,3 +1226,7 @@ template class MWWorld::TypedDynamicStore<ESM::Spell>;
template class MWWorld::TypedDynamicStore<ESM::StartScript>;
template class MWWorld::TypedDynamicStore<ESM::Static>;
template class MWWorld::TypedDynamicStore<ESM::Weapon>;
template class MWWorld::TypedDynamicStore<ESM4::Static>;
template class MWWorld::TypedDynamicStore<ESM4::Reference>;
template class MWWorld::TypedDynamicStore<ESM4::Cell>;

2
apps/openmw/mwworld/store.hpp
View File

@ -517,6 +517,8 @@ namespace MWWorld
const MWDialogue::KeywordSearch<std::string, int>& getDialogIdKeywordSearch() const;
};
ESM::FixedString<6> getRecNameString(ESM::RecNameInts recName);
} // end namespace
#endif

41
apps/openmw_test_suite/mwworld/test_store.cpp
View File

@ -8,6 +8,12 @@
#include <components/esm/records.hpp>
#include <components/esm3/esmreader.hpp>
#include <components/esm3/esmwriter.hpp>
#include <components/esm4/common.hpp>
#include <components/esm4/loadcell.hpp>
#include <components/esm4/loadrefr.hpp>
#include <components/esm4/loadstat.hpp>
#include <components/esm4/reader.hpp>
#include <components/esm4/readerutils.hpp>
#include <components/files/configurationmanager.hpp>
#include <components/files/conversion.hpp>
#include <components/loadinglistener/loadinglistener.hpp>
@ -289,6 +295,41 @@ TEST_F(StoreTest, delete_test)
ASSERT_TRUE(mEsmStore.get<RecordType>().getSize() == 1);
}
template <typename T>
static unsigned int hasSameRecordId(const MWWorld::Store<T>& store, ESM::RecNameInts RecName)
{
if constexpr (MWWorld::HasRecordId<T>::value)
{
return T::sRecordId == RecName ? 1 : 0;
}
else
{
return 0;
}
}
template <typename T>
static void testRecNameIntCount(const MWWorld::Store<T>& store, const MWWorld::ESMStore::StoreTuple& stores)
{
if constexpr (MWWorld::HasRecordId<T>::value)
{
const unsigned int recordIdCount
= std::apply([](auto&&... x) { return (hasSameRecordId(x, T::sRecordId) + ...); }, stores);
ASSERT_EQ(recordIdCount, static_cast<unsigned int>(1))
<< "The same RecNameInt is used twice ESM::REC_" << MWWorld::getRecNameString(T::sRecordId).toStringView();
}
}
static void testAllRecNameIntUnique(const MWWorld::ESMStore::StoreTuple& stores)
{
std::apply([&stores](auto&&... x) { (testRecNameIntCount(x, stores), ...); }, stores);
}
TEST_F(StoreTest, eachRecordTypeShouldHaveUniqueRecordId)
{
testAllRecNameIntUnique(MWWorld::ESMStore::StoreTuple());
}
/// Tests overwriting of records.
TEST_F(StoreTest, overwrite_test)
{

1
components/CMakeLists.txt
View File

@ -193,6 +193,7 @@ add_component_dir (esm4
reader
reference
script
readerutils
)
add_component_dir (misc

5
components/esm/defs.hpp
View File

@ -333,6 +333,11 @@ namespace ESM
REC_MSET4 = esm4Recname(ESM4::REC_MSET) // Media Set
};
constexpr bool isESM4Rec(RecNameInts RecName)
{
return RecName & sEsm4RecnameFlag;
}
/// Common subrecords
enum SubRecNameInts
{

5
components/esm/refid.cpp
View File

@ -29,6 +29,11 @@ namespace ESM
return newRefId;
}
RefId RefId::formIdRefId(const ESM4::FormId id)
{
return ESM::RefId::stringRefId(ESM4::formIdToString(id));
}
bool RefId::operator==(std::string_view rhs) const
{
return Misc::StringUtils::ciEqual(mId, rhs);

3
components/esm/refid.hpp
View File

@ -6,6 +6,8 @@
#include <string>
#include <string_view>
#include <components/esm4/formid.hpp>
namespace ESM
{
// RefId is used to represent an Id that identifies an ESM record. These Ids can then be used in
@ -27,6 +29,7 @@ namespace ESM
// RefIds that are as string in the code. For serialization, and display. Using explicit conversions make it
// very clear where in the code we need to convert from string to RefId and Vice versa.
static RefId stringRefId(std::string_view id);
static RefId formIdRefId(const ESM4::FormId id);
const std::string& getRefIdString() const { return mId; }
private:

14
components/esm4/loadcell.cpp
View File

@ -32,12 +32,13 @@
#include <cassert>
#include <cfloat> // FLT_MAX for gcc
#include <iostream> // FIXME: debug only
#include <stdexcept>
#include <iostream> // FIXME: debug only
#include "reader.hpp"
//#include "writer.hpp"
// #include "writer.hpp"
#include <components/esm/refid.hpp>
// TODO: Try loading only EDID and XCLC (along with mFormId, mFlags and mParent)
//
@ -48,8 +49,9 @@
// longer/shorter/same as loading the subrecords.
void ESM4::Cell::load(ESM4::Reader& reader)
{
mFormId = reader.hdr().record.id;
reader.adjustFormId(mFormId);
auto formId = reader.hdr().record.id;
reader.adjustFormId(formId);
mId = ESM::RefId::formIdRefId(formId);
mFlags = reader.hdr().record.flags;
mParent = reader.currWorld();
@ -71,7 +73,7 @@ void ESM4::Cell::load(ESM4::Reader& reader)
// WARN: we need to call setCurrCell (and maybe setCurrCellGrid?) again before loading
// cell child groups if we are loading them after restoring the context
// (may be easier to update the context before saving?)
reader.setCurrCell(mFormId); // save for LAND (and other children) to access later
reader.setCurrCell(formId); // save for LAND (and other children) to access later
std::uint32_t esmVer = reader.esmVersion();
bool isFONV = esmVer == ESM::VER_132 || esmVer == ESM::VER_133 || esmVer == ESM::VER_134;

7
components/esm4/loadcell.hpp
View File

@ -34,6 +34,9 @@
#include "formid.hpp"
#include "lighting.hpp"
#include <components/esm/defs.hpp>
#include <components/esm/refid.hpp>
namespace ESM4
{
class Reader;
@ -61,7 +64,7 @@ namespace ESM4
{
FormId mParent; // world formId (for grouping cells), from the loading sequence
FormId mFormId; // from the header
ESM::RefId mId; // from the header
std::uint32_t mFlags; // from the header, see enum type RecordFlag for details
std::string mEditorId;
@ -95,6 +98,8 @@ namespace ESM4
// void save(ESM4::Writer& writer) const;
void blank();
static constexpr ESM::RecNameInts sRecordId = ESM::REC_CELL4;
};
}

13
components/esm4/loadrefr.cpp
View File

@ -30,14 +30,15 @@
#include <stdexcept>
#include "reader.hpp"
//#include "writer.hpp"
// #include "writer.hpp"
void ESM4::Reference::load(ESM4::Reader& reader)
{
mFormId = reader.hdr().record.id;
reader.adjustFormId(mFormId);
auto formId = reader.hdr().record.id;
reader.adjustFormId(formId);
mId = ESM::RefId::formIdRefId(formId);
mFlags = reader.hdr().record.flags;
mParent = reader.currCell(); // NOTE: only for persistent refs?
mParent = ESM::RefId::formIdRefId(reader.currCell()); // NOTE: only for persistent refs?
// TODO: Let the engine apply this? Saved games?
// mInitiallyDisabled = ((mFlags & ESM4::Rec_Disabled) != 0) ? true : false;
@ -60,7 +61,9 @@ void ESM4::Reference::load(ESM4::Reader& reader)
break;
case ESM4::SUB_NAME:
{
reader.getFormId(mBaseObj);
FormId BaseId;
reader.getFormId(BaseId);
mBaseObj = ESM::RefId::formIdRefId(BaseId);
#if 0
if (mFlags & ESM4::Rec_Disabled)
std::cout << "REFR disable at start " << formIdToString(mFormId) <<

13
components/esm4/loadrefr.hpp
View File

@ -31,6 +31,9 @@
#include "reference.hpp" // FormId, Placement, EnableParent
#include <components/esm/defs.hpp>
#include <components/esm/refid.hpp>
namespace ESM4
{
class Reader;
@ -71,15 +74,15 @@ namespace ESM4
struct Reference
{
FormId mParent; // cell FormId (currently persistent refs only), from the loading sequence
// NOTE: for exterior cells it will be the dummy cell FormId
ESM::RefId mParent; // cell FormId (currently persistent refs only), from the loading sequence
// NOTE: for exterior cells it will be the dummy cell FormId
FormId mFormId; // from the header
ESM::RefId mId; // from the header
std::uint32_t mFlags; // from the header, see enum type RecordFlag for details
std::string mEditorId;
std::string mFullName;
FormId mBaseObj;
ESM::RefId mBaseObj;
Placement mPlacement;
float mScale = 1.0f;
@ -110,6 +113,8 @@ namespace ESM4
// void save(ESM4::Writer& writer) const;
void blank();
static constexpr ESM::RecNameInts sRecordId = ESM::REC_REFR4;
};
}

7
components/esm4/loadstat.cpp
View File

@ -30,12 +30,13 @@
#include <stdexcept>
#include "reader.hpp"
//#include "writer.hpp"
// #include "writer.hpp"
void ESM4::Static::load(ESM4::Reader& reader)
{
mFormId = reader.hdr().record.id;
reader.adjustFormId(mFormId);
FormId formId = reader.hdr().record.id;
reader.adjustFormId(formId);
mId = ESM::RefId::formIdRefId(formId);
mFlags = reader.hdr().record.flags;
while (reader.getSubRecordHeader())

7
components/esm4/loadstat.hpp
View File

@ -33,6 +33,9 @@
#include "formid.hpp"
#include <components/esm/defs.hpp>
#include <components/esm/refid.hpp>
namespace ESM4
{
class Reader;
@ -40,7 +43,7 @@ namespace ESM4
struct Static
{
FormId mFormId; // from the header
ESM::RefId mId; // from the header
std::uint32_t mFlags; // from the header, see enum type RecordFlag for details
std::string mEditorId;
@ -53,6 +56,8 @@ namespace ESM4
// void save(ESM4::Writer& writer) const;
// void blank();
static constexpr ESM::RecNameInts sRecordId = ESM::REC_STAT4;
};
}

84
components/esm4/readerutils.hpp Normal file
View File

@ -0,0 +1,84 @@
#ifndef OPENMW_COMPONENTS_ESM4_READERUTILS
#define OPENMW_COMPONENTS_ESM4_READERUTILS
#include <components/esm4/reader.hpp>
namespace ESM4
{
struct ReaderUtils
{
/* RecordInvocable must be an invocable, takes an ESM4::Reader& as input, and outputs a boolean that indicates
if the record was read or ignored. Will be invoked for every record
GroupInvocable's invocable must take a ESM4::Reader& as input, doesn't need to output anything. Will be invoked
for every group*/
template <typename RecordInvocable, typename GroupInvocable>
static void readAll(ESM4::Reader& reader, RecordInvocable&& recordInvocable, GroupInvocable&& groupInvocable)
{
while (reader.hasMoreRecs())
{
reader.exitGroupCheck();
if (!readItem(reader, recordInvocable, groupInvocable))
break;
}
}
template <typename RecordInvocable>
static void readRecord(ESM4::Reader& reader, RecordInvocable&& recordInvocable)
{
if (!recordInvocable(reader))
reader.skipRecordData();
}
template <typename RecordInvocable, typename GroupInvocable>
static bool readGroup(ESM4::Reader& reader, RecordInvocable&& recordInvocable, GroupInvocable&& groupInvocable)
{
const ESM4::RecordHeader& header = reader.hdr();
groupInvocable(reader);
switch (static_cast<ESM4::GroupType>(header.group.type))
{
case ESM4::Grp_RecordType:
case ESM4::Grp_InteriorCell:
case ESM4::Grp_InteriorSubCell:
case ESM4::Grp_ExteriorCell:
case ESM4::Grp_ExteriorSubCell:
reader.enterGroup();
return readItem(reader, recordInvocable, groupInvocable);
case ESM4::Grp_WorldChild:
case ESM4::Grp_CellChild:
case ESM4::Grp_TopicChild:
case ESM4::Grp_CellPersistentChild:
case ESM4::Grp_CellTemporaryChild:
case ESM4::Grp_CellVisibleDistChild:
reader.adjustGRUPFormId();
reader.enterGroup();
if (!reader.hasMoreRecs())
return false;
return readItem(reader, recordInvocable, groupInvocable);
}
reader.skipGroup();
return true;
}
template <typename RecordInvocable, typename GroupInvocable>
static bool readItem(ESM4::Reader& reader, RecordInvocable&& recordInvocable, GroupInvocable&& groupInvocable)
{
if (!reader.getRecordHeader() || !reader.hasMoreRecs())
return false;
const ESM4::RecordHeader& header = reader.hdr();
if (header.record.typeId == ESM4::REC_GRUP)
return readGroup(reader, recordInvocable, groupInvocable);
readRecord(reader, recordInvocable);
return true;
}
};
}
#endif // !OPENMW_COMPONENTS_ESM4_READERUTILS

2
components/to_utf8/to_utf8.hpp
View File

@ -68,6 +68,8 @@ namespace ToUTF8
/// ASCII-only string. Otherwise returns a view to the input.
std::string_view getLegacyEnc(std::string_view input);
StatelessUtf8Encoder getStatelessEncoder() const { return mImpl; }
private:
std::string mBuffer;
StatelessUtf8Encoder mImpl;