#include "esmstore.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "../mwmechanics/spelllist.hpp" namespace { struct Ref { ESM::RefNum mRefNum; std::size_t mRefID; Ref(ESM::RefNum refNum, std::size_t refID) : mRefNum(refNum) , mRefID(refID) { } }; constexpr std::size_t deletedRefID = std::numeric_limits::max(); void readRefs(const ESM::Cell& cell, std::vector& refs, std::vector& refIDs, std::set& keyIDs, ESM::ReadersCache& readers) { // TODO: we have many similar copies of this code. for (size_t i = 0; i < cell.mContextList.size(); i++) { const std::size_t index = static_cast(cell.mContextList[i].index); const ESM::ReadersCache::BusyItem reader = readers.get(index); cell.restore(*reader, i); ESM::CellRef ref; bool deleted = false; while (cell.getNextRef(*reader, ref, deleted)) { if (deleted) refs.emplace_back(ref.mRefNum, deletedRefID); else if (std::find(cell.mMovedRefs.begin(), cell.mMovedRefs.end(), ref.mRefNum) == cell.mMovedRefs.end()) { if (!ref.mKey.empty()) keyIDs.insert(std::move(ref.mKey)); refs.emplace_back(ref.mRefNum, refIDs.size()); refIDs.push_back(std::move(ref.mRefID)); } } } for (const auto& [value, deleted] : cell.mLeasedRefs) { if (deleted) refs.emplace_back(value.mRefNum, deletedRefID); else { if (!value.mKey.empty()) keyIDs.insert(std::move(value.mKey)); refs.emplace_back(value.mRefNum, refIDs.size()); refIDs.push_back(value.mRefID); } } } const ESM::RefId& getDefaultClass(const MWWorld::Store& classes) { auto it = classes.begin(); if (it != classes.end()) return it->mId; throw std::runtime_error("List of NPC classes is empty!"); } std::vector getNPCsToReplace(const MWWorld::Store& factions, const MWWorld::Store& classes, const std::unordered_map& npcs) { // Cache first class from store - we will use it if current class is not found const ESM::RefId& defaultCls = getDefaultClass(classes); // Validate NPCs for non-existing class and faction. // We will replace invalid entries by fixed ones std::vector npcsToReplace; for (const auto& npcIter : npcs) { ESM::NPC npc = npcIter.second; bool changed = false; const ESM::RefId& npcFaction = npc.mFaction; if (!npcFaction.empty()) { const ESM::Faction* fact = factions.search(npcFaction); if (!fact) { Log(Debug::Verbose) << "NPC " << npc.mId << " (" << npc.mName << ") has nonexistent faction " << npc.mFaction << ", ignoring it."; npc.mFaction = ESM::RefId(); npc.mNpdt.mRank = 0; changed = true; } } const ESM::RefId& npcClass = npc.mClass; const ESM::Class* cls = classes.search(npcClass); if (!cls) { Log(Debug::Verbose) << "NPC " << npc.mId << " (" << npc.mName << ") has nonexistent class " << npc.mClass << ", using " << defaultCls << " class as replacement."; npc.mClass = defaultCls; changed = true; } if (changed) npcsToReplace.push_back(npc); } return npcsToReplace; } // Custom enchanted items can reference scripts that no longer exist, this doesn't necessarily mean the base item no // longer exists however. So instead of removing the item altogether, we're only removing the script. template void removeMissingScripts(const MWWorld::Store& scripts, MapT& items) { for (auto& [id, item] : items) { if (!item.mScript.empty() && !scripts.search(item.mScript)) { item.mScript = ESM::RefId(); Log(Debug::Verbose) << "Item " << id << " (" << item.mName << ") has nonexistent script " << item.mScript << ", ignoring it."; } } } } namespace MWWorld { using IDMap = std::unordered_map; struct ESMStoreImp { ESMStore::StoreTuple mStores; std::map mRecNameToStore; // Lookup of all IDs. Makes looking up references faster. Just // maps the id name to the record type. IDMap mIds; IDMap mStaticIds; template static void assignStoreToIndex(ESMStore& stores, Store& store) { const std::size_t storeIndex = ESMStore::getTypeIndex(); if (stores.mStores.size() <= storeIndex) stores.mStores.resize(storeIndex + 1); assert(&store == &std::get>(stores.mStoreImp->mStores)); stores.mStores[storeIndex] = &store; if constexpr (std::is_convertible_v*, DynamicStore*>) { stores.mDynamicStores.push_back(&store); constexpr ESM::RecNameInts recName = T::sRecordId; if constexpr (recName != ESM::REC_INTERNAL_PLAYER) { stores.mStoreImp->mRecNameToStore[recName] = &store; } } } template static bool typedReadRecordESM4(ESM4::Reader& reader, Store& store) { auto recordType = static_cast(reader.hdr().record.typeId); ESM::RecNameInts esm4RecName = static_cast(ESM::esm4Recname(recordType)); if constexpr (HasRecordId::value) { if constexpr (ESM::isESM4Rec(T::sRecordId)) { if (T::sRecordId == esm4RecName) { reader.getRecordData(); T value; value.load(reader); store.insertStatic(value); return true; } } } return false; } static bool readRecord(ESM4::Reader& reader, ESMStore& store) { return std::apply( [&reader](auto&... x) { return (typedReadRecordESM4(reader, x) || ...); }, store.mStoreImp->mStores); } }; int ESMStore::find(const ESM::RefId& id) const { IDMap::const_iterator it = mStoreImp->mIds.find(id); if (it == mStoreImp->mIds.end()) { return 0; } return it->second; } int ESMStore::findStatic(const ESM::RefId& id) const { IDMap::const_iterator it = mStoreImp->mStaticIds.find(id); if (it == mStoreImp->mStaticIds.end()) { return 0; } return it->second; } ESMStore::ESMStore() { mStoreImp = std::make_unique(); std::apply([this](auto&... x) { (ESMStoreImp::assignStoreToIndex(*this, x), ...); }, mStoreImp->mStores); mDynamicCount = 0; getWritable().setCells(getWritable()); } ESMStore::~ESMStore() = default; void ESMStore::clearDynamic() { for (const auto& store : mDynamicStores) store->clearDynamic(); mStoreImp->mIds = mStoreImp->mStaticIds; movePlayerRecord(); } static bool isCacheableRecord(int id) { switch (id) { case ESM::REC_ACTI: case ESM::REC_ALCH: case ESM::REC_APPA: case ESM::REC_ARMO: case ESM::REC_BOOK: case ESM::REC_CLOT: case ESM::REC_CONT: case ESM::REC_CREA: case ESM::REC_DOOR: case ESM::REC_INGR: case ESM::REC_LEVC: case ESM::REC_LEVI: case ESM::REC_LIGH: case ESM::REC_LOCK: case ESM::REC_MISC: case ESM::REC_NPC_: case ESM::REC_PROB: case ESM::REC_REPA: case ESM::REC_STAT: case ESM::REC_WEAP: case ESM::REC_BODY: case ESM::REC_STAT4: case ESM::REC_LIGH4: case ESM::REC_ACTI4: case ESM::REC_ALCH4: case ESM::REC_AMMO4: case ESM::REC_ARMO4: case ESM::REC_BOOK4: case ESM::REC_CONT4: case ESM::REC_DOOR4: case ESM::REC_FURN4: case ESM::REC_INGR4: case ESM::REC_MISC4: case ESM::REC_TREE4: case ESM::REC_WEAP4: return true; break; } return false; } void ESMStore::load(ESM::ESMReader& esm, Loading::Listener* listener, ESM::Dialogue*& dialogue) { if (listener != nullptr) listener->setProgressRange(::EsmLoader::fileProgress); // Land texture loading needs to use a separate internal store for each plugin. // We set the number of plugins here so we can properly verify if valid plugin // indices are being passed to the LandTexture Store retrieval methods. getWritable().resize(esm.getIndex() + 1); // Loop through all records while (esm.hasMoreRecs()) { ESM::NAME n = esm.getRecName(); esm.getRecHeader(); if (esm.getRecordFlags() & ESM::FLAG_Ignored) { esm.skipRecord(); continue; } // Look up the record type. ESM::RecNameInts recName = static_cast(n.toInt()); const auto& it = mStoreImp->mRecNameToStore.find(recName); if (it == mStoreImp->mRecNameToStore.end()) { if (recName == ESM::REC_INFO) { if (dialogue) { dialogue->readInfo(esm); } else { Log(Debug::Error) << "Error: info record without dialog"; esm.skipRecord(); } } else if (n.toInt() == ESM::REC_MGEF) { getWritable().load(esm); } else if (n.toInt() == ESM::REC_SKIL) { getWritable().load(esm); } else if (n.toInt() == ESM::REC_FILT || n.toInt() == ESM::REC_DBGP) { // ignore project file only records esm.skipRecord(); } else if (n.toInt() == ESM::REC_LUAL) { ESM::LuaScriptsCfg cfg; cfg.load(esm); cfg.adjustRefNums(esm); mLuaContent.push_back(std::move(cfg)); } else { throw std::runtime_error("Unknown record: " + n.toString()); } } else { RecordId id = it->second->load(esm); if (id.mIsDeleted) { it->second->eraseStatic(id.mId); continue; } if (n.toInt() == ESM::REC_DIAL) { dialogue = const_cast(getWritable().find(id.mId)); } else { dialogue = nullptr; } } if (listener != nullptr) listener->setProgress(::EsmLoader::fileProgress * esm.getFileOffset() / esm.getFileSize()); } } void ESMStore::loadESM4(ESM4::Reader& reader) { auto visitorRec = [this](ESM4::Reader& reader) { return ESMStoreImp::readRecord(reader, *this); }; ESM4::ReaderUtils::readAll(reader, visitorRec, [](ESM4::Reader&) {}); } void ESMStore::setIdType(const ESM::RefId& id, ESM::RecNameInts type) { mStoreImp->mIds[id] = type; } ESM::LuaScriptsCfg ESMStore::getLuaScriptsCfg() const { ESM::LuaScriptsCfg cfg; for (const LuaContent& c : mLuaContent) { if (std::holds_alternative(c)) { // *.omwscripts are intentionally reloaded every time when `getLuaScriptsCfg` is called. // It is important for the `reloadlua` console command. try { auto file = std::ifstream(std::get(c)); std::string fileContent(std::istreambuf_iterator(file), {}); LuaUtil::parseOMWScripts(cfg, fileContent); } catch (std::exception& e) { Log(Debug::Error) << e.what(); } } else { const ESM::LuaScriptsCfg& addition = std::get(c); cfg.mScripts.insert(cfg.mScripts.end(), addition.mScripts.begin(), addition.mScripts.end()); } } return cfg; } void ESMStore::setUp() { if (mIsSetUpDone) throw std::logic_error("ESMStore::setUp() is called twice"); mIsSetUpDone = true; for (const auto& [_, store] : mStoreImp->mRecNameToStore) store->setUp(); getWritable().setUp(get()); getWritable().setUp(); getWritable().setUp(get()); getWritable().updateLandPositions(get()); getWritable().preprocessReferences(get()); rebuildIdsIndex(); mStoreImp->mStaticIds = mStoreImp->mIds; } void ESMStore::rebuildIdsIndex() { mStoreImp->mIds.clear(); for (const auto& [recordType, store] : mStoreImp->mRecNameToStore) { if (isCacheableRecord(recordType)) { std::vector identifiers; store->listIdentifier(identifiers); for (auto& record : identifiers) mStoreImp->mIds[record] = recordType; } } } void ESMStore::validateRecords(ESM::ReadersCache& readers) { validate(); countAllCellRefsAndMarkKeys(readers); } void ESMStore::countAllCellRefsAndMarkKeys(ESM::ReadersCache& readers) { // TODO: We currently need to read entire files here again. // We should consider consolidating or deferring this reading. if (!mRefCount.empty()) return; std::vector refs; std::set keyIDs; std::vector refIDs; Store Cells = get(); for (auto it = Cells.intBegin(); it != Cells.intEnd(); ++it) readRefs(*it, refs, refIDs, keyIDs, readers); for (auto it = Cells.extBegin(); it != Cells.extEnd(); ++it) readRefs(*it, refs, refIDs, keyIDs, readers); const auto lessByRefNum = [](const Ref& l, const Ref& r) { return l.mRefNum < r.mRefNum; }; std::stable_sort(refs.begin(), refs.end(), lessByRefNum); const auto equalByRefNum = [](const Ref& l, const Ref& r) { return l.mRefNum == r.mRefNum; }; const auto incrementRefCount = [&](const Ref& value) { if (value.mRefID != deletedRefID) { ESM::RefId& refId = refIDs[value.mRefID]; ++mRefCount[std::move(refId)]; } }; Misc::forEachUnique(refs.rbegin(), refs.rend(), equalByRefNum, incrementRefCount); auto& store = getWritable().mStatic; for (const auto& id : keyIDs) { auto it = store.find(id); if (it != store.end()) it->second.mData.mFlags |= ESM::Miscellaneous::Key; } } int ESMStore::getRefCount(const ESM::RefId& id) const { auto it = mRefCount.find(id); if (it == mRefCount.end()) return 0; return it->second; } void ESMStore::validate() { auto& npcs = getWritable(); std::vector npcsToReplace = getNPCsToReplace(getWritable(), getWritable(), npcs.mStatic); for (const ESM::NPC& npc : npcsToReplace) { npcs.eraseStatic(npc.mId); npcs.insertStatic(npc); } // Validate spell effects for invalid arguments std::vector spellsToReplace; auto& spells = getWritable(); for (ESM::Spell spell : spells) { if (spell.mEffects.mList.empty()) continue; bool changed = false; auto iter = spell.mEffects.mList.begin(); while (iter != spell.mEffects.mList.end()) { const ESM::MagicEffect* mgef = getWritable().search(iter->mEffectID); if (!mgef) { Log(Debug::Verbose) << "Spell '" << spell.mId << "' has an invalid effect (index " << iter->mEffectID << ") present. Dropping the effect."; iter = spell.mEffects.mList.erase(iter); changed = true; continue; } if (mgef->mData.mFlags & ESM::MagicEffect::TargetSkill) { if (iter->mAttribute != -1) { iter->mAttribute = -1; Log(Debug::Verbose) << ESM::MagicEffect::indexToGmstString(iter->mEffectID) << " effect of spell '" << spell.mId << "' has an attribute argument present. Dropping the argument."; changed = true; } } else if (mgef->mData.mFlags & ESM::MagicEffect::TargetAttribute) { if (iter->mSkill != -1) { iter->mSkill = -1; Log(Debug::Verbose) << ESM::MagicEffect::indexToGmstString(iter->mEffectID) << " effect of spell '" << spell.mId << "' has a skill argument present. Dropping the argument."; changed = true; } } else if (iter->mSkill != -1 || iter->mAttribute != -1) { iter->mSkill = -1; iter->mAttribute = -1; Log(Debug::Verbose) << ESM::MagicEffect::indexToGmstString(iter->mEffectID) << " effect of spell '" << spell.mId << "' has argument(s) present. Dropping the argument(s)."; changed = true; } ++iter; } if (changed) spellsToReplace.emplace_back(spell); } for (const ESM::Spell& spell : spellsToReplace) { spells.eraseStatic(spell.mId); spells.insertStatic(spell); } } void ESMStore::movePlayerRecord() { auto& npcs = getWritable(); auto player = npcs.find(ESM::RefId::stringRefId("Player")); npcs.insert(*player); } void ESMStore::validateDynamic() { auto& npcs = getWritable(); auto& scripts = getWritable(); std::vector npcsToReplace = getNPCsToReplace(getWritable(), getWritable(), npcs.mDynamic); for (const ESM::NPC& npc : npcsToReplace) npcs.insert(npc); removeMissingScripts(scripts, getWritable().mDynamic); removeMissingScripts(scripts, getWritable().mDynamic); removeMissingScripts(scripts, getWritable().mDynamic); removeMissingScripts(scripts, getWritable().mDynamic); removeMissingObjects(getWritable()); removeMissingObjects(getWritable()); } // Leveled lists can be modified by scripts. This removes items that no longer exist (presumably because the // plugin was removed) from modified lists template void ESMStore::removeMissingObjects(Store& store) { for (auto& entry : store.mDynamic) { auto first = std::remove_if(entry.second.mList.begin(), entry.second.mList.end(), [&](const auto& item) { if (!find(item.mId)) { Log(Debug::Verbose) << "Leveled list " << entry.first << " has nonexistent object " << item.mId << ", ignoring it."; return true; } return false; }); entry.second.mList.erase(first, entry.second.mList.end()); } } int ESMStore::countSavedGameRecords() const { return 1 // DYNA (dynamic name counter) + get().getDynamicSize() + get().getDynamicSize() + get().getDynamicSize() + get().getDynamicSize() + get().getDynamicSize() + get().getDynamicSize() + get().getDynamicSize() + get().getDynamicSize() + get().getDynamicSize() + get().getDynamicSize() + get().getDynamicSize() + get().getDynamicSize() + get().getDynamicSize() + get().getDynamicSize() + get().getDynamicSize(); } void ESMStore::write(ESM::ESMWriter& writer, Loading::Listener& progress) const { writer.startRecord(ESM::REC_DYNA); writer.startSubRecord("COUN"); writer.writeT(mDynamicCount); writer.endRecord("COUN"); writer.endRecord(ESM::REC_DYNA); get().write(writer, progress); get().write(writer, progress); get().write(writer, progress); get().write(writer, progress); get().write(writer, progress); get().write(writer, progress); get().write(writer, progress); get().write(writer, progress); get().write(writer, progress); get().write(writer, progress); get().write(writer, progress); get().write(writer, progress); get().write(writer, progress); get().write(writer, progress); get().write(writer, progress); } bool ESMStore::readRecord(ESM::ESMReader& reader, uint32_t type_id) { ESM::RecNameInts type = (ESM::RecNameInts)type_id; switch (type) { case ESM::REC_ALCH: case ESM::REC_MISC: case ESM::REC_ACTI: case ESM::REC_ARMO: case ESM::REC_BOOK: case ESM::REC_CLAS: case ESM::REC_CLOT: case ESM::REC_ENCH: case ESM::REC_SPEL: case ESM::REC_WEAP: case ESM::REC_LEVI: case ESM::REC_LEVC: mStoreImp->mRecNameToStore[type]->read(reader); return true; case ESM::REC_NPC_: case ESM::REC_CREA: case ESM::REC_CONT: mStoreImp->mRecNameToStore[type]->read(reader, true); return true; case ESM::REC_DYNA: reader.getSubNameIs("COUN"); reader.getHT(mDynamicCount); return true; default: return false; } } void ESMStore::checkPlayer() { const ESM::NPC* player = get().find(ESM::RefId::stringRefId("Player")); if (!get().find(player->mRace) || !get().find(player->mClass)) throw std::runtime_error("Invalid player record (race or class unavailable"); } std::pair, bool> ESMStore::getSpellList(const ESM::RefId& id) const { auto result = mSpellListCache.find(id); std::shared_ptr ptr; if (result != mSpellListCache.end()) ptr = result->second.lock(); if (!ptr) { int type = find(id); ptr = std::make_shared(id, type); if (result != mSpellListCache.end()) result->second = ptr; else mSpellListCache.insert({ id, ptr }); return { ptr, false }; } return { ptr, true }; } template <> const ESM::Cell* ESMStore::insert(const ESM::Cell& cell) { return getWritable().insert(cell); } template <> const ESM::NPC* ESMStore::insert(const ESM::NPC& npc) { auto& npcs = getWritable(); if (npc.mId == "Player") { return npcs.insert(npc); } const ESM::RefId id = ESM::RefId::generated(mDynamicCount++); if (npcs.search(id) != nullptr) throw std::runtime_error("Try to override existing record: " + id.toDebugString()); ESM::NPC record = npc; record.mId = id; ESM::NPC* ptr = npcs.insert(record); mStoreImp->mIds[ptr->mId] = ESM::REC_NPC_; return ptr; } } // end namespace