From 42eefaf36ff371d8bdecdca54e5aa26269d7ba6b Mon Sep 17 00:00:00 2001 From: Mark Siewert Date: Sat, 10 Nov 2012 21:43:41 +0100 Subject: [PATCH] - Add support for loading references from multiple esm/esp files. Full reference ID mangling coming soon (currently, moved references are simply cloned). - Reference loader now (partially) supports MVRF tag. --- apps/esmtool/esmtool.cpp | 5 ++++- apps/openmw/mwbase/world.hpp | 2 +- apps/openmw/mwworld/cells.cpp | 3 ++- apps/openmw/mwworld/cells.hpp | 5 +++-- apps/openmw/mwworld/cellstore.cpp | 34 +++++++++++++++---------------- apps/openmw/mwworld/cellstore.hpp | 10 ++++----- apps/openmw/mwworld/worldimp.cpp | 24 ++++++++++++++-------- apps/openmw/mwworld/worldimp.hpp | 4 ++-- components/esm/esmcommon.hpp | 4 ++++ components/esm/esmreader.hpp | 2 +- components/esm/loadcell.cpp | 31 +++++++++++++++++++++------- components/esm/loadcell.hpp | 2 +- components/esm_store/reclists.hpp | 19 +++++++++++++++-- 13 files changed, 95 insertions(+), 50 deletions(-) diff --git a/apps/esmtool/esmtool.cpp b/apps/esmtool/esmtool.cpp index 4f6d9dbfc0..25c4f3a7a9 100644 --- a/apps/esmtool/esmtool.cpp +++ b/apps/esmtool/esmtool.cpp @@ -220,7 +220,10 @@ void loadCell(ESM::Cell &cell, ESM::ESMReader &esm, Arguments& info) bool save = (info.mode == "clone"); // Skip back to the beginning of the reference list - cell.restore(esm); + // FIXME: Changes to the references backend required to support multiple plugins have + // almost certainly broken this following line. I'll leave it as is for now, so that + // the compiler does not complain. + cell.restore(esm, 0); // Loop through all the references ESM::CellRef ref; diff --git a/apps/openmw/mwbase/world.hpp b/apps/openmw/mwbase/world.hpp index 6710fc68b0..06c3002c7e 100644 --- a/apps/openmw/mwbase/world.hpp +++ b/apps/openmw/mwbase/world.hpp @@ -106,7 +106,7 @@ namespace MWBase virtual const ESMS::ESMStore& getStore() const = 0; - virtual ESM::ESMReader& getEsmReader() = 0; + virtual std::vector& getEsmReader() = 0; virtual MWWorld::LocalScripts& getLocalScripts() = 0; diff --git a/apps/openmw/mwworld/cells.cpp b/apps/openmw/mwworld/cells.cpp index e5a38d4be1..e49835f412 100644 --- a/apps/openmw/mwworld/cells.cpp +++ b/apps/openmw/mwworld/cells.cpp @@ -85,7 +85,7 @@ MWWorld::Ptr MWWorld::Cells::getPtrAndCache (const std::string& name, Ptr::CellS return ptr; } -MWWorld::Cells::Cells (const ESMS::ESMStore& store, ESM::ESMReader& reader) +MWWorld::Cells::Cells (const ESMS::ESMStore& store, std::vector& reader) : mStore (store), mReader (reader), mIdCache (20, std::pair ("", (Ptr::CellStore*)0)), /// \todo make cache size configurable mIdCacheIndex (0) @@ -120,6 +120,7 @@ MWWorld::Ptr::CellStore *MWWorld::Cells::getExterior (int x, int y) if (result->second.mState!=Ptr::CellStore::State_Loaded) { + // Multiple plugin support for landscape data is much easier than for references. The last plugin wins. result->second.load (mStore, mReader); fillContainers (result->second); } diff --git a/apps/openmw/mwworld/cells.hpp b/apps/openmw/mwworld/cells.hpp index 3e13831665..3941399f89 100644 --- a/apps/openmw/mwworld/cells.hpp +++ b/apps/openmw/mwworld/cells.hpp @@ -2,6 +2,7 @@ #define GAME_MWWORLD_CELLS_H #include +#include #include #include "ptr.hpp" @@ -22,7 +23,7 @@ namespace MWWorld class Cells { const ESMS::ESMStore& mStore; - ESM::ESMReader& mReader; + std::vector& mReader; std::map mInteriors; std::map, CellStore> mExteriors; std::vector > mIdCache; @@ -39,7 +40,7 @@ namespace MWWorld public: - Cells (const ESMS::ESMStore& store, ESM::ESMReader& reader); + Cells (const ESMS::ESMStore& store, std::vector& reader); ///< \todo pass the dynamic part of the ESMStore isntead (once it is written) of the whole /// world diff --git a/apps/openmw/mwworld/cellstore.cpp b/apps/openmw/mwworld/cellstore.cpp index 322b43a00d..91c0afe26f 100644 --- a/apps/openmw/mwworld/cellstore.cpp +++ b/apps/openmw/mwworld/cellstore.cpp @@ -16,7 +16,7 @@ namespace MWWorld mWaterLevel = cell->mWater; } - void CellStore::load (const ESMS::ESMStore &store, ESM::ESMReader &esm) + void CellStore::load (const ESMS::ESMStore &store, std::vector &esm) { if (mState!=State_Loaded) { @@ -31,7 +31,7 @@ namespace MWWorld } } - void CellStore::preload (const ESMS::ESMStore &store, ESM::ESMReader &esm) + void CellStore::preload (const ESMS::ESMStore &store, std::vector &esm) { if (mState==State_Unloaded) { @@ -41,7 +41,7 @@ namespace MWWorld } } - void CellStore::listRefs(const ESMS::ESMStore &store, ESM::ESMReader &esm) + void CellStore::listRefs(const ESMS::ESMStore &store, std::vector &esm) { assert (cell); @@ -49,26 +49,24 @@ namespace MWWorld return; // this is a dynamically generated cell -> skipping. // Load references from all plugins that do something with this cell. - // HACK: only use first entry for now, full support requires some more work - //for (int i = 0; i < cell->mContextList.size(); i++) - for (int i = 0; i < 1; i++) + for (size_t i = 0; i < cell->mContextList.size(); i++) { // Reopen the ESM reader and seek to the right position. - // TODO: we will need to intoduce separate "esm"s, one per plugin! - cell->restore (esm); + int index = cell->mContextList.at(i).index; + cell->restore (esm[index], i); ESM::CellRef ref; // Get each reference in turn - while (cell->getNextRef (esm, ref)) + while (cell->getNextRef (esm[index], ref)) { std::string lowerCase; std::transform (ref.mRefID.begin(), ref.mRefID.end(), std::back_inserter (lowerCase), (int(*)(int)) std::tolower); - // TODO: support deletion / moving references out of the cell. no simple "push_back", - // but see what the plugin wants to do. + // TODO: Fully support deletion / moving references out of the cell. no simple "push_back", + // but make sure that the reference exists only once. mIds.push_back (lowerCase); } } @@ -76,7 +74,7 @@ namespace MWWorld std::sort (mIds.begin(), mIds.end()); } - void CellStore::loadRefs(const ESMS::ESMStore &store, ESM::ESMReader &esm) + void CellStore::loadRefs(const ESMS::ESMStore &store, std::vector &esm) { assert (cell); @@ -84,24 +82,24 @@ namespace MWWorld return; // this is a dynamically generated cell -> skipping. // Load references from all plugins that do something with this cell. - // HACK: only use first entry for now, full support requires some more work - //for (int i = 0; i < cell->mContextList.size(); i++) - for (int i = 0; i < 1; i++) + for (size_t i = 0; i < cell->mContextList.size(); i++) { // Reopen the ESM reader and seek to the right position. - // TODO: we will need to intoduce separate "esm"s, one per plugin! - cell->restore(esm); + int index = cell->mContextList.at(i).index; + cell->restore (esm[index], i); ESM::CellRef ref; // Get each reference in turn - while(cell->getNextRef(esm, ref)) + while(cell->getNextRef(esm[index], ref)) { std::string lowerCase; std::transform (ref.mRefID.begin(), ref.mRefID.end(), std::back_inserter (lowerCase), (int(*)(int)) std::tolower); + // TODO: Fully support deletion / moving references out of the cell. No simple loading, + // but make sure that the reference exists only once. Current code clones references. int rec = store.find(ref.mRefID); ref.mRefID = lowerCase; diff --git a/apps/openmw/mwworld/cellstore.hpp b/apps/openmw/mwworld/cellstore.hpp index fd739b5651..69c4cf9b4b 100644 --- a/apps/openmw/mwworld/cellstore.hpp +++ b/apps/openmw/mwworld/cellstore.hpp @@ -3,7 +3,7 @@ #include -#include +#include #include #include "refdata.hpp" @@ -126,9 +126,9 @@ namespace MWWorld CellRefList statics; CellRefList weapons; - void load (const ESMS::ESMStore &store, ESM::ESMReader &esm); + void load (const ESMS::ESMStore &store, std::vector &esm); - void preload (const ESMS::ESMStore &store, ESM::ESMReader &esm); + void preload (const ESMS::ESMStore &store, std::vector &esm); /// Call functor (ref) for each reference. functor must return a bool. Returning /// false will abort the iteration. @@ -187,9 +187,9 @@ namespace MWWorld } /// Run through references and store IDs - void listRefs(const ESMS::ESMStore &store, ESM::ESMReader &esm); + void listRefs(const ESMS::ESMStore &store, std::vector &esm); - void loadRefs(const ESMS::ESMStore &store, ESM::ESMReader &esm); + void loadRefs(const ESMS::ESMStore &store, std::vector &esm); }; } diff --git a/apps/openmw/mwworld/worldimp.cpp b/apps/openmw/mwworld/worldimp.cpp index 6cf831b534..bbc44f5e10 100644 --- a/apps/openmw/mwworld/worldimp.cpp +++ b/apps/openmw/mwworld/worldimp.cpp @@ -179,6 +179,8 @@ namespace MWWorld mWeatherManager = new MWWorld::WeatherManager(mRendering); int idx = 0; + // NOTE: We might need to reserve one more for the running game / save. + mEsm.resize(master.size() + plugins.size()); for (std::vector::size_type i = 0; i < master.size(); i++, idx++) { boost::filesystem::path masterPath (fileCollections.getCollection (".esm").getPath (master[i])); @@ -186,10 +188,12 @@ namespace MWWorld std::cout << "Loading ESM " << masterPath.string() << "\n"; // This parses the ESM file - mEsm.setEncoding(encoding); - mEsm.open (masterPath.string()); - mEsm.setIndex(idx); - mStore.load (mEsm); + ESM::ESMReader lEsm; + lEsm.setEncoding(encoding); + lEsm.open (masterPath.string()); + lEsm.setIndex(idx); + mEsm[idx] = lEsm; + mStore.load (mEsm[idx]); } for (std::vector::size_type i = 0; i < plugins.size(); i++, idx++) @@ -199,10 +203,12 @@ namespace MWWorld std::cout << "Loading ESP " << pluginPath.string() << "\n"; // This parses the ESP file - mEsm.setEncoding(encoding); - mEsm.open (pluginPath.string()); - mEsm.setIndex(idx); - mStore.load (mEsm); + ESM::ESMReader lEsm; + lEsm.setEncoding(encoding); + lEsm.open (pluginPath.string()); + lEsm.setIndex(idx); + mEsm[idx] = lEsm; + mStore.load (mEsm[idx]); } mPlayer = new MWWorld::Player (mStore.npcs.find ("player"), *this); @@ -282,7 +288,7 @@ namespace MWWorld return mStore; } - ESM::ESMReader& World::getEsmReader() + std::vector& World::getEsmReader() { return mEsm; } diff --git a/apps/openmw/mwworld/worldimp.hpp b/apps/openmw/mwworld/worldimp.hpp index c8c56f130e..ffb2137c7b 100644 --- a/apps/openmw/mwworld/worldimp.hpp +++ b/apps/openmw/mwworld/worldimp.hpp @@ -55,7 +55,7 @@ namespace MWWorld MWWorld::Scene *mWorldScene; MWWorld::Player *mPlayer; - ESM::ESMReader mEsm; + std::vector mEsm; ESMS::ESMStore mStore; LocalScripts mLocalScripts; MWWorld::Globals *mGlobalVariables; @@ -127,7 +127,7 @@ namespace MWWorld virtual const ESMS::ESMStore& getStore() const; - virtual ESM::ESMReader& getEsmReader(); + virtual std::vector& getEsmReader(); virtual LocalScripts& getLocalScripts(); diff --git a/components/esm/esmcommon.hpp b/components/esm/esmcommon.hpp index e0c5c08afe..d61564c676 100644 --- a/components/esm/esmcommon.hpp +++ b/components/esm/esmcommon.hpp @@ -113,6 +113,10 @@ struct ESM_Context size_t leftFile; NAME recName, subName; HEDRstruct header; + // When working with multiple esX files, we will generate lists of all files that + // actually contribute to a specific cell. Therefore, we need to store the index + // of the file belonging to this contest. See CellStore::(list/load)refs for details. + int index; // True if subName has been read but not used. bool subCached; diff --git a/components/esm/esmreader.hpp b/components/esm/esmreader.hpp index 96b164d5e9..f09442a577 100644 --- a/components/esm/esmreader.hpp +++ b/components/esm/esmreader.hpp @@ -81,7 +81,7 @@ public: // to the individual load() methods. This hack allows to pass this reference // indirectly to the load() method. int idx; - void setIndex(const int index) {idx = index;} + void setIndex(const int index) {idx = index; mCtx.index = index;} const int getIndex() {return idx;} /************************************************************************* diff --git a/components/esm/loadcell.cpp b/components/esm/loadcell.cpp index 65c9c23e7a..beedd3cacf 100644 --- a/components/esm/loadcell.cpp +++ b/components/esm/loadcell.cpp @@ -113,6 +113,8 @@ void Cell::load(ESMReader &esm) // Save position of the cell references and move on mContextList.push_back(esm.getContext()); + if (mContextList.size() > 1) + std::cout << "found two plugins" << std::endl; esm.skipRecord(); } @@ -147,10 +149,9 @@ void Cell::save(ESMWriter &esm) esm.writeHNT("NAM0", mNAM0); } -void Cell::restore(ESMReader &esm) const +void Cell::restore(ESMReader &esm, int iCtx) const { - // TODO: support all contexts in the list! - esm.restoreContext(mContextList[0]); + esm.restoreContext(mContextList[iCtx]); } std::string Cell::getDescription() const @@ -169,15 +170,28 @@ std::string Cell::getDescription() const bool Cell::getNextRef(ESMReader &esm, CellRef &ref) { - // TODO: Add support for moved references. References moved without crossing a cell boundary simply - // overwrite old data. References moved across cell boundaries are using a different set of keywords, - // and I'll have to think more about how this can be done. // TODO: Add support for multiple plugins. This requires a tricky renaming scheme for "ref.mRefnum". // I'll probably add something to "ESMReader", we will need one per plugin anyway. // TODO: Try and document reference numbering, I don't think this has been done anywhere else. if (!esm.hasMoreSubs()) return false; - + + if (esm.isNextSub("MVRF")) { + // Moved existing reference across cell boundaries, so interpret the blocks correctly. + // FIXME: Right now, we don't do anything with this data. This might result in weird behaviour, + // where a moved reference does not appear because the owning cell (i.e. this cell) is not + // loaded in memory. + int movedRefnum = 0; + int destCell[2]; + esm.getHT(movedRefnum); + esm.getHNT(destCell, "CNDT"); + // TODO: Figure out what happens when a reference has moved into an interior cell. This might + // be required for NPCs following the player. + } + // If we have just parsed a MVRF entry, there should be a regular FRMR entry following right there. + // With the exception that this bock technically belongs to a different cell than this one. + // TODO: Figure out a way to handle these weird references that do not belong to this cell. + // This may require some not-so-small behing-the-scenes updates. esm.getHNT(ref.mRefnum, "FRMR"); ref.mRefID = esm.getHNString("NAME"); @@ -228,6 +242,9 @@ bool Cell::getNextRef(ESMReader &esm, CellRef &ref) // Number of references in the cell? Maximum once in each cell, // but not always at the beginning, and not always right. In other // words, completely useless. + // Update: Well, maybe not completely useless. This might actually be + // number_of_references + number_of_references_moved_here_Across_boundaries, + // and could be helpful for collecting these weird moved references. ref.mNam0 = 0; if (esm.isNextSub("NAM0")) { diff --git a/components/esm/loadcell.hpp b/components/esm/loadcell.hpp index 9511ae0e74..ccfdcadd87 100644 --- a/components/esm/loadcell.hpp +++ b/components/esm/loadcell.hpp @@ -152,7 +152,7 @@ struct Cell // somewhere other than the file system, you need to pre-open the // ESMReader, and the filename must match the stored filename // exactly. - void restore(ESMReader &esm) const; + void restore(ESMReader &esm, int iCtx) const; std::string getDescription() const; ///< Return a short string describing the cell (mostly used for debugging/logging purpose) diff --git a/components/esm_store/reclists.hpp b/components/esm_store/reclists.hpp index eac1b5b724..44b7e6569e 100644 --- a/components/esm_store/reclists.hpp +++ b/components/esm_store/reclists.hpp @@ -496,6 +496,11 @@ namespace ESMS { count++; + // Don't automatically assume that a new cell must be spawned. Multiple plugins write to the same cell, + // and we merge all this data into one Cell object. However, we can't simply search for the cell id, + // as many exterior cells do not have a name. Instead, we need to search by (x,y) coordinates - and they + // are not available until both cells have been loaded! So first, proceed as usual. + // All cells have a name record, even nameless exterior cells. ESM::Cell *cell = new ESM::Cell; cell->mName = id; @@ -505,12 +510,22 @@ namespace ESMS if(cell->mData.mFlags & ESM::Cell::Interior) { - // Store interior cell by name + // Store interior cell by name, try to merge with existing parent data. + ESM::Cell *oldcell = const_cast(searchInt(id)); + if (oldcell) { + cell->mContextList.push_back(oldcell->mContextList.at(0)); + delete oldcell; + } intCells[id] = cell; } else { - // Store exterior cells by grid position + // Store exterior cells by grid position, try to merge with existing parent data. + ESM::Cell *oldcell = const_cast(searchExt(cell->getGridX(), cell->getGridY())); + if (oldcell) { + cell->mContextList.push_back(oldcell->mContextList.at(0)); + delete oldcell; + } extCells[std::make_pair (cell->mData.mX, cell->mData.mY)] = cell; } }