Convert the CellRef record index lookup maps to use integer keys rather than strings.

- Morrowind load over 300,000 references, so even small inefficiencies add up to longer loading times.
- std::map is used, but should try others, std::unordered_map or even std::vector

(copied the changes from commit SHA-1: 86945d19128468ad987b5bf4332602d613d25d9f)

apps/opencs/model/world/data.hpp

@@ -118,7 +118,7 @@ namespace CSMWorld
             const ESM::Dialogue *mDialogue; // last loaded dialogue
             bool mBase;
             bool mProject;
-            std::map<std::string, std::map<unsigned int, std::string> > mRefLoadCache;
+            std::map<std::string, std::map<unsigned int, unsigned int> > mRefLoadCache;
             int mReaderIndex;
 
             bool mFsStrict;

apps/opencs/model/world/ref.cpp

@@ -2,7 +2,7 @@
 
 #include "cellcoordinates.hpp"
 
-CSMWorld::CellRef::CellRef() : mNew (true)
+CSMWorld::CellRef::CellRef() : mNew (true), mIdNum(0)
 {
     mRefNum.mIndex = 0;
     mRefNum.mContentFile = 0;

apps/opencs/model/world/ref.hpp

@@ -14,6 +14,7 @@ namespace CSMWorld
         std::string mCell;
         std::string mOriginalCell;
         bool mNew; // new reference, not counted yet, ref num not assigned yet
+        unsigned int mIdNum;
 
         CellRef();
 

apps/opencs/model/world/refcollection.cpp

@@ -7,8 +7,37 @@
 #include "universalid.hpp"
 #include "record.hpp"
 
+namespace CSMWorld
+{
+    template<>
+    void Collection<CellRef, IdAccessor<CellRef> >::removeRows (int index, int count)
+    {
+        mRecords.erase(mRecords.begin()+index, mRecords.begin()+index+count);
+
+        // index map is updated in RefCollection::removeRows()
+    }
+
+    template<>
+    void Collection<CellRef, IdAccessor<CellRef> >::insertRecord (std::unique_ptr<RecordBase> record, int index,
+        UniversalId::Type type)
+    {
+        int size = static_cast<int>(mRecords.size());
+        if (index < 0 || index > size)
+            throw std::runtime_error("index out of range");
+
+        std::unique_ptr<Record<CellRef> > record2(static_cast<Record<CellRef>*>(record.release()));
+
+        if (index == size)
+            mRecords.push_back(std::move(record2));
+        else
+            mRecords.insert(mRecords.begin()+index, std::move(record2));
+
+        // index map is updated in RefCollection::insertRecord()
+    }
+}
+
 void CSMWorld::RefCollection::load (ESM::ESMReader& reader, int cellIndex, bool base,
-    std::map<unsigned int, std::string>& cache, CSMDoc::Messages& messages)
+    std::map<unsigned int, unsigned int>& cache, CSMDoc::Messages& messages)
 {
     Record<Cell> cell = mCells.getRecord (cellIndex);
 
@@ -63,7 +92,7 @@ void CSMWorld::RefCollection::load (ESM::ESMReader& reader, int cellIndex, bool
         unsigned int  refNum = (ref.mRefNum.mIndex & 0x00ffffff) |
             (ref.mRefNum.hasContentFile() ? ref.mRefNum.mContentFile : 0xff) << 24;
 
-        std::map<unsigned int, std::string>::iterator iter = cache.find(refNum);
+        std::map<unsigned int, unsigned int>::iterator iter = cache.find(refNum);
 
         if (ref.mRefNum.mContentFile != -1 && !base) ref.mRefNum.mContentFile = ref.mRefNum.mIndex >> 24;
 
@@ -74,11 +103,15 @@ void CSMWorld::RefCollection::load (ESM::ESMReader& reader, int cellIndex, bool
                 CSMWorld::UniversalId id (CSMWorld::UniversalId::Type_Cell,
                     mCells.getId (cellIndex));
 
-                messages.add (id, "Attempt to delete a non-existent reference");
+                messages.add (id, "Attempt to delete a non-existent reference - RefNum index "
+                        + std::to_string(ref.mRefNum.mIndex) + ", refID " + ref.mRefID + ", content file index "
+                        + std::to_string(ref.mRefNum.mContentFile),
+                        /*hint*/"",
+                        CSMDoc::Message::Severity_Warning);
                 continue;
             }
 
-            int index = getIndex (iter->second);
+            int index = getIntIndex (iter->second);
 
             if (base)
             {
@@ -98,9 +131,10 @@ void CSMWorld::RefCollection::load (ESM::ESMReader& reader, int cellIndex, bool
         if (iter==cache.end())
         {
             // new reference
+            ref.mIdNum = mNextId; // FIXME: fragile
             ref.mId = getNewId();
 
-            cache.emplace(refNum, ref.mId);
+            cache.emplace(refNum, ref.mIdNum);
 
             std::unique_ptr<Record<CellRef> > record(new Record<CellRef>);
             record->mState = base ? RecordBase::State_BaseOnly : RecordBase::State_ModifiedOnly;
@@ -111,9 +145,27 @@ void CSMWorld::RefCollection::load (ESM::ESMReader& reader, int cellIndex, bool
         else
         {
             // old reference -> merge
-            ref.mId = iter->second;
+            int index = getIntIndex(iter->second);
+#if 0
+            // ref.mRefNum.mIndex     : the key
+            // iter->second           : previously cached idNum for the key
+            // index                  : position of the record for that idNum
+            // getRecord(index).get() : record in the index position
+            assert(iter->second != getRecord(index).get().mIdNum); // sanity check
 
-            int index = getIndex (ref.mId);
+            // check if the plugin used the same RefNum index for a different record
+            if (ref.mRefID != getRecord(index).get().mRefID)
+            {
+                CSMWorld::UniversalId id(CSMWorld::UniversalId::Type_Cell, mCells.getId(cellIndex));
+                messages.add(id,
+                    "RefNum renamed from RefID \"" + getRecord(index).get().mRefID + "\" to \""
+                    + ref.mRefID + "\" (RefNum index " + std::to_string(ref.mRefNum.mIndex) + ")",
+                    /*hint*/"",
+                    CSMDoc::Message::Severity_Info);
+            }
+#endif
+            ref.mId = getRecord(index).get().mId;
+            ref.mIdNum = extractIdNum(ref.mId);
 
             std::unique_ptr<Record<CellRef> > record(new Record<CellRef>(getRecord(index)));
             record->mState = base ? RecordBase::State_BaseOnly : RecordBase::State_Modified;
@@ -128,3 +180,117 @@ std::string CSMWorld::RefCollection::getNewId()
 {
     return "ref#" + std::to_string(mNextId++);
 }
+
+unsigned int CSMWorld::RefCollection::extractIdNum (const std::string& id) const
+{
+    std::string::size_type separator = id.find_last_of('#');
+
+    if (separator == std::string::npos)
+        throw std::runtime_error("invalid ref ID: " + id);
+
+    return static_cast<unsigned int>(std::stoi(id.substr(separator+1)));
+}
+
+int CSMWorld::RefCollection::getIntIndex (unsigned int id) const
+{
+    int index = searchId(id);
+
+    if (index == -1)
+        throw std::runtime_error("invalid RefNum: " + std::to_string(id));
+
+    return index;
+}
+
+int CSMWorld::RefCollection::searchId (unsigned int id) const
+{
+    std::map<unsigned int, int>::const_iterator iter = mRefIndex.find(id);
+
+    if (iter == mRefIndex.end())
+        return -1;
+
+    return iter->second;
+}
+
+void CSMWorld::RefCollection::removeRows (int index, int count)
+{
+    Collection<CellRef, IdAccessor<CellRef> >::removeRows(index, count); // erase records only
+
+    std::map<unsigned int, int>::iterator iter = mRefIndex.begin();
+    while (iter != mRefIndex.end())
+    {
+        if (iter->second>=index)
+        {
+            if (iter->second >= index+count)
+            {
+                iter->second -= count;
+                ++iter;
+            }
+            else
+                mRefIndex.erase(iter++);
+        }
+        else
+            ++iter;
+    }
+}
+
+void  CSMWorld::RefCollection::appendBlankRecord (const std::string& id, UniversalId::Type type)
+{
+    std::unique_ptr<Record<CellRef> > record2(new Record<CellRef>);
+
+    record2->mState = Record<CellRef>::State_ModifiedOnly;
+    record2->mModified.blank();
+
+    record2->get().mId = id;
+    record2->get().mIdNum = extractIdNum(id);
+
+    Collection<CellRef, IdAccessor<CellRef> >::appendRecord(std::move(record2));
+}
+
+void CSMWorld::RefCollection::cloneRecord (const std::string& origin,
+                                           const std::string& destination,
+                                           const UniversalId::Type type)
+{
+   std::unique_ptr<Record<CellRef> > copy(new Record<CellRef>);
+
+   copy->mModified = getRecord(origin).get();
+   copy->mState = RecordBase::State_ModifiedOnly;
+
+   copy->get().mId = destination;
+   copy->get().mIdNum = extractIdNum(destination);
+
+   insertRecord(std::move(copy), getAppendIndex(destination, type)); // call RefCollection::insertRecord()
+}
+
+int CSMWorld::RefCollection::searchId (const std::string& id) const
+{
+    return searchId(extractIdNum(id));
+}
+
+void CSMWorld::RefCollection::appendRecord (std::unique_ptr<RecordBase> record, UniversalId::Type type)
+{
+    int index = getAppendIndex(/*id*/"", type); // for CellRef records id is ignored
+
+    mRefIndex.insert(std::make_pair(static_cast<Record<CellRef>*>(record.get())->get().mIdNum, index));
+
+    Collection<CellRef, IdAccessor<CellRef> >::insertRecord(std::move(record), index, type); // add records only
+}
+
+void CSMWorld::RefCollection::insertRecord (std::unique_ptr<RecordBase> record, int index,
+    UniversalId::Type type)
+{
+    int size = getAppendIndex(/*id*/"", type); // for CellRef records id is ignored
+    unsigned int idNum = static_cast<Record<CellRef>*>(record.get())->get().mIdNum;
+
+    Collection<CellRef, IdAccessor<CellRef> >::insertRecord(std::move(record), index, type); // add records only
+
+    if (index < size-1)
+    {
+        for (std::map<unsigned int, int>::iterator iter(mRefIndex.begin()); iter != mRefIndex.end(); ++iter)
+        {
+            if (iter->second >= index)
+                ++(iter->second);
+        }
+    }
+
+    mRefIndex.insert(std::make_pair(idNum, index));
+}

apps/opencs/model/world/refcollection.hpp

@@ -14,12 +14,27 @@ namespace CSMWorld
     struct Cell;
     class UniversalId;
 
+    template<>
+    void Collection<CellRef, IdAccessor<CellRef> >::removeRows (int index, int count);
+
+    template<>
+    void Collection<CellRef, IdAccessor<CellRef> >::insertRecord (std::unique_ptr<RecordBase> record, int index,
+        UniversalId::Type type);
+
     /// \brief References in cells
     class RefCollection : public Collection<CellRef>
     {
             Collection<Cell>& mCells;
+            std::map<unsigned int, int> mRefIndex;
+
             int mNextId;
 
+            unsigned int extractIdNum(const std::string& id) const;
+
+            int getIntIndex (unsigned int id) const;
+
+            int searchId (unsigned int id) const;
+
         public:
             // MSVC needs the constructor for a class inheriting a template to be defined in header
             RefCollection (Collection<Cell>& cells)
@@ -27,10 +42,28 @@ namespace CSMWorld
             {}
 
             void load (ESM::ESMReader& reader, int cellIndex, bool base,
-                std::map<unsigned int, std::string>& cache, CSMDoc::Messages& messages);
+                std::map<unsigned int, unsigned int>& cache, CSMDoc::Messages& messages);
             ///< Load a sequence of references.
 
             std::string getNewId();
+
+            virtual void removeRows (int index, int count);
+
+            virtual void appendBlankRecord (const std::string& id,
+                                            UniversalId::Type type = UniversalId::Type_None);
+
+            virtual void cloneRecord (const std::string& origin,
+                                      const std::string& destination,
+                                      const UniversalId::Type type);
+
+            virtual int searchId (const std::string& id) const;
+
+            virtual void appendRecord (std::unique_ptr<RecordBase> record,
+                                       UniversalId::Type type = UniversalId::Type_None);
+
+            virtual void insertRecord (std::unique_ptr<RecordBase> record,
+                                       int index,
+                                       UniversalId::Type type = UniversalId::Type_None);
     };
 }