diff --git a/apps/openmw/mwphysics/mtphysics.cpp b/apps/openmw/mwphysics/mtphysics.cpp
index cc302f8937..0f05ccdbb1 100644
--- a/apps/openmw/mwphysics/mtphysics.cpp
+++ b/apps/openmw/mwphysics/mtphysics.cpp
@@ -550,7 +550,7 @@ namespace MWPhysics
         mCollisionWorld->removeCollisionObject(collisionObject);
     }
 
-    void PhysicsTaskScheduler::updateSingleAabb(std::shared_ptr<PtrHolder> ptr, bool immediate)
+    void PhysicsTaskScheduler::updateSingleAabb(const std::shared_ptr<PtrHolder>& ptr, bool immediate)
     {
         if (immediate || mNumThreads == 0)
         {
@@ -559,7 +559,7 @@ namespace MWPhysics
         else
         {
             MaybeExclusiveLock lock(mUpdateAabbMutex, mNumThreads);
-            mUpdateAabb.insert(std::move(ptr));
+            mUpdateAabb.insert(ptr);
         }
     }
 
@@ -602,7 +602,12 @@ namespace MWPhysics
     {
         MaybeExclusiveLock lock(mUpdateAabbMutex, mNumThreads);
         std::for_each(mUpdateAabb.begin(), mUpdateAabb.end(),
-            [this](const std::shared_ptr<PtrHolder>& ptr) { updatePtrAabb(ptr); });
+            [this](const std::weak_ptr<PtrHolder>& ptr)
+            {
+                auto p = ptr.lock();
+                if (p != nullptr)
+                    updatePtrAabb(p);
+            });
         mUpdateAabb.clear();
     }
 
diff --git a/apps/openmw/mwphysics/mtphysics.hpp b/apps/openmw/mwphysics/mtphysics.hpp
index 9ded1262d6..685d4eefab 100644
--- a/apps/openmw/mwphysics/mtphysics.hpp
+++ b/apps/openmw/mwphysics/mtphysics.hpp
@@ -54,7 +54,7 @@ namespace MWPhysics
             void setCollisionFilterMask(btCollisionObject* collisionObject, int collisionFilterMask);
             void addCollisionObject(btCollisionObject* collisionObject, int collisionFilterGroup, int collisionFilterMask);
             void removeCollisionObject(btCollisionObject* collisionObject);
-            void updateSingleAabb(std::shared_ptr<PtrHolder> ptr, bool immediate=false);
+            void updateSingleAabb(const std::shared_ptr<PtrHolder>& ptr, bool immediate=false);
             bool getLineOfSight(const std::shared_ptr<Actor>& actor1, const std::shared_ptr<Actor>& actor2);
             void debugDraw();
             void* getUserPointer(const btCollisionObject* object) const;
@@ -85,7 +85,7 @@ namespace MWPhysics
             btCollisionWorld* mCollisionWorld;
             MWRender::DebugDrawer* mDebugDrawer;
             std::vector<LOSRequest> mLOSCache;
-            std::set<std::shared_ptr<PtrHolder>> mUpdateAabb;
+            std::set<std::weak_ptr<PtrHolder>, std::owner_less<std::weak_ptr<PtrHolder>>> mUpdateAabb;
 
             // TODO: use std::experimental::flex_barrier or std::barrier once it becomes a thing
             std::unique_ptr<Misc::Barrier> mPreStepBarrier;