Merge branch 'navmesh_disk_rm_unused_tiles' into 'master'

Add navmeshtool flag to remove unused tiles from navmesh disk cache

See merge request OpenMW/openmw!1671

apps/navmeshtool/main.cpp

@@ -83,6 +83,9 @@ namespace NavMeshTool
 
                 ("process-interior-cells", bpo::value<bool>()->implicit_value(true)
                     ->default_value(false), "build navmesh for interior cells")
+
+                ("remove-unused-tiles", bpo::value<bool>()->implicit_value(true)
+                    ->default_value(false), "remove tiles from cache that will not be used with current content profile")
             ;
             Files::ConfigurationManager::addCommonOptions(result);
 
@@ -141,6 +144,7 @@ namespace NavMeshTool
             }
 
             const bool processInteriorCells = variables["process-interior-cells"].as<bool>();
+            const bool removeUnusedTiles = variables["remove-unused-tiles"].as<bool>();
 
             Fallback::Map::init(variables["fallback"].as<Fallback::FallbackMap>().mMap);
 
@@ -177,7 +181,8 @@ namespace NavMeshTool
             WorldspaceData cellsData = gatherWorldspaceData(navigatorSettings, readers, vfs, bulletShapeManager,
                                                             esmData, processInteriorCells);
 
-            generateAllNavMeshTiles(agentHalfExtents, navigatorSettings, threadsNumber, cellsData, std::move(db));
+            generateAllNavMeshTiles(agentHalfExtents, navigatorSettings, threadsNumber, removeUnusedTiles,
+                                    cellsData, std::move(db));
 
             Log(Debug::Info) << "Done";
 

apps/navmeshtool/navmesh.cpp

@@ -24,6 +24,7 @@
 #include <utility>
 #include <vector>
 #include <random>
+#include <string_view>
 
 namespace NavMeshTool
 {
@@ -40,6 +41,7 @@ namespace NavMeshTool
         using DetourNavigator::TileId;
         using DetourNavigator::TilePosition;
         using DetourNavigator::TileVersion;
+        using DetourNavigator::TilesPositionsRange;
         using Sqlite3::Transaction;
 
         void logGeneratedTiles(std::size_t provided, std::size_t expected)
@@ -62,8 +64,9 @@ namespace NavMeshTool
         public:
             std::atomic_size_t mExpected {0};
 
-            explicit NavMeshTileConsumer(NavMeshDb&& db)
+            explicit NavMeshTileConsumer(NavMeshDb&& db, bool removeUnusedTiles)
                 : mDb(std::move(db))
+                , mRemoveUnusedTiles(removeUnusedTiles)
                 , mTransaction(mDb.startTransaction())
                 , mNextTileId(mDb.getMaxTileId() + 1)
                 , mNextShapeId(mDb.getMaxShapeId() + 1)
@@ -75,13 +78,19 @@ namespace NavMeshTool
 
             std::size_t getUpdated() const { return mUpdated.load(); }
 
+            std::size_t getDeleted() const
+            {
+                const std::lock_guard lock(mMutex);
+                return mDeleted;
+            }
+
             std::int64_t resolveMeshSource(const MeshSource& source) override
             {
                 const std::lock_guard lock(mMutex);
                 return DetourNavigator::resolveMeshSource(mDb, source, mNextShapeId);
             }
 
-            std::optional<NavMeshTileInfo> find(const std::string& worldspace, const TilePosition &tilePosition,
+            std::optional<NavMeshTileInfo> find(std::string_view worldspace, const TilePosition &tilePosition,
                 const std::vector<std::byte> &input) override
             {
                 std::optional<NavMeshTileInfo> result;
@@ -96,11 +105,34 @@ namespace NavMeshTool
                 return result;
             }
 
-            void ignore() override { report(); }
-
-            void insert(const std::string& worldspace, const TilePosition& tilePosition, std::int64_t version,
-                const std::vector<std::byte>& input, PreparedNavMeshData& data) override
+            void ignore(std::string_view worldspace, const TilePosition& tilePosition) override
             {
+                if (mRemoveUnusedTiles)
+                {
+                    std::lock_guard lock(mMutex);
+                    mDeleted += static_cast<std::size_t>(mDb.deleteTilesAt(worldspace, tilePosition));
+                }
+                report();
+            }
+
+            void identity(std::string_view worldspace, const TilePosition& tilePosition, std::int64_t tileId) override
+            {
+                if (mRemoveUnusedTiles)
+                {
+                    std::lock_guard lock(mMutex);
+                    mDeleted += static_cast<std::size_t>(mDb.deleteTilesAtExcept(worldspace, tilePosition, TileId {tileId}));
+                }
+                report();
+            }
+
+            void insert(std::string_view worldspace, const TilePosition& tilePosition,
+                std::int64_t version, const std::vector<std::byte>& input, PreparedNavMeshData& data) override
+            {
+                if (mRemoveUnusedTiles)
+                {
+                    std::lock_guard lock(mMutex);
+                    mDeleted += static_cast<std::size_t>(mDb.deleteTilesAt(worldspace, tilePosition));
+                }
                 data.mUserId = static_cast<unsigned>(mNextTileId);
                 {
                     std::lock_guard lock(mMutex);
@@ -111,11 +143,14 @@ namespace NavMeshTool
                 report();
             }
 
-            void update(std::int64_t tileId, std::int64_t version, PreparedNavMeshData& data) override
+            void update(std::string_view worldspace, const TilePosition& tilePosition,
+                std::int64_t tileId, std::int64_t version, PreparedNavMeshData& data) override
             {
                 data.mUserId = static_cast<unsigned>(tileId);
                 {
                     std::lock_guard lock(mMutex);
+                    if (mRemoveUnusedTiles)
+                        mDeleted += static_cast<std::size_t>(mDb.deleteTilesAtExcept(worldspace, tilePosition, TileId {tileId}));
                     mDb.updateTile(TileId {tileId}, TileVersion {version}, serialize(data));
                 }
                 ++mUpdated;
@@ -140,49 +175,66 @@ namespace NavMeshTool
 
             void commit() { mTransaction.commit(); }
 
+            void vacuum() { mDb.vacuum(); }
+
+            void removeTilesOutsideRange(std::string_view worldspace, const TilesPositionsRange& range)
+            {
+                const std::lock_guard lock(mMutex);
+                mTransaction.commit();
+                Log(Debug::Info) << "Removing tiles outside processed range for worldspace \"" << worldspace << "\"...";
+                mDeleted += static_cast<std::size_t>(mDb.deleteTilesOutsideRange(worldspace, range));
+                mTransaction = mDb.startTransaction();
+            }
+
         private:
             std::atomic_size_t mProvided {0};
             std::atomic_size_t mInserted {0};
             std::atomic_size_t mUpdated {0};
-            std::mutex mMutex;
+            std::size_t mDeleted = 0;
+            mutable std::mutex mMutex;
             NavMeshDb mDb;
+            const bool mRemoveUnusedTiles;
             Transaction mTransaction;
             TileId mNextTileId;
             std::condition_variable mHasTile;
             Misc::ProgressReporter<LogGeneratedTiles> mReporter;
             ShapeId mNextShapeId;
+            std::mutex mReportMutex;
 
             void report()
             {
-                mReporter(mProvided + 1, mExpected);
-                ++mProvided;
+                const std::size_t provided = mProvided.fetch_add(1, std::memory_order_relaxed) + 1;
+                mReporter(provided, mExpected);
                 mHasTile.notify_one();
             }
         };
     }
 
     void generateAllNavMeshTiles(const osg::Vec3f& agentHalfExtents, const Settings& settings,
-        const std::size_t threadsNumber, WorldspaceData& data, NavMeshDb&& db)
+        std::size_t threadsNumber, bool removeUnusedTiles, WorldspaceData& data, NavMeshDb&& db)
     {
         Log(Debug::Info) << "Generating navmesh tiles by " << threadsNumber << " parallel workers...";
 
         SceneUtil::WorkQueue workQueue(threadsNumber);
-        auto navMeshTileConsumer = std::make_shared<NavMeshTileConsumer>(std::move(db));
+        auto navMeshTileConsumer = std::make_shared<NavMeshTileConsumer>(std::move(db), removeUnusedTiles);
         std::size_t tiles = 0;
         std::mt19937_64 random;
 
         for (const std::unique_ptr<WorldspaceNavMeshInput>& input : data.mNavMeshInputs)
         {
+            const auto range = DetourNavigator::makeTilesPositionsRange(
+                Misc::Convert::toOsgXY(input->mAabb.m_min),
+                Misc::Convert::toOsgXY(input->mAabb.m_max),
+                settings.mRecast
+            );
+
+            if (removeUnusedTiles)
+                navMeshTileConsumer->removeTilesOutsideRange(input->mWorldspace, range);
+
             std::vector<TilePosition> worldspaceTiles;
 
-            DetourNavigator::getTilesPositions(
-                DetourNavigator::makeTilesPositionsRange(
-                    Misc::Convert::toOsgXY(input->mAabb.m_min),
-                    Misc::Convert::toOsgXY(input->mAabb.m_max),
-                    settings.mRecast
-                ),
-                [&] (const TilePosition& tilePosition) { worldspaceTiles.push_back(tilePosition); }
-            );
+            DetourNavigator::getTilesPositions(range,
+                [&] (const TilePosition& tilePosition) { worldspaceTiles.push_back(tilePosition); });
 
             tiles += worldspaceTiles.size();
 
@@ -204,8 +256,19 @@ namespace NavMeshTool
         navMeshTileConsumer->wait();
         navMeshTileConsumer->commit();
 
+        const auto inserted = navMeshTileConsumer->getInserted();
+        const auto updated = navMeshTileConsumer->getUpdated();
+        const auto deleted = navMeshTileConsumer->getDeleted();
+
         Log(Debug::Info) << "Generated navmesh for " << navMeshTileConsumer->getProvided() << " tiles, "
-            << navMeshTileConsumer->getInserted() << " are inserted and "
-            << navMeshTileConsumer->getUpdated() << " updated";
+            << inserted << " are inserted, "
+            << updated << " updated and "
+            << deleted << " deleted";
+
+        if (inserted + updated + deleted > 0)
+        {
+            Log(Debug::Info) << "Vacuuming the database...";
+            navMeshTileConsumer->vacuum();
+        }
     }
 }

apps/navmeshtool/navmesh.hpp

@@ -4,7 +4,6 @@
 #include <osg/Vec3f>
 
 #include <cstddef>
-#include <string_view>
 
 namespace DetourNavigator
 {
@@ -17,7 +16,8 @@ namespace NavMeshTool
     struct WorldspaceData;
 
     void generateAllNavMeshTiles(const osg::Vec3f& agentHalfExtents, const DetourNavigator::Settings& settings,
-        const std::size_t threadsNumber, WorldspaceData& cellsData, DetourNavigator::NavMeshDb&& db);
+        std::size_t threadsNumber, bool removeUnusedTiles, WorldspaceData& cellsData,
+        DetourNavigator::NavMeshDb&& db);
 }
 
 #endif

apps/openmw_test_suite/detournavigator/navmeshdb.cpp

@@ -109,4 +109,61 @@ namespace
         EXPECT_THROW(mDb.insertTile(tileId, worldspace, tilePosition, version, input, data), std::runtime_error);
         EXPECT_NO_THROW(insertTile(TileId {54}, version));
     }
+
+    TEST_F(DetourNavigatorNavMeshDbTest, delete_tiles_at_should_remove_all_tiles_with_given_worldspace_and_position)
+    {
+        const TileVersion version {1};
+        const std::string worldspace = "sys::default";
+        const TilePosition tilePosition {3, 4};
+        const std::vector<std::byte> input1 = generateData();
+        const std::vector<std::byte> input2 = generateData();
+        const std::vector<std::byte> data = generateData();
+        ASSERT_EQ(mDb.insertTile(TileId {53}, worldspace, tilePosition, version, input1, data), 1);
+        ASSERT_EQ(mDb.insertTile(TileId {54}, worldspace, tilePosition, version, input2, data), 1);
+        ASSERT_EQ(mDb.deleteTilesAt(worldspace, tilePosition), 2);
+        EXPECT_FALSE(mDb.findTile(worldspace, tilePosition, input1).has_value());
+        EXPECT_FALSE(mDb.findTile(worldspace, tilePosition, input2).has_value());
+    }
+
+    TEST_F(DetourNavigatorNavMeshDbTest, delete_tiles_at_except_should_leave_tile_with_given_id)
+    {
+        const TileId leftTileId {53};
+        const TileId removedTileId {54};
+        const TileVersion version {1};
+        const std::string worldspace = "sys::default";
+        const TilePosition tilePosition {3, 4};
+        const std::vector<std::byte> leftInput = generateData();
+        const std::vector<std::byte> removedInput = generateData();
+        const std::vector<std::byte> data = generateData();
+        ASSERT_EQ(mDb.insertTile(leftTileId, worldspace, tilePosition, version, leftInput, data), 1);
+        ASSERT_EQ(mDb.insertTile(removedTileId, worldspace, tilePosition, version, removedInput, data), 1);
+        ASSERT_EQ(mDb.deleteTilesAtExcept(worldspace, tilePosition, leftTileId), 1);
+        const auto left = mDb.findTile(worldspace, tilePosition, leftInput);
+        ASSERT_TRUE(left.has_value());
+        EXPECT_EQ(left->mTileId, leftTileId);
+        EXPECT_FALSE(mDb.findTile(worldspace, tilePosition, removedInput).has_value());
+    }
+
+    TEST_F(DetourNavigatorNavMeshDbTest, delete_tiles_outside_range_should_leave_tiles_inside_given_rectangle)
+    {
+        TileId tileId {1};
+        const TileVersion version {1};
+        const std::string worldspace = "sys::default";
+        const std::vector<std::byte> input = generateData();
+        const std::vector<std::byte> data = generateData();
+        for (int x = -2; x <= 2; ++x)
+        {
+            for (int y = -2; y <= 2; ++y)
+            {
+                ASSERT_EQ(mDb.insertTile(tileId, worldspace, TilePosition {x, y}, version, input, data), 1);
+                ++tileId.t;
+            }
+        }
+        const TilesPositionsRange range {TilePosition {-1, -1}, TilePosition {2, 2}};
+        ASSERT_EQ(mDb.deleteTilesOutsideRange(worldspace, range), 16);
+        for (int x = -2; x <= 2; ++x)
+            for (int y = -2; y <= 2; ++y)
+                ASSERT_EQ(mDb.findTile(worldspace, TilePosition {x, y}, input).has_value(),
+                          -1 <= x && x <= 1 && -1 <= y && y <= 1) << "x=" << x << " y=" << y;
+    }
 }

apps/openmw_test_suite/serialization/format.hpp

@@ -5,6 +5,7 @@
 
 #include <utility>
 #include <type_traits>
+#include <cstdint>
 
 namespace SerializationTesting
 {
@@ -20,7 +21,7 @@ namespace SerializationTesting
         }
     };
 
-    enum Enum
+    enum Enum : std::int32_t
     {
         A,
         B,
@@ -30,7 +31,7 @@ namespace SerializationTesting
     struct Composite
     {
         short mFloatArray[3] = {0};
-        std::vector<int> mIntVector;
+        std::vector<std::int32_t> mIntVector;
         std::vector<Enum> mEnumVector;
         std::vector<Pod> mPodVector;
         std::size_t mPodDataSize = 0;

components/detournavigator/generatenavmeshtile.cpp

@@ -25,12 +25,14 @@ namespace DetourNavigator
     {
         struct Ignore
         {
+            std::string_view mWorldspace;
+            const TilePosition& mTilePosition;
             std::shared_ptr<NavMeshTileConsumer> mConsumer;
 
             ~Ignore() noexcept
             {
                 if (mConsumer != nullptr)
-                    mConsumer->ignore();
+                    mConsumer->ignore(mWorldspace, mTilePosition);
             }
         };
     }
@@ -59,7 +61,7 @@ namespace DetourNavigator
 
         try
         {
-            Ignore ignore {consumer};
+            Ignore ignore {mWorldspace, mTilePosition, consumer};
 
             const std::shared_ptr<RecastMesh> recastMesh = mRecastMeshProvider.getMesh(mWorldspace, mTilePosition);
 
@@ -72,7 +74,11 @@ namespace DetourNavigator
             const std::optional<NavMeshTileInfo> info = consumer->find(mWorldspace, mTilePosition, input);
 
             if (info.has_value() && info->mVersion == mSettings.mNavMeshVersion)
+            {
+                consumer->identity(mWorldspace, mTilePosition, info->mTileId);
+                ignore.mConsumer = nullptr;
                 return;
+            }
 
             const auto data = prepareNavMeshTileData(*recastMesh, mTilePosition, mAgentHalfExtents, mSettings.mRecast);
 
@@ -80,7 +86,7 @@ namespace DetourNavigator
                 return;
 
             if (info.has_value())
-                consumer->update(info->mTileId, mSettings.mNavMeshVersion, *data);
+                consumer->update(mWorldspace, mTilePosition, info->mTileId, mSettings.mNavMeshVersion, *data);
             else
                 consumer->insert(mWorldspace, mTilePosition, mSettings.mNavMeshVersion, input, *data);
 

components/detournavigator/generatenavmeshtile.hpp

@@ -36,15 +36,19 @@ namespace DetourNavigator
 
         virtual std::int64_t resolveMeshSource(const MeshSource& source) = 0;
 
-        virtual std::optional<NavMeshTileInfo> find(const std::string& worldspace, const TilePosition& tilePosition,
+        virtual std::optional<NavMeshTileInfo> find(std::string_view worldspace, const TilePosition& tilePosition,
             const std::vector<std::byte>& input) = 0;
 
-        virtual void ignore() = 0;
+        virtual void ignore(std::string_view worldspace, const TilePosition& tilePosition) = 0;
 
-        virtual void insert(const std::string& worldspace, const TilePosition& tilePosition,
+        virtual void identity(std::string_view worldspace, const TilePosition& tilePosition,
+                              std::int64_t tileId) = 0;
+
+        virtual void insert(std::string_view worldspace, const TilePosition& tilePosition,
                             std::int64_t version, const std::vector<std::byte>& input, PreparedNavMeshData& data) = 0;
 
-        virtual void update(std::int64_t tileId, std::int64_t version, PreparedNavMeshData& data) = 0;
+        virtual void update(std::string_view worldspace, const TilePosition& tilePosition,
+                            std::int64_t tileId, std::int64_t version, PreparedNavMeshData& data) = 0;
     };
 
     class GenerateNavMeshTile final : public SceneUtil::WorkItem

components/detournavigator/gettilespositions.hpp

@@ -3,6 +3,7 @@
 
 #include "tilebounds.hpp"
 #include "tileposition.hpp"
+#include "tilespositionsrange.hpp"
 
 class btVector3;
 class btTransform;
@@ -17,12 +18,6 @@ namespace DetourNavigator
 {
     struct RecastSettings;
 
-    struct TilesPositionsRange
-    {
-        TilePosition mBegin;
-        TilePosition mEnd;
-    };
-
     TilesPositionsRange makeTilesPositionsRange(const osg::Vec2f& aabbMin,
         const osg::Vec2f& aabbMax, const RecastSettings& settings);
 

components/detournavigator/navmeshdb.cpp

@@ -10,7 +10,6 @@
 #include <sqlite3.h>
 
 #include <cstddef>
-#include <string>
 #include <string_view>
 #include <vector>
 
@@ -35,6 +34,9 @@ namespace DetourNavigator
             CREATE UNIQUE INDEX IF NOT EXISTS index_unique_tiles_by_worldspace_and_tile_position_and_input
                 ON tiles (worldspace, tile_position_x, tile_position_y, input);
 
+            CREATE INDEX IF NOT EXISTS index_tiles_by_worldspace_and_tile_position
+                ON tiles (worldspace, tile_position_x, tile_position_y);
+
             CREATE TABLE IF NOT EXISTS shapes (
                 shape_id INTEGER PRIMARY KEY,
                 name TEXT NOT NULL,
@@ -83,6 +85,31 @@ namespace DetourNavigator
              WHERE tile_id = :tile_id
         )";
 
+        constexpr std::string_view deleteTilesAtQuery = R"(
+            DELETE FROM tiles
+             WHERE worldspace = :worldspace
+               AND tile_position_x = :tile_position_x
+               AND tile_position_y = :tile_position_y
+        )";
+
+        constexpr std::string_view deleteTilesAtExceptQuery = R"(
+            DELETE FROM tiles
+             WHERE worldspace = :worldspace
+               AND tile_position_x = :tile_position_x
+               AND tile_position_y = :tile_position_y
+               AND tile_id != :exclude_tile_id
+        )";
+
+        constexpr std::string_view deleteTilesOutsideRangeQuery = R"(
+            DELETE FROM tiles
+             WHERE worldspace = :worldspace
+               AND (   tile_position_x < :begin_tile_position_x
+                    OR tile_position_y < :begin_tile_position_y
+                    OR tile_position_x >= :end_tile_position_x
+                    OR tile_position_y >= :end_tile_position_y
+                   )
+        )";
+
         constexpr std::string_view getMaxShapeIdQuery = R"(
             SELECT max(shape_id) FROM shapes
         )";
@@ -99,6 +126,10 @@ namespace DetourNavigator
             INSERT INTO shapes ( shape_id,  name,  type,  hash)
                    VALUES      (:shape_id, :name, :type, :hash)
         )";
+
+        constexpr std::string_view vacuumQuery = R"(
+            VACUUM;
+        )";
     }
 
     std::ostream& operator<<(std::ostream& stream, ShapeType value)
@@ -118,9 +149,13 @@ namespace DetourNavigator
         , mGetTileData(*mDb, DbQueries::GetTileData {})
         , mInsertTile(*mDb, DbQueries::InsertTile {})
         , mUpdateTile(*mDb, DbQueries::UpdateTile {})
+        , mDeleteTilesAt(*mDb, DbQueries::DeleteTilesAt {})
+        , mDeleteTilesAtExcept(*mDb, DbQueries::DeleteTilesAtExcept {})
+        , mDeleteTilesOutsideRange(*mDb, DbQueries::DeleteTilesOutsideRange {})
         , mGetMaxShapeId(*mDb, DbQueries::GetMaxShapeId {})
         , mFindShapeId(*mDb, DbQueries::FindShapeId {})
         , mInsertShape(*mDb, DbQueries::InsertShape {})
+        , mVacuum(*mDb, DbQueries::Vacuum {})
     {
     }
 
@@ -136,7 +171,7 @@ namespace DetourNavigator
         return tileId;
     }
 
-    std::optional<Tile> NavMeshDb::findTile(const std::string& worldspace,
+    std::optional<Tile> NavMeshDb::findTile(std::string_view worldspace,
         const TilePosition& tilePosition, const std::vector<std::byte>& input)
     {
         Tile result;
@@ -147,7 +182,7 @@ namespace DetourNavigator
         return result;
     }
 
-    std::optional<TileData> NavMeshDb::getTileData(const std::string& worldspace,
+    std::optional<TileData> NavMeshDb::getTileData(std::string_view worldspace,
         const TilePosition& tilePosition, const std::vector<std::byte>& input)
     {
         TileData result;
@@ -159,7 +194,7 @@ namespace DetourNavigator
         return result;
     }
 
-    int NavMeshDb::insertTile(TileId tileId, const std::string& worldspace, const TilePosition& tilePosition,
+    int NavMeshDb::insertTile(TileId tileId, std::string_view worldspace, const TilePosition& tilePosition,
         TileVersion version, const std::vector<std::byte>& input, const std::vector<std::byte>& data)
     {
         const std::vector<std::byte> compressedInput = Misc::compress(input);
@@ -173,6 +208,21 @@ namespace DetourNavigator
         return execute(*mDb, mUpdateTile, tileId, version, compressedData);
     }
 
+    int NavMeshDb::deleteTilesAt(std::string_view worldspace, const TilePosition& tilePosition)
+    {
+        return execute(*mDb, mDeleteTilesAt, worldspace, tilePosition);
+    }
+
+    int NavMeshDb::deleteTilesAtExcept(std::string_view worldspace, const TilePosition& tilePosition, TileId excludeTileId)
+    {
+        return execute(*mDb, mDeleteTilesAtExcept, worldspace, tilePosition, excludeTileId);
+    }
+
+    int NavMeshDb::deleteTilesOutsideRange(std::string_view worldspace, const TilesPositionsRange& range)
+    {
+        return execute(*mDb, mDeleteTilesOutsideRange, worldspace, range);
+    }
+
     ShapeId NavMeshDb::getMaxShapeId()
     {
         ShapeId shapeId {0};
@@ -180,7 +230,7 @@ namespace DetourNavigator
         return shapeId;
     }
 
-    std::optional<ShapeId> NavMeshDb::findShapeId(const std::string& name, ShapeType type,
+    std::optional<ShapeId> NavMeshDb::findShapeId(std::string_view name, ShapeType type,
         const Sqlite3::ConstBlob& hash)
     {
         ShapeId shapeId;
@@ -189,12 +239,17 @@ namespace DetourNavigator
         return shapeId;
     }
 
-    int NavMeshDb::insertShape(ShapeId shapeId, const std::string& name, ShapeType type,
+    int NavMeshDb::insertShape(ShapeId shapeId, std::string_view name, ShapeType type,
         const Sqlite3::ConstBlob& hash)
     {
         return execute(*mDb, mInsertShape, shapeId, name, type, hash);
     }
 
+    void NavMeshDb::vacuum()
+    {
+        execute(*mDb, mVacuum);
+    }
+
     namespace DbQueries
     {
         std::string_view GetMaxTileId::text() noexcept
@@ -207,7 +262,7 @@ namespace DetourNavigator
             return findTileQuery;
         }
 
-        void FindTile::bind(sqlite3& db, sqlite3_stmt& statement, const std::string& worldspace,
+        void FindTile::bind(sqlite3& db, sqlite3_stmt& statement, std::string_view worldspace,
             const TilePosition& tilePosition, const std::vector<std::byte>& input)
         {
             Sqlite3::bindParameter(db, statement, ":worldspace", worldspace);
@@ -221,7 +276,7 @@ namespace DetourNavigator
             return getTileDataQuery;
         }
 
-        void GetTileData::bind(sqlite3& db, sqlite3_stmt& statement, const std::string& worldspace,
+        void GetTileData::bind(sqlite3& db, sqlite3_stmt& statement, std::string_view worldspace,
             const TilePosition& tilePosition, const std::vector<std::byte>& input)
         {
             Sqlite3::bindParameter(db, statement, ":worldspace", worldspace);
@@ -235,7 +290,7 @@ namespace DetourNavigator
             return insertTileQuery;
         }
 
-        void InsertTile::bind(sqlite3& db, sqlite3_stmt& statement, TileId tileId, const std::string& worldspace,
+        void InsertTile::bind(sqlite3& db, sqlite3_stmt& statement, TileId tileId, std::string_view worldspace,
             const TilePosition& tilePosition, TileVersion version, const std::vector<std::byte>& input,
             const std::vector<std::byte>& data)
         {
@@ -261,6 +316,48 @@ namespace DetourNavigator
             Sqlite3::bindParameter(db, statement, ":data", data);
         }
 
+        std::string_view DeleteTilesAt::text() noexcept
+        {
+            return deleteTilesAtQuery;
+        }
+
+        void DeleteTilesAt::bind(sqlite3& db, sqlite3_stmt& statement, std::string_view worldspace,
+            const TilePosition& tilePosition)
+        {
+            Sqlite3::bindParameter(db, statement, ":worldspace", worldspace);
+            Sqlite3::bindParameter(db, statement, ":tile_position_x", tilePosition.x());
+            Sqlite3::bindParameter(db, statement, ":tile_position_y", tilePosition.y());
+        }
+
+        std::string_view DeleteTilesAtExcept::text() noexcept
+        {
+            return deleteTilesAtExceptQuery;
+        }
+
+        void DeleteTilesAtExcept::bind(sqlite3& db, sqlite3_stmt& statement, std::string_view worldspace,
+            const TilePosition& tilePosition, TileId excludeTileId)
+        {
+            Sqlite3::bindParameter(db, statement, ":worldspace", worldspace);
+            Sqlite3::bindParameter(db, statement, ":tile_position_x", tilePosition.x());
+            Sqlite3::bindParameter(db, statement, ":tile_position_y", tilePosition.y());
+            Sqlite3::bindParameter(db, statement, ":exclude_tile_id", excludeTileId);
+        }
+
+        std::string_view DeleteTilesOutsideRange::text() noexcept
+        {
+            return deleteTilesOutsideRangeQuery;
+        }
+
+        void DeleteTilesOutsideRange::bind(sqlite3& db, sqlite3_stmt& statement, std::string_view worldspace,
+            const TilesPositionsRange& range)
+        {
+            Sqlite3::bindParameter(db, statement, ":worldspace", worldspace);
+            Sqlite3::bindParameter(db, statement, ":begin_tile_position_x", range.mBegin.x());
+            Sqlite3::bindParameter(db, statement, ":begin_tile_position_y", range.mBegin.y());
+            Sqlite3::bindParameter(db, statement, ":end_tile_position_x", range.mEnd.x());
+            Sqlite3::bindParameter(db, statement, ":end_tile_position_y", range.mEnd.y());
+        }
+
         std::string_view GetMaxShapeId::text() noexcept
         {
             return getMaxShapeIdQuery;
@@ -271,7 +368,7 @@ namespace DetourNavigator
             return findShapeIdQuery;
         }
 
-        void FindShapeId::bind(sqlite3& db, sqlite3_stmt& statement, const std::string& name,
+        void FindShapeId::bind(sqlite3& db, sqlite3_stmt& statement, std::string_view name,
             ShapeType type, const Sqlite3::ConstBlob& hash)
         {
             Sqlite3::bindParameter(db, statement, ":name", name);
@@ -284,7 +381,7 @@ namespace DetourNavigator
             return insertShapeQuery;
         }
 
-        void InsertShape::bind(sqlite3& db, sqlite3_stmt& statement, ShapeId shapeId, const std::string& name,
+        void InsertShape::bind(sqlite3& db, sqlite3_stmt& statement, ShapeId shapeId, std::string_view name,
             ShapeType type, const Sqlite3::ConstBlob& hash)
         {
             Sqlite3::bindParameter(db, statement, ":shape_id", shapeId);
@@ -292,5 +389,10 @@ namespace DetourNavigator
             Sqlite3::bindParameter(db, statement, ":type", static_cast<int>(type));
             Sqlite3::bindParameter(db, statement, ":hash", hash);
         }
+
+        std::string_view Vacuum::text() noexcept
+        {
+            return vacuumQuery;
+        }
     }
 }

components/detournavigator/navmeshdb.hpp

@@ -3,6 +3,8 @@
 
 #include "tileposition.hpp"
 
+#include <components/detournavigator/tilespositionsrange.hpp>
+
 #include <components/sqlite3/db.hpp>
 #include <components/sqlite3/statement.hpp>
 #include <components/sqlite3/transaction.hpp>
@@ -15,7 +17,6 @@
 #include <cstring>
 #include <optional>
 #include <stdexcept>
-#include <string>
 #include <string_view>
 #include <tuple>
 #include <utility>
@@ -64,21 +65,21 @@ namespace DetourNavigator
         struct FindTile
         {
             static std::string_view text() noexcept;
-            static void bind(sqlite3& db, sqlite3_stmt& statement, const std::string& worldspace,
+            static void bind(sqlite3& db, sqlite3_stmt& statement, std::string_view worldspace,
                 const TilePosition& tilePosition, const std::vector<std::byte>& input);
         };
 
         struct GetTileData
         {
             static std::string_view text() noexcept;
-            static void bind(sqlite3& db, sqlite3_stmt& statement, const std::string& worldspace,
+            static void bind(sqlite3& db, sqlite3_stmt& statement, std::string_view worldspace,
                 const TilePosition& tilePosition, const std::vector<std::byte>& input);
         };
 
         struct InsertTile
         {
             static std::string_view text() noexcept;
-            static void bind(sqlite3& db, sqlite3_stmt& statement, TileId tileId, const std::string& worldspace,
+            static void bind(sqlite3& db, sqlite3_stmt& statement, TileId tileId, std::string_view worldspace,
                 const TilePosition& tilePosition, TileVersion version, const std::vector<std::byte>& input,
                 const std::vector<std::byte>& data);
         };
@@ -90,6 +91,27 @@ namespace DetourNavigator
                 const std::vector<std::byte>& data);
         };
 
+        struct DeleteTilesAt
+        {
+            static std::string_view text() noexcept;
+            static void bind(sqlite3& db, sqlite3_stmt& statement, std::string_view worldspace,
+                const TilePosition& tilePosition);
+        };
+
+        struct DeleteTilesAtExcept
+        {
+            static std::string_view text() noexcept;
+            static void bind(sqlite3& db, sqlite3_stmt& statement, std::string_view worldspace,
+                const TilePosition& tilePosition, TileId excludeTileId);
+        };
+
+        struct DeleteTilesOutsideRange
+        {
+            static std::string_view text() noexcept;
+            static void bind(sqlite3& db, sqlite3_stmt& statement, std::string_view worldspace,
+                const TilesPositionsRange& range);
+        };
+
         struct GetMaxShapeId
         {
             static std::string_view text() noexcept;
@@ -99,16 +121,22 @@ namespace DetourNavigator
         struct FindShapeId
         {
             static std::string_view text() noexcept;
-            static void bind(sqlite3& db, sqlite3_stmt& statement, const std::string& name,
+            static void bind(sqlite3& db, sqlite3_stmt& statement, std::string_view name,
                 ShapeType type, const Sqlite3::ConstBlob& hash);
         };
 
         struct InsertShape
         {
             static std::string_view text() noexcept;
-            static void bind(sqlite3& db, sqlite3_stmt& statement, ShapeId shapeId, const std::string& name,
+            static void bind(sqlite3& db, sqlite3_stmt& statement, ShapeId shapeId, std::string_view name,
                 ShapeType type, const Sqlite3::ConstBlob& hash);
         };
+
+        struct Vacuum
+        {
+            static std::string_view text() noexcept;
+            static void bind(sqlite3&, sqlite3_stmt&) {}
+        };
     }
 
     class NavMeshDb
@@ -120,22 +148,30 @@ namespace DetourNavigator
 
         TileId getMaxTileId();
 
-        std::optional<Tile> findTile(const std::string& worldspace,
+        std::optional<Tile> findTile(std::string_view worldspace,
             const TilePosition& tilePosition, const std::vector<std::byte>& input);
 
-        std::optional<TileData> getTileData(const std::string& worldspace,
+        std::optional<TileData> getTileData(std::string_view worldspace,
             const TilePosition& tilePosition, const std::vector<std::byte>& input);
 
-        int insertTile(TileId tileId, const std::string& worldspace, const TilePosition& tilePosition,
+        int insertTile(TileId tileId, std::string_view worldspace, const TilePosition& tilePosition,
             TileVersion version, const std::vector<std::byte>& input, const std::vector<std::byte>& data);
 
         int updateTile(TileId tileId, TileVersion version, const std::vector<std::byte>& data);
 
+        int deleteTilesAt(std::string_view worldspace, const TilePosition& tilePosition);
+
+        int deleteTilesAtExcept(std::string_view worldspace, const TilePosition& tilePosition, TileId excludeTileId);
+
+        int deleteTilesOutsideRange(std::string_view worldspace, const TilesPositionsRange& range);
+
         ShapeId getMaxShapeId();
 
-        std::optional<ShapeId> findShapeId(const std::string& name, ShapeType type, const Sqlite3::ConstBlob& hash);
+        std::optional<ShapeId> findShapeId(std::string_view name, ShapeType type, const Sqlite3::ConstBlob& hash);
 
-        int insertShape(ShapeId shapeId, const std::string& name, ShapeType type, const Sqlite3::ConstBlob& hash);
+        int insertShape(ShapeId shapeId, std::string_view name, ShapeType type, const Sqlite3::ConstBlob& hash);
+
+        void vacuum();
 
     private:
         Sqlite3::Db mDb;
@@ -144,9 +180,13 @@ namespace DetourNavigator
         Sqlite3::Statement<DbQueries::GetTileData> mGetTileData;
         Sqlite3::Statement<DbQueries::InsertTile> mInsertTile;
         Sqlite3::Statement<DbQueries::UpdateTile> mUpdateTile;
+        Sqlite3::Statement<DbQueries::DeleteTilesAt> mDeleteTilesAt;
+        Sqlite3::Statement<DbQueries::DeleteTilesAtExcept> mDeleteTilesAtExcept;
+        Sqlite3::Statement<DbQueries::DeleteTilesOutsideRange> mDeleteTilesOutsideRange;
         Sqlite3::Statement<DbQueries::GetMaxShapeId> mGetMaxShapeId;
         Sqlite3::Statement<DbQueries::FindShapeId> mFindShapeId;
         Sqlite3::Statement<DbQueries::InsertShape> mInsertShape;
+        Sqlite3::Statement<DbQueries::Vacuum> mVacuum;
     };
 }
 

components/detournavigator/navmeshdbutils.cpp

@@ -6,19 +6,20 @@
 
 #include <cassert>
 #include <optional>
+#include <string_view>
 
 namespace DetourNavigator
 {
     namespace
     {
-        std::optional<ShapeId> findShapeId(NavMeshDb& db, const std::string& name, ShapeType type,
+        std::optional<ShapeId> findShapeId(NavMeshDb& db, std::string_view name, ShapeType type,
             const std::string& hash)
         {
             const Sqlite3::ConstBlob hashData {hash.data(), static_cast<int>(hash.size())};
             return db.findShapeId(name, type, hashData);
         }
 
-        ShapeId getShapeId(NavMeshDb& db, const std::string& name, ShapeType type,
+        ShapeId getShapeId(NavMeshDb& db, std::string_view name, ShapeType type,
             const std::string& hash, ShapeId& nextShapeId)
         {
             const Sqlite3::ConstBlob hashData {hash.data(), static_cast<int>(hash.size())};

components/detournavigator/tilespositionsrange.hpp

@@ -0,0 +1,15 @@
+#ifndef OPENMW_COMPONENTS_DETOURNAVIGATOR_TILESPOSITIONSRANGE_H
+#define OPENMW_COMPONENTS_DETOURNAVIGATOR_TILESPOSITIONSRANGE_H
+
+#include "tileposition.hpp"
+
+namespace DetourNavigator
+{
+    struct TilesPositionsRange
+    {
+        TilePosition mBegin;
+        TilePosition mEnd;
+    };
+}
+
+#endif

components/serialization/format.hpp

@@ -7,6 +7,7 @@
 #include <type_traits>
 #include <utility>
 #include <vector>
+#include <cstdint>
 
 namespace Serialization
 {
@@ -26,7 +27,7 @@ namespace Serialization
     struct IsContiguousContainer<std::array<T, n>> : std::true_type {};
 
     template <class T>
-    constexpr bool isContiguousContainer = IsContiguousContainer<std::decay_t<T>>::value;
+    inline constexpr bool isContiguousContainer = IsContiguousContainer<std::decay_t<T>>::value;
 
     template <Mode mode, class Derived>
     struct Format
@@ -51,13 +52,13 @@ namespace Serialization
             -> std::enable_if_t<isContiguousContainer<T>>
         {
             if constexpr (mode == Mode::Write)
-                visitor(self(), value.size());
+                visitor(self(), static_cast<std::uint64_t>(value.size()));
             else
             {
                 static_assert(mode == Mode::Read);
-                std::size_t size = 0;
+                std::uint64_t size = 0;
                 visitor(self(), size);
-                value.resize(size);
+                value.resize(static_cast<std::size_t>(size));
             }
             self()(std::forward<Visitor>(visitor), value.data(), value.size());
         }