From 9584cb7ac2083ab01e8c05cb7cde00cd5a77b55f Mon Sep 17 00:00:00 2001 From: elsid Date: Sat, 5 Feb 2022 16:24:57 +0100 Subject: [PATCH 1/4] Use fixed size types for serialization --- apps/openmw_test_suite/serialization/format.hpp | 5 +++-- components/serialization/format.hpp | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/apps/openmw_test_suite/serialization/format.hpp b/apps/openmw_test_suite/serialization/format.hpp index 8f61838fde..603d2790e0 100644 --- a/apps/openmw_test_suite/serialization/format.hpp +++ b/apps/openmw_test_suite/serialization/format.hpp @@ -5,6 +5,7 @@ #include #include +#include namespace SerializationTesting { @@ -20,7 +21,7 @@ namespace SerializationTesting } }; - enum Enum + enum Enum : std::int32_t { A, B, @@ -30,7 +31,7 @@ namespace SerializationTesting struct Composite { short mFloatArray[3] = {0}; - std::vector mIntVector; + std::vector mIntVector; std::vector mEnumVector; std::vector mPodVector; std::size_t mPodDataSize = 0; diff --git a/components/serialization/format.hpp b/components/serialization/format.hpp index 595afd0dad..956345149c 100644 --- a/components/serialization/format.hpp +++ b/components/serialization/format.hpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace Serialization { @@ -51,13 +52,13 @@ namespace Serialization -> std::enable_if_t> { if constexpr (mode == Mode::Write) - visitor(self(), value.size()); + visitor(self(), static_cast(value.size())); else { static_assert(mode == Mode::Read); - std::size_t size = 0; + std::uint64_t size = 0; visitor(self(), size); - value.resize(size); + value.resize(static_cast(size)); } self()(std::forward(visitor), value.data(), value.size()); } From 52b3a87dae5df9b4fcacd2dc2e61a730b7352abd Mon Sep 17 00:00:00 2001 From: elsid Date: Sat, 5 Feb 2022 16:32:40 +0100 Subject: [PATCH 2/4] Make constexpr variable defined in header inline --- components/serialization/format.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/serialization/format.hpp b/components/serialization/format.hpp index 956345149c..29fc0ec42d 100644 --- a/components/serialization/format.hpp +++ b/components/serialization/format.hpp @@ -27,7 +27,7 @@ namespace Serialization struct IsContiguousContainer> : std::true_type {}; template - constexpr bool isContiguousContainer = IsContiguousContainer>::value; + inline constexpr bool isContiguousContainer = IsContiguousContainer>::value; template struct Format From 67741402b5ef92ca8aad11325bea7878dbf3f7b9 Mon Sep 17 00:00:00 2001 From: elsid Date: Fri, 18 Feb 2022 21:45:50 +0100 Subject: [PATCH 3/4] Replace reference to const std::string by std::string_view for navmeshdb related arguments --- apps/navmeshtool/navmesh.cpp | 5 +++-- apps/navmeshtool/navmesh.hpp | 1 - .../detournavigator/generatenavmeshtile.hpp | 4 ++-- components/detournavigator/navmeshdb.cpp | 21 +++++++++---------- components/detournavigator/navmeshdb.hpp | 21 +++++++++---------- components/detournavigator/navmeshdbutils.cpp | 5 +++-- 6 files changed, 28 insertions(+), 29 deletions(-) diff --git a/apps/navmeshtool/navmesh.cpp b/apps/navmeshtool/navmesh.cpp index ca614d0cf6..cf18d7edc3 100644 --- a/apps/navmeshtool/navmesh.cpp +++ b/apps/navmeshtool/navmesh.cpp @@ -24,6 +24,7 @@ #include #include #include +#include namespace NavMeshTool { @@ -81,7 +82,7 @@ namespace NavMeshTool return DetourNavigator::resolveMeshSource(mDb, source, mNextShapeId); } - std::optional find(const std::string& worldspace, const TilePosition &tilePosition, + std::optional find(std::string_view worldspace, const TilePosition &tilePosition, const std::vector &input) override { std::optional result; @@ -98,7 +99,7 @@ namespace NavMeshTool void ignore() override { report(); } - void insert(const std::string& worldspace, const TilePosition& tilePosition, std::int64_t version, + void insert(std::string_view worldspace, const TilePosition& tilePosition, std::int64_t version, const std::vector& input, PreparedNavMeshData& data) override { data.mUserId = static_cast(mNextTileId); diff --git a/apps/navmeshtool/navmesh.hpp b/apps/navmeshtool/navmesh.hpp index 725f0cd6a4..ff837eebe7 100644 --- a/apps/navmeshtool/navmesh.hpp +++ b/apps/navmeshtool/navmesh.hpp @@ -4,7 +4,6 @@ #include #include -#include namespace DetourNavigator { diff --git a/components/detournavigator/generatenavmeshtile.hpp b/components/detournavigator/generatenavmeshtile.hpp index 511b8dfb8f..69afe426bc 100644 --- a/components/detournavigator/generatenavmeshtile.hpp +++ b/components/detournavigator/generatenavmeshtile.hpp @@ -36,12 +36,12 @@ namespace DetourNavigator virtual std::int64_t resolveMeshSource(const MeshSource& source) = 0; - virtual std::optional find(const std::string& worldspace, const TilePosition& tilePosition, + virtual std::optional find(std::string_view worldspace, const TilePosition& tilePosition, const std::vector& input) = 0; virtual void ignore() = 0; - virtual void insert(const std::string& worldspace, const TilePosition& tilePosition, + virtual void insert(std::string_view worldspace, const TilePosition& tilePosition, std::int64_t version, const std::vector& input, PreparedNavMeshData& data) = 0; virtual void update(std::int64_t tileId, std::int64_t version, PreparedNavMeshData& data) = 0; diff --git a/components/detournavigator/navmeshdb.cpp b/components/detournavigator/navmeshdb.cpp index ebff250ee0..425cf7d434 100644 --- a/components/detournavigator/navmeshdb.cpp +++ b/components/detournavigator/navmeshdb.cpp @@ -10,7 +10,6 @@ #include #include -#include #include #include @@ -136,7 +135,7 @@ namespace DetourNavigator return tileId; } - std::optional NavMeshDb::findTile(const std::string& worldspace, + std::optional NavMeshDb::findTile(std::string_view worldspace, const TilePosition& tilePosition, const std::vector& input) { Tile result; @@ -147,7 +146,7 @@ namespace DetourNavigator return result; } - std::optional NavMeshDb::getTileData(const std::string& worldspace, + std::optional NavMeshDb::getTileData(std::string_view worldspace, const TilePosition& tilePosition, const std::vector& input) { TileData result; @@ -159,7 +158,7 @@ namespace DetourNavigator return result; } - int NavMeshDb::insertTile(TileId tileId, const std::string& worldspace, const TilePosition& tilePosition, + int NavMeshDb::insertTile(TileId tileId, std::string_view worldspace, const TilePosition& tilePosition, TileVersion version, const std::vector& input, const std::vector& data) { const std::vector compressedInput = Misc::compress(input); @@ -180,7 +179,7 @@ namespace DetourNavigator return shapeId; } - std::optional NavMeshDb::findShapeId(const std::string& name, ShapeType type, + std::optional NavMeshDb::findShapeId(std::string_view name, ShapeType type, const Sqlite3::ConstBlob& hash) { ShapeId shapeId; @@ -189,7 +188,7 @@ namespace DetourNavigator return shapeId; } - int NavMeshDb::insertShape(ShapeId shapeId, const std::string& name, ShapeType type, + int NavMeshDb::insertShape(ShapeId shapeId, std::string_view name, ShapeType type, const Sqlite3::ConstBlob& hash) { return execute(*mDb, mInsertShape, shapeId, name, type, hash); @@ -207,7 +206,7 @@ namespace DetourNavigator return findTileQuery; } - void FindTile::bind(sqlite3& db, sqlite3_stmt& statement, const std::string& worldspace, + void FindTile::bind(sqlite3& db, sqlite3_stmt& statement, std::string_view worldspace, const TilePosition& tilePosition, const std::vector& input) { Sqlite3::bindParameter(db, statement, ":worldspace", worldspace); @@ -221,7 +220,7 @@ namespace DetourNavigator return getTileDataQuery; } - void GetTileData::bind(sqlite3& db, sqlite3_stmt& statement, const std::string& worldspace, + void GetTileData::bind(sqlite3& db, sqlite3_stmt& statement, std::string_view worldspace, const TilePosition& tilePosition, const std::vector& input) { Sqlite3::bindParameter(db, statement, ":worldspace", worldspace); @@ -235,7 +234,7 @@ namespace DetourNavigator return insertTileQuery; } - void InsertTile::bind(sqlite3& db, sqlite3_stmt& statement, TileId tileId, const std::string& worldspace, + void InsertTile::bind(sqlite3& db, sqlite3_stmt& statement, TileId tileId, std::string_view worldspace, const TilePosition& tilePosition, TileVersion version, const std::vector& input, const std::vector& data) { @@ -271,7 +270,7 @@ namespace DetourNavigator return findShapeIdQuery; } - void FindShapeId::bind(sqlite3& db, sqlite3_stmt& statement, const std::string& name, + void FindShapeId::bind(sqlite3& db, sqlite3_stmt& statement, std::string_view name, ShapeType type, const Sqlite3::ConstBlob& hash) { Sqlite3::bindParameter(db, statement, ":name", name); @@ -284,7 +283,7 @@ namespace DetourNavigator return insertShapeQuery; } - void InsertShape::bind(sqlite3& db, sqlite3_stmt& statement, ShapeId shapeId, const std::string& name, + void InsertShape::bind(sqlite3& db, sqlite3_stmt& statement, ShapeId shapeId, std::string_view name, ShapeType type, const Sqlite3::ConstBlob& hash) { Sqlite3::bindParameter(db, statement, ":shape_id", shapeId); diff --git a/components/detournavigator/navmeshdb.hpp b/components/detournavigator/navmeshdb.hpp index 636f1de000..09604e5706 100644 --- a/components/detournavigator/navmeshdb.hpp +++ b/components/detournavigator/navmeshdb.hpp @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -64,21 +63,21 @@ namespace DetourNavigator struct FindTile { static std::string_view text() noexcept; - static void bind(sqlite3& db, sqlite3_stmt& statement, const std::string& worldspace, + static void bind(sqlite3& db, sqlite3_stmt& statement, std::string_view worldspace, const TilePosition& tilePosition, const std::vector& input); }; struct GetTileData { static std::string_view text() noexcept; - static void bind(sqlite3& db, sqlite3_stmt& statement, const std::string& worldspace, + static void bind(sqlite3& db, sqlite3_stmt& statement, std::string_view worldspace, const TilePosition& tilePosition, const std::vector& input); }; struct InsertTile { static std::string_view text() noexcept; - static void bind(sqlite3& db, sqlite3_stmt& statement, TileId tileId, const std::string& worldspace, + static void bind(sqlite3& db, sqlite3_stmt& statement, TileId tileId, std::string_view worldspace, const TilePosition& tilePosition, TileVersion version, const std::vector& input, const std::vector& data); }; @@ -99,14 +98,14 @@ namespace DetourNavigator struct FindShapeId { static std::string_view text() noexcept; - static void bind(sqlite3& db, sqlite3_stmt& statement, const std::string& name, + static void bind(sqlite3& db, sqlite3_stmt& statement, std::string_view name, ShapeType type, const Sqlite3::ConstBlob& hash); }; struct InsertShape { static std::string_view text() noexcept; - static void bind(sqlite3& db, sqlite3_stmt& statement, ShapeId shapeId, const std::string& name, + static void bind(sqlite3& db, sqlite3_stmt& statement, ShapeId shapeId, std::string_view name, ShapeType type, const Sqlite3::ConstBlob& hash); }; } @@ -120,22 +119,22 @@ namespace DetourNavigator TileId getMaxTileId(); - std::optional findTile(const std::string& worldspace, + std::optional findTile(std::string_view worldspace, const TilePosition& tilePosition, const std::vector& input); - std::optional getTileData(const std::string& worldspace, + std::optional getTileData(std::string_view worldspace, const TilePosition& tilePosition, const std::vector& input); - int insertTile(TileId tileId, const std::string& worldspace, const TilePosition& tilePosition, + int insertTile(TileId tileId, std::string_view worldspace, const TilePosition& tilePosition, TileVersion version, const std::vector& input, const std::vector& data); int updateTile(TileId tileId, TileVersion version, const std::vector& data); ShapeId getMaxShapeId(); - std::optional findShapeId(const std::string& name, ShapeType type, const Sqlite3::ConstBlob& hash); + std::optional findShapeId(std::string_view name, ShapeType type, const Sqlite3::ConstBlob& hash); - int insertShape(ShapeId shapeId, const std::string& name, ShapeType type, const Sqlite3::ConstBlob& hash); + int insertShape(ShapeId shapeId, std::string_view name, ShapeType type, const Sqlite3::ConstBlob& hash); private: Sqlite3::Db mDb; diff --git a/components/detournavigator/navmeshdbutils.cpp b/components/detournavigator/navmeshdbutils.cpp index 86f81bfc51..71873972b9 100644 --- a/components/detournavigator/navmeshdbutils.cpp +++ b/components/detournavigator/navmeshdbutils.cpp @@ -6,19 +6,20 @@ #include #include +#include namespace DetourNavigator { namespace { - std::optional findShapeId(NavMeshDb& db, const std::string& name, ShapeType type, + std::optional findShapeId(NavMeshDb& db, std::string_view name, ShapeType type, const std::string& hash) { const Sqlite3::ConstBlob hashData {hash.data(), static_cast(hash.size())}; return db.findShapeId(name, type, hashData); } - ShapeId getShapeId(NavMeshDb& db, const std::string& name, ShapeType type, + ShapeId getShapeId(NavMeshDb& db, std::string_view name, ShapeType type, const std::string& hash, ShapeId& nextShapeId) { const Sqlite3::ConstBlob hashData {hash.data(), static_cast(hash.size())}; From ab1a6e034ee822d9ca6b664e5b72a3a45bde97e0 Mon Sep 17 00:00:00 2001 From: elsid Date: Fri, 18 Feb 2022 21:35:09 +0100 Subject: [PATCH 4/4] Add navmeshtool flag to remove unused tiles from navmesh disk cache * Remove tiles outside processing range. Useful when new content profile map has different bounds. * Remove ignored tiles. For a case when content profile maps have intersection but there is no more data for navmesh. * Remove older tiles at the same worldspace position. If navmesh tile data has changed with new content, the old ones unlikely to be used. * Vacuum the database when there are modifications. SQLite leaves empty pages in the file on database modification. Vacuum cleans up unused pages reducing the file size. --- apps/navmeshtool/main.cpp | 7 +- apps/navmeshtool/navmesh.cpp | 104 ++++++++++++++---- apps/navmeshtool/navmesh.hpp | 3 +- .../detournavigator/navmeshdb.cpp | 57 ++++++++++ .../detournavigator/generatenavmeshtile.cpp | 12 +- .../detournavigator/generatenavmeshtile.hpp | 8 +- .../detournavigator/gettilespositions.hpp | 7 +- components/detournavigator/navmeshdb.cpp | 103 +++++++++++++++++ components/detournavigator/navmeshdb.hpp | 41 +++++++ .../detournavigator/tilespositionsrange.hpp | 15 +++ 10 files changed, 323 insertions(+), 34 deletions(-) create mode 100644 components/detournavigator/tilespositionsrange.hpp diff --git a/apps/navmeshtool/main.cpp b/apps/navmeshtool/main.cpp index f89e80e542..894ec6b3b1 100644 --- a/apps/navmeshtool/main.cpp +++ b/apps/navmeshtool/main.cpp @@ -83,6 +83,9 @@ namespace NavMeshTool ("process-interior-cells", bpo::value()->implicit_value(true) ->default_value(false), "build navmesh for interior cells") + + ("remove-unused-tiles", bpo::value()->implicit_value(true) + ->default_value(false), "remove tiles from cache that will not be used with current content profile") ; Files::ConfigurationManager::addCommonOptions(result); @@ -141,6 +144,7 @@ namespace NavMeshTool } const bool processInteriorCells = variables["process-interior-cells"].as(); + const bool removeUnusedTiles = variables["remove-unused-tiles"].as(); Fallback::Map::init(variables["fallback"].as().mMap); @@ -177,7 +181,8 @@ namespace NavMeshTool WorldspaceData cellsData = gatherWorldspaceData(navigatorSettings, readers, vfs, bulletShapeManager, esmData, processInteriorCells); - generateAllNavMeshTiles(agentHalfExtents, navigatorSettings, threadsNumber, cellsData, std::move(db)); + generateAllNavMeshTiles(agentHalfExtents, navigatorSettings, threadsNumber, removeUnusedTiles, + cellsData, std::move(db)); Log(Debug::Info) << "Done"; diff --git a/apps/navmeshtool/navmesh.cpp b/apps/navmeshtool/navmesh.cpp index cf18d7edc3..3161192cf9 100644 --- a/apps/navmeshtool/navmesh.cpp +++ b/apps/navmeshtool/navmesh.cpp @@ -41,6 +41,7 @@ namespace NavMeshTool using DetourNavigator::TileId; using DetourNavigator::TilePosition; using DetourNavigator::TileVersion; + using DetourNavigator::TilesPositionsRange; using Sqlite3::Transaction; void logGeneratedTiles(std::size_t provided, std::size_t expected) @@ -63,8 +64,9 @@ namespace NavMeshTool public: std::atomic_size_t mExpected {0}; - explicit NavMeshTileConsumer(NavMeshDb&& db) + explicit NavMeshTileConsumer(NavMeshDb&& db, bool removeUnusedTiles) : mDb(std::move(db)) + , mRemoveUnusedTiles(removeUnusedTiles) , mTransaction(mDb.startTransaction()) , mNextTileId(mDb.getMaxTileId() + 1) , mNextShapeId(mDb.getMaxShapeId() + 1) @@ -76,6 +78,12 @@ namespace NavMeshTool std::size_t getUpdated() const { return mUpdated.load(); } + std::size_t getDeleted() const + { + const std::lock_guard lock(mMutex); + return mDeleted; + } + std::int64_t resolveMeshSource(const MeshSource& source) override { const std::lock_guard lock(mMutex); @@ -97,11 +105,34 @@ namespace NavMeshTool return result; } - void ignore() override { report(); } - - void insert(std::string_view worldspace, const TilePosition& tilePosition, std::int64_t version, - const std::vector& input, PreparedNavMeshData& data) override + void ignore(std::string_view worldspace, const TilePosition& tilePosition) override { + if (mRemoveUnusedTiles) + { + std::lock_guard lock(mMutex); + mDeleted += static_cast(mDb.deleteTilesAt(worldspace, tilePosition)); + } + report(); + } + + void identity(std::string_view worldspace, const TilePosition& tilePosition, std::int64_t tileId) override + { + if (mRemoveUnusedTiles) + { + std::lock_guard lock(mMutex); + mDeleted += static_cast(mDb.deleteTilesAtExcept(worldspace, tilePosition, TileId {tileId})); + } + report(); + } + + void insert(std::string_view worldspace, const TilePosition& tilePosition, + std::int64_t version, const std::vector& input, PreparedNavMeshData& data) override + { + if (mRemoveUnusedTiles) + { + std::lock_guard lock(mMutex); + mDeleted += static_cast(mDb.deleteTilesAt(worldspace, tilePosition)); + } data.mUserId = static_cast(mNextTileId); { std::lock_guard lock(mMutex); @@ -112,11 +143,14 @@ namespace NavMeshTool report(); } - void update(std::int64_t tileId, std::int64_t version, PreparedNavMeshData& data) override + void update(std::string_view worldspace, const TilePosition& tilePosition, + std::int64_t tileId, std::int64_t version, PreparedNavMeshData& data) override { data.mUserId = static_cast(tileId); { std::lock_guard lock(mMutex); + if (mRemoveUnusedTiles) + mDeleted += static_cast(mDb.deleteTilesAtExcept(worldspace, tilePosition, TileId {tileId})); mDb.updateTile(TileId {tileId}, TileVersion {version}, serialize(data)); } ++mUpdated; @@ -141,49 +175,66 @@ namespace NavMeshTool void commit() { mTransaction.commit(); } + void vacuum() { mDb.vacuum(); } + + void removeTilesOutsideRange(std::string_view worldspace, const TilesPositionsRange& range) + { + const std::lock_guard lock(mMutex); + mTransaction.commit(); + Log(Debug::Info) << "Removing tiles outside processed range for worldspace \"" << worldspace << "\"..."; + mDeleted += static_cast(mDb.deleteTilesOutsideRange(worldspace, range)); + mTransaction = mDb.startTransaction(); + } + private: std::atomic_size_t mProvided {0}; std::atomic_size_t mInserted {0}; std::atomic_size_t mUpdated {0}; - std::mutex mMutex; + std::size_t mDeleted = 0; + mutable std::mutex mMutex; NavMeshDb mDb; + const bool mRemoveUnusedTiles; Transaction mTransaction; TileId mNextTileId; std::condition_variable mHasTile; Misc::ProgressReporter mReporter; ShapeId mNextShapeId; + std::mutex mReportMutex; void report() { - mReporter(mProvided + 1, mExpected); - ++mProvided; + const std::size_t provided = mProvided.fetch_add(1, std::memory_order_relaxed) + 1; + mReporter(provided, mExpected); mHasTile.notify_one(); } }; } void generateAllNavMeshTiles(const osg::Vec3f& agentHalfExtents, const Settings& settings, - const std::size_t threadsNumber, WorldspaceData& data, NavMeshDb&& db) + std::size_t threadsNumber, bool removeUnusedTiles, WorldspaceData& data, NavMeshDb&& db) { Log(Debug::Info) << "Generating navmesh tiles by " << threadsNumber << " parallel workers..."; SceneUtil::WorkQueue workQueue(threadsNumber); - auto navMeshTileConsumer = std::make_shared(std::move(db)); + auto navMeshTileConsumer = std::make_shared(std::move(db), removeUnusedTiles); std::size_t tiles = 0; std::mt19937_64 random; for (const std::unique_ptr& input : data.mNavMeshInputs) { + const auto range = DetourNavigator::makeTilesPositionsRange( + Misc::Convert::toOsgXY(input->mAabb.m_min), + Misc::Convert::toOsgXY(input->mAabb.m_max), + settings.mRecast + ); + + if (removeUnusedTiles) + navMeshTileConsumer->removeTilesOutsideRange(input->mWorldspace, range); + std::vector worldspaceTiles; - DetourNavigator::getTilesPositions( - DetourNavigator::makeTilesPositionsRange( - Misc::Convert::toOsgXY(input->mAabb.m_min), - Misc::Convert::toOsgXY(input->mAabb.m_max), - settings.mRecast - ), - [&] (const TilePosition& tilePosition) { worldspaceTiles.push_back(tilePosition); } - ); + DetourNavigator::getTilesPositions(range, + [&] (const TilePosition& tilePosition) { worldspaceTiles.push_back(tilePosition); }); tiles += worldspaceTiles.size(); @@ -205,8 +256,19 @@ namespace NavMeshTool navMeshTileConsumer->wait(); navMeshTileConsumer->commit(); + const auto inserted = navMeshTileConsumer->getInserted(); + const auto updated = navMeshTileConsumer->getUpdated(); + const auto deleted = navMeshTileConsumer->getDeleted(); + Log(Debug::Info) << "Generated navmesh for " << navMeshTileConsumer->getProvided() << " tiles, " - << navMeshTileConsumer->getInserted() << " are inserted and " - << navMeshTileConsumer->getUpdated() << " updated"; + << inserted << " are inserted, " + << updated << " updated and " + << deleted << " deleted"; + + if (inserted + updated + deleted > 0) + { + Log(Debug::Info) << "Vacuuming the database..."; + navMeshTileConsumer->vacuum(); + } } } diff --git a/apps/navmeshtool/navmesh.hpp b/apps/navmeshtool/navmesh.hpp index ff837eebe7..3d0e9e4665 100644 --- a/apps/navmeshtool/navmesh.hpp +++ b/apps/navmeshtool/navmesh.hpp @@ -16,7 +16,8 @@ namespace NavMeshTool struct WorldspaceData; void generateAllNavMeshTiles(const osg::Vec3f& agentHalfExtents, const DetourNavigator::Settings& settings, - const std::size_t threadsNumber, WorldspaceData& cellsData, DetourNavigator::NavMeshDb&& db); + std::size_t threadsNumber, bool removeUnusedTiles, WorldspaceData& cellsData, + DetourNavigator::NavMeshDb&& db); } #endif diff --git a/apps/openmw_test_suite/detournavigator/navmeshdb.cpp b/apps/openmw_test_suite/detournavigator/navmeshdb.cpp index ba008f50ff..a17f5132c5 100644 --- a/apps/openmw_test_suite/detournavigator/navmeshdb.cpp +++ b/apps/openmw_test_suite/detournavigator/navmeshdb.cpp @@ -109,4 +109,61 @@ namespace EXPECT_THROW(mDb.insertTile(tileId, worldspace, tilePosition, version, input, data), std::runtime_error); EXPECT_NO_THROW(insertTile(TileId {54}, version)); } + + TEST_F(DetourNavigatorNavMeshDbTest, delete_tiles_at_should_remove_all_tiles_with_given_worldspace_and_position) + { + const TileVersion version {1}; + const std::string worldspace = "sys::default"; + const TilePosition tilePosition {3, 4}; + const std::vector input1 = generateData(); + const std::vector input2 = generateData(); + const std::vector data = generateData(); + ASSERT_EQ(mDb.insertTile(TileId {53}, worldspace, tilePosition, version, input1, data), 1); + ASSERT_EQ(mDb.insertTile(TileId {54}, worldspace, tilePosition, version, input2, data), 1); + ASSERT_EQ(mDb.deleteTilesAt(worldspace, tilePosition), 2); + EXPECT_FALSE(mDb.findTile(worldspace, tilePosition, input1).has_value()); + EXPECT_FALSE(mDb.findTile(worldspace, tilePosition, input2).has_value()); + } + + TEST_F(DetourNavigatorNavMeshDbTest, delete_tiles_at_except_should_leave_tile_with_given_id) + { + const TileId leftTileId {53}; + const TileId removedTileId {54}; + const TileVersion version {1}; + const std::string worldspace = "sys::default"; + const TilePosition tilePosition {3, 4}; + const std::vector leftInput = generateData(); + const std::vector removedInput = generateData(); + const std::vector data = generateData(); + ASSERT_EQ(mDb.insertTile(leftTileId, worldspace, tilePosition, version, leftInput, data), 1); + ASSERT_EQ(mDb.insertTile(removedTileId, worldspace, tilePosition, version, removedInput, data), 1); + ASSERT_EQ(mDb.deleteTilesAtExcept(worldspace, tilePosition, leftTileId), 1); + const auto left = mDb.findTile(worldspace, tilePosition, leftInput); + ASSERT_TRUE(left.has_value()); + EXPECT_EQ(left->mTileId, leftTileId); + EXPECT_FALSE(mDb.findTile(worldspace, tilePosition, removedInput).has_value()); + } + + TEST_F(DetourNavigatorNavMeshDbTest, delete_tiles_outside_range_should_leave_tiles_inside_given_rectangle) + { + TileId tileId {1}; + const TileVersion version {1}; + const std::string worldspace = "sys::default"; + const std::vector input = generateData(); + const std::vector data = generateData(); + for (int x = -2; x <= 2; ++x) + { + for (int y = -2; y <= 2; ++y) + { + ASSERT_EQ(mDb.insertTile(tileId, worldspace, TilePosition {x, y}, version, input, data), 1); + ++tileId.t; + } + } + const TilesPositionsRange range {TilePosition {-1, -1}, TilePosition {2, 2}}; + ASSERT_EQ(mDb.deleteTilesOutsideRange(worldspace, range), 16); + for (int x = -2; x <= 2; ++x) + for (int y = -2; y <= 2; ++y) + ASSERT_EQ(mDb.findTile(worldspace, TilePosition {x, y}, input).has_value(), + -1 <= x && x <= 1 && -1 <= y && y <= 1) << "x=" << x << " y=" << y; + } } diff --git a/components/detournavigator/generatenavmeshtile.cpp b/components/detournavigator/generatenavmeshtile.cpp index ad8978cd4b..360c05931f 100644 --- a/components/detournavigator/generatenavmeshtile.cpp +++ b/components/detournavigator/generatenavmeshtile.cpp @@ -25,12 +25,14 @@ namespace DetourNavigator { struct Ignore { + std::string_view mWorldspace; + const TilePosition& mTilePosition; std::shared_ptr mConsumer; ~Ignore() noexcept { if (mConsumer != nullptr) - mConsumer->ignore(); + mConsumer->ignore(mWorldspace, mTilePosition); } }; } @@ -59,7 +61,7 @@ namespace DetourNavigator try { - Ignore ignore {consumer}; + Ignore ignore {mWorldspace, mTilePosition, consumer}; const std::shared_ptr recastMesh = mRecastMeshProvider.getMesh(mWorldspace, mTilePosition); @@ -72,7 +74,11 @@ namespace DetourNavigator const std::optional info = consumer->find(mWorldspace, mTilePosition, input); if (info.has_value() && info->mVersion == mSettings.mNavMeshVersion) + { + consumer->identity(mWorldspace, mTilePosition, info->mTileId); + ignore.mConsumer = nullptr; return; + } const auto data = prepareNavMeshTileData(*recastMesh, mTilePosition, mAgentHalfExtents, mSettings.mRecast); @@ -80,7 +86,7 @@ namespace DetourNavigator return; if (info.has_value()) - consumer->update(info->mTileId, mSettings.mNavMeshVersion, *data); + consumer->update(mWorldspace, mTilePosition, info->mTileId, mSettings.mNavMeshVersion, *data); else consumer->insert(mWorldspace, mTilePosition, mSettings.mNavMeshVersion, input, *data); diff --git a/components/detournavigator/generatenavmeshtile.hpp b/components/detournavigator/generatenavmeshtile.hpp index 69afe426bc..e6d9e26c1d 100644 --- a/components/detournavigator/generatenavmeshtile.hpp +++ b/components/detournavigator/generatenavmeshtile.hpp @@ -39,12 +39,16 @@ namespace DetourNavigator virtual std::optional find(std::string_view worldspace, const TilePosition& tilePosition, const std::vector& input) = 0; - virtual void ignore() = 0; + virtual void ignore(std::string_view worldspace, const TilePosition& tilePosition) = 0; + + virtual void identity(std::string_view worldspace, const TilePosition& tilePosition, + std::int64_t tileId) = 0; virtual void insert(std::string_view worldspace, const TilePosition& tilePosition, std::int64_t version, const std::vector& input, PreparedNavMeshData& data) = 0; - virtual void update(std::int64_t tileId, std::int64_t version, PreparedNavMeshData& data) = 0; + virtual void update(std::string_view worldspace, const TilePosition& tilePosition, + std::int64_t tileId, std::int64_t version, PreparedNavMeshData& data) = 0; }; class GenerateNavMeshTile final : public SceneUtil::WorkItem diff --git a/components/detournavigator/gettilespositions.hpp b/components/detournavigator/gettilespositions.hpp index 79188868dc..33c1131176 100644 --- a/components/detournavigator/gettilespositions.hpp +++ b/components/detournavigator/gettilespositions.hpp @@ -3,6 +3,7 @@ #include "tilebounds.hpp" #include "tileposition.hpp" +#include "tilespositionsrange.hpp" class btVector3; class btTransform; @@ -17,12 +18,6 @@ namespace DetourNavigator { struct RecastSettings; - struct TilesPositionsRange - { - TilePosition mBegin; - TilePosition mEnd; - }; - TilesPositionsRange makeTilesPositionsRange(const osg::Vec2f& aabbMin, const osg::Vec2f& aabbMax, const RecastSettings& settings); diff --git a/components/detournavigator/navmeshdb.cpp b/components/detournavigator/navmeshdb.cpp index 425cf7d434..621c97f390 100644 --- a/components/detournavigator/navmeshdb.cpp +++ b/components/detournavigator/navmeshdb.cpp @@ -34,6 +34,9 @@ namespace DetourNavigator CREATE UNIQUE INDEX IF NOT EXISTS index_unique_tiles_by_worldspace_and_tile_position_and_input ON tiles (worldspace, tile_position_x, tile_position_y, input); + CREATE INDEX IF NOT EXISTS index_tiles_by_worldspace_and_tile_position + ON tiles (worldspace, tile_position_x, tile_position_y); + CREATE TABLE IF NOT EXISTS shapes ( shape_id INTEGER PRIMARY KEY, name TEXT NOT NULL, @@ -82,6 +85,31 @@ namespace DetourNavigator WHERE tile_id = :tile_id )"; + constexpr std::string_view deleteTilesAtQuery = R"( + DELETE FROM tiles + WHERE worldspace = :worldspace + AND tile_position_x = :tile_position_x + AND tile_position_y = :tile_position_y + )"; + + constexpr std::string_view deleteTilesAtExceptQuery = R"( + DELETE FROM tiles + WHERE worldspace = :worldspace + AND tile_position_x = :tile_position_x + AND tile_position_y = :tile_position_y + AND tile_id != :exclude_tile_id + )"; + + constexpr std::string_view deleteTilesOutsideRangeQuery = R"( + DELETE FROM tiles + WHERE worldspace = :worldspace + AND ( tile_position_x < :begin_tile_position_x + OR tile_position_y < :begin_tile_position_y + OR tile_position_x >= :end_tile_position_x + OR tile_position_y >= :end_tile_position_y + ) + )"; + constexpr std::string_view getMaxShapeIdQuery = R"( SELECT max(shape_id) FROM shapes )"; @@ -98,6 +126,10 @@ namespace DetourNavigator INSERT INTO shapes ( shape_id, name, type, hash) VALUES (:shape_id, :name, :type, :hash) )"; + + constexpr std::string_view vacuumQuery = R"( + VACUUM; + )"; } std::ostream& operator<<(std::ostream& stream, ShapeType value) @@ -117,9 +149,13 @@ namespace DetourNavigator , mGetTileData(*mDb, DbQueries::GetTileData {}) , mInsertTile(*mDb, DbQueries::InsertTile {}) , mUpdateTile(*mDb, DbQueries::UpdateTile {}) + , mDeleteTilesAt(*mDb, DbQueries::DeleteTilesAt {}) + , mDeleteTilesAtExcept(*mDb, DbQueries::DeleteTilesAtExcept {}) + , mDeleteTilesOutsideRange(*mDb, DbQueries::DeleteTilesOutsideRange {}) , mGetMaxShapeId(*mDb, DbQueries::GetMaxShapeId {}) , mFindShapeId(*mDb, DbQueries::FindShapeId {}) , mInsertShape(*mDb, DbQueries::InsertShape {}) + , mVacuum(*mDb, DbQueries::Vacuum {}) { } @@ -172,6 +208,21 @@ namespace DetourNavigator return execute(*mDb, mUpdateTile, tileId, version, compressedData); } + int NavMeshDb::deleteTilesAt(std::string_view worldspace, const TilePosition& tilePosition) + { + return execute(*mDb, mDeleteTilesAt, worldspace, tilePosition); + } + + int NavMeshDb::deleteTilesAtExcept(std::string_view worldspace, const TilePosition& tilePosition, TileId excludeTileId) + { + return execute(*mDb, mDeleteTilesAtExcept, worldspace, tilePosition, excludeTileId); + } + + int NavMeshDb::deleteTilesOutsideRange(std::string_view worldspace, const TilesPositionsRange& range) + { + return execute(*mDb, mDeleteTilesOutsideRange, worldspace, range); + } + ShapeId NavMeshDb::getMaxShapeId() { ShapeId shapeId {0}; @@ -194,6 +245,11 @@ namespace DetourNavigator return execute(*mDb, mInsertShape, shapeId, name, type, hash); } + void NavMeshDb::vacuum() + { + execute(*mDb, mVacuum); + } + namespace DbQueries { std::string_view GetMaxTileId::text() noexcept @@ -260,6 +316,48 @@ namespace DetourNavigator Sqlite3::bindParameter(db, statement, ":data", data); } + std::string_view DeleteTilesAt::text() noexcept + { + return deleteTilesAtQuery; + } + + void DeleteTilesAt::bind(sqlite3& db, sqlite3_stmt& statement, std::string_view worldspace, + const TilePosition& tilePosition) + { + Sqlite3::bindParameter(db, statement, ":worldspace", worldspace); + Sqlite3::bindParameter(db, statement, ":tile_position_x", tilePosition.x()); + Sqlite3::bindParameter(db, statement, ":tile_position_y", tilePosition.y()); + } + + std::string_view DeleteTilesAtExcept::text() noexcept + { + return deleteTilesAtExceptQuery; + } + + void DeleteTilesAtExcept::bind(sqlite3& db, sqlite3_stmt& statement, std::string_view worldspace, + const TilePosition& tilePosition, TileId excludeTileId) + { + Sqlite3::bindParameter(db, statement, ":worldspace", worldspace); + Sqlite3::bindParameter(db, statement, ":tile_position_x", tilePosition.x()); + Sqlite3::bindParameter(db, statement, ":tile_position_y", tilePosition.y()); + Sqlite3::bindParameter(db, statement, ":exclude_tile_id", excludeTileId); + } + + std::string_view DeleteTilesOutsideRange::text() noexcept + { + return deleteTilesOutsideRangeQuery; + } + + void DeleteTilesOutsideRange::bind(sqlite3& db, sqlite3_stmt& statement, std::string_view worldspace, + const TilesPositionsRange& range) + { + Sqlite3::bindParameter(db, statement, ":worldspace", worldspace); + Sqlite3::bindParameter(db, statement, ":begin_tile_position_x", range.mBegin.x()); + Sqlite3::bindParameter(db, statement, ":begin_tile_position_y", range.mBegin.y()); + Sqlite3::bindParameter(db, statement, ":end_tile_position_x", range.mEnd.x()); + Sqlite3::bindParameter(db, statement, ":end_tile_position_y", range.mEnd.y()); + } + std::string_view GetMaxShapeId::text() noexcept { return getMaxShapeIdQuery; @@ -291,5 +389,10 @@ namespace DetourNavigator Sqlite3::bindParameter(db, statement, ":type", static_cast(type)); Sqlite3::bindParameter(db, statement, ":hash", hash); } + + std::string_view Vacuum::text() noexcept + { + return vacuumQuery; + } } } diff --git a/components/detournavigator/navmeshdb.hpp b/components/detournavigator/navmeshdb.hpp index 09604e5706..f10a3a3288 100644 --- a/components/detournavigator/navmeshdb.hpp +++ b/components/detournavigator/navmeshdb.hpp @@ -3,6 +3,8 @@ #include "tileposition.hpp" +#include + #include #include #include @@ -89,6 +91,27 @@ namespace DetourNavigator const std::vector& data); }; + struct DeleteTilesAt + { + static std::string_view text() noexcept; + static void bind(sqlite3& db, sqlite3_stmt& statement, std::string_view worldspace, + const TilePosition& tilePosition); + }; + + struct DeleteTilesAtExcept + { + static std::string_view text() noexcept; + static void bind(sqlite3& db, sqlite3_stmt& statement, std::string_view worldspace, + const TilePosition& tilePosition, TileId excludeTileId); + }; + + struct DeleteTilesOutsideRange + { + static std::string_view text() noexcept; + static void bind(sqlite3& db, sqlite3_stmt& statement, std::string_view worldspace, + const TilesPositionsRange& range); + }; + struct GetMaxShapeId { static std::string_view text() noexcept; @@ -108,6 +131,12 @@ namespace DetourNavigator static void bind(sqlite3& db, sqlite3_stmt& statement, ShapeId shapeId, std::string_view name, ShapeType type, const Sqlite3::ConstBlob& hash); }; + + struct Vacuum + { + static std::string_view text() noexcept; + static void bind(sqlite3&, sqlite3_stmt&) {} + }; } class NavMeshDb @@ -130,12 +159,20 @@ namespace DetourNavigator int updateTile(TileId tileId, TileVersion version, const std::vector& data); + int deleteTilesAt(std::string_view worldspace, const TilePosition& tilePosition); + + int deleteTilesAtExcept(std::string_view worldspace, const TilePosition& tilePosition, TileId excludeTileId); + + int deleteTilesOutsideRange(std::string_view worldspace, const TilesPositionsRange& range); + ShapeId getMaxShapeId(); std::optional findShapeId(std::string_view name, ShapeType type, const Sqlite3::ConstBlob& hash); int insertShape(ShapeId shapeId, std::string_view name, ShapeType type, const Sqlite3::ConstBlob& hash); + void vacuum(); + private: Sqlite3::Db mDb; Sqlite3::Statement mGetMaxTileId; @@ -143,9 +180,13 @@ namespace DetourNavigator Sqlite3::Statement mGetTileData; Sqlite3::Statement mInsertTile; Sqlite3::Statement mUpdateTile; + Sqlite3::Statement mDeleteTilesAt; + Sqlite3::Statement mDeleteTilesAtExcept; + Sqlite3::Statement mDeleteTilesOutsideRange; Sqlite3::Statement mGetMaxShapeId; Sqlite3::Statement mFindShapeId; Sqlite3::Statement mInsertShape; + Sqlite3::Statement mVacuum; }; } diff --git a/components/detournavigator/tilespositionsrange.hpp b/components/detournavigator/tilespositionsrange.hpp new file mode 100644 index 0000000000..d5f2622ba1 --- /dev/null +++ b/components/detournavigator/tilespositionsrange.hpp @@ -0,0 +1,15 @@ +#ifndef OPENMW_COMPONENTS_DETOURNAVIGATOR_TILESPOSITIONSRANGE_H +#define OPENMW_COMPONENTS_DETOURNAVIGATOR_TILESPOSITIONSRANGE_H + +#include "tileposition.hpp" + +namespace DetourNavigator +{ + struct TilesPositionsRange + { + TilePosition mBegin; + TilePosition mEnd; + }; +} + +#endif