From a665a38aca554515b7e6e2d071183ba325134317 Mon Sep 17 00:00:00 2001 From: elsid Date: Mon, 15 Nov 2021 17:40:22 +0100 Subject: [PATCH] Use MurmurHash3_x64_128 for file hash --- apps/openmw_test_suite/CMakeLists.txt | 2 + apps/openmw_test_suite/files/hash.cpp | 55 +++++++++++++++++++ .../nifloader/testbulletnifloader.cpp | 4 +- components/CMakeLists.txt | 1 + components/files/hash.cpp | 19 ++++--- components/files/hash.hpp | 3 +- components/nif/niffile.cpp | 3 +- components/nif/niffile.hpp | 6 +- components/nifosg/nifloader.cpp | 4 +- components/resource/bulletshape.hpp | 3 +- components/resource/bulletshapemanager.cpp | 5 +- components/resource/scenemanager.cpp | 4 +- 12 files changed, 85 insertions(+), 24 deletions(-) create mode 100644 apps/openmw_test_suite/files/hash.cpp diff --git a/apps/openmw_test_suite/CMakeLists.txt b/apps/openmw_test_suite/CMakeLists.txt index 29564ef191..ed1d6c413b 100644 --- a/apps/openmw_test_suite/CMakeLists.txt +++ b/apps/openmw_test_suite/CMakeLists.txt @@ -62,6 +62,8 @@ if (GTEST_FOUND AND GMOCK_FOUND) esmloader/load.cpp esmloader/esmdata.cpp + + files/hash.cpp ) source_group(apps\\openmw_test_suite FILES openmw_test_suite.cpp ${UNITTEST_SRC_FILES}) diff --git a/apps/openmw_test_suite/files/hash.cpp b/apps/openmw_test_suite/files/hash.cpp new file mode 100644 index 0000000000..e6dbc8f6cc --- /dev/null +++ b/apps/openmw_test_suite/files/hash.cpp @@ -0,0 +1,55 @@ +#include +#include + +#include +#include + +#include +#include +#include +#include + +namespace +{ + using namespace testing; + using namespace Files; + + struct Params + { + std::size_t mSize; + std::array mHash; + }; + + struct FilesGetHash : TestWithParam {}; + + TEST_P(FilesGetHash, shouldReturnHashForStringStream) + { + const std::string fileName = "fileName"; + std::string content; + std::fill_n(std::back_inserter(content), GetParam().mSize, 'a'); + std::istringstream stream(content); + EXPECT_EQ(getHash(fileName, stream), GetParam().mHash); + } + + TEST_P(FilesGetHash, shouldReturnHashForConstrainedFileStream) + { + std::string fileName(UnitTest::GetInstance()->current_test_info()->name()); + std::replace(fileName.begin(), fileName.end(), '/', '_'); + std::string content; + std::fill_n(std::back_inserter(content), GetParam().mSize, 'a'); + std::fstream(fileName, std::ios_base::out | std::ios_base::binary) + .write(content.data(), static_cast(content.size())); + const auto stream = Files::openConstrainedFileStream(fileName.data(), 0, content.size()); + EXPECT_EQ(getHash(fileName, *stream), GetParam().mHash); + } + + INSTANTIATE_TEST_SUITE_P(Params, FilesGetHash, Values( + Params {0, {0, 0}}, + Params {1, {9607679276477937801ull, 16624257681780017498ull}}, + Params {128, {15287858148353394424ull, 16818615825966581310ull}}, + Params {1000, {11018119256083894017ull, 6631144854802791578ull}}, + Params {4096, {11972283295181039100ull, 16027670129106775155ull}}, + Params {4097, {16717956291025443060ull, 12856404199748778153ull}}, + Params {5000, {15775925571142117787ull, 10322955217889622896ull}} + )); +} diff --git a/apps/openmw_test_suite/nifloader/testbulletnifloader.cpp b/apps/openmw_test_suite/nifloader/testbulletnifloader.cpp index 92aece9075..8fbf5c1b5b 100644 --- a/apps/openmw_test_suite/nifloader/testbulletnifloader.cpp +++ b/apps/openmw_test_suite/nifloader/testbulletnifloader.cpp @@ -335,7 +335,7 @@ namespace MOCK_METHOD(void, setUseSkinning, (bool), (override)); MOCK_METHOD(bool, getUseSkinning, (), (const, override)); MOCK_METHOD(std::string, getFilename, (), (const, override)); - MOCK_METHOD(std::uint64_t, getHash, (), (const, override)); + MOCK_METHOD(std::string, getHash, (), (const, override)); MOCK_METHOD(unsigned int, getVersion, (), (const, override)); MOCK_METHOD(unsigned int, getUserVersion, (), (const, override)); MOCK_METHOD(unsigned int, getBethVersion, (), (const, override)); @@ -382,7 +382,7 @@ namespace ), btVector3(4, 8, 12) }; - const std::uint64_t mHash = 42; + const std::string mHash = "hash"; TestBulletNifLoader() { diff --git a/components/CMakeLists.txt b/components/CMakeLists.txt index 56371a8c02..3fc9cb92a1 100644 --- a/components/CMakeLists.txt +++ b/components/CMakeLists.txt @@ -289,6 +289,7 @@ target_link_libraries(components Base64 SQLite::SQLite3 + smhasher ) target_link_libraries(components ${BULLET_LIBRARIES}) diff --git a/components/files/hash.cpp b/components/files/hash.cpp index 91af1e9283..079a169ae5 100644 --- a/components/files/hash.cpp +++ b/components/files/hash.cpp @@ -1,17 +1,17 @@ #include "hash.hpp" -#include +#include +#include #include -#include #include #include namespace Files { - std::uint64_t getHash(const std::string& fileName, std::istream& stream) + std::array getHash(const std::string& fileName, std::istream& stream) { - std::uint64_t hash = std::hash {}(fileName); + std::array hash {0, 0}; try { const auto start = stream.tellg(); @@ -19,9 +19,14 @@ namespace Files stream.exceptions(std::ios_base::badbit); while (stream) { - std::uint64_t value = 0; - stream.read(reinterpret_cast(&value), sizeof(value)); - Misc::hashCombine(hash, value); + std::array value; + stream.read(value.data(), value.size()); + const std::streamsize read = stream.gcount(); + if (read == 0) + break; + std::array blockHash {0, 0}; + MurmurHash3_x64_128(value.data(), static_cast(read), hash.data(), blockHash.data()); + hash = blockHash; } stream.exceptions(exceptions); stream.clear(); diff --git a/components/files/hash.hpp b/components/files/hash.hpp index 46784ee9be..13d56d5824 100644 --- a/components/files/hash.hpp +++ b/components/files/hash.hpp @@ -1,13 +1,14 @@ #ifndef COMPONENTS_FILES_HASH_H #define COMPONENTS_FILES_HASH_H +#include #include #include #include namespace Files { - std::uint64_t getHash(const std::string& fileName, std::istream& stream); + std::array getHash(const std::string& fileName, std::istream& stream); } #endif diff --git a/components/nif/niffile.cpp b/components/nif/niffile.cpp index f11b75d218..f70b9024f9 100644 --- a/components/nif/niffile.cpp +++ b/components/nif/niffile.cpp @@ -173,7 +173,8 @@ std::string NIFFile::printVersion(unsigned int version) void NIFFile::parse(Files::IStreamPtr stream) { - hash = Files::getHash(filename, *stream); + const std::array fileHash = Files::getHash(filename, *stream); + hash.append(reinterpret_cast(fileHash.data()), fileHash.size() * sizeof(std::uint64_t)); NIFStream nif (this, stream); diff --git a/components/nif/niffile.hpp b/components/nif/niffile.hpp index eb851a74ff..6884f51d58 100644 --- a/components/nif/niffile.hpp +++ b/components/nif/niffile.hpp @@ -34,7 +34,7 @@ struct File virtual std::string getFilename() const = 0; - virtual std::uint64_t getHash() const = 0; + virtual std::string getHash() const = 0; virtual unsigned int getVersion() const = 0; @@ -52,7 +52,7 @@ class NIFFile final : public File /// File name, used for error messages and opening the file std::string filename; - std::uint64_t hash = 0; + std::string hash; /// Record list std::vector records; @@ -144,7 +144,7 @@ public: /// Get the name of the file std::string getFilename() const override { return filename; } - std::uint64_t getHash() const override { return hash; } + std::string getHash() const override { return hash; } /// Get the version of the NIF format used unsigned int getVersion() const override { return ver; } diff --git a/components/nifosg/nifloader.cpp b/components/nifosg/nifloader.cpp index 91d2300161..99aaaa3323 100644 --- a/components/nifosg/nifloader.cpp +++ b/components/nifosg/nifloader.cpp @@ -325,9 +325,7 @@ namespace NifOsg if (!textkeys->mTextKeys.empty()) created->getOrCreateUserDataContainer()->addUserObject(textkeys); - const std::uint64_t nifHash = nif->getHash(); - created->setUserValue(Misc::OsgUserValues::sFileHash, - std::string(reinterpret_cast(&nifHash), sizeof(nifHash))); + created->setUserValue(Misc::OsgUserValues::sFileHash, nif->getHash()); return created; } diff --git a/components/resource/bulletshape.hpp b/components/resource/bulletshape.hpp index 6dfa37aeda..cd8922ec8e 100644 --- a/components/resource/bulletshape.hpp +++ b/components/resource/bulletshape.hpp @@ -1,6 +1,7 @@ #ifndef OPENMW_COMPONENTS_RESOURCE_BULLETSHAPE_H #define OPENMW_COMPONENTS_RESOURCE_BULLETSHAPE_H +#include #include #include @@ -53,7 +54,7 @@ namespace Resource std::map mAnimatedShapes; std::string mFileName; - std::uint64_t mFileHash = 0; + std::string mFileHash; void setLocalScaling(const btVector3& scale); diff --git a/components/resource/bulletshapemanager.cpp b/components/resource/bulletshapemanager.cpp index 3803fbf669..da4672757a 100644 --- a/components/resource/bulletshapemanager.cpp +++ b/components/resource/bulletshapemanager.cpp @@ -169,10 +169,7 @@ osg::ref_ptr BulletShapeManager::getShape(const std::string & if (shape != nullptr) { shape->mFileName = normalized; - std::string fileHash; - constNode->getUserValue(Misc::OsgUserValues::sFileHash, fileHash); - if (!fileHash.empty()) - std::memcpy(&shape->mFileHash, fileHash.data(), std::min(fileHash.size(), sizeof(shape->mFileHash))); + constNode->getUserValue(Misc::OsgUserValues::sFileHash, shape->mFileHash); } } diff --git a/components/resource/scenemanager.cpp b/components/resource/scenemanager.cpp index 16d942ecec..5f2d78d2ed 100644 --- a/components/resource/scenemanager.cpp +++ b/components/resource/scenemanager.cpp @@ -506,7 +506,7 @@ namespace Resource options->setReadFileCallback(new ImageReadCallback(imageManager)); if (ext == "dae") options->setOptionString("daeUseSequencedTextureUnits"); - const std::uint64_t fileHash = Files::getHash(normalizedFilename, model); + const std::array fileHash = Files::getHash(normalizedFilename, model); osgDB::ReaderWriter::ReadResult result = reader->readNode(model, options); if (!result.success()) @@ -538,7 +538,7 @@ namespace Resource } node->setUserValue(Misc::OsgUserValues::sFileHash, - std::string(reinterpret_cast(&fileHash), sizeof(fileHash))); + std::string(reinterpret_cast(fileHash.data()), fileHash.size() * sizeof(std::uint64_t))); return node; }