mirror/OpenMW
1
0
Fork 0
mirror of https://gitlab.com/OpenMW/openmw.git synced 2025-01-30 03:32:36 +00:00
Code Issues Projects Releases Wiki Activity

Use MurmurHash3_x64_128 for file hash

Browse Source
This commit is contained in:
elsid 2021-11-15 17:40:22 +01:00
parent f85053d78c
commit a665a38aca
No known key found for this signature in database
GPG Key ID: B845CB9FEE18AB40
12 changed files with 85 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/openmw_test_suite/CMakeLists.txt
View File

@ -62,6 +62,8 @@ if (GTEST_FOUND AND GMOCK_FOUND)
esmloader/load.cpp
esmloader/esmdata.cpp
files/hash.cpp
)
source_group(apps\\openmw_test_suite FILES openmw_test_suite.cpp ${UNITTEST_SRC_FILES})

55
apps/openmw_test_suite/files/hash.cpp Normal file
View File

@ -0,0 +1,55 @@
#include <components/files/hash.hpp>
#include <components/files/constrainedfilestream.hpp>
#include <gtest/gtest.h>
#include <gmock/gmock.h>
#include <algorithm>
#include <fstream>
#include <sstream>
#include <string>
namespace
{
using namespace testing;
using namespace Files;
struct Params
{
std::size_t mSize;
std::array<std::uint64_t, 2> mHash;
};
struct FilesGetHash : TestWithParam<Params> {};
TEST_P(FilesGetHash, shouldReturnHashForStringStream)
{
const std::string fileName = "fileName";
std::string content;
std::fill_n(std::back_inserter(content), GetParam().mSize, 'a');
std::istringstream stream(content);
EXPECT_EQ(getHash(fileName, stream), GetParam().mHash);
}
TEST_P(FilesGetHash, shouldReturnHashForConstrainedFileStream)
{
std::string fileName(UnitTest::GetInstance()->current_test_info()->name());
std::replace(fileName.begin(), fileName.end(), '/', '_');
std::string content;
std::fill_n(std::back_inserter(content), GetParam().mSize, 'a');
std::fstream(fileName, std::ios_base::out | std::ios_base::binary)
.write(content.data(), static_cast<std::streamsize>(content.size()));
const auto stream = Files::openConstrainedFileStream(fileName.data(), 0, content.size());
EXPECT_EQ(getHash(fileName, *stream), GetParam().mHash);
}
INSTANTIATE_TEST_SUITE_P(Params, FilesGetHash, Values(
Params {0, {0, 0}},
Params {1, {9607679276477937801ull, 16624257681780017498ull}},
Params {128, {15287858148353394424ull, 16818615825966581310ull}},
Params {1000, {11018119256083894017ull, 6631144854802791578ull}},
Params {4096, {11972283295181039100ull, 16027670129106775155ull}},
Params {4097, {16717956291025443060ull, 12856404199748778153ull}},
Params {5000, {15775925571142117787ull, 10322955217889622896ull}}
));
}

4
apps/openmw_test_suite/nifloader/testbulletnifloader.cpp
View File

@ -335,7 +335,7 @@ namespace
MOCK_METHOD(void, setUseSkinning, (bool), (override));
MOCK_METHOD(bool, getUseSkinning, (), (const, override));
MOCK_METHOD(std::string, getFilename, (), (const, override));
MOCK_METHOD(std::uint64_t, getHash, (), (const, override));
MOCK_METHOD(std::string, getHash, (), (const, override));
MOCK_METHOD(unsigned int, getVersion, (), (const, override));
MOCK_METHOD(unsigned int, getUserVersion, (), (const, override));
MOCK_METHOD(unsigned int, getBethVersion, (), (const, override));
@ -382,7 +382,7 @@ namespace
),
btVector3(4, 8, 12)
};
const std::uint64_t mHash = 42;
const std::string mHash = "hash";
TestBulletNifLoader()
{

1
components/CMakeLists.txt
View File

@ -289,6 +289,7 @@ target_link_libraries(components
Base64
SQLite::SQLite3
smhasher
)
target_link_libraries(components ${BULLET_LIBRARIES})

19
components/files/hash.cpp
View File

@ -1,17 +1,17 @@
#include "hash.hpp"
#include <components/misc/hash.hpp>
#include <extern/smhasher/MurmurHash3.h>
#include <array>
#include <cstdint>
#include <functional>
#include <istream>
#include <string>
namespace Files
{
std::uint64_t getHash(const std::string& fileName, std::istream& stream)
std::array<std::uint64_t, 2> getHash(const std::string& fileName, std::istream& stream)
{
std::uint64_t hash = std::hash<std::string> {}(fileName);
std::array<std::uint64_t, 2> hash {0, 0};
try
{
const auto start = stream.tellg();
@ -19,9 +19,14 @@ namespace Files
stream.exceptions(std::ios_base::badbit);
while (stream)
{
std::uint64_t value = 0;
stream.read(reinterpret_cast<char*>(&value), sizeof(value));
Misc::hashCombine(hash, value);
std::array<char, 4096> value;
stream.read(value.data(), value.size());
const std::streamsize read = stream.gcount();
if (read == 0)
break;
std::array<std::uint64_t, 2> blockHash {0, 0};
MurmurHash3_x64_128(value.data(), static_cast<int>(read), hash.data(), blockHash.data());
hash = blockHash;
}
stream.exceptions(exceptions);
stream.clear();

3
components/files/hash.hpp
View File

@ -1,13 +1,14 @@
#ifndef COMPONENTS_FILES_HASH_H
#define COMPONENTS_FILES_HASH_H
#include <array>
#include <cstdint>
#include <istream>
#include <string>
namespace Files
{
std::uint64_t getHash(const std::string& fileName, std::istream& stream);
std::array<std::uint64_t, 2> getHash(const std::string& fileName, std::istream& stream);
}
#endif

3
components/nif/niffile.cpp
View File

@ -173,7 +173,8 @@ std::string NIFFile::printVersion(unsigned int version)
void NIFFile::parse(Files::IStreamPtr stream)
{
hash = Files::getHash(filename, *stream);
const std::array<std::uint64_t, 2> fileHash = Files::getHash(filename, *stream);
hash.append(reinterpret_cast<const char*>(fileHash.data()), fileHash.size() * sizeof(std::uint64_t));
NIFStream nif (this, stream);

6
components/nif/niffile.hpp
View File

@ -34,7 +34,7 @@ struct File
virtual std::string getFilename() const = 0;
virtual std::uint64_t getHash() const = 0;
virtual std::string getHash() const = 0;
virtual unsigned int getVersion() const = 0;
@ -52,7 +52,7 @@ class NIFFile final : public File
/// File name, used for error messages and opening the file
std::string filename;
std::uint64_t hash = 0;
std::string hash;
/// Record list
std::vector<Record*> records;
@ -144,7 +144,7 @@ public:
/// Get the name of the file
std::string getFilename() const override { return filename; }
std::uint64_t getHash() const override { return hash; }
std::string getHash() const override { return hash; }
/// Get the version of the NIF format used
unsigned int getVersion() const override { return ver; }

4
components/nifosg/nifloader.cpp
View File

@ -325,9 +325,7 @@ namespace NifOsg
if (!textkeys->mTextKeys.empty())
created->getOrCreateUserDataContainer()->addUserObject(textkeys);
const std::uint64_t nifHash = nif->getHash();
created->setUserValue(Misc::OsgUserValues::sFileHash,
std::string(reinterpret_cast<const char*>(&nifHash), sizeof(nifHash)));
created->setUserValue(Misc::OsgUserValues::sFileHash, nif->getHash());
return created;
}

3
components/resource/bulletshape.hpp
View File

@ -1,6 +1,7 @@
#ifndef OPENMW_COMPONENTS_RESOURCE_BULLETSHAPE_H
#define OPENMW_COMPONENTS_RESOURCE_BULLETSHAPE_H
#include <array>
#include <map>
#include <memory>
@ -53,7 +54,7 @@ namespace Resource
std::map<int, int> mAnimatedShapes;
std::string mFileName;
std::uint64_t mFileHash = 0;
std::string mFileHash;
void setLocalScaling(const btVector3& scale);

5
components/resource/bulletshapemanager.cpp
View File

@ -169,10 +169,7 @@ osg::ref_ptr<const BulletShape> BulletShapeManager::getShape(const std::string &
if (shape != nullptr)
{
shape->mFileName = normalized;
std::string fileHash;
constNode->getUserValue(Misc::OsgUserValues::sFileHash, fileHash);
if (!fileHash.empty())
std::memcpy(&shape->mFileHash, fileHash.data(), std::min(fileHash.size(), sizeof(shape->mFileHash)));
constNode->getUserValue(Misc::OsgUserValues::sFileHash, shape->mFileHash);
}
}

4
components/resource/scenemanager.cpp
View File

@ -506,7 +506,7 @@ namespace Resource
options->setReadFileCallback(new ImageReadCallback(imageManager));
if (ext == "dae") options->setOptionString("daeUseSequencedTextureUnits");
const std::uint64_t fileHash = Files::getHash(normalizedFilename, model);
const std::array<std::uint64_t, 2> fileHash = Files::getHash(normalizedFilename, model);
osgDB::ReaderWriter::ReadResult result = reader->readNode(model, options);
if (!result.success())
@ -538,7 +538,7 @@ namespace Resource
}
node->setUserValue(Misc::OsgUserValues::sFileHash,
std::string(reinterpret_cast<const char*>(&fileHash), sizeof(fileHash)));
std::string(reinterpret_cast<const char*>(fileHash.data()), fileHash.size() * sizeof(std::uint64_t)));
return node;
}