Merge branch 'murmur_hash' into 'master'

Use MurmurHash3_x64_128 for Files::getHash

See merge request OpenMW/openmw!1404

apps/openmw_test_suite/CMakeLists.txt

@@ -62,6 +62,8 @@ if (GTEST_FOUND AND GMOCK_FOUND)
 
         esmloader/load.cpp
         esmloader/esmdata.cpp
+
+        files/hash.cpp
     )
 
     source_group(apps\\openmw_test_suite FILES openmw_test_suite.cpp ${UNITTEST_SRC_FILES})

apps/openmw_test_suite/files/hash.cpp

@@ -0,0 +1,55 @@
+#include <components/files/hash.hpp>
+#include <components/files/constrainedfilestream.hpp>
+
+#include <gtest/gtest.h>
+#include <gmock/gmock.h>
+
+#include <algorithm>
+#include <fstream>
+#include <sstream>
+#include <string>
+
+namespace
+{
+    using namespace testing;
+    using namespace Files;
+
+    struct Params
+    {
+        std::size_t mSize;
+        std::array<std::uint64_t, 2> mHash;
+    };
+
+    struct FilesGetHash : TestWithParam<Params> {};
+
+    TEST_P(FilesGetHash, shouldReturnHashForStringStream)
+    {
+        const std::string fileName = "fileName";
+        std::string content;
+        std::fill_n(std::back_inserter(content), GetParam().mSize, 'a');
+        std::istringstream stream(content);
+        EXPECT_EQ(getHash(fileName, stream), GetParam().mHash);
+    }
+
+    TEST_P(FilesGetHash, shouldReturnHashForConstrainedFileStream)
+    {
+        std::string fileName(UnitTest::GetInstance()->current_test_info()->name());
+        std::replace(fileName.begin(), fileName.end(), '/', '_');
+        std::string content;
+        std::fill_n(std::back_inserter(content), GetParam().mSize, 'a');
+        std::fstream(fileName, std::ios_base::out | std::ios_base::binary)
+            .write(content.data(), static_cast<std::streamsize>(content.size()));
+        const auto stream = Files::openConstrainedFileStream(fileName.data(), 0, content.size());
+        EXPECT_EQ(getHash(fileName, *stream), GetParam().mHash);
+    }
+
+    INSTANTIATE_TEST_SUITE_P(Params, FilesGetHash, Values(
+        Params {0, {0, 0}},
+        Params {1, {9607679276477937801ull, 16624257681780017498ull}},
+        Params {128, {15287858148353394424ull, 16818615825966581310ull}},
+        Params {1000, {11018119256083894017ull, 6631144854802791578ull}},
+        Params {4096, {11972283295181039100ull, 16027670129106775155ull}},
+        Params {4097, {16717956291025443060ull, 12856404199748778153ull}},
+        Params {5000, {15775925571142117787ull, 10322955217889622896ull}}
+    ));
+}

apps/openmw_test_suite/nifloader/testbulletnifloader.cpp

@@ -335,7 +335,7 @@ namespace
         MOCK_METHOD(void, setUseSkinning, (bool), (override));
         MOCK_METHOD(bool, getUseSkinning, (), (const, override));
         MOCK_METHOD(std::string, getFilename, (), (const, override));
-        MOCK_METHOD(std::uint64_t, getHash, (), (const, override));
+        MOCK_METHOD(std::string, getHash, (), (const, override));
         MOCK_METHOD(unsigned int, getVersion, (), (const, override));
         MOCK_METHOD(unsigned int, getUserVersion, (), (const, override));
         MOCK_METHOD(unsigned int, getBethVersion, (), (const, override));
@@ -382,7 +382,7 @@ namespace
             ),
             btVector3(4, 8, 12)
         };
-        const std::uint64_t mHash = 42;
+        const std::string mHash = "hash";
 
         TestBulletNifLoader()
         {

components/CMakeLists.txt

@@ -290,6 +290,7 @@ target_link_libraries(components
 
     Base64
     SQLite::SQLite3
+    smhasher
     )
 
 target_link_libraries(components ${BULLET_LIBRARIES})

components/files/hash.cpp

@@ -1,17 +1,17 @@
 #include "hash.hpp"
 
-#include <components/misc/hash.hpp>
+#include <extern/smhasher/MurmurHash3.h>
 
+#include <array>
 #include <cstdint>
-#include <functional>
 #include <istream>
 #include <string>
 
 namespace Files
 {
-    std::uint64_t getHash(const std::string& fileName, std::istream& stream)
+    std::array<std::uint64_t, 2> getHash(const std::string& fileName, std::istream& stream)
     {
-        std::uint64_t hash = std::hash<std::string> {}(fileName);
+        std::array<std::uint64_t, 2> hash {0, 0};
         try
         {
             const auto start = stream.tellg();
@@ -19,9 +19,14 @@ namespace Files
             stream.exceptions(std::ios_base::badbit);
             while (stream)
             {
-                std::uint64_t value = 0;
-                stream.read(reinterpret_cast<char*>(&value), sizeof(value));
-                Misc::hashCombine(hash, value);
+                std::array<char, 4096> value;
+                stream.read(value.data(), value.size());
+                const std::streamsize read = stream.gcount();
+                if (read == 0)
+                    break;
+                std::array<std::uint64_t, 2> blockHash {0, 0};
+                MurmurHash3_x64_128(value.data(), static_cast<int>(read), hash.data(), blockHash.data());
+                hash = blockHash;
             }
             stream.exceptions(exceptions);
             stream.clear();

components/files/hash.hpp

@@ -1,13 +1,14 @@
 #ifndef COMPONENTS_FILES_HASH_H
 #define COMPONENTS_FILES_HASH_H
 
+#include <array>
 #include <cstdint>
 #include <istream>
 #include <string>
 
 namespace Files
 {
-    std::uint64_t getHash(const std::string& fileName, std::istream& stream);
+    std::array<std::uint64_t, 2> getHash(const std::string& fileName, std::istream& stream);
 }
 
 #endif

components/nif/niffile.cpp

@@ -173,7 +173,8 @@ std::string NIFFile::printVersion(unsigned int version)
 
 void NIFFile::parse(Files::IStreamPtr stream)
 {
-    hash = Files::getHash(filename, *stream);
+    const std::array<std::uint64_t, 2> fileHash = Files::getHash(filename, *stream);
+    hash.append(reinterpret_cast<const char*>(fileHash.data()), fileHash.size() * sizeof(std::uint64_t));
 
     NIFStream nif (this, stream);
 

components/nif/niffile.hpp

@@ -34,7 +34,7 @@ struct File
 
     virtual std::string getFilename() const = 0;
 
-    virtual std::uint64_t getHash() const = 0;
+    virtual std::string getHash() const = 0;
 
     virtual unsigned int getVersion() const = 0;
 
@@ -52,7 +52,7 @@ class NIFFile final : public File
 
     /// File name, used for error messages and opening the file
     std::string filename;
-    std::uint64_t hash = 0;
+    std::string hash;
 
     /// Record list
     std::vector<Record*> records;
@@ -144,7 +144,7 @@ public:
     /// Get the name of the file
     std::string getFilename() const override { return filename; }
 
-    std::uint64_t getHash() const override { return hash; }
+    std::string getHash() const override { return hash; }
 
     /// Get the version of the NIF format used
     unsigned int getVersion() const override { return ver; }

components/nifosg/nifloader.cpp

@@ -325,9 +325,7 @@ namespace NifOsg
             if (!textkeys->mTextKeys.empty())
                 created->getOrCreateUserDataContainer()->addUserObject(textkeys);
 
-            const std::uint64_t nifHash = nif->getHash();
-            created->setUserValue(Misc::OsgUserValues::sFileHash,
-                std::string(reinterpret_cast<const char*>(&nifHash), sizeof(nifHash)));
+            created->setUserValue(Misc::OsgUserValues::sFileHash, nif->getHash());
 
             return created;
         }

components/resource/bulletshape.hpp

@@ -1,6 +1,7 @@
 #ifndef OPENMW_COMPONENTS_RESOURCE_BULLETSHAPE_H
 #define OPENMW_COMPONENTS_RESOURCE_BULLETSHAPE_H
 
+#include <array>
 #include <map>
 #include <memory>
 
@@ -53,7 +54,7 @@ namespace Resource
         std::map<int, int> mAnimatedShapes;
 
         std::string mFileName;
-        std::uint64_t mFileHash = 0;
+        std::string mFileHash;
 
         void setLocalScaling(const btVector3& scale);
 

components/resource/bulletshapemanager.cpp

@@ -169,10 +169,7 @@ osg::ref_ptr<const BulletShape> BulletShapeManager::getShape(const std::string &
             if (shape != nullptr)
             {
                 shape->mFileName = normalized;
-                std::string fileHash;
-                constNode->getUserValue(Misc::OsgUserValues::sFileHash, fileHash);
-                if (!fileHash.empty())
-                    std::memcpy(&shape->mFileHash, fileHash.data(), std::min(fileHash.size(), sizeof(shape->mFileHash)));
+                constNode->getUserValue(Misc::OsgUserValues::sFileHash, shape->mFileHash);
             }
         }
 

components/resource/scenemanager.cpp

@@ -506,7 +506,7 @@ namespace Resource
             options->setReadFileCallback(new ImageReadCallback(imageManager));
             if (ext == "dae") options->setOptionString("daeUseSequencedTextureUnits");
 
-            const std::uint64_t fileHash = Files::getHash(normalizedFilename, model);
+            const std::array<std::uint64_t, 2> fileHash = Files::getHash(normalizedFilename, model);
 
             osgDB::ReaderWriter::ReadResult result = reader->readNode(model, options);
             if (!result.success())
@@ -538,7 +538,7 @@ namespace Resource
             }
 
             node->setUserValue(Misc::OsgUserValues::sFileHash,
-                std::string(reinterpret_cast<const char*>(&fileHash), sizeof(fileHash)));
+                std::string(reinterpret_cast<const char*>(fileHash.data()), fileHash.size() * sizeof(std::uint64_t)));
 
             return node;
         }

extern/CMakeLists.txt

@@ -208,3 +208,5 @@ if (NOT OPENMW_USE_SYSTEM_SQLITE3)
     set(SQLite3_INCLUDE_DIR ${sqlite3_SOURCE_DIR}/ PARENT_SCOPE)
     set(SQLite3_LIBRARY sqlite3 PARENT_SCOPE)
 endif()
+
+add_subdirectory(smhasher)

extern/smhasher/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_library(smhasher STATIC MurmurHash3.cpp)
+target_include_directories(smhasher INTERFACE .)

extern/smhasher/MurmurHash3.cpp

@@ -0,0 +1,152 @@
+//-----------------------------------------------------------------------------
+// MurmurHash3 was written by Austin Appleby, and is placed in the public
+// domain. The author hereby disclaims copyright to this source code.
+
+// Note - The x86 and x64 versions do _not_ produce the same results, as the
+// algorithms are optimized for their respective platforms. You can still
+// compile and run any of them on any platform, but your performance with the
+// non-native version will be less than optimal.
+
+#include "MurmurHash3.h"
+
+#include <cstring>
+
+//-----------------------------------------------------------------------------
+// Platform-specific functions and macros
+
+// Microsoft Visual Studio
+
+#if defined(_MSC_VER)
+
+#define FORCE_INLINE	__forceinline
+
+#include <stdlib.h>
+
+#define ROTL64(x,y)	_rotl64(x,y)
+
+#define BIG_CONSTANT(x) (x)
+
+// Other compilers
+
+#else	// defined(_MSC_VER)
+
+#define	FORCE_INLINE inline __attribute__((always_inline))
+
+inline uint64_t rotl64 ( uint64_t x, int8_t r )
+{
+  return (x << r) | (x >> (64 - r));
+}
+
+#define ROTL64(x,y)	rotl64(x,y)
+
+#define BIG_CONSTANT(x) (x##LLU)
+
+#endif // !defined(_MSC_VER)
+
+//-----------------------------------------------------------------------------
+// Block read - if your platform needs to do endian-swapping or can only
+// handle aligned reads, do the conversion here
+
+FORCE_INLINE uint64_t getblock64 ( const uint64_t * p, int i )
+{
+  uint64_t result = 0;
+  std::memcpy(&result, p + i, sizeof(result));
+  return result;
+}
+
+//----------
+
+FORCE_INLINE uint64_t fmix64 ( uint64_t k )
+{
+  k ^= k >> 33;
+  k *= BIG_CONSTANT(0xff51afd7ed558ccd);
+  k ^= k >> 33;
+  k *= BIG_CONSTANT(0xc4ceb9fe1a85ec53);
+  k ^= k >> 33;
+
+  return k;
+}
+
+//-----------------------------------------------------------------------------
+
+void MurmurHash3_x64_128 ( const void * key, const int len,
+                           const uint64_t * seed, void * out )
+{
+  const uint8_t * data = (const uint8_t*)key;
+  const int nblocks = len / 16;
+
+  uint64_t h1 = seed[0];
+  uint64_t h2 = seed[1];
+
+  const uint64_t c1 = BIG_CONSTANT(0x87c37b91114253d5);
+  const uint64_t c2 = BIG_CONSTANT(0x4cf5ad432745937f);
+
+  //----------
+  // body
+
+  const uint64_t * blocks = (const uint64_t *)(data);
+
+  for(int i = 0; i < nblocks; i++)
+  {
+    uint64_t k1 = getblock64(blocks,i*2+0);
+    uint64_t k2 = getblock64(blocks,i*2+1);
+
+    k1 *= c1; k1  = ROTL64(k1,31); k1 *= c2; h1 ^= k1;
+
+    h1 = ROTL64(h1,27); h1 += h2; h1 = h1*5+0x52dce729;
+
+    k2 *= c2; k2  = ROTL64(k2,33); k2 *= c1; h2 ^= k2;
+
+    h2 = ROTL64(h2,31); h2 += h1; h2 = h2*5+0x38495ab5;
+  }
+
+  //----------
+  // tail
+
+  const uint8_t * tail = (const uint8_t*)(data + nblocks*16);
+
+  uint64_t k1 = 0;
+  uint64_t k2 = 0;
+
+  switch(len & 15)
+  {
+  case 15: k2 ^= ((uint64_t)tail[14]) << 48;
+  case 14: k2 ^= ((uint64_t)tail[13]) << 40;
+  case 13: k2 ^= ((uint64_t)tail[12]) << 32;
+  case 12: k2 ^= ((uint64_t)tail[11]) << 24;
+  case 11: k2 ^= ((uint64_t)tail[10]) << 16;
+  case 10: k2 ^= ((uint64_t)tail[ 9]) << 8;
+  case  9: k2 ^= ((uint64_t)tail[ 8]) << 0;
+           k2 *= c2; k2  = ROTL64(k2,33); k2 *= c1; h2 ^= k2;
+
+  case  8: k1 ^= ((uint64_t)tail[ 7]) << 56;
+  case  7: k1 ^= ((uint64_t)tail[ 6]) << 48;
+  case  6: k1 ^= ((uint64_t)tail[ 5]) << 40;
+  case  5: k1 ^= ((uint64_t)tail[ 4]) << 32;
+  case  4: k1 ^= ((uint64_t)tail[ 3]) << 24;
+  case  3: k1 ^= ((uint64_t)tail[ 2]) << 16;
+  case  2: k1 ^= ((uint64_t)tail[ 1]) << 8;
+  case  1: k1 ^= ((uint64_t)tail[ 0]) << 0;
+           k1 *= c1; k1  = ROTL64(k1,31); k1 *= c2; h1 ^= k1;
+  };
+
+  //----------
+  // finalization
+
+  h1 ^= len; h2 ^= len;
+
+  h1 += h2;
+  h2 += h1;
+
+  h1 = fmix64(h1);
+  h2 = fmix64(h2);
+
+  h1 += h2;
+  h2 += h1;
+
+  ((uint64_t*)out)[0] = h1;
+  ((uint64_t*)out)[1] = h2;
+}
+
+//-----------------------------------------------------------------------------
+

extern/smhasher/MurmurHash3.h

@@ -0,0 +1,33 @@
+//-----------------------------------------------------------------------------
+// MurmurHash3 was written by Austin Appleby, and is placed in the public
+// domain. The author hereby disclaims copyright to this source code.
+
+#ifndef _MURMURHASH3_H_
+#define _MURMURHASH3_H_
+
+//-----------------------------------------------------------------------------
+// Platform-specific functions and macros
+
+// Microsoft Visual Studio
+
+#if defined(_MSC_VER) && (_MSC_VER < 1600)
+
+typedef unsigned char uint8_t;
+typedef unsigned int uint32_t;
+typedef unsigned __int64 uint64_t;
+
+// Other compilers
+
+#else	// defined(_MSC_VER)
+
+#include <stdint.h>
+
+#endif // !defined(_MSC_VER)
+
+//-----------------------------------------------------------------------------
+
+void MurmurHash3_x64_128 ( const void * key, int len, const uint64_t * seed, void * out );
+
+//-----------------------------------------------------------------------------
+
+#endif // _MURMURHASH3_H_