Merge branch 'tes4_bsa' into 'master'

Read BSA files in TES4 / TES5 formats

See merge request OpenMW/openmw!56

.gitlab-ci.yml

@@ -13,7 +13,7 @@ Debian:
   before_script:
     - export APT_CACHE_DIR=`pwd`/apt-cache && mkdir -pv $APT_CACHE_DIR
     - apt-get update -yq 
-    - apt-get -o dir::cache::archives="$APT_CACHE_DIR" install -y cmake libboost-filesystem-dev libboost-program-options-dev libboost-system-dev libavcodec-dev libavformat-dev libavutil-dev libswscale-dev libswresample-dev libsdl2-dev libqt4-dev libopenal-dev libopenscenegraph-3.4-dev libunshield-dev libtinyxml-dev
+    - apt-get -o dir::cache::archives="$APT_CACHE_DIR" install -y cmake libboost-filesystem-dev libboost-program-options-dev libboost-system-dev libboost-iostreams-dev libavcodec-dev libavformat-dev libavutil-dev libswscale-dev libswresample-dev libsdl2-dev libqt4-dev libopenal-dev libopenscenegraph-3.4-dev libunshield-dev libtinyxml-dev
   # - apt-get install -y libmygui-dev libbullet-dev  # to be updated to latest below because stretch is too old
     - curl -L http://archive.ubuntu.com/ubuntu/pool/universe/b/bullet/libbullet-dev_2.87+dfsg-2_amd64.deb -o libbullet-dev_2.87+dfsg-2_amd64.deb
     - curl -L http://archive.ubuntu.com/ubuntu/pool/universe/b/bullet/libbullet2.87_2.87+dfsg-2_amd64.deb -o libbullet2.87_2.87+dfsg-2_amd64.deb

CMakeLists.txt

@@ -296,9 +296,9 @@ if(QT_STATIC)
 endif()
 
 
-set(BOOST_COMPONENTS system filesystem program_options)
+set(BOOST_COMPONENTS system filesystem program_options iostreams)
 if(WIN32)
-    set(BOOST_COMPONENTS ${BOOST_COMPONENTS} locale)
+    set(BOOST_COMPONENTS ${BOOST_COMPONENTS} locale zlib)
 endif(WIN32)
 
 IF(BOOST_STATIC)
@@ -610,6 +610,7 @@ add_subdirectory (extern/oics)
 if (BUILD_OPENCS)
     add_subdirectory (extern/osgQt)
 endif()
+add_subdirectory (extern/BSAOpt)
 
 # Components
 add_subdirectory (components)

components/CMakeLists.txt

@@ -33,7 +33,7 @@ add_component_dir (settings
     )
 
 add_component_dir (bsa
-    bsa_file
+    bsa_file compressedbsafile memorystream
     )
 
 add_component_dir (vfs
@@ -218,6 +218,7 @@ target_link_libraries(components
     ${Boost_FILESYSTEM_LIBRARY}
     ${Boost_THREAD_LIBRARY}
     ${Boost_PROGRAM_OPTIONS_LIBRARY}
+    ${Boost_IOSTREAMS_LIBRARY}
     ${OSG_LIBRARIES}
     ${OPENTHREADS_LIBRARIES}
     ${OSGPARTICLE_LIBRARIES}
@@ -231,6 +232,7 @@ target_link_libraries(components
     ${SDL2_LIBRARIES}
     ${OPENGL_gl_LIBRARY}
     ${MyGUI_LIBRARIES}
+    ${BSAOPTHASH_LIBRARIES}
     RecastNavigation::DebugUtils
     RecastNavigation::Detour
     RecastNavigation::Recast
@@ -238,7 +240,8 @@ target_link_libraries(components
 
 if (WIN32)
     target_link_libraries(components
-    ${Boost_LOCALE_LIBRARY})
+    ${Boost_LOCALE_LIBRARY}
+    ${Boost_ZLIB_LIBRARY})
 endif()
 
 if (USE_QT)

components/bsa/bsa_file.cpp

@@ -35,7 +35,7 @@ using namespace Bsa;
 /// Error handling
 void BSAFile::fail(const string &msg)
 {
-    throw std::runtime_error("BSA Error: " + msg + "\nArchive: " + filename);
+    throw std::runtime_error("BSA Error: " + msg + "\nArchive: " + mFilename);
 }
 
 /// Read header information from the input source
@@ -71,10 +71,10 @@ void BSAFile::readHeader()
      *   the beginning of this buffer.
      *
      */
-    assert(!isLoaded);
+    assert(!mIsLoaded);
 
     namespace bfs = boost::filesystem;
-    bfs::ifstream input(bfs::path(filename), std::ios_base::binary);
+    bfs::ifstream input(bfs::path(mFilename), std::ios_base::binary);
 
     // Total archive size
     std::streamoff fsize = 0;
@@ -117,8 +117,8 @@ void BSAFile::readHeader()
     input.read(reinterpret_cast<char*>(&offsets[0]), 12*filenum);
 
     // Read the string table
-    stringBuf.resize(dirsize-12*filenum);
-    input.read(&stringBuf[0], stringBuf.size());
+    mStringBuf.resize(dirsize-12*filenum);
+    input.read(&mStringBuf[0], mStringBuf.size());
 
     // Check our position
     assert(input.tellg() == std::streampos(12+dirsize));
@@ -129,40 +129,40 @@ void BSAFile::readHeader()
     size_t fileDataOffset = 12 + dirsize + 8*filenum;
 
     // Set up the the FileStruct table
-    files.resize(filenum);
+    mFiles.resize(filenum);
     for(size_t i=0;i<filenum;i++)
     {
-        FileStruct &fs = files[i];
+        FileStruct &fs = mFiles[i];
         fs.fileSize = offsets[i*2];
         fs.offset = offsets[i*2+1] + fileDataOffset;
-        fs.name = &stringBuf[offsets[2*filenum+i]];
+        fs.name = &mStringBuf[offsets[2*filenum+i]];
 
         if(fs.offset + fs.fileSize > fsize)
             fail("Archive contains offsets outside itself");
 
         // Add the file name to the lookup
-        lookup[fs.name] = i;
+        mLookup[fs.name] = i;
     }
 
-    isLoaded = true;
+    mIsLoaded = true;
 }
 
 /// Get the index of a given file name, or -1 if not found
 int BSAFile::getIndex(const char *str) const
 {
-    Lookup::const_iterator it = lookup.find(str);
-    if(it == lookup.end())
+    Lookup::const_iterator it = mLookup.find(str);
+    if(it == mLookup.end())
         return -1;
 
     int res = it->second;
-    assert(res >= 0 && (size_t)res < files.size());
+    assert(res >= 0 && (size_t)res < mFiles.size());
     return res;
 }
 
 /// Open an archive file.
 void BSAFile::open(const string &file)
 {
-    filename = file;
+    mFilename = file;
     readHeader();
 }
 
@@ -173,12 +173,12 @@ Files::IStreamPtr BSAFile::getFile(const char *file)
     if(i == -1)
         fail("File not found: " + string(file));
 
-    const FileStruct &fs = files[i];
+    const FileStruct &fs = mFiles[i];
 
-    return Files::openConstrainedFileStream (filename.c_str (), fs.offset, fs.fileSize);
+    return Files::openConstrainedFileStream (mFilename.c_str (), fs.offset, fs.fileSize);
 }
 
 Files::IStreamPtr BSAFile::getFile(const FileStruct *file)
 {
-    return Files::openConstrainedFileStream (filename.c_str (), file->offset, file->fileSize);
+    return Files::openConstrainedFileStream (mFilename.c_str (), file->offset, file->fileSize);
 }

components/bsa/bsa_file.hpp

@@ -56,18 +56,18 @@ public:
     };
     typedef std::vector<FileStruct> FileList;
 
-private:
+protected:
     /// Table of files in this archive
-    FileList files;
+    FileList mFiles;
 
     /// Filename string buffer
-    std::vector<char> stringBuf;
+    std::vector<char> mStringBuf;
 
     /// True when an archive has been loaded
-    bool isLoaded;
+    bool mIsLoaded;
 
     /// Used for error messages
-    std::string filename;
+    std::string mFilename;
 
     /// Case insensitive string comparison
     struct iltstr
@@ -81,13 +81,16 @@ private:
         checks are case insensitive.
     */
     typedef std::map<const char*, int, iltstr> Lookup;
-    Lookup lookup;
+    Lookup mLookup;
 
     /// Error handling
     void fail(const std::string &msg);
 
     /// Read header information from the input source
-    void readHeader();
+    virtual void readHeader();
+
+    /// Read header information from the input source
+
 
     /// Get the index of a given file name, or -1 if not found
     /// @note Thread safe.
@@ -100,7 +103,10 @@ public:
      */
 
     BSAFile()
-      : isLoaded(false)
+      : mIsLoaded(false)
+    { }
+
+    virtual ~BSAFile()
     { }
 
     /// Open an archive file.
@@ -112,24 +118,24 @@ public:
      */
 
     /// Check if a file exists
-    bool exists(const char *file) const
+    virtual bool exists(const char *file) const
     { return getIndex(file) != -1; }
 
     /** Open a file contained in the archive. Throws an exception if the
         file doesn't exist.
      * @note Thread safe.
     */
-    Files::IStreamPtr getFile(const char *file);
+    virtual Files::IStreamPtr getFile(const char *file);
 
     /** Open a file contained in the archive.
      * @note Thread safe.
     */
-    Files::IStreamPtr getFile(const FileStruct* file);
+    virtual Files::IStreamPtr getFile(const FileStruct* file);
 
     /// Get a list of all files
     /// @note Thread safe.
     const FileList &getList() const
-    { return files; }
+    { return mFiles; }
 };
 
 }

components/bsa/compressedbsafile.cpp

@@ -0,0 +1,428 @@
+/*
+  OpenMW - The completely unofficial reimplementation of Morrowind
+  Copyright (C) 2008-2010  Nicolay Korslund
+  Email: < korslund@gmail.com >
+  WWW: http://openmw.sourceforge.net/
+
+  This file (compressedbsafile.cpp) is part of the OpenMW package.
+
+  OpenMW is distributed as free software: you can redistribute it
+  and/or modify it under the terms of the GNU General Public License
+  version 3, as published by the Free Software Foundation.
+
+  This program is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  version 3 along with this program. If not, see
+  http://www.gnu.org/licenses/ .
+
+  Compressed BSA stuff added by cc9cii 2018
+
+ */
+#include "compressedbsafile.hpp"
+
+#include <stdexcept>
+#include <cassert>
+
+#include <boost/scoped_array.hpp>
+#include <boost/algorithm/string.hpp>
+#include <boost/filesystem/path.hpp>
+#include <boost/filesystem/fstream.hpp>
+
+#include <extern/BSAOpt/hash.hpp> // see: http://en.uesp.net/wiki/Tes4Mod:Hash_Calculation
+
+#include <boost/iostreams/filtering_streambuf.hpp>
+#include <boost/iostreams/copy.hpp>
+#include <boost/iostreams/filter/zlib.hpp>
+#include <boost/iostreams/stream.hpp>
+#include <boost/iostreams/device/array.hpp>
+#include <components/bsa/memorystream.hpp>
+
+namespace Bsa
+{
+//special marker for invalid records,
+//equal to max uint32_t value
+const uint32_t CompressedBSAFile::sInvalidOffset = std::numeric_limits<uint32_t>::max();
+
+//bit marking compression on file size
+const uint32_t CompressedBSAFile::sCompressedFlag = 1u << 30u;
+
+
+CompressedBSAFile::FileRecord::FileRecord() : size(0), offset(sInvalidOffset)
+{ }
+
+bool CompressedBSAFile::FileRecord::isValid() const
+{
+    return offset != sInvalidOffset;
+}
+
+bool CompressedBSAFile::FileRecord::isCompressed(bool bsaCompressedByDefault) const
+{
+    bool recordCompressionFlagEnabled = ((size & sCompressedFlag) == sCompressedFlag);
+
+    //record is compressed when:
+    //- bsaCompressedByDefault flag is set and 30th bit is NOT set, or
+    //- bsaCompressedByDefault flag is NOT set and 30th bit is set
+    //record is NOT compressed when:
+    //- bsaCompressedByDefault flag is NOT set and 30th bit is NOT set, or
+    //- bsaCompressedByDefault flag is set and 30th bit is set
+    return (bsaCompressedByDefault != recordCompressionFlagEnabled);
+}
+
+std::uint32_t CompressedBSAFile::FileRecord::getSizeWithoutCompressionFlag() const {
+    return size & (~sCompressedFlag);
+}
+
+void CompressedBSAFile::getBZString(std::string& str, std::istream& filestream)
+{
+    char size = 0;
+    filestream.read(&size, 1);
+
+    boost::scoped_array<char> buf(new char[size]);
+    filestream.read(buf.get(), size);
+
+    if (buf[size - 1] != 0)
+    {
+        str.assign(buf.get(), size);
+        if (str.size() != ((size_t)size)) {
+            fail("getBZString string size mismatch");
+        }
+    }
+    else
+    {
+        str.assign(buf.get(), size - 1); // don't copy null terminator
+        if (str.size() != ((size_t)size - 1)) {
+            fail("getBZString string size mismatch (null terminator)");
+        }
+    }
+}
+
+CompressedBSAFile::CompressedBSAFile()
+    : mCompressedByDefault(false), mEmbeddedFileNames(false)
+{ }
+
+CompressedBSAFile::~CompressedBSAFile()
+{ }
+
+/// Read header information from the input source
+void CompressedBSAFile::readHeader()
+{
+    assert(!mIsLoaded);
+
+    namespace bfs = boost::filesystem;
+    bfs::ifstream input(bfs::path(mFilename), std::ios_base::binary);
+
+    // Total archive size
+    std::streamoff fsize = 0;
+    if(input.seekg(0, std::ios_base::end))
+    {
+        fsize = input.tellg();
+        input.seekg(0);
+    }
+
+    if(fsize < 36) // header is 36 bytes
+        fail("File too small to be a valid BSA archive");
+
+    // Get essential header numbers
+    //size_t dirsize, filenum;
+    std::uint32_t archiveFlags, folderCount, totalFileNameLength;
+    {
+        // First 36 bytes
+        std::uint32_t header[9];
+
+        input.read(reinterpret_cast<char*>(header), 36);
+
+        if(header[0] != 0x00415342 /*"BSA\x00"*/ || (header[1] != 0x67 /*TES4*/ && header[1] != 0x68 /*TES5*/))
+            fail("Unrecognized TES4 BSA header");
+
+        // header[2] is offset, should be 36 = 0x24 which is the size of the header
+
+        // Oblivion - Meshes.bsa
+        //
+        // 0111 1000 0111 = 0x0787
+        //  ^^^ ^     ^^^
+        //  ||| |     ||+-- has names for dirs  (mandatory?)
+        //  ||| |     |+--- has names for files (mandatory?)
+        //  ||| |     +---- files are compressed by default
+        //  ||| |
+        //  ||| +---------- unknown (TES5: retain strings during startup)
+        //  ||+------------ unknown (TES5: embedded file names)
+        //  |+------------- unknown
+        //  +-------------- unknown
+        //
+        archiveFlags          = header[3];
+        folderCount           = header[4];
+        // header[5] - fileCount
+        // totalFolderNameLength = header[6];
+        totalFileNameLength   = header[7];
+        // header[8]; // fileFlags : an opportunity to optimize here
+
+        mCompressedByDefault = (archiveFlags & 0x4) != 0;
+        mEmbeddedFileNames = header[1] == 0x68 /*TES5*/ && (archiveFlags & 0x100) != 0;
+    }
+
+    // folder records
+    std::uint64_t hash;
+    FolderRecord fr;
+    for (std::uint32_t i = 0; i < folderCount; ++i)
+    {
+        input.read(reinterpret_cast<char*>(&hash), 8);
+        input.read(reinterpret_cast<char*>(&fr.count), 4); // not sure purpose of count
+        input.read(reinterpret_cast<char*>(&fr.offset), 4); // not sure purpose of offset
+
+        std::map<std::uint64_t, FolderRecord>::const_iterator lb = mFolders.lower_bound(hash);
+        if (lb != mFolders.end() && !(mFolders.key_comp()(hash, lb->first)))
+            fail("Archive found duplicate folder name hash");
+        else
+            mFolders.insert(lb, std::pair<std::uint64_t, FolderRecord>(hash, fr));
+    }
+
+    // file record blocks
+    std::uint64_t fileHash;
+    FileRecord file;
+
+    std::string folder("");
+    std::uint64_t folderHash;
+    if ((archiveFlags & 0x1) == 0)
+        folderCount = 1; // TODO: not tested - unit test necessary
+
+    mFiles.clear();
+    std::vector<std::string> fullPaths;
+    
+    for (std::uint32_t i = 0; i < folderCount; ++i)
+    {
+        if ((archiveFlags & 0x1) != 0)
+            getBZString(folder, input);
+
+        std::string emptyString;
+        folderHash = GenOBHash(folder, emptyString);
+
+        std::map<std::uint64_t, FolderRecord>::iterator iter = mFolders.find(folderHash);
+        if (iter == mFolders.end())
+            fail("Archive folder name hash not found");
+
+        for (std::uint32_t j = 0; j < iter->second.count; ++j)
+        {
+            input.read(reinterpret_cast<char*>(&fileHash), 8);
+            input.read(reinterpret_cast<char*>(&file.size), 4);
+            input.read(reinterpret_cast<char*>(&file.offset), 4);
+
+            std::map<std::uint64_t, FileRecord>::const_iterator lb = iter->second.files.lower_bound(fileHash);
+            if (lb != iter->second.files.end() && !(iter->second.files.key_comp()(fileHash, lb->first)))
+                fail("Archive found duplicate file name hash");
+
+            iter->second.files.insert(lb, std::pair<std::uint64_t, FileRecord>(fileHash, file));
+
+            FileStruct fileStruct;
+            fileStruct.fileSize = file.getSizeWithoutCompressionFlag();
+            fileStruct.offset = file.offset;
+            fileStruct.name = nullptr;
+            mFiles.push_back(fileStruct);
+
+            fullPaths.push_back(folder);
+        }
+    }
+
+    // file record blocks
+    if ((archiveFlags & 0x2) != 0)
+    {
+        mStringBuf.resize(totalFileNameLength);
+        input.read(&mStringBuf[0], mStringBuf.size()); // TODO: maybe useful in building a lookup map?
+    }
+
+    size_t mStringBuffOffset = 0;
+    size_t totalStringsSize = 0;
+    for (std::uint32_t fileIndex = 0; fileIndex < mFiles.size(); ++fileIndex) {
+
+        if (mStringBuffOffset >= totalFileNameLength) {
+            fail("Corrupted names record in BSA file");
+        }
+
+        //The vector guarantees that its elements occupy contiguous memory
+        mFiles[fileIndex].name = reinterpret_cast<char*>(mStringBuf.data() + mStringBuffOffset);
+
+        fullPaths.at(fileIndex) += "\\" + std::string(mStringBuf.data() + mStringBuffOffset);
+
+        while (mStringBuffOffset < totalFileNameLength) {
+            if (mStringBuf[mStringBuffOffset] != '\0') {
+                mStringBuffOffset++;
+            }
+            else {
+                mStringBuffOffset++;
+                break;
+            }
+        }
+        //we want to keep one more 0 character at the end of each string
+        totalStringsSize += fullPaths.at(fileIndex).length() + 1u;
+    }
+    mStringBuf.resize(totalStringsSize);
+
+    mStringBuffOffset = 0;
+    for (std::uint32_t fileIndex = 0u; fileIndex < mFiles.size(); fileIndex++) {
+        size_t stringLength = fullPaths.at(fileIndex).length();
+
+        std::copy(fullPaths.at(fileIndex).c_str(),
+            //plus 1 because we also want to copy 0 at the end of the string
+            fullPaths.at(fileIndex).c_str() + stringLength + 1u,
+            mStringBuf.data() + mStringBuffOffset);
+
+        mFiles[fileIndex].name = reinterpret_cast<char*>(mStringBuf.data() + mStringBuffOffset);
+
+        mLookup[reinterpret_cast<char*>(mStringBuf.data() + mStringBuffOffset)] = fileIndex;
+        mStringBuffOffset += stringLength + 1u;
+    }
+
+    if (mStringBuffOffset != mStringBuf.size()) {
+        fail("Could not resolve names of files in BSA file");
+    }
+
+    convertCompressedSizesToUncompressed();
+    mIsLoaded = true;
+}
+
+CompressedBSAFile::FileRecord CompressedBSAFile::getFileRecord(const std::string& str) const
+{
+    boost::filesystem::path p(str);
+    std::string stem = p.stem().string();
+    std::string ext = p.extension().string();
+    std::string filename = p.filename().string();
+    p.remove_filename();
+
+    std::string folder = p.string();
+    // GenOBHash already converts to lowercase and replaces file separators but not for path
+    boost::algorithm::to_lower(folder);
+    std::replace(folder.begin(), folder.end(), '/', '\\');
+
+    std::string emptyString;
+    std::uint64_t folderHash = GenOBHash(folder, emptyString);
+
+    std::map<std::uint64_t, FolderRecord>::const_iterator it = mFolders.find(folderHash);
+    if (it == mFolders.end())
+        return FileRecord(); // folder not found, return default which has offset of sInvalidOffset
+
+    boost::algorithm::to_lower(stem);
+    boost::algorithm::to_lower(ext);
+    std::uint64_t fileHash = GenOBHashPair(stem, ext);
+    std::map<std::uint64_t, FileRecord>::const_iterator iter = it->second.files.find(fileHash);
+    if (iter == it->second.files.end())
+        return FileRecord(); // file not found, return default which has offset of sInvalidOffset
+
+    return iter->second;
+}
+
+Files::IStreamPtr CompressedBSAFile::getFile(const FileStruct* file) 
+{
+    FileRecord fileRec = getFileRecord(file->name);
+    if (!fileRec.isValid()) {
+        fail("File not found: " + std::string(file->name));
+    }
+    return getFile(fileRec);
+}
+
+Files::IStreamPtr CompressedBSAFile::getFile(const char* file)
+{
+    FileRecord fileRec = getFileRecord(file);
+    if (!fileRec.isValid()) {
+        fail("File not found: " + std::string(file));
+    }
+    return getFile(fileRec);
+}
+
+Files::IStreamPtr CompressedBSAFile::getFile(const FileRecord& fileRecord)
+{
+    if (fileRecord.isCompressed(mCompressedByDefault)) {
+        Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, fileRecord.getSizeWithoutCompressionFlag());
+
+        std::istream* fileStream = streamPtr.get();
+
+        if (mEmbeddedFileNames) {
+            std::string embeddedFileName;
+            getBZString(embeddedFileName, *fileStream);
+        }
+
+        uint32_t uncompressedSize = 0u;
+        fileStream->read(reinterpret_cast<char*>(&uncompressedSize), sizeof(uncompressedSize));
+
+        boost::iostreams::filtering_streambuf<boost::iostreams::input> inputStreamBuf;
+        inputStreamBuf.push(boost::iostreams::zlib_decompressor());
+        inputStreamBuf.push(*fileStream);
+
+        std::shared_ptr<Bsa::MemoryInputStream> memoryStreamPtr = std::make_shared<MemoryInputStream>(uncompressedSize);
+
+        boost::iostreams::basic_array_sink<char> sr(memoryStreamPtr->getRawData(), uncompressedSize);
+        boost::iostreams::copy(inputStreamBuf, sr);
+
+        return std::shared_ptr<std::istream>(memoryStreamPtr, (std::istream*)memoryStreamPtr.get());
+    }
+
+    return Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, fileRecord.size);
+}
+
+BsaVersion CompressedBSAFile::detectVersion(std::string filePath)
+{
+    namespace bfs = boost::filesystem;
+    bfs::ifstream input(bfs::path(filePath), std::ios_base::binary);
+
+    // Total archive size
+    std::streamoff fsize = 0;
+    if (input.seekg(0, std::ios_base::end))
+    {
+        fsize = input.tellg();
+        input.seekg(0);
+    }
+
+    if (fsize < 12) {
+        return BSAVER_UNKNOWN;
+    }
+
+    // Get essential header numbers
+
+    // First 12 bytes
+    uint32_t head[3];
+
+    input.read(reinterpret_cast<char*>(head), 12);
+
+    if (head[0] == static_cast<uint32_t>(BSAVER_UNCOMPRESSED)) {
+        return BSAVER_UNCOMPRESSED;
+    }
+
+    if (head[0] == static_cast<uint32_t>(BSAVER_COMPRESSED)) {
+        return BSAVER_COMPRESSED;
+    }
+
+    return BSAVER_UNKNOWN;
+}
+
+//mFiles used by OpenMW expects uncompressed sizes
+void CompressedBSAFile::convertCompressedSizesToUncompressed()
+{
+    for (auto iter = mFiles.begin(); iter != mFiles.end(); ++iter)
+    {
+        const FileRecord& fileRecord = getFileRecord(iter->name);
+        if (!fileRecord.isValid())
+        {
+            fail("Could not find file " + std::string(iter->name) + " in BSA");
+        }
+
+        if (!fileRecord.isCompressed(mCompressedByDefault))
+        {
+            //no need to fix fileSize in mFiles - uncompressed size already set
+            continue;
+        }
+
+        Files::IStreamPtr dataBegin = Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, fileRecord.getSizeWithoutCompressionFlag());
+
+        if (mEmbeddedFileNames)
+        {
+            std::string embeddedFileName;
+            getBZString(embeddedFileName, *(dataBegin.get()));
+        }
+
+        dataBegin->read(reinterpret_cast<char*>(&(iter->fileSize)), sizeof(iter->fileSize));
+    }
+}
+
+} //namespace Bsa

components/bsa/compressedbsafile.hpp

@@ -0,0 +1,101 @@
+/*
+  OpenMW - The completely unofficial reimplementation of Morrowind
+  Copyright (C) 2008-2010  Nicolay Korslund
+  Email: < korslund@gmail.com >
+  WWW: http://openmw.sourceforge.net/
+
+  This file (compressedbsafile.hpp) is part of the OpenMW package.
+
+  OpenMW is distributed as free software: you can redistribute it
+  and/or modify it under the terms of the GNU General Public License
+  version 3, as published by the Free Software Foundation.
+
+  This program is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  version 3 along with this program. If not, see
+  http://www.gnu.org/licenses/ .
+
+  Compressed BSA stuff added by cc9cii 2018
+
+ */
+
+#ifndef BSA_COMPRESSED_BSA_FILE_H
+#define BSA_COMPRESSED_BSA_FILE_H
+
+#include <stdint.h>
+#include <string>
+#include <vector>
+#include <map>
+#include <components/bsa/bsa_file.hpp>
+
+namespace Bsa
+{
+    enum BsaVersion
+    {
+        BSAVER_UNKNOWN = 0x0,
+        BSAVER_UNCOMPRESSED = 0x100,
+        BSAVER_COMPRESSED = 0x415342 //B, S, A
+    };
+
+    class CompressedBSAFile : public BSAFile
+    {
+    private:
+        //special marker for invalid records,
+        //equal to max uint32_t value
+        static const uint32_t sInvalidOffset;
+
+        //bit marking compression on file size
+        static const uint32_t sCompressedFlag;
+
+        struct FileRecord
+        {
+            std::uint32_t size;
+            std::uint32_t offset;
+
+            FileRecord();
+            bool isCompressed(bool bsaCompressedByDefault) const;
+            bool isValid() const;
+            std::uint32_t getSizeWithoutCompressionFlag() const;
+        };
+        
+        //if files in BSA  without 30th bit enabled are compressed
+        bool mCompressedByDefault;
+
+        //if each file record begins with BZ string with file name
+        bool mEmbeddedFileNames;
+
+        struct FolderRecord
+        {
+            std::uint32_t count;
+            std::uint32_t offset;
+            std::map<std::uint64_t, FileRecord> files;
+        };
+        std::map<std::uint64_t, FolderRecord> mFolders;
+
+        FileRecord getFileRecord(const std::string& str) const;
+        
+        void getBZString(std::string& str, std::istream& filestream);
+        //mFiles used by OpenMW will contain uncompressed file sizes
+        void convertCompressedSizesToUncompressed();
+        Files::IStreamPtr getFile(const FileRecord& fileRecord);
+    public:
+        CompressedBSAFile();
+        virtual ~CompressedBSAFile();
+
+        //checks version of BSA from file header
+        static BsaVersion detectVersion(std::string filePath);
+
+        /// Read header information from the input source
+        virtual void readHeader();
+       
+        Files::IStreamPtr getFile(const char* filePath);
+        Files::IStreamPtr getFile(const FileStruct* fileStruct);
+        
+    };
+}
+
+#endif

components/bsa/memorystream.cpp

@@ -0,0 +1,48 @@
+/*
+  OpenMW - The completely unofficial reimplementation of Morrowind
+  Copyright (C) 2008-2010  Nicolay Korslund
+  Email: < korslund@gmail.com >
+  WWW: http://openmw.sourceforge.net/
+
+  This file (memorystream.cpp) is part of the OpenMW package.
+
+  OpenMW is distributed as free software: you can redistribute it
+  and/or modify it under the terms of the GNU General Public License
+  version 3, as published by the Free Software Foundation.
+
+  This program is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  version 3 along with this program. If not, see
+  http://www.gnu.org/licenses/ .
+
+  Compressed BSA upgrade added by Azdul 2019
+
+ */
+#include "memorystream.hpp"
+
+
+namespace Bsa
+{
+MemoryInputStreamBuf::MemoryInputStreamBuf(size_t bufferSize) : mBufferPtr(bufferSize)
+{
+    this->setg(mBufferPtr.data(), mBufferPtr.data(), mBufferPtr.data() + bufferSize);
+}
+
+char* MemoryInputStreamBuf::getRawData() {
+    return mBufferPtr.data();
+}
+
+MemoryInputStream::MemoryInputStream(size_t bufferSize) : 
+                   MemoryInputStreamBuf(bufferSize),
+    std::istream(static_cast<std::streambuf*>(this)) {
+
+}
+
+char* MemoryInputStream::getRawData() {
+    return MemoryInputStreamBuf::getRawData();
+}
+}

components/bsa/memorystream.hpp

@@ -0,0 +1,62 @@
+/*
+  OpenMW - The completely unofficial reimplementation of Morrowind
+  Copyright (C) 2008-2010  Nicolay Korslund
+  Email: < korslund@gmail.com >
+  WWW: http://openmw.sourceforge.net/
+
+  This file (memorystream.hpp) is part of the OpenMW package.
+
+  OpenMW is distributed as free software: you can redistribute it
+  and/or modify it under the terms of the GNU General Public License
+  version 3, as published by the Free Software Foundation.
+
+  This program is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+  You should have received a copy of the GNU General Public License
+  version 3 along with this program. If not, see
+  http://www.gnu.org/licenses/ .
+
+  Compressed BSA upgrade added by Azdul 2019
+
+ */
+
+#ifndef BSA_MEMORY_STREAM_H
+#define BSA_MEMORY_STREAM_H
+
+#include <vector>
+#include <iostream>
+
+namespace Bsa
+{
+/**
+Class used internally by MemoryInputStream.
+*/
+class MemoryInputStreamBuf : public std::streambuf {
+
+public:
+    MemoryInputStreamBuf(size_t bufferSize);
+    char* getRawData();
+private:
+    //correct call to delete [] on C++ 11
+    std::vector<char> mBufferPtr;
+};
+
+/**
+    Class replaces Ogre memory streams without introducing any new external dependencies
+    beyond standard library.
+
+    Allows to pass memory buffer as Files::IStreamPtr.
+
+    Memory buffer is freed once the class instance is destroyed.
+ */
+class MemoryInputStream : virtual MemoryInputStreamBuf, std::istream {
+public:
+    MemoryInputStream(size_t bufferSize);
+    char* getRawData();
+};
+
+}
+#endif

components/vfs/bsaarchive.cpp

@@ -1,20 +1,33 @@
 #include "bsaarchive.hpp"
+#include <components/bsa/compressedbsafile.hpp>
+#include <memory>
 
 namespace VFS
 {
 
-
 BsaArchive::BsaArchive(const std::string &filename)
 {
-    mFile.open(filename);
+    Bsa::BsaVersion bsaVersion = Bsa::CompressedBSAFile::detectVersion(filename);
 
-    const Bsa::BSAFile::FileList &filelist = mFile.getList();
+    if (bsaVersion == Bsa::BSAVER_COMPRESSED) {
+        mFile = std::unique_ptr<Bsa::CompressedBSAFile>(new Bsa::CompressedBSAFile());
+    }
+    else {
+        mFile = std::unique_ptr<Bsa::BSAFile>(new Bsa::BSAFile());
+    }
+
+    mFile->open(filename);
+
+    const Bsa::BSAFile::FileList &filelist = mFile->getList();
     for(Bsa::BSAFile::FileList::const_iterator it = filelist.begin();it != filelist.end();++it)
     {
-        mResources.push_back(BsaArchiveFile(&*it, &mFile));
+        mResources.push_back(BsaArchiveFile(&*it, mFile.get()));
     }
 }
 
+BsaArchive::~BsaArchive() {
+}
+
 void BsaArchive::listResources(std::map<std::string, File *> &out, char (*normalize_function)(char))
 {
     for (std::vector<BsaArchiveFile>::iterator it = mResources.begin(); it != mResources.end(); ++it)

components/vfs/bsaarchive.hpp

@@ -7,7 +7,6 @@
 
 namespace VFS
 {
-
     class BsaArchiveFile : public File
     {
     public:
@@ -23,15 +22,13 @@ namespace VFS
     {
     public:
         BsaArchive(const std::string& filename);
-
+        virtual ~BsaArchive();
         virtual void listResources(std::map<std::string, File*>& out, char (*normalize_function) (char));
 
     private:
-        Bsa::BSAFile mFile;
-
+        std::unique_ptr<Bsa::BSAFile> mFile;
         std::vector<BsaArchiveFile> mResources;
     };
-
 }
 
 #endif

extern/BSAOpt/CMakeLists.txt

@@ -0,0 +1,17 @@
+cmake_minimum_required(VERSION 2.8)
+
+# This is NOT intended as a stand-alone build system! Instead, you should include this from the main CMakeLists of your project.
+
+set(BSAOPTHASH_LIBRARY "bsaopthash")
+
+# Sources
+set(SOURCE_FILES
+    hash.cpp
+)
+
+add_library(${BSAOPTHASH_LIBRARY} STATIC ${SOURCE_FILES})
+
+set(BSAOPTHASH_LIBRARIES ${BSAOPTHASH_LIBRARY})
+
+link_directories(${CMAKE_CURRENT_BINARY_DIR})
+set(BSAOPTHASH_LIBRARIES ${BSAOPTHASH_LIBRARIES} PARENT_SCOPE)

extern/BSAOpt/hash.cpp

@@ -0,0 +1,109 @@
+/* Version: MPL 1.1/LGPL 3.0
+ *
+ * "The contents of this file are subject to the Mozilla Public License
+ * Version 1.1 (the "License"); you may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
+ * License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * The Original Code is BSAopt.
+ *
+ * The Initial Developer of the Original Code is
+ * Ethatron <niels@paradice-insight.us>. Portions created by The Initial
+ * Developer are Copyright (C) 2011 The Initial Developer.
+ * All Rights Reserved.
+ *
+ * Alternatively, the contents of this file may be used under the terms
+ * of the GNU Library General Public License Version 3 license (the
+ * "LGPL License"), in which case the provisions of LGPL License are
+ * applicable instead of those above. If you wish to allow use of your
+ * version of this file only under the terms of the LGPL License and not
+ * to allow others to use your version of this file under the MPL,
+ * indicate your decision by deleting the provisions above and replace
+ * them with the notice and other provisions required by the LGPL License.
+ * If you do not delete the provisions above, a recipient may use your
+ * version of this file under either the MPL or the LGPL License."
+ */
+
+#include <cstdint>
+#include <algorithm>
+#include <cstring>
+
+#include "hash.hpp"
+
+std::uint32_t GenOBHashStr(const std::string& s) {
+    std::uint32_t hash = 0;
+
+  for (std::size_t i = 0; i < s.length(); i++) {
+    hash *= 0x1003F;
+    hash += (unsigned char)s[i];
+  }
+
+  return hash;
+}
+
+std::uint64_t GenOBHashPair(const std::string& fle, const std::string& ext) {
+    std::uint64_t hash = 0;
+
+  if (fle.length() > 0) {
+    hash = (std::uint64_t)(
+      (((unsigned char)fle[fle.length() - 1]) * 0x1) +
+      ((fle.length() > 2 ? (unsigned char)fle[fle.length() - 2] : (unsigned char)0) * 0x100) +
+      (fle.length() * 0x10000) +
+      (((unsigned char)fle[0]) * 0x1000000)
+    );
+
+    if (fle.length() > 3) {
+      hash += (std::uint64_t)(GenOBHashStr(fle.substr(1, fle.length() - 3)) * 0x100000000);
+    }
+  }
+
+  if (ext.length() > 0) {
+    hash += (std::uint64_t)(GenOBHashStr(ext) * 0x100000000LL);
+
+    unsigned char i = 0;
+    if (ext == ".nif") i = 1;
+    if (ext == ".kf" ) i = 2;
+    if (ext == ".dds") i = 3;
+    if (ext == ".wav") i = 4;
+
+    if (i != 0) {
+      unsigned char a = (unsigned char)(((i & 0xfc ) << 5) + (unsigned char)((hash & 0xff000000) >> 24));
+      unsigned char b = (unsigned char)(((i & 0xfe ) << 6) + (unsigned char)( hash & 0x000000ff)       );
+      unsigned char c = (unsigned char)(( i          << 7) + (unsigned char)((hash & 0x0000ff00) >>  8));
+
+      hash -= hash & 0xFF00FFFF;
+      hash += (std::uint32_t)((a << 24) + b + (c << 8));
+    }
+  }
+
+  return hash;
+}
+
+std::uint64_t GenOBHash(const std::string& path, std::string& file) {
+  std::transform(file.begin(), file.end(), file.begin(), ::tolower);
+  std::replace(file.begin(), file.end(), '/', '\\');
+
+  std::string fle;
+  std::string ext;
+
+  const char *_fle = file.data();
+  const char *_ext = strrchr(_fle, '.');
+  if (_ext) {
+    ext = file.substr((0 + _ext) - _fle);
+    fle = file.substr(0, ( _ext) - _fle);
+  }
+  else {
+    ext = "";
+    fle = file;
+  }
+
+  if (path.length() && fle.length())
+    return GenOBHashPair(path + "\\" + fle, ext);
+  else
+    return GenOBHashPair(path +        fle, ext);
+}

extern/BSAOpt/hash.hpp

@@ -0,0 +1,42 @@
+/* Version: MPL 1.1/LGPL 3.0
+ *
+ * "The contents of this file are subject to the Mozilla Public License
+ * Version 1.1 (the "License"); you may not use this file except in
+ * compliance with the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS"
+ * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
+ * License for the specific language governing rights and limitations
+ * under the License.
+ *
+ * The Original Code is BSAopt.
+ *
+ * The Initial Developer of the Original Code is
+ * Ethatron <niels@paradice-insight.us>. Portions created by The Initial
+ * Developer are Copyright (C) 2011 The Initial Developer.
+ * All Rights Reserved.
+ *
+ * Alternatively, the contents of this file may be used under the terms
+ * of the GNU Library General Public License Version 3 license (the
+ * "LGPL License"), in which case the provisions of LGPL License are
+ * applicable instead of those above. If you wish to allow use of your
+ * version of this file only under the terms of the LGPL License and not
+ * to allow others to use your version of this file under the MPL,
+ * indicate your decision by deleting the provisions above and replace
+ * them with the notice and other provisions required by the LGPL License.
+ * If you do not delete the provisions above, a recipient may use your
+ * version of this file under either the MPL or the LGPL License."
+ */
+#ifndef BSAOPT_HASH_H
+#define BSAOPT_HASH_H
+
+#include <string>
+
+std::uint32_t GenOBHashStr(const std::string& s);
+
+std::uint64_t GenOBHashPair(const std::string& fle, const std::string& ext);
+
+std::uint64_t GenOBHash(const std::string& path, std::string& file);
+
+#endif // BSAOPT_HASH_H