diff --git a/CMakeLists.txt b/CMakeLists.txt index f34e3c83cf..1c54376c8c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -296,7 +296,7 @@ if(QT_STATIC) endif() -set(BOOST_COMPONENTS system filesystem program_options) +set(BOOST_COMPONENTS system filesystem program_options iostreams zlib) if(WIN32) set(BOOST_COMPONENTS ${BOOST_COMPONENTS} locale) endif(WIN32) @@ -610,6 +610,7 @@ add_subdirectory (extern/oics) if (BUILD_OPENCS) add_subdirectory (extern/osgQt) endif() +add_subdirectory (extern/bsaopthash) # Components add_subdirectory (components) diff --git a/components/CMakeLists.txt b/components/CMakeLists.txt index 7a88dc18e8..7ab08292ee 100644 --- a/components/CMakeLists.txt +++ b/components/CMakeLists.txt @@ -33,7 +33,7 @@ add_component_dir (settings ) add_component_dir (bsa - bsa_file + bsa_file tes4bsa_file memorystream ) add_component_dir (vfs @@ -218,6 +218,8 @@ target_link_libraries(components ${Boost_FILESYSTEM_LIBRARY} ${Boost_THREAD_LIBRARY} ${Boost_PROGRAM_OPTIONS_LIBRARY} + ${Boost_IOSTREAMS_LIBRARY} + ${Boost_ZLIB_LIBRARY} ${OSG_LIBRARIES} ${OPENTHREADS_LIBRARIES} ${OSGPARTICLE_LIBRARIES} @@ -231,6 +233,7 @@ target_link_libraries(components ${SDL2_LIBRARIES} ${OPENGL_gl_LIBRARY} ${MyGUI_LIBRARIES} + ${BSAOPTHASH_LIBRARIES} RecastNavigation::DebugUtils RecastNavigation::Detour RecastNavigation::Recast diff --git a/components/bsa/bsa_file.cpp b/components/bsa/bsa_file.cpp index 8905a86a10..abeca5326c 100644 --- a/components/bsa/bsa_file.cpp +++ b/components/bsa/bsa_file.cpp @@ -35,7 +35,7 @@ using namespace Bsa; /// Error handling void BSAFile::fail(const string &msg) { - throw std::runtime_error("BSA Error: " + msg + "\nArchive: " + filename); + throw std::runtime_error("BSA Error: " + msg + "\nArchive: " + mFilename); } /// Read header information from the input source @@ -71,10 +71,10 @@ void BSAFile::readHeader() * the beginning of this buffer. * */ - assert(!isLoaded); + assert(!mIsLoaded); namespace bfs = boost::filesystem; - bfs::ifstream input(bfs::path(filename), std::ios_base::binary); + bfs::ifstream input(bfs::path(mFilename), std::ios_base::binary); // Total archive size std::streamoff fsize = 0; @@ -117,8 +117,8 @@ void BSAFile::readHeader() input.read(reinterpret_cast(&offsets[0]), 12*filenum); // Read the string table - stringBuf.resize(dirsize-12*filenum); - input.read(&stringBuf[0], stringBuf.size()); + mStringBuf.resize(dirsize-12*filenum); + input.read(&mStringBuf[0], mStringBuf.size()); // Check our position assert(input.tellg() == std::streampos(12+dirsize)); @@ -129,40 +129,40 @@ void BSAFile::readHeader() size_t fileDataOffset = 12 + dirsize + 8*filenum; // Set up the the FileStruct table - files.resize(filenum); + mFiles.resize(filenum); for(size_t i=0;i fsize) fail("Archive contains offsets outside itself"); // Add the file name to the lookup - lookup[fs.name] = i; + mLookup[fs.name] = i; } - isLoaded = true; + mIsLoaded = true; } /// Get the index of a given file name, or -1 if not found int BSAFile::getIndex(const char *str) const { - Lookup::const_iterator it = lookup.find(str); - if(it == lookup.end()) + Lookup::const_iterator it = mLookup.find(str); + if(it == mLookup.end()) return -1; int res = it->second; - assert(res >= 0 && (size_t)res < files.size()); + assert(res >= 0 && (size_t)res < mFiles.size()); return res; } /// Open an archive file. void BSAFile::open(const string &file) { - filename = file; + mFilename = file; readHeader(); } @@ -173,12 +173,12 @@ Files::IStreamPtr BSAFile::getFile(const char *file) if(i == -1) fail("File not found: " + string(file)); - const FileStruct &fs = files[i]; + const FileStruct &fs = mFiles[i]; - return Files::openConstrainedFileStream (filename.c_str (), fs.offset, fs.fileSize); + return Files::openConstrainedFileStream (mFilename.c_str (), fs.offset, fs.fileSize); } Files::IStreamPtr BSAFile::getFile(const FileStruct *file) { - return Files::openConstrainedFileStream (filename.c_str (), file->offset, file->fileSize); + return Files::openConstrainedFileStream (mFilename.c_str (), file->offset, file->fileSize); } diff --git a/components/bsa/bsa_file.hpp b/components/bsa/bsa_file.hpp index 196dc30fbb..d12d01b0c4 100644 --- a/components/bsa/bsa_file.hpp +++ b/components/bsa/bsa_file.hpp @@ -56,18 +56,18 @@ public: }; typedef std::vector FileList; -private: +protected: /// Table of files in this archive - FileList files; + FileList mFiles; /// Filename string buffer - std::vector stringBuf; + std::vector mStringBuf; /// True when an archive has been loaded - bool isLoaded; + bool mIsLoaded; /// Used for error messages - std::string filename; + std::string mFilename; /// Case insensitive string comparison struct iltstr @@ -81,13 +81,16 @@ private: checks are case insensitive. */ typedef std::map Lookup; - Lookup lookup; + Lookup mLookup; /// Error handling void fail(const std::string &msg); /// Read header information from the input source - void readHeader(); + virtual void readHeader(); + + /// Read header information from the input source + /// Get the index of a given file name, or -1 if not found /// @note Thread safe. @@ -100,7 +103,10 @@ public: */ BSAFile() - : isLoaded(false) + : mIsLoaded(false) + { } + + virtual ~BSAFile() { } /// Open an archive file. @@ -112,24 +118,24 @@ public: */ /// Check if a file exists - bool exists(const char *file) const + virtual bool exists(const char *file) const { return getIndex(file) != -1; } /** Open a file contained in the archive. Throws an exception if the file doesn't exist. * @note Thread safe. */ - Files::IStreamPtr getFile(const char *file); + virtual Files::IStreamPtr getFile(const char *file); /** Open a file contained in the archive. * @note Thread safe. */ - Files::IStreamPtr getFile(const FileStruct* file); + virtual Files::IStreamPtr getFile(const FileStruct* file); /// Get a list of all files /// @note Thread safe. const FileList &getList() const - { return files; } + { return mFiles; } }; } diff --git a/components/bsa/memorystream.cpp b/components/bsa/memorystream.cpp new file mode 100644 index 0000000000..3cacad60f7 --- /dev/null +++ b/components/bsa/memorystream.cpp @@ -0,0 +1,48 @@ +/* + OpenMW - The completely unofficial reimplementation of Morrowind + Copyright (C) 2008-2010 Nicolay Korslund + Email: < korslund@gmail.com > + WWW: http://openmw.sourceforge.net/ + + This file (memorystream.cpp) is part of the OpenMW package. + + OpenMW is distributed as free software: you can redistribute it + and/or modify it under the terms of the GNU General Public License + version 3, as published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + version 3 along with this program. If not, see + http://www.gnu.org/licenses/ . + + TES4 stuff upgrade added by Azdul 2019 + + */ +#include "memorystream.hpp" + + +namespace Bsa +{ +MemoryInputStreamBuf::MemoryInputStreamBuf(size_t bufferSize) : mBufferPtr(bufferSize) +{ + this->setg(mBufferPtr.data(), mBufferPtr.data(), mBufferPtr.data() + bufferSize); +} + +char* MemoryInputStreamBuf::getRawData() { + return mBufferPtr.data(); +} + +MemoryInputStream::MemoryInputStream(size_t bufferSize) : + MemoryInputStreamBuf(bufferSize), + std::istream(static_cast(this)) { + +} + +char* MemoryInputStream::getRawData() { + return MemoryInputStreamBuf::getRawData(); +} +} diff --git a/components/bsa/memorystream.hpp b/components/bsa/memorystream.hpp new file mode 100644 index 0000000000..aed39e6d6f --- /dev/null +++ b/components/bsa/memorystream.hpp @@ -0,0 +1,62 @@ +/* + OpenMW - The completely unofficial reimplementation of Morrowind + Copyright (C) 2008-2010 Nicolay Korslund + Email: < korslund@gmail.com > + WWW: http://openmw.sourceforge.net/ + + This file (memorystream.h) is part of the OpenMW package. + + OpenMW is distributed as free software: you can redistribute it + and/or modify it under the terms of the GNU General Public License + version 3, as published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + version 3 along with this program. If not, see + http://www.gnu.org/licenses/ . + + TES4 stuff upgrade added by Azdul 2019 + + */ + +#ifndef BSA_MEMORY_STREAM_H +#define BSA_MEMORY_STREAM_H + +#include +#include + +namespace Bsa +{ +/** +Class used internally by MemoryInputStream. +*/ +class MemoryInputStreamBuf : public std::streambuf { + +public: + MemoryInputStreamBuf(size_t bufferSize); + char* getRawData(); +private: + //correct call to delete [] on C++ 11 + std::vector mBufferPtr; +}; + +/** + Class replaces Ogre memory streams without introducing any new external dependencies + beyond standard library. + + Allows to pass memory buffer as Files::IStreamPtr. + + Memory buffer is freed once the class instance is destroyed. + */ +class MemoryInputStream : virtual MemoryInputStreamBuf, std::istream { +public: + MemoryInputStream(size_t bufferSize); + char* getRawData(); +}; + +} +#endif diff --git a/components/bsa/tes4bsa_file.cpp b/components/bsa/tes4bsa_file.cpp new file mode 100644 index 0000000000..5c36061d5c --- /dev/null +++ b/components/bsa/tes4bsa_file.cpp @@ -0,0 +1,434 @@ +/* + OpenMW - The completely unofficial reimplementation of Morrowind + Copyright (C) 2008-2010 Nicolay Korslund + Email: < korslund@gmail.com > + WWW: http://openmw.sourceforge.net/ + + This file (tes4bsa_file.cpp) is part of the OpenMW package. + + OpenMW is distributed as free software: you can redistribute it + and/or modify it under the terms of the GNU General Public License + version 3, as published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + version 3 along with this program. If not, see + http://www.gnu.org/licenses/ . + + TES4 stuff added by cc9cii 2018 + + */ +#include "tes4bsa_file.hpp" + +#include +#include +#include +#include +#include +#include + +#include // see: http://en.uesp.net/wiki/Tes4Mod:Hash_Calculation +#include +#include +#include +#include +#include +#include +#include + +namespace Bsa +{ + +//special marker for invalid records, +//equal to max uint32_t value +const uint32_t TES4BSAFile::sInvalidOffset = std::numeric_limits::max(); + +//bit marking compression on file size +const uint32_t TES4BSAFile::sCompressedFlag = 1u << 30u; + + +TES4BSAFile::FileRecord::FileRecord() : size(0), offset(sInvalidOffset) +{ } + +bool TES4BSAFile::FileRecord::isValid() const +{ + return offset != sInvalidOffset; +} + +bool TES4BSAFile::FileRecord::isCompressed(bool bsaCompressedByDefault) const +{ + bool compressionFlagEnabled = ((size & sCompressedFlag) == sCompressedFlag); + + if (bsaCompressedByDefault) { + return !compressionFlagEnabled; + } + return compressionFlagEnabled; +} + +std::uint32_t TES4BSAFile::FileRecord::getSizeWithoutCompressionFlag() const { + return size & (~sCompressedFlag); +} + +TES4BSAFile::TES4BSAFile() + : mCompressedByDefault(false), mEmbeddedFileNames(false) +{ } + +TES4BSAFile::~TES4BSAFile() +{ } + +void TES4BSAFile::getBZString(std::string& str, std::istream& filestream) +{ + char size = 0; + filestream.read(&size, 1); + + boost::scoped_array buf(new char[size]); + filestream.read(buf.get(), size); + + if (buf[size - 1] != 0) + { + str.assign(buf.get(), size); + if (str.size() != ((size_t)size)) { + fail("getBZString string size mismatch"); + } + } + else + { + str.assign(buf.get(), size - 1); // don't copy null terminator + if (str.size() != ((size_t)size - 1)) { + fail("getBZString string size mismatch (null terminator)"); + } + } + + return; +} + +/// Read header information from the input source +void TES4BSAFile::readHeader() +{ + assert(!mIsLoaded); + + namespace bfs = boost::filesystem; + bfs::ifstream input(bfs::path(mFilename), std::ios_base::binary); + + // Total archive size + std::streamoff fsize = 0; + if (input.seekg(0, std::ios_base::end)) + { + fsize = input.tellg(); + input.seekg(0); + } + + if (fsize < 36) // header is 36 bytes + fail("File too small to be a valid BSA archive"); + + // Get essential header numbers + //size_t dirsize, filenum; + std::uint32_t archiveFlags, folderCount, fileCount, totalFileNameLength; + { + // First 36 bytes + std::uint32_t header[9]; + + input.read(reinterpret_cast(header), 36); + + if (header[0] != 0x00415342 /*"BSA\x00"*/ || (header[1] != 0x67 /*TES4*/ && header[1] != 0x68 /*TES5*/)) + fail("Unrecognized TES4 BSA header"); + + // header[2] is offset, should be 36 = 0x24 which is the size of the header + + // Oblivion - Meshes.bsa + // + // 0111 1000 0111 = 0x0787 + // ^^^ ^ ^^^ + // ||| | ||+-- has names for dirs (mandatory?) + // ||| | |+--- has names for files (mandatory?) + // ||| | +---- files are compressed by default + // ||| | + // ||| +---------- unknown (TES5: retain strings during startup) + // ||+------------ unknown (TES5: embedded file names) + // |+------------- unknown + // +-------------- unknown + // + archiveFlags = header[3]; + folderCount = header[4]; + fileCount = header[5]; + totalFileNameLength = header[7]; + + mCompressedByDefault = (archiveFlags & 0x4) != 0; + mEmbeddedFileNames = header[1] == 0x68 /*TES5*/ && (archiveFlags & 0x100) != 0; + } + + // folder records + std::uint64_t hash; + FolderRecord fr; + for (std::uint32_t i = 0; i < folderCount; ++i) + { + input.read(reinterpret_cast(&hash), 8); + input.read(reinterpret_cast(&fr.count), 4); + input.read(reinterpret_cast(&fr.offset), 4); + + std::map::const_iterator lb = mFolders.lower_bound(hash); + if (lb != mFolders.end() && !(mFolders.key_comp()(hash, lb->first))) + fail("Archive found duplicate folder name hash"); + else + mFolders.insert(lb, std::pair(hash, fr)); + } + + // file record blocks + std::uint64_t fileHash; + FileRecord file; + + std::string folder(""); + std::uint64_t folderHash; + if ((archiveFlags & 0x1) == 0) + { + folderCount = 1; + } + + mFiles.clear(); + std::vector fullPaths; + + for (std::uint32_t i = 0; i < folderCount; ++i) + { + if ((archiveFlags & 0x1) != 0) + getBZString(folder, input); + + std::string emptyString; + folderHash = GenOBHash(folder, emptyString); + + std::map::iterator iter = mFolders.find(folderHash); + if (iter == mFolders.end()) + fail("Archive folder name hash not found"); + + for (std::uint32_t j = 0; j < iter->second.count; ++j) + { + input.read(reinterpret_cast(&fileHash), 8); + input.read(reinterpret_cast(&file.size), 4); + input.read(reinterpret_cast(&file.offset), 4); + + std::map::const_iterator lb = iter->second.files.lower_bound(fileHash); + if (lb != iter->second.files.end() && !(iter->second.files.key_comp()(fileHash, lb->first))) + { + fail("Archive found duplicate file name hash"); + } + + iter->second.files.insert(lb, std::pair(fileHash, file)); + + FileStruct fileStruct; + fileStruct.fileSize = file.getSizeWithoutCompressionFlag(); + fileStruct.offset = file.offset; + fileStruct.name = nullptr; + mFiles.push_back(fileStruct); + + fullPaths.push_back(folder); + } + } + + // file record blocks + if ((archiveFlags & 0x2) != 0) + { + mStringBuf.resize(totalFileNameLength); + input.read(&mStringBuf[0], mStringBuf.size()); + } + + size_t mStringBuffOffset = 0; + size_t totalStringsSize = 0; + for (std::uint32_t fileIndex = 0; fileIndex < mFiles.size(); ++fileIndex) { + + if (mStringBuffOffset >= totalFileNameLength) { + fail("Corrupted names record in BSA file"); + } + + //The vector guarantees that its elements occupy contiguous memory + mFiles[fileIndex].name = reinterpret_cast(mStringBuf.data() + mStringBuffOffset); + + fullPaths.at(fileIndex) += "\\" + std::string(mStringBuf.data() + mStringBuffOffset); + + while (mStringBuffOffset < totalFileNameLength) { + if (mStringBuf[mStringBuffOffset] != '\0') { + mStringBuffOffset++; + } + else { + mStringBuffOffset++; + break; + } + } + //we want to keep one more 0 character at the end of each string + totalStringsSize += fullPaths.at(fileIndex).length() + 1u; + } + mStringBuf.resize(totalStringsSize); + + mStringBuffOffset = 0; + for (std::uint32_t fileIndex = 0u; fileIndex < mFiles.size(); fileIndex++) { + size_t stringLength = fullPaths.at(fileIndex).length(); + + std::copy(fullPaths.at(fileIndex).c_str(), + //plus 1 because we also want to copy 0 at the end of the string + fullPaths.at(fileIndex).c_str() + stringLength + 1u, + mStringBuf.data() + mStringBuffOffset); + + mFiles[fileIndex].name = reinterpret_cast(mStringBuf.data() + mStringBuffOffset); + + mLookup[reinterpret_cast(mStringBuf.data() + mStringBuffOffset)] = fileIndex; + mStringBuffOffset += stringLength + 1u; + } + + if (mStringBuffOffset != mStringBuf.size()) { + fail("Could not resolve names of files in BSA file"); + } + + convertCompressedSizesToUncompressed(); + mIsLoaded = true; +} + +TES4BSAFile::FileRecord TES4BSAFile::getFileRecord(const std::string& filePath) const +{ + boost::filesystem::path p(filePath); + std::string stem = p.stem().string(); + std::string ext = p.extension().string(); + std::string filename = p.filename().string(); + p.remove_filename(); + + std::string folder = p.string(); + // GenOBHash already converts to lowercase and replaces file separators but not for path + boost::algorithm::to_lower(folder); + std::replace(folder.begin(), folder.end(), '/', '\\'); + + std::string emptyString; + std::uint64_t folderHash = GenOBHash(folder, emptyString); + + std::map::const_iterator it = mFolders.find(folderHash); + if (it == mFolders.end()) + { + return FileRecord(); // folder not found, return default which has offset of -1 + } + + boost::algorithm::to_lower(stem); + boost::algorithm::to_lower(ext); + std::uint64_t fileHash = GenOBHashPair(stem, ext); + std::map::const_iterator fileInDirIter = it->second.files.find(fileHash); + if (fileInDirIter == it->second.files.end()) + { + return FileRecord(); // file not found, return default which has offset of -1 + } + + return fileInDirIter->second; +} + +Files::IStreamPtr TES4BSAFile::getFile(const FileStruct* file) { + + FileRecord fileRec = getFileRecord(file->name); + return getFile(fileRec); +} + +Files::IStreamPtr TES4BSAFile::getFile(const char* file) +{ + FileRecord fileRec = getFileRecord(file); + if (!fileRec.isValid()) { + fail("File not found: " + std::string(file)); + } + + return getFile(fileRec); +} + +Files::IStreamPtr TES4BSAFile::getFile(const FileRecord& fileRecord) { + + if (fileRecord.isCompressed(mCompressedByDefault)) { + Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, fileRecord.getSizeWithoutCompressionFlag()); + + std::istream* fileStream = streamPtr.get(); + + if (mEmbeddedFileNames) { + std::string embeddedFileName; + getBZString(embeddedFileName,*fileStream); + } + + uint32_t uncompressedSize = 0u; + fileStream->read(reinterpret_cast(&uncompressedSize), sizeof(uncompressedSize)); + + boost::iostreams::filtering_streambuf inputStreamBuf; + inputStreamBuf.push(boost::iostreams::zlib_decompressor()); + inputStreamBuf.push(*fileStream); + + std::shared_ptr > bufferVec = std::make_shared>(uncompressedSize); + Bsa::MemoryInputStream* result = new MemoryInputStream(uncompressedSize); + + boost::iostreams::basic_array_sink sr(result->getRawData(), uncompressedSize); + boost::iostreams::copy(inputStreamBuf, sr); + std::istream* pPtr = (std::istream*) result; + return std::shared_ptr(pPtr); + } + + return Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, fileRecord.size); +} + +BsaVersion TES4BSAFile::detectVersion(std::string filePath) +{ + namespace bfs = boost::filesystem; + bfs::ifstream input(bfs::path(filePath), std::ios_base::binary); + + // Total archive size + std::streamoff fsize = 0; + if (input.seekg(0, std::ios_base::end)) + { + fsize = input.tellg(); + input.seekg(0); + } + + if (fsize < 12) { + return BSAVER_UNKNOWN; + } + + + // Get essential header numbers + size_t dirsize, filenum; + { + // First 12 bytes + uint32_t head[3]; + + input.read(reinterpret_cast(head), 12); + + if (head[0] == static_cast(BSAVER_TES3)) { + return BSAVER_TES3; + } + + if (head[0] = static_cast(BSAVER_TES4PLUS)) { + return BSAVER_TES4PLUS; + } + } + return BSAVER_UNKNOWN; +} + +//mFiles used by OpenMW expects uncompressed sizes +void TES4BSAFile::convertCompressedSizesToUncompressed() +{ + for (auto iter = mFiles.begin(); iter != mFiles.end(); ++iter) + { + const FileRecord& fileRecord = getFileRecord(iter->name); + if (!fileRecord.isValid()) + { + fail("Could not find file " + std::string(iter->name) + " in BSA"); + } + + if (!fileRecord.isCompressed(mCompressedByDefault)) + { + //no need to fix fileSize in mFiles - uncompressed size already set + continue; + } + + Files::IStreamPtr dataBegin = Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, fileRecord.getSizeWithoutCompressionFlag()); + + if (mEmbeddedFileNames) + { + std::string embeddedFileName; + getBZString(embeddedFileName, *(dataBegin.get())); + } + + dataBegin->read(reinterpret_cast(&(iter->fileSize)), sizeof(iter->fileSize)); + } +} + +} //namespace Bsa diff --git a/components/bsa/tes4bsa_file.hpp b/components/bsa/tes4bsa_file.hpp new file mode 100644 index 0000000000..53a08ffc52 --- /dev/null +++ b/components/bsa/tes4bsa_file.hpp @@ -0,0 +1,108 @@ +/* + OpenMW - The completely unofficial reimplementation of Morrowind + Copyright (C) 2008-2010 Nicolay Korslund + Email: < korslund@gmail.com > + WWW: http://openmw.sourceforge.net/ + + This file (tes4bsa_file.h) is part of the OpenMW package. + + OpenMW is distributed as free software: you can redistribute it + and/or modify it under the terms of the GNU General Public License + version 3, as published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + version 3 along with this program. If not, see + http://www.gnu.org/licenses/ . + + TES4 stuff added by cc9cii 2018 + + */ + +#ifndef BSA_TES4BSA_FILE_H +#define BSA_TES4BSA_FILE_H + +#include +#include +#include +#include +#include + +namespace Bsa +{ + enum BsaVersion + { + BSAVER_UNKNOWN = 0x0, + BSAVER_TES3 = 0x100, + BSAVER_TES4PLUS = 0x415342 //B, S, A + }; + +/** + This class is used to read "Bethesda Archive Files", or BSAs + in newer formats first introduced in TES IV Oblivion + */ +class TES4BSAFile : public BSAFile +{ +public: + //checks version of BSA from file header + static BsaVersion detectVersion(std::string filePath); + +private: + //special marker for invalid records, + //equal to max uint32_t value + static const uint32_t sInvalidOffset; + + //bit marking compression on file size + static const uint32_t sCompressedFlag; + + struct FileRecord + { + //size of file, with 30th bit containing compression flag + std::uint32_t size; + std::uint32_t offset; + + FileRecord(); + bool isCompressed(bool bsaCompressedByDefault) const; + bool isValid() const; + std::uint32_t getSizeWithoutCompressionFlag() const; + }; + + //if files in BSA without 30th bit enabled are compressed + bool mCompressedByDefault; + + //if file names are not present in file header, + //and we need to look for them at given file offset + bool mEmbeddedFileNames; + + struct FolderRecord + { + std::uint32_t count; + std::uint32_t offset; + std::map files; + }; + std::map mFolders; + + FileRecord getFileRecord(const std::string& filePath) const; + + /// Read header information from the input source + virtual void readHeader(); +public: + TES4BSAFile(); + virtual ~TES4BSAFile(); + + virtual Files::IStreamPtr getFile(const char* file); + virtual Files::IStreamPtr getFile(const FileStruct* file); + +private: + Files::IStreamPtr getFile(const FileRecord& file); + //mFiles used by OpenMW expects uncompressed sizes + void convertCompressedSizesToUncompressed(); + void getBZString(std::string& str, std::istream& filestream); +}; +} + +#endif diff --git a/components/vfs/bsaarchive.cpp b/components/vfs/bsaarchive.cpp index a527a6ad92..5b9aad437a 100644 --- a/components/vfs/bsaarchive.cpp +++ b/components/vfs/bsaarchive.cpp @@ -1,4 +1,5 @@ #include "bsaarchive.hpp" +#include namespace VFS { @@ -6,15 +7,30 @@ namespace VFS BsaArchive::BsaArchive(const std::string &filename) { - mFile.open(filename); + mFile = new Bsa::BSAFile(); - const Bsa::BSAFile::FileList &filelist = mFile.getList(); + Bsa::BsaVersion bsaVersion = Bsa::TES4BSAFile::detectVersion(filename); + + if (bsaVersion == Bsa::BSAVER_TES4PLUS) { + mFile = new Bsa::TES4BSAFile(); + } + else { + mFile = new Bsa::BSAFile(); + } + + mFile->open(filename); + + const Bsa::BSAFile::FileList &filelist = mFile->getList(); for(Bsa::BSAFile::FileList::const_iterator it = filelist.begin();it != filelist.end();++it) { - mResources.push_back(BsaArchiveFile(&*it, &mFile)); + mResources.push_back(BsaArchiveFile(&*it, mFile)); } } +BsaArchive::~BsaArchive() { + delete mFile; +} + void BsaArchive::listResources(std::map &out, char (*normalize_function)(char)) { for (std::vector::iterator it = mResources.begin(); it != mResources.end(); ++it) diff --git a/components/vfs/bsaarchive.hpp b/components/vfs/bsaarchive.hpp index a6e2740378..0af9650132 100644 --- a/components/vfs/bsaarchive.hpp +++ b/components/vfs/bsaarchive.hpp @@ -7,7 +7,6 @@ namespace VFS { - class BsaArchiveFile : public File { public: @@ -23,15 +22,13 @@ namespace VFS { public: BsaArchive(const std::string& filename); - + virtual ~BsaArchive(); virtual void listResources(std::map& out, char (*normalize_function) (char)); private: - Bsa::BSAFile mFile; - + Bsa::BSAFile* mFile; std::vector mResources; }; - } #endif diff --git a/extern/bsaopthash/CMakeLists.txt b/extern/bsaopthash/CMakeLists.txt new file mode 100644 index 0000000000..a7f1dcf74a --- /dev/null +++ b/extern/bsaopthash/CMakeLists.txt @@ -0,0 +1,17 @@ +cmake_minimum_required(VERSION 2.8) + +# This is NOT intended as a stand-alone build system! Instead, you should include this from the main CMakeLists of your project. + +set(BSAOPTHASH_LIBRARY "bsaopthash") + +# Sources +set(SOURCE_FILES + hash.cpp +) + +add_library(${BSAOPTHASH_LIBRARY} STATIC ${SOURCE_FILES}) + +set(BSAOPTHASH_LIBRARIES ${BSAOPTHASH_LIBRARY}) + +link_directories(${CMAKE_CURRENT_BINARY_DIR}) +set(BSAOPTHASH_LIBRARIES ${BSAOPTHASH_LIBRARIES} PARENT_SCOPE) diff --git a/extern/bsaopthash/hash.cpp b/extern/bsaopthash/hash.cpp new file mode 100644 index 0000000000..029c542799 --- /dev/null +++ b/extern/bsaopthash/hash.cpp @@ -0,0 +1,108 @@ +/* Version: MPL 1.1/LGPL 3.0 + * + * "The contents of this file are subject to the Mozilla Public License + * Version 1.1 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the + * License for the specific language governing rights and limitations + * under the License. + * + * The Original Code is BSAopt. + * + * The Initial Developer of the Original Code is + * Ethatron . Portions created by The Initial + * Developer are Copyright (C) 2011 The Initial Developer. + * All Rights Reserved. + * + * Alternatively, the contents of this file may be used under the terms + * of the GNU Library General Public License Version 3 license (the + * "LGPL License"), in which case the provisions of LGPL License are + * applicable instead of those above. If you wish to allow use of your + * version of this file only under the terms of the LGPL License and not + * to allow others to use your version of this file under the MPL, + * indicate your decision by deleting the provisions above and replace + * them with the notice and other provisions required by the LGPL License. + * If you do not delete the provisions above, a recipient may use your + * version of this file under either the MPL or the LGPL License." + */ + +#include +#include + +#include "hash.hpp" + +std::uint32_t GenOBHashStr(const std::string& s) { + std::uint32_t hash = 0; + + for (std::size_t i = 0; i < s.length(); i++) { + hash *= 0x1003F; + hash += (unsigned char)s[i]; + } + + return hash; +} + +std::uint64_t GenOBHashPair(const std::string& fle, const std::string& ext) { + std::uint64_t hash = 0; + + if (fle.length() > 0) { + hash = (std::uint64_t)( + (((unsigned char)fle[fle.length() - 1]) * 0x1) + + ((fle.length() > 2 ? (unsigned char)fle[fle.length() - 2] : (unsigned char)0) * 0x100) + + (fle.length() * 0x10000) + + (((unsigned char)fle[0]) * 0x1000000) + ); + + if (fle.length() > 3) { + hash += (std::uint64_t)(GenOBHashStr(fle.substr(1, fle.length() - 3)) * 0x100000000); + } + } + + if (ext.length() > 0) { + hash += (std::uint64_t)(GenOBHashStr(ext) * 0x100000000LL); + + unsigned char i = 0; + if (ext == ".nif") i = 1; + if (ext == ".kf" ) i = 2; + if (ext == ".dds") i = 3; + if (ext == ".wav") i = 4; + + if (i != 0) { + unsigned char a = (unsigned char)(((i & 0xfc ) << 5) + (unsigned char)((hash & 0xff000000) >> 24)); + unsigned char b = (unsigned char)(((i & 0xfe ) << 6) + (unsigned char)( hash & 0x000000ff) ); + unsigned char c = (unsigned char)(( i << 7) + (unsigned char)((hash & 0x0000ff00) >> 8)); + + hash -= hash & 0xFF00FFFF; + hash += (std::uint32_t)((a << 24) + b + (c << 8)); + } + } + + return hash; +} + +std::uint64_t GenOBHash(const std::string& path, std::string& file) { + std::transform(file.begin(), file.end(), file.begin(), ::tolower); + std::replace(file.begin(), file.end(), '/', '\\'); + + std::string fle; + std::string ext; + + const char *_fle = file.data(); + const char *_ext = strrchr(_fle, '.'); + if (_ext) { + ext = file.substr((0 + _ext) - _fle); + fle = file.substr(0, ( _ext) - _fle); + } + else { + ext = ""; + fle = file; + } + + if (path.length() && fle.length()) + return GenOBHashPair(path + "\\" + fle, ext); + else + return GenOBHashPair(path + fle, ext); +} diff --git a/extern/bsaopthash/hash.hpp b/extern/bsaopthash/hash.hpp new file mode 100644 index 0000000000..cd936530a8 --- /dev/null +++ b/extern/bsaopthash/hash.hpp @@ -0,0 +1,42 @@ +/* Version: MPL 1.1/LGPL 3.0 + * + * "The contents of this file are subject to the Mozilla Public License + * Version 1.1 (the "License"); you may not use this file except in + * compliance with the License. You may obtain a copy of the License at + * http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS IS" + * basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the + * License for the specific language governing rights and limitations + * under the License. + * + * The Original Code is BSAopt. + * + * The Initial Developer of the Original Code is + * Ethatron . Portions created by The Initial + * Developer are Copyright (C) 2011 The Initial Developer. + * All Rights Reserved. + * + * Alternatively, the contents of this file may be used under the terms + * of the GNU Library General Public License Version 3 license (the + * "LGPL License"), in which case the provisions of LGPL License are + * applicable instead of those above. If you wish to allow use of your + * version of this file only under the terms of the LGPL License and not + * to allow others to use your version of this file under the MPL, + * indicate your decision by deleting the provisions above and replace + * them with the notice and other provisions required by the LGPL License. + * If you do not delete the provisions above, a recipient may use your + * version of this file under either the MPL or the LGPL License." + */ +#ifndef BSAOPT_HASH_H +#define BSAOPT_HASH_H + +#include + +std::uint32_t GenOBHashStr(const std::string& s); + +std::uint64_t GenOBHashPair(const std::string& fle, const std::string& ext); + +std::uint64_t GenOBHash(const std::string& path, std::string& file); + +#endif // BSAOPT_HASH_H