Merge branch 'bsa' into 'master'

Compressed BSA support revisions

See merge request OpenMW/openmw!350

.gitlab-ci.yml

@@ -13,7 +13,7 @@ stages:
   before_script:
     - export APT_CACHE_DIR=`pwd`/apt-cache && mkdir -pv $APT_CACHE_DIR
     - apt-get update -yq 
-    - apt-get -o dir::cache::archives="$APT_CACHE_DIR" install -y cmake build-essential libboost-filesystem-dev libboost-program-options-dev libboost-system-dev libboost-iostreams-dev libavcodec-dev libavformat-dev libavutil-dev libswscale-dev libswresample-dev libsdl2-dev libqt5opengl5-dev libopenal-dev libopenscenegraph-dev libunshield-dev libtinyxml-dev libmygui-dev libbullet-dev ccache git clang
+    - apt-get -o dir::cache::archives="$APT_CACHE_DIR" install -y cmake build-essential libboost-filesystem-dev libboost-program-options-dev libboost-system-dev libboost-iostreams-dev libavcodec-dev libavformat-dev libavutil-dev libswscale-dev libswresample-dev libsdl2-dev libqt5opengl5-dev libopenal-dev libopenscenegraph-dev libunshield-dev libtinyxml-dev libmygui-dev libbullet-dev liblz4-dev ccache git clang
   stage: build
   script:
     - export CCACHE_BASEDIR="`pwd`"

.travis.yml

@@ -23,7 +23,7 @@ addons:
       # FFmpeg
       libavcodec-dev, libavformat-dev, libavutil-dev, libswresample-dev, libswscale-dev,
       # Audio, Video and Misc. deps
-      libsdl2-dev, libqt5opengl5-dev, libopenal-dev, libunshield-dev, libtinyxml-dev,
+      libsdl2-dev, libqt5opengl5-dev, libopenal-dev, libunshield-dev, libtinyxml-dev, liblz4-dev
       # The other ones from OpenMW ppa
       libbullet-dev, libopenscenegraph-dev, libmygui-dev
     ]

CHANGELOG.md

@@ -73,6 +73,7 @@
     Feature #5579: MCP SetAngle enhancement
     Feature #5610: Actors movement should be smoother
     Feature #5642: Ability to attach arrows to actor skeleton instead of bow mesh
+    Feature #5649: Skyrim SE compressed BSA format support
     Task #5480: Drop Qt4 support
     Task #5520: Improve cell name autocompleter implementation
 

CI/before_install.osx.sh

@@ -5,5 +5,5 @@ command -v ccache >/dev/null 2>&1 || brew install ccache
 command -v cmake >/dev/null 2>&1 || brew install cmake
 command -v qmake >/dev/null 2>&1 || brew install qt
 
-curl -fSL -R -J https://downloads.openmw.org/osx/dependencies/openmw-deps-ef2462c.zip -o ~/openmw-deps.zip
+curl -fSL -R -J https://downloads.openmw.org/osx/dependencies/openmw-deps-20201016.zip -o ~/openmw-deps.zip
 unzip -o ~/openmw-deps.zip -d /private/tmp/openmw-deps > /dev/null

CI/before_script.msvc.sh

@@ -571,6 +571,11 @@ if [ -z $SKIP_DOWNLOAD ]; then
 		"https://gitlab.com/OpenMW/openmw-deps/-/raw/main/windows/SDL2-2.0.12.zip" \
 		"SDL2-2.0.12.zip"
 
+	# LZ4
+	download "LZ4 1.9.2" \
+		"https://gitlab.com/OpenMW/openmw-deps/-/raw/main/windows/lz4_win${BITS}_v1_9_2.7z" \
+		"lz4_win${BITS}_v1_9_2.7z"
+
 	# Google test and mock
 	if [ ! -z $TEST_FRAMEWORK ]; then
 		echo "Google test 1.10.0..."
@@ -901,6 +906,31 @@ printf "SDL 2.0.12... "
 }
 cd $DEPS
 echo
+# LZ4
+printf "LZ4 1.9.2... "
+{
+	if [ -d LZ4_1.9.2 ]; then
+		printf "Exists. "
+	elif [ -z $SKIP_EXTRACT ]; then
+		rm -rf LZ4_1.9.2
+		eval 7z x -y lz4_win${BITS}_v1_9_2.7z -o./LZ4_1.9.2 $STRIP
+	fi
+	export LZ4DIR="$(real_pwd)/LZ4_1.9.2"
+	add_cmake_opts -DLZ4_INCLUDE_DIR="${LZ4DIR}/include" \
+		-DLZ4_LIBRARY="${LZ4DIR}/lib/liblz4.lib"
+	for CONFIGURATION in ${CONFIGURATIONS[@]}; do
+		if [ $CONFIGURATION == "Debug" ]; then
+			LZ4_CONFIGURATION="Debug"
+		else
+			SUFFIX=""
+			LZ4_CONFIGURATION="Release"
+		fi
+		add_runtime_dlls $CONFIGURATION "$(pwd)/LZ4_1.9.2/bin/${LZ4_CONFIGURATION}/liblz4.dll"
+	done
+	echo Done.
+}
+cd $DEPS
+echo
 # Google Test and Google Mock
 if [ ! -z $TEST_FRAMEWORK ]; then
 	printf "Google test 1.10.0 ..."

CMakeLists.txt

@@ -152,6 +152,8 @@ endif()
 # Dependencies
 find_package(OpenGL REQUIRED)
 
+find_package(LZ4 REQUIRED)
+
 if (USE_QT)
     find_package(Qt5Core 5.12 REQUIRED)
     find_package(Qt5Widgets REQUIRED)

cmake/FindLZ4.cmake

@@ -0,0 +1,130 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+#[=======================================================================[.rst:
+FindLZ4
+-------
+
+Find the LZ4 include directory and library.
+
+Use this module by invoking find_package with the form::
+
+.. code-block:: cmake
+
+  find_package(LZ4
+    [version]              # Minimum version e.g. 1.8.0
+    [REQUIRED]             # Fail with error if LZ4 is not found
+  )
+
+Imported targets
+^^^^^^^^^^^^^^^^
+
+This module defines the following :prop_tgt:`IMPORTED` targets:
+
+.. variable:: LZ4::LZ4
+
+  Imported target for using the LZ4 library, if found.
+
+Result variables
+^^^^^^^^^^^^^^^^
+
+.. variable:: LZ4_FOUND
+
+  Set to true if LZ4 library found, otherwise false or undefined.
+
+.. variable:: LZ4_INCLUDE_DIRS
+
+  Paths to include directories listed in one variable for use by LZ4 client.
+
+.. variable:: LZ4_LIBRARIES
+
+  Paths to libraries to linked against to use LZ4.
+
+.. variable:: LZ4_VERSION
+
+  The version string of LZ4 found.
+
+Cache variables
+^^^^^^^^^^^^^^^
+
+For users who wish to edit and control the module behavior, this module
+reads hints about search locations from the following variables::
+
+.. variable:: LZ4_INCLUDE_DIR
+
+  Path to LZ4 include directory with ``lz4.h`` header.
+
+.. variable:: LZ4_LIBRARY
+
+  Path to LZ4 library to be linked.
+
+NOTE: The variables above should not usually be used in CMakeLists.txt files!
+
+#]=======================================================================]
+
+### Find library ##############################################################
+
+if(NOT LZ4_LIBRARY)
+    find_library(LZ4_LIBRARY_RELEASE NAMES lz4)
+    find_library(LZ4_LIBRARY_DEBUG NAMES lz4d)
+
+    include(SelectLibraryConfigurations)
+    select_library_configurations(LZ4)
+else()
+    file(TO_CMAKE_PATH "${LZ4_LIBRARY}" LZ4_LIBRARY)
+endif()
+
+### Find include directory ####################################################
+find_path(LZ4_INCLUDE_DIR NAMES lz4.h)
+
+if(LZ4_INCLUDE_DIR AND EXISTS "${LZ4_INCLUDE_DIR}/lz4.h")
+    file(STRINGS "${LZ4_INCLUDE_DIR}/lz4.h" _lz4_h_contents
+            REGEX "#define LZ4_VERSION_[A-Z]+[ ]+[0-9]+")
+    string(REGEX REPLACE "#define LZ4_VERSION_MAJOR[ ]+([0-9]+).+" "\\1"
+            LZ4_VERSION_MAJOR "${_lz4_h_contents}")
+    string(REGEX REPLACE ".+#define LZ4_VERSION_MINOR[ ]+([0-9]+).+" "\\1"
+            LZ4_VERSION_MINOR "${_lz4_h_contents}")
+    string(REGEX REPLACE ".+#define LZ4_VERSION_RELEASE[ ]+([0-9]+).*" "\\1"
+            LZ4_VERSION_RELEASE "${_lz4_h_contents}")
+    set(LZ4_VERSION "${LZ4_VERSION_MAJOR}.${LZ4_VERSION_MINOR}.${LZ4_VERSION_RELEASE}")
+    unset(_lz4_h_contents)
+endif()
+
+### Set result variables ######################################################
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(LZ4 DEFAULT_MSG
+        LZ4_LIBRARY LZ4_INCLUDE_DIR LZ4_VERSION)
+
+mark_as_advanced(LZ4_INCLUDE_DIR LZ4_LIBRARY)
+
+set(LZ4_LIBRARIES ${LZ4_LIBRARY})
+set(LZ4_INCLUDE_DIRS ${LZ4_INCLUDE_DIR})
+
+### Import targets ############################################################
+if(LZ4_FOUND)
+    if(NOT TARGET LZ4::LZ4)
+        add_library(LZ4::LZ4 UNKNOWN IMPORTED)
+        set_target_properties(LZ4::LZ4 PROPERTIES
+                IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+                INTERFACE_INCLUDE_DIRECTORIES "${LZ4_INCLUDE_DIR}")
+
+        if(LZ4_LIBRARY_RELEASE)
+            set_property(TARGET LZ4::LZ4 APPEND PROPERTY
+                    IMPORTED_CONFIGURATIONS RELEASE)
+            set_target_properties(LZ4::LZ4 PROPERTIES
+                    IMPORTED_LOCATION_RELEASE "${LZ4_LIBRARY_RELEASE}")
+        endif()
+
+        if(LZ4_LIBRARY_DEBUG)
+            set_property(TARGET LZ4::LZ4 APPEND PROPERTY
+                    IMPORTED_CONFIGURATIONS DEBUG)
+            set_target_properties(LZ4::LZ4 PROPERTIES
+                    IMPORTED_LOCATION_DEBUG "${LZ4_LIBRARY_DEBUG}")
+        endif()
+
+        if(NOT LZ4_LIBRARY_RELEASE AND NOT LZ4_LIBRARY_DEBUG)
+            set_property(TARGET LZ4::LZ4 APPEND PROPERTY
+                    IMPORTED_LOCATION "${LZ4_LIBRARY}")
+        endif()
+    endif()
+endif()

components/CMakeLists.txt

@@ -233,7 +233,7 @@ target_link_libraries(components
     ${SDL2_LIBRARIES}
     ${OPENGL_gl_LIBRARY}
     ${MyGUI_LIBRARIES}
-    ${BSAOPTHASH_LIBRARIES}
+    LZ4::LZ4
     RecastNavigation::DebugUtils
     RecastNavigation::Detour
     RecastNavigation::Recast

components/bsa/compressedbsafile.cpp

@@ -27,6 +27,8 @@
 #include <stdexcept>
 #include <cassert>
 
+#include <lz4frame.h>
+
 #include <boost/scoped_array.hpp>
 #include <boost/filesystem/path.hpp>
 #include <boost/filesystem/fstream.hpp>
@@ -132,8 +134,11 @@ void CompressedBSAFile::readHeader()
 
         input.read(reinterpret_cast<char*>(header), 36);
 
-        if(header[0] != 0x00415342 /*"BSA\x00"*/ || (header[1] != 0x67 /*TES4*/ && header[1] != 0x68 /*TES5*/))
+        if (header[0] != 0x00415342) /*"BSA\x00"*/
-            fail("Unrecognized TES4 BSA header");
+            fail("Unrecognized compressed BSA format");
+        mVersion = header[1];
+        if (mVersion != 0x67 /*TES4*/ && mVersion != 0x68 /*FO3, FNV, TES5*/ && mVersion != 0x69 /*SSE*/)
+            fail("Unrecognized compressed BSA version");
 
         // header[2] is offset, should be 36 = 0x24 which is the size of the header
 
@@ -158,7 +163,8 @@ void CompressedBSAFile::readHeader()
         // header[8]; // fileFlags : an opportunity to optimize here
 
         mCompressedByDefault = (archiveFlags & 0x4) != 0;
-        mEmbeddedFileNames = header[1] == 0x68 /*TES5*/ && (archiveFlags & 0x100) != 0;
+        if (mVersion == 0x68 || mVersion == 0x69) /*FO3, FNV, TES5, SSE*/
+            mEmbeddedFileNames = (archiveFlags & 0x100) != 0;
     }
 
     // folder records
@@ -168,7 +174,14 @@ void CompressedBSAFile::readHeader()
     {
         input.read(reinterpret_cast<char*>(&hash), 8);
         input.read(reinterpret_cast<char*>(&fr.count), 4); // not sure purpose of count
-        input.read(reinterpret_cast<char*>(&fr.offset), 4); // not sure purpose of offset
+        if (mVersion == 0x69) // SSE
+        {
+            std::uint32_t unknown;
+            input.read(reinterpret_cast<char*>(&unknown), 4);
+            input.read(reinterpret_cast<char*>(&fr.offset), 8);
+        }
+        else
+            input.read(reinterpret_cast<char*>(&fr.offset), 4); // not sure purpose of offset
 
         std::map<std::uint64_t, FolderRecord>::const_iterator lb = mFolders.lower_bound(hash);
         if (lb != mFolders.end() && !(mFolders.key_comp()(hash, lb->first)))
@@ -327,32 +340,56 @@ Files::IStreamPtr CompressedBSAFile::getFile(const char* file)
 
 Files::IStreamPtr CompressedBSAFile::getFile(const FileRecord& fileRecord)
 {
-    if (fileRecord.isCompressed(mCompressedByDefault)) {
+    size_t size = fileRecord.getSizeWithoutCompressionFlag();
-        Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, fileRecord.getSizeWithoutCompressionFlag());
+    size_t uncompressedSize = size;
+    bool compressed = fileRecord.isCompressed(mCompressedByDefault);
+    Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, size);
+    std::istream* fileStream = streamPtr.get();
+    if (mEmbeddedFileNames)
+    {
+        // Skip over the embedded file name
+        char length = 0;
+        fileStream->read(&length, 1);
+        fileStream->ignore(length);
+        size -= length + sizeof(char);
+    }
+    if (compressed)
+    {
+        fileStream->read(reinterpret_cast<char*>(&uncompressedSize), sizeof(uint32_t));
+        size -= sizeof(uint32_t);
+    }
+    std::shared_ptr<Bsa::MemoryInputStream> memoryStreamPtr = std::make_shared<MemoryInputStream>(uncompressedSize);
 
-        std::istream* fileStream = streamPtr.get();
+    if (compressed)
+    {
+        if (mVersion != 0x69) // Non-SSE: zlib
+        {
+            boost::iostreams::filtering_streambuf<boost::iostreams::input> inputStreamBuf;
+            inputStreamBuf.push(boost::iostreams::zlib_decompressor());
+            inputStreamBuf.push(*fileStream);
 
-        if (mEmbeddedFileNames) {
+            boost::iostreams::basic_array_sink<char> sr(memoryStreamPtr->getRawData(), uncompressedSize);
-            std::string embeddedFileName;
+            boost::iostreams::copy(inputStreamBuf, sr);
-            getBZString(embeddedFileName, *fileStream);
         }
-
+        else // SSE: lz4
-        uint32_t uncompressedSize = 0u;
+        {
-        fileStream->read(reinterpret_cast<char*>(&uncompressedSize), sizeof(uncompressedSize));
+            boost::scoped_array<char> buffer(new char[size]);
-
+            fileStream->read(buffer.get(), size);
-        boost::iostreams::filtering_streambuf<boost::iostreams::input> inputStreamBuf;
+            LZ4F_dctx* context = nullptr;
-        inputStreamBuf.push(boost::iostreams::zlib_decompressor());
+            LZ4F_createDecompressionContext(&context, LZ4F_VERSION);
-        inputStreamBuf.push(*fileStream);
+            LZ4F_decompressOptions_t options = {};
-
+            LZ4F_decompress(context, memoryStreamPtr->getRawData(), &uncompressedSize, buffer.get(), &size, &options);
-        std::shared_ptr<Bsa::MemoryInputStream> memoryStreamPtr = std::make_shared<MemoryInputStream>(uncompressedSize);
+            LZ4F_errorCode_t errorCode = LZ4F_freeDecompressionContext(context);
-
+            if (LZ4F_isError(errorCode))
-        boost::iostreams::basic_array_sink<char> sr(memoryStreamPtr->getRawData(), uncompressedSize);
+                fail("LZ4 decompression error (file " + mFilename + "): " + LZ4F_getErrorName(errorCode));
-        boost::iostreams::copy(inputStreamBuf, sr);
+        }
-
+    }
-        return std::shared_ptr<std::istream>(memoryStreamPtr, (std::istream*)memoryStreamPtr.get());
+    else
+    {
+        fileStream->read(memoryStreamPtr->getRawData(), size);
     }
 
-    return Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, fileRecord.size);
+    return std::shared_ptr<std::istream>(memoryStreamPtr, (std::istream*)memoryStreamPtr.get());
 }
 
 BsaVersion CompressedBSAFile::detectVersion(std::string filePath)

components/bsa/compressedbsafile.hpp

@@ -64,10 +64,12 @@ namespace Bsa
         //if each file record begins with BZ string with file name
         bool mEmbeddedFileNames;
 
+        std::uint32_t mVersion{0u};
+
         struct FolderRecord
         {
             std::uint32_t count;
-            std::uint32_t offset;
+            std::uint64_t offset;
             std::map<std::uint64_t, FileRecord> files;
         };
         std::map<std::uint64_t, FolderRecord> mFolders;