1
0
mirror of https://gitlab.com/OpenMW/openmw.git synced 2025-03-16 07:20:54 +00:00

Merge branch 'bsa' into 'master'

Compressed BSA support revisions

See merge request OpenMW/openmw!350
This commit is contained in:
psi29a 2020-10-20 09:13:17 +00:00
commit 992c82f298
10 changed files with 232 additions and 30 deletions

View File

@ -13,7 +13,7 @@ stages:
before_script:
- export APT_CACHE_DIR=`pwd`/apt-cache && mkdir -pv $APT_CACHE_DIR
- apt-get update -yq
- apt-get -o dir::cache::archives="$APT_CACHE_DIR" install -y cmake build-essential libboost-filesystem-dev libboost-program-options-dev libboost-system-dev libboost-iostreams-dev libavcodec-dev libavformat-dev libavutil-dev libswscale-dev libswresample-dev libsdl2-dev libqt5opengl5-dev libopenal-dev libopenscenegraph-dev libunshield-dev libtinyxml-dev libmygui-dev libbullet-dev ccache git clang
- apt-get -o dir::cache::archives="$APT_CACHE_DIR" install -y cmake build-essential libboost-filesystem-dev libboost-program-options-dev libboost-system-dev libboost-iostreams-dev libavcodec-dev libavformat-dev libavutil-dev libswscale-dev libswresample-dev libsdl2-dev libqt5opengl5-dev libopenal-dev libopenscenegraph-dev libunshield-dev libtinyxml-dev libmygui-dev libbullet-dev liblz4-dev ccache git clang
stage: build
script:
- export CCACHE_BASEDIR="`pwd`"

View File

@ -23,7 +23,7 @@ addons:
# FFmpeg
libavcodec-dev, libavformat-dev, libavutil-dev, libswresample-dev, libswscale-dev,
# Audio, Video and Misc. deps
libsdl2-dev, libqt5opengl5-dev, libopenal-dev, libunshield-dev, libtinyxml-dev,
libsdl2-dev, libqt5opengl5-dev, libopenal-dev, libunshield-dev, libtinyxml-dev, liblz4-dev
# The other ones from OpenMW ppa
libbullet-dev, libopenscenegraph-dev, libmygui-dev
]

View File

@ -73,6 +73,7 @@
Feature #5579: MCP SetAngle enhancement
Feature #5610: Actors movement should be smoother
Feature #5642: Ability to attach arrows to actor skeleton instead of bow mesh
Feature #5649: Skyrim SE compressed BSA format support
Task #5480: Drop Qt4 support
Task #5520: Improve cell name autocompleter implementation

View File

@ -5,5 +5,5 @@ command -v ccache >/dev/null 2>&1 || brew install ccache
command -v cmake >/dev/null 2>&1 || brew install cmake
command -v qmake >/dev/null 2>&1 || brew install qt
curl -fSL -R -J https://downloads.openmw.org/osx/dependencies/openmw-deps-ef2462c.zip -o ~/openmw-deps.zip
curl -fSL -R -J https://downloads.openmw.org/osx/dependencies/openmw-deps-20201016.zip -o ~/openmw-deps.zip
unzip -o ~/openmw-deps.zip -d /private/tmp/openmw-deps > /dev/null

View File

@ -571,6 +571,11 @@ if [ -z $SKIP_DOWNLOAD ]; then
"https://gitlab.com/OpenMW/openmw-deps/-/raw/main/windows/SDL2-2.0.12.zip" \
"SDL2-2.0.12.zip"
# LZ4
download "LZ4 1.9.2" \
"https://gitlab.com/OpenMW/openmw-deps/-/raw/main/windows/lz4_win${BITS}_v1_9_2.7z" \
"lz4_win${BITS}_v1_9_2.7z"
# Google test and mock
if [ ! -z $TEST_FRAMEWORK ]; then
echo "Google test 1.10.0..."
@ -901,6 +906,31 @@ printf "SDL 2.0.12... "
}
cd $DEPS
echo
# LZ4
printf "LZ4 1.9.2... "
{
if [ -d LZ4_1.9.2 ]; then
printf "Exists. "
elif [ -z $SKIP_EXTRACT ]; then
rm -rf LZ4_1.9.2
eval 7z x -y lz4_win${BITS}_v1_9_2.7z -o./LZ4_1.9.2 $STRIP
fi
export LZ4DIR="$(real_pwd)/LZ4_1.9.2"
add_cmake_opts -DLZ4_INCLUDE_DIR="${LZ4DIR}/include" \
-DLZ4_LIBRARY="${LZ4DIR}/lib/liblz4.lib"
for CONFIGURATION in ${CONFIGURATIONS[@]}; do
if [ $CONFIGURATION == "Debug" ]; then
LZ4_CONFIGURATION="Debug"
else
SUFFIX=""
LZ4_CONFIGURATION="Release"
fi
add_runtime_dlls $CONFIGURATION "$(pwd)/LZ4_1.9.2/bin/${LZ4_CONFIGURATION}/liblz4.dll"
done
echo Done.
}
cd $DEPS
echo
# Google Test and Google Mock
if [ ! -z $TEST_FRAMEWORK ]; then
printf "Google test 1.10.0 ..."

View File

@ -152,6 +152,8 @@ endif()
# Dependencies
find_package(OpenGL REQUIRED)
find_package(LZ4 REQUIRED)
if (USE_QT)
find_package(Qt5Core 5.12 REQUIRED)
find_package(Qt5Widgets REQUIRED)

130
cmake/FindLZ4.cmake Normal file
View File

@ -0,0 +1,130 @@
# Distributed under the OSI-approved BSD 3-Clause License. See accompanying
# file Copyright.txt or https://cmake.org/licensing for details.
#[=======================================================================[.rst:
FindLZ4
-------
Find the LZ4 include directory and library.
Use this module by invoking find_package with the form::
.. code-block:: cmake
find_package(LZ4
[version] # Minimum version e.g. 1.8.0
[REQUIRED] # Fail with error if LZ4 is not found
)
Imported targets
^^^^^^^^^^^^^^^^
This module defines the following :prop_tgt:`IMPORTED` targets:
.. variable:: LZ4::LZ4
Imported target for using the LZ4 library, if found.
Result variables
^^^^^^^^^^^^^^^^
.. variable:: LZ4_FOUND
Set to true if LZ4 library found, otherwise false or undefined.
.. variable:: LZ4_INCLUDE_DIRS
Paths to include directories listed in one variable for use by LZ4 client.
.. variable:: LZ4_LIBRARIES
Paths to libraries to linked against to use LZ4.
.. variable:: LZ4_VERSION
The version string of LZ4 found.
Cache variables
^^^^^^^^^^^^^^^
For users who wish to edit and control the module behavior, this module
reads hints about search locations from the following variables::
.. variable:: LZ4_INCLUDE_DIR
Path to LZ4 include directory with ``lz4.h`` header.
.. variable:: LZ4_LIBRARY
Path to LZ4 library to be linked.
NOTE: The variables above should not usually be used in CMakeLists.txt files!
#]=======================================================================]
### Find library ##############################################################
if(NOT LZ4_LIBRARY)
find_library(LZ4_LIBRARY_RELEASE NAMES lz4)
find_library(LZ4_LIBRARY_DEBUG NAMES lz4d)
include(SelectLibraryConfigurations)
select_library_configurations(LZ4)
else()
file(TO_CMAKE_PATH "${LZ4_LIBRARY}" LZ4_LIBRARY)
endif()
### Find include directory ####################################################
find_path(LZ4_INCLUDE_DIR NAMES lz4.h)
if(LZ4_INCLUDE_DIR AND EXISTS "${LZ4_INCLUDE_DIR}/lz4.h")
file(STRINGS "${LZ4_INCLUDE_DIR}/lz4.h" _lz4_h_contents
REGEX "#define LZ4_VERSION_[A-Z]+[ ]+[0-9]+")
string(REGEX REPLACE "#define LZ4_VERSION_MAJOR[ ]+([0-9]+).+" "\\1"
LZ4_VERSION_MAJOR "${_lz4_h_contents}")
string(REGEX REPLACE ".+#define LZ4_VERSION_MINOR[ ]+([0-9]+).+" "\\1"
LZ4_VERSION_MINOR "${_lz4_h_contents}")
string(REGEX REPLACE ".+#define LZ4_VERSION_RELEASE[ ]+([0-9]+).*" "\\1"
LZ4_VERSION_RELEASE "${_lz4_h_contents}")
set(LZ4_VERSION "${LZ4_VERSION_MAJOR}.${LZ4_VERSION_MINOR}.${LZ4_VERSION_RELEASE}")
unset(_lz4_h_contents)
endif()
### Set result variables ######################################################
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(LZ4 DEFAULT_MSG
LZ4_LIBRARY LZ4_INCLUDE_DIR LZ4_VERSION)
mark_as_advanced(LZ4_INCLUDE_DIR LZ4_LIBRARY)
set(LZ4_LIBRARIES ${LZ4_LIBRARY})
set(LZ4_INCLUDE_DIRS ${LZ4_INCLUDE_DIR})
### Import targets ############################################################
if(LZ4_FOUND)
if(NOT TARGET LZ4::LZ4)
add_library(LZ4::LZ4 UNKNOWN IMPORTED)
set_target_properties(LZ4::LZ4 PROPERTIES
IMPORTED_LINK_INTERFACE_LANGUAGES "C"
INTERFACE_INCLUDE_DIRECTORIES "${LZ4_INCLUDE_DIR}")
if(LZ4_LIBRARY_RELEASE)
set_property(TARGET LZ4::LZ4 APPEND PROPERTY
IMPORTED_CONFIGURATIONS RELEASE)
set_target_properties(LZ4::LZ4 PROPERTIES
IMPORTED_LOCATION_RELEASE "${LZ4_LIBRARY_RELEASE}")
endif()
if(LZ4_LIBRARY_DEBUG)
set_property(TARGET LZ4::LZ4 APPEND PROPERTY
IMPORTED_CONFIGURATIONS DEBUG)
set_target_properties(LZ4::LZ4 PROPERTIES
IMPORTED_LOCATION_DEBUG "${LZ4_LIBRARY_DEBUG}")
endif()
if(NOT LZ4_LIBRARY_RELEASE AND NOT LZ4_LIBRARY_DEBUG)
set_property(TARGET LZ4::LZ4 APPEND PROPERTY
IMPORTED_LOCATION "${LZ4_LIBRARY}")
endif()
endif()
endif()

View File

@ -233,7 +233,7 @@ target_link_libraries(components
${SDL2_LIBRARIES}
${OPENGL_gl_LIBRARY}
${MyGUI_LIBRARIES}
${BSAOPTHASH_LIBRARIES}
LZ4::LZ4
RecastNavigation::DebugUtils
RecastNavigation::Detour
RecastNavigation::Recast

View File

@ -27,6 +27,8 @@
#include <stdexcept>
#include <cassert>
#include <lz4frame.h>
#include <boost/scoped_array.hpp>
#include <boost/filesystem/path.hpp>
#include <boost/filesystem/fstream.hpp>
@ -132,8 +134,11 @@ void CompressedBSAFile::readHeader()
input.read(reinterpret_cast<char*>(header), 36);
if(header[0] != 0x00415342 /*"BSA\x00"*/ || (header[1] != 0x67 /*TES4*/ && header[1] != 0x68 /*TES5*/))
fail("Unrecognized TES4 BSA header");
if (header[0] != 0x00415342) /*"BSA\x00"*/
fail("Unrecognized compressed BSA format");
mVersion = header[1];
if (mVersion != 0x67 /*TES4*/ && mVersion != 0x68 /*FO3, FNV, TES5*/ && mVersion != 0x69 /*SSE*/)
fail("Unrecognized compressed BSA version");
// header[2] is offset, should be 36 = 0x24 which is the size of the header
@ -158,7 +163,8 @@ void CompressedBSAFile::readHeader()
// header[8]; // fileFlags : an opportunity to optimize here
mCompressedByDefault = (archiveFlags & 0x4) != 0;
mEmbeddedFileNames = header[1] == 0x68 /*TES5*/ && (archiveFlags & 0x100) != 0;
if (mVersion == 0x68 || mVersion == 0x69) /*FO3, FNV, TES5, SSE*/
mEmbeddedFileNames = (archiveFlags & 0x100) != 0;
}
// folder records
@ -168,7 +174,14 @@ void CompressedBSAFile::readHeader()
{
input.read(reinterpret_cast<char*>(&hash), 8);
input.read(reinterpret_cast<char*>(&fr.count), 4); // not sure purpose of count
input.read(reinterpret_cast<char*>(&fr.offset), 4); // not sure purpose of offset
if (mVersion == 0x69) // SSE
{
std::uint32_t unknown;
input.read(reinterpret_cast<char*>(&unknown), 4);
input.read(reinterpret_cast<char*>(&fr.offset), 8);
}
else
input.read(reinterpret_cast<char*>(&fr.offset), 4); // not sure purpose of offset
std::map<std::uint64_t, FolderRecord>::const_iterator lb = mFolders.lower_bound(hash);
if (lb != mFolders.end() && !(mFolders.key_comp()(hash, lb->first)))
@ -327,32 +340,56 @@ Files::IStreamPtr CompressedBSAFile::getFile(const char* file)
Files::IStreamPtr CompressedBSAFile::getFile(const FileRecord& fileRecord)
{
if (fileRecord.isCompressed(mCompressedByDefault)) {
Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, fileRecord.getSizeWithoutCompressionFlag());
size_t size = fileRecord.getSizeWithoutCompressionFlag();
size_t uncompressedSize = size;
bool compressed = fileRecord.isCompressed(mCompressedByDefault);
Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, size);
std::istream* fileStream = streamPtr.get();
if (mEmbeddedFileNames)
{
// Skip over the embedded file name
char length = 0;
fileStream->read(&length, 1);
fileStream->ignore(length);
size -= length + sizeof(char);
}
if (compressed)
{
fileStream->read(reinterpret_cast<char*>(&uncompressedSize), sizeof(uint32_t));
size -= sizeof(uint32_t);
}
std::shared_ptr<Bsa::MemoryInputStream> memoryStreamPtr = std::make_shared<MemoryInputStream>(uncompressedSize);
std::istream* fileStream = streamPtr.get();
if (compressed)
{
if (mVersion != 0x69) // Non-SSE: zlib
{
boost::iostreams::filtering_streambuf<boost::iostreams::input> inputStreamBuf;
inputStreamBuf.push(boost::iostreams::zlib_decompressor());
inputStreamBuf.push(*fileStream);
if (mEmbeddedFileNames) {
std::string embeddedFileName;
getBZString(embeddedFileName, *fileStream);
boost::iostreams::basic_array_sink<char> sr(memoryStreamPtr->getRawData(), uncompressedSize);
boost::iostreams::copy(inputStreamBuf, sr);
}
uint32_t uncompressedSize = 0u;
fileStream->read(reinterpret_cast<char*>(&uncompressedSize), sizeof(uncompressedSize));
boost::iostreams::filtering_streambuf<boost::iostreams::input> inputStreamBuf;
inputStreamBuf.push(boost::iostreams::zlib_decompressor());
inputStreamBuf.push(*fileStream);
std::shared_ptr<Bsa::MemoryInputStream> memoryStreamPtr = std::make_shared<MemoryInputStream>(uncompressedSize);
boost::iostreams::basic_array_sink<char> sr(memoryStreamPtr->getRawData(), uncompressedSize);
boost::iostreams::copy(inputStreamBuf, sr);
return std::shared_ptr<std::istream>(memoryStreamPtr, (std::istream*)memoryStreamPtr.get());
else // SSE: lz4
{
boost::scoped_array<char> buffer(new char[size]);
fileStream->read(buffer.get(), size);
LZ4F_dctx* context = nullptr;
LZ4F_createDecompressionContext(&context, LZ4F_VERSION);
LZ4F_decompressOptions_t options = {};
LZ4F_decompress(context, memoryStreamPtr->getRawData(), &uncompressedSize, buffer.get(), &size, &options);
LZ4F_errorCode_t errorCode = LZ4F_freeDecompressionContext(context);
if (LZ4F_isError(errorCode))
fail("LZ4 decompression error (file " + mFilename + "): " + LZ4F_getErrorName(errorCode));
}
}
else
{
fileStream->read(memoryStreamPtr->getRawData(), size);
}
return Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, fileRecord.size);
return std::shared_ptr<std::istream>(memoryStreamPtr, (std::istream*)memoryStreamPtr.get());
}
BsaVersion CompressedBSAFile::detectVersion(std::string filePath)

View File

@ -64,10 +64,12 @@ namespace Bsa
//if each file record begins with BZ string with file name
bool mEmbeddedFileNames;
std::uint32_t mVersion{0u};
struct FolderRecord
{
std::uint32_t count;
std::uint32_t offset;
std::uint64_t offset;
std::map<std::uint64_t, FileRecord> files;
};
std::map<std::uint64_t, FolderRecord> mFolders;