mirror of
https://gitlab.com/OpenMW/openmw.git
synced 2025-03-16 07:20:54 +00:00
Merge branch 'bsa' into 'master'
Compressed BSA support revisions See merge request OpenMW/openmw!350
This commit is contained in:
psi29a
commit
992c82f298
@ -13,7 +13,7 @@ stages:
|
||||
before_script:
|
||||
- export APT_CACHE_DIR=`pwd`/apt-cache && mkdir -pv $APT_CACHE_DIR
|
||||
- apt-get update -yq
|
||||
- apt-get -o dir::cache::archives="$APT_CACHE_DIR" install -y cmake build-essential libboost-filesystem-dev libboost-program-options-dev libboost-system-dev libboost-iostreams-dev libavcodec-dev libavformat-dev libavutil-dev libswscale-dev libswresample-dev libsdl2-dev libqt5opengl5-dev libopenal-dev libopenscenegraph-dev libunshield-dev libtinyxml-dev libmygui-dev libbullet-dev ccache git clang
|
||||
- apt-get -o dir::cache::archives="$APT_CACHE_DIR" install -y cmake build-essential libboost-filesystem-dev libboost-program-options-dev libboost-system-dev libboost-iostreams-dev libavcodec-dev libavformat-dev libavutil-dev libswscale-dev libswresample-dev libsdl2-dev libqt5opengl5-dev libopenal-dev libopenscenegraph-dev libunshield-dev libtinyxml-dev libmygui-dev libbullet-dev liblz4-dev ccache git clang
|
||||
stage: build
|
||||
script:
|
||||
- export CCACHE_BASEDIR="`pwd`"
|
||||
|
||||
@ -23,7 +23,7 @@ addons:
|
||||
# FFmpeg
|
||||
libavcodec-dev, libavformat-dev, libavutil-dev, libswresample-dev, libswscale-dev,
|
||||
# Audio, Video and Misc. deps
|
||||
libsdl2-dev, libqt5opengl5-dev, libopenal-dev, libunshield-dev, libtinyxml-dev,
|
||||
libsdl2-dev, libqt5opengl5-dev, libopenal-dev, libunshield-dev, libtinyxml-dev, liblz4-dev
|
||||
# The other ones from OpenMW ppa
|
||||
libbullet-dev, libopenscenegraph-dev, libmygui-dev
|
||||
]
|
||||
|
||||
@ -73,6 +73,7 @@
|
||||
Feature #5579: MCP SetAngle enhancement
|
||||
Feature #5610: Actors movement should be smoother
|
||||
Feature #5642: Ability to attach arrows to actor skeleton instead of bow mesh
|
||||
Feature #5649: Skyrim SE compressed BSA format support
|
||||
Task #5480: Drop Qt4 support
|
||||
Task #5520: Improve cell name autocompleter implementation
|
||||
|
||||
|
||||
@ -5,5 +5,5 @@ command -v ccache >/dev/null 2>&1 || brew install ccache
|
||||
command -v cmake >/dev/null 2>&1 || brew install cmake
|
||||
command -v qmake >/dev/null 2>&1 || brew install qt
|
||||
|
||||
curl -fSL -R -J https://downloads.openmw.org/osx/dependencies/openmw-deps-ef2462c.zip -o ~/openmw-deps.zip
|
||||
curl -fSL -R -J https://downloads.openmw.org/osx/dependencies/openmw-deps-20201016.zip -o ~/openmw-deps.zip
|
||||
unzip -o ~/openmw-deps.zip -d /private/tmp/openmw-deps > /dev/null
|
||||
|
||||
@ -571,6 +571,11 @@ if [ -z $SKIP_DOWNLOAD ]; then
|
||||
"https://gitlab.com/OpenMW/openmw-deps/-/raw/main/windows/SDL2-2.0.12.zip" \
|
||||
"SDL2-2.0.12.zip"
|
||||
|
||||
# LZ4
|
||||
download "LZ4 1.9.2" \
|
||||
"https://gitlab.com/OpenMW/openmw-deps/-/raw/main/windows/lz4_win${BITS}_v1_9_2.7z" \
|
||||
"lz4_win${BITS}_v1_9_2.7z"
|
||||
|
||||
# Google test and mock
|
||||
if [ ! -z $TEST_FRAMEWORK ]; then
|
||||
echo "Google test 1.10.0..."
|
||||
@ -901,6 +906,31 @@ printf "SDL 2.0.12... "
|
||||
}
|
||||
cd $DEPS
|
||||
echo
|
||||
# LZ4
|
||||
printf "LZ4 1.9.2... "
|
||||
{
|
||||
if [ -d LZ4_1.9.2 ]; then
|
||||
printf "Exists. "
|
||||
elif [ -z $SKIP_EXTRACT ]; then
|
||||
rm -rf LZ4_1.9.2
|
||||
eval 7z x -y lz4_win${BITS}_v1_9_2.7z -o./LZ4_1.9.2 $STRIP
|
||||
fi
|
||||
export LZ4DIR="$(real_pwd)/LZ4_1.9.2"
|
||||
add_cmake_opts -DLZ4_INCLUDE_DIR="${LZ4DIR}/include" \
|
||||
-DLZ4_LIBRARY="${LZ4DIR}/lib/liblz4.lib"
|
||||
for CONFIGURATION in ${CONFIGURATIONS[@]}; do
|
||||
if [ $CONFIGURATION == "Debug" ]; then
|
||||
LZ4_CONFIGURATION="Debug"
|
||||
else
|
||||
SUFFIX=""
|
||||
LZ4_CONFIGURATION="Release"
|
||||
fi
|
||||
add_runtime_dlls $CONFIGURATION "$(pwd)/LZ4_1.9.2/bin/${LZ4_CONFIGURATION}/liblz4.dll"
|
||||
done
|
||||
echo Done.
|
||||
}
|
||||
cd $DEPS
|
||||
echo
|
||||
# Google Test and Google Mock
|
||||
if [ ! -z $TEST_FRAMEWORK ]; then
|
||||
printf "Google test 1.10.0 ..."
|
||||
|
||||
@ -152,6 +152,8 @@ endif()
|
||||
# Dependencies
|
||||
find_package(OpenGL REQUIRED)
|
||||
|
||||
find_package(LZ4 REQUIRED)
|
||||
|
||||
if (USE_QT)
|
||||
find_package(Qt5Core 5.12 REQUIRED)
|
||||
find_package(Qt5Widgets REQUIRED)
|
||||
|
||||
130
cmake/FindLZ4.cmake
Normal file
130
cmake/FindLZ4.cmake
Normal file
@ -0,0 +1,130 @@
|
||||
# Distributed under the OSI-approved BSD 3-Clause License. See accompanying
|
||||
# file Copyright.txt or https://cmake.org/licensing for details.
|
||||
|
||||
#[=======================================================================[.rst:
|
||||
FindLZ4
|
||||
-------
|
||||
|
||||
Find the LZ4 include directory and library.
|
||||
|
||||
Use this module by invoking find_package with the form::
|
||||
|
||||
.. code-block:: cmake
|
||||
|
||||
find_package(LZ4
|
||||
[version] # Minimum version e.g. 1.8.0
|
||||
[REQUIRED] # Fail with error if LZ4 is not found
|
||||
)
|
||||
|
||||
Imported targets
|
||||
^^^^^^^^^^^^^^^^
|
||||
|
||||
This module defines the following :prop_tgt:`IMPORTED` targets:
|
||||
|
||||
.. variable:: LZ4::LZ4
|
||||
|
||||
Imported target for using the LZ4 library, if found.
|
||||
|
||||
Result variables
|
||||
^^^^^^^^^^^^^^^^
|
||||
|
||||
.. variable:: LZ4_FOUND
|
||||
|
||||
Set to true if LZ4 library found, otherwise false or undefined.
|
||||
|
||||
.. variable:: LZ4_INCLUDE_DIRS
|
||||
|
||||
Paths to include directories listed in one variable for use by LZ4 client.
|
||||
|
||||
.. variable:: LZ4_LIBRARIES
|
||||
|
||||
Paths to libraries to linked against to use LZ4.
|
||||
|
||||
.. variable:: LZ4_VERSION
|
||||
|
||||
The version string of LZ4 found.
|
||||
|
||||
Cache variables
|
||||
^^^^^^^^^^^^^^^
|
||||
|
||||
For users who wish to edit and control the module behavior, this module
|
||||
reads hints about search locations from the following variables::
|
||||
|
||||
.. variable:: LZ4_INCLUDE_DIR
|
||||
|
||||
Path to LZ4 include directory with ``lz4.h`` header.
|
||||
|
||||
.. variable:: LZ4_LIBRARY
|
||||
|
||||
Path to LZ4 library to be linked.
|
||||
|
||||
NOTE: The variables above should not usually be used in CMakeLists.txt files!
|
||||
|
||||
#]=======================================================================]
|
||||
|
||||
### Find library ##############################################################
|
||||
|
||||
if(NOT LZ4_LIBRARY)
|
||||
find_library(LZ4_LIBRARY_RELEASE NAMES lz4)
|
||||
find_library(LZ4_LIBRARY_DEBUG NAMES lz4d)
|
||||
|
||||
include(SelectLibraryConfigurations)
|
||||
select_library_configurations(LZ4)
|
||||
else()
|
||||
file(TO_CMAKE_PATH "${LZ4_LIBRARY}" LZ4_LIBRARY)
|
||||
endif()
|
||||
|
||||
### Find include directory ####################################################
|
||||
find_path(LZ4_INCLUDE_DIR NAMES lz4.h)
|
||||
|
||||
if(LZ4_INCLUDE_DIR AND EXISTS "${LZ4_INCLUDE_DIR}/lz4.h")
|
||||
file(STRINGS "${LZ4_INCLUDE_DIR}/lz4.h" _lz4_h_contents
|
||||
REGEX "#define LZ4_VERSION_[A-Z]+[ ]+[0-9]+")
|
||||
string(REGEX REPLACE "#define LZ4_VERSION_MAJOR[ ]+([0-9]+).+" "\\1"
|
||||
LZ4_VERSION_MAJOR "${_lz4_h_contents}")
|
||||
string(REGEX REPLACE ".+#define LZ4_VERSION_MINOR[ ]+([0-9]+).+" "\\1"
|
||||
LZ4_VERSION_MINOR "${_lz4_h_contents}")
|
||||
string(REGEX REPLACE ".+#define LZ4_VERSION_RELEASE[ ]+([0-9]+).*" "\\1"
|
||||
LZ4_VERSION_RELEASE "${_lz4_h_contents}")
|
||||
set(LZ4_VERSION "${LZ4_VERSION_MAJOR}.${LZ4_VERSION_MINOR}.${LZ4_VERSION_RELEASE}")
|
||||
unset(_lz4_h_contents)
|
||||
endif()
|
||||
|
||||
### Set result variables ######################################################
|
||||
include(FindPackageHandleStandardArgs)
|
||||
find_package_handle_standard_args(LZ4 DEFAULT_MSG
|
||||
LZ4_LIBRARY LZ4_INCLUDE_DIR LZ4_VERSION)
|
||||
|
||||
mark_as_advanced(LZ4_INCLUDE_DIR LZ4_LIBRARY)
|
||||
|
||||
set(LZ4_LIBRARIES ${LZ4_LIBRARY})
|
||||
set(LZ4_INCLUDE_DIRS ${LZ4_INCLUDE_DIR})
|
||||
|
||||
### Import targets ############################################################
|
||||
if(LZ4_FOUND)
|
||||
if(NOT TARGET LZ4::LZ4)
|
||||
add_library(LZ4::LZ4 UNKNOWN IMPORTED)
|
||||
set_target_properties(LZ4::LZ4 PROPERTIES
|
||||
IMPORTED_LINK_INTERFACE_LANGUAGES "C"
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${LZ4_INCLUDE_DIR}")
|
||||
|
||||
if(LZ4_LIBRARY_RELEASE)
|
||||
set_property(TARGET LZ4::LZ4 APPEND PROPERTY
|
||||
IMPORTED_CONFIGURATIONS RELEASE)
|
||||
set_target_properties(LZ4::LZ4 PROPERTIES
|
||||
IMPORTED_LOCATION_RELEASE "${LZ4_LIBRARY_RELEASE}")
|
||||
endif()
|
||||
|
||||
if(LZ4_LIBRARY_DEBUG)
|
||||
set_property(TARGET LZ4::LZ4 APPEND PROPERTY
|
||||
IMPORTED_CONFIGURATIONS DEBUG)
|
||||
set_target_properties(LZ4::LZ4 PROPERTIES
|
||||
IMPORTED_LOCATION_DEBUG "${LZ4_LIBRARY_DEBUG}")
|
||||
endif()
|
||||
|
||||
if(NOT LZ4_LIBRARY_RELEASE AND NOT LZ4_LIBRARY_DEBUG)
|
||||
set_property(TARGET LZ4::LZ4 APPEND PROPERTY
|
||||
IMPORTED_LOCATION "${LZ4_LIBRARY}")
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
@ -233,7 +233,7 @@ target_link_libraries(components
|
||||
${SDL2_LIBRARIES}
|
||||
${OPENGL_gl_LIBRARY}
|
||||
${MyGUI_LIBRARIES}
|
||||
${BSAOPTHASH_LIBRARIES}
|
||||
LZ4::LZ4
|
||||
RecastNavigation::DebugUtils
|
||||
RecastNavigation::Detour
|
||||
RecastNavigation::Recast
|
||||
|
||||
@ -27,6 +27,8 @@
|
||||
#include <stdexcept>
|
||||
#include <cassert>
|
||||
|
||||
#include <lz4frame.h>
|
||||
|
||||
#include <boost/scoped_array.hpp>
|
||||
#include <boost/filesystem/path.hpp>
|
||||
#include <boost/filesystem/fstream.hpp>
|
||||
@ -132,8 +134,11 @@ void CompressedBSAFile::readHeader()
|
||||
|
||||
input.read(reinterpret_cast<char*>(header), 36);
|
||||
|
||||
if(header[0] != 0x00415342 /*"BSA\x00"*/ || (header[1] != 0x67 /*TES4*/ && header[1] != 0x68 /*TES5*/))
|
||||
fail("Unrecognized TES4 BSA header");
|
||||
if (header[0] != 0x00415342) /*"BSA\x00"*/
|
||||
fail("Unrecognized compressed BSA format");
|
||||
mVersion = header[1];
|
||||
if (mVersion != 0x67 /*TES4*/ && mVersion != 0x68 /*FO3, FNV, TES5*/ && mVersion != 0x69 /*SSE*/)
|
||||
fail("Unrecognized compressed BSA version");
|
||||
|
||||
// header[2] is offset, should be 36 = 0x24 which is the size of the header
|
||||
|
||||
@ -158,7 +163,8 @@ void CompressedBSAFile::readHeader()
|
||||
// header[8]; // fileFlags : an opportunity to optimize here
|
||||
|
||||
mCompressedByDefault = (archiveFlags & 0x4) != 0;
|
||||
mEmbeddedFileNames = header[1] == 0x68 /*TES5*/ && (archiveFlags & 0x100) != 0;
|
||||
if (mVersion == 0x68 || mVersion == 0x69) /*FO3, FNV, TES5, SSE*/
|
||||
mEmbeddedFileNames = (archiveFlags & 0x100) != 0;
|
||||
}
|
||||
|
||||
// folder records
|
||||
@ -168,7 +174,14 @@ void CompressedBSAFile::readHeader()
|
||||
{
|
||||
input.read(reinterpret_cast<char*>(&hash), 8);
|
||||
input.read(reinterpret_cast<char*>(&fr.count), 4); // not sure purpose of count
|
||||
input.read(reinterpret_cast<char*>(&fr.offset), 4); // not sure purpose of offset
|
||||
if (mVersion == 0x69) // SSE
|
||||
{
|
||||
std::uint32_t unknown;
|
||||
input.read(reinterpret_cast<char*>(&unknown), 4);
|
||||
input.read(reinterpret_cast<char*>(&fr.offset), 8);
|
||||
}
|
||||
else
|
||||
input.read(reinterpret_cast<char*>(&fr.offset), 4); // not sure purpose of offset
|
||||
|
||||
std::map<std::uint64_t, FolderRecord>::const_iterator lb = mFolders.lower_bound(hash);
|
||||
if (lb != mFolders.end() && !(mFolders.key_comp()(hash, lb->first)))
|
||||
@ -327,32 +340,56 @@ Files::IStreamPtr CompressedBSAFile::getFile(const char* file)
|
||||
|
||||
Files::IStreamPtr CompressedBSAFile::getFile(const FileRecord& fileRecord)
|
||||
{
|
||||
if (fileRecord.isCompressed(mCompressedByDefault)) {
|
||||
Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, fileRecord.getSizeWithoutCompressionFlag());
|
||||
size_t size = fileRecord.getSizeWithoutCompressionFlag();
|
||||
size_t uncompressedSize = size;
|
||||
bool compressed = fileRecord.isCompressed(mCompressedByDefault);
|
||||
Files::IStreamPtr streamPtr = Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, size);
|
||||
std::istream* fileStream = streamPtr.get();
|
||||
if (mEmbeddedFileNames)
|
||||
{
|
||||
// Skip over the embedded file name
|
||||
char length = 0;
|
||||
fileStream->read(&length, 1);
|
||||
fileStream->ignore(length);
|
||||
size -= length + sizeof(char);
|
||||
}
|
||||
if (compressed)
|
||||
{
|
||||
fileStream->read(reinterpret_cast<char*>(&uncompressedSize), sizeof(uint32_t));
|
||||
size -= sizeof(uint32_t);
|
||||
}
|
||||
std::shared_ptr<Bsa::MemoryInputStream> memoryStreamPtr = std::make_shared<MemoryInputStream>(uncompressedSize);
|
||||
|
||||
std::istream* fileStream = streamPtr.get();
|
||||
if (compressed)
|
||||
{
|
||||
if (mVersion != 0x69) // Non-SSE: zlib
|
||||
{
|
||||
boost::iostreams::filtering_streambuf<boost::iostreams::input> inputStreamBuf;
|
||||
inputStreamBuf.push(boost::iostreams::zlib_decompressor());
|
||||
inputStreamBuf.push(*fileStream);
|
||||
|
||||
if (mEmbeddedFileNames) {
|
||||
std::string embeddedFileName;
|
||||
getBZString(embeddedFileName, *fileStream);
|
||||
boost::iostreams::basic_array_sink<char> sr(memoryStreamPtr->getRawData(), uncompressedSize);
|
||||
boost::iostreams::copy(inputStreamBuf, sr);
|
||||
}
|
||||
|
||||
uint32_t uncompressedSize = 0u;
|
||||
fileStream->read(reinterpret_cast<char*>(&uncompressedSize), sizeof(uncompressedSize));
|
||||
|
||||
boost::iostreams::filtering_streambuf<boost::iostreams::input> inputStreamBuf;
|
||||
inputStreamBuf.push(boost::iostreams::zlib_decompressor());
|
||||
inputStreamBuf.push(*fileStream);
|
||||
|
||||
std::shared_ptr<Bsa::MemoryInputStream> memoryStreamPtr = std::make_shared<MemoryInputStream>(uncompressedSize);
|
||||
|
||||
boost::iostreams::basic_array_sink<char> sr(memoryStreamPtr->getRawData(), uncompressedSize);
|
||||
boost::iostreams::copy(inputStreamBuf, sr);
|
||||
|
||||
return std::shared_ptr<std::istream>(memoryStreamPtr, (std::istream*)memoryStreamPtr.get());
|
||||
else // SSE: lz4
|
||||
{
|
||||
boost::scoped_array<char> buffer(new char[size]);
|
||||
fileStream->read(buffer.get(), size);
|
||||
LZ4F_dctx* context = nullptr;
|
||||
LZ4F_createDecompressionContext(&context, LZ4F_VERSION);
|
||||
LZ4F_decompressOptions_t options = {};
|
||||
LZ4F_decompress(context, memoryStreamPtr->getRawData(), &uncompressedSize, buffer.get(), &size, &options);
|
||||
LZ4F_errorCode_t errorCode = LZ4F_freeDecompressionContext(context);
|
||||
if (LZ4F_isError(errorCode))
|
||||
fail("LZ4 decompression error (file " + mFilename + "): " + LZ4F_getErrorName(errorCode));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
fileStream->read(memoryStreamPtr->getRawData(), size);
|
||||
}
|
||||
|
||||
return Files::openConstrainedFileStream(mFilename.c_str(), fileRecord.offset, fileRecord.size);
|
||||
return std::shared_ptr<std::istream>(memoryStreamPtr, (std::istream*)memoryStreamPtr.get());
|
||||
}
|
||||
|
||||
BsaVersion CompressedBSAFile::detectVersion(std::string filePath)
|
||||
|
||||
@ -64,10 +64,12 @@ namespace Bsa
|
||||
//if each file record begins with BZ string with file name
|
||||
bool mEmbeddedFileNames;
|
||||
|
||||
std::uint32_t mVersion{0u};
|
||||
|
||||
struct FolderRecord
|
||||
{
|
||||
std::uint32_t count;
|
||||
std::uint32_t offset;
|
||||
std::uint64_t offset;
|
||||
std::map<std::uint64_t, FileRecord> files;
|
||||
};
|
||||
std::map<std::uint64_t, FolderRecord> mFolders;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user