diff --git a/CI/before_script.msvc.sh b/CI/before_script.msvc.sh index 5b79f4304d..e665579dc6 100644 --- a/CI/before_script.msvc.sh +++ b/CI/before_script.msvc.sh @@ -652,6 +652,10 @@ if [ -z $SKIP_DOWNLOAD ]; then download "ICU ${ICU_VER/_/.}"\ "https://github.com/unicode-org/icu/releases/download/release-${ICU_VER/_/-}/icu4c-${ICU_VER}-Win${BITS}-MSVC2019.zip" \ "icu4c-${ICU_VER}-Win${BITS}-MSVC2019.zip" + + download "zlib 1.2.11"\ + "https://gitlab.com/OpenMW/openmw-deps/-/raw/main/windows/zlib-1.2.11-msvc2017-win64.7z" \ + "zlib-1.2.11-msvc2017-win64.7z" fi cd .. #/.. @@ -849,10 +853,10 @@ printf "${OSG_ARCHIVE_NAME}... " fi if ! [ -z $OSG_MULTIVIEW_BUILD ]; then - add_runtime_dlls $CONFIGURATION "$(pwd)/OSG/bin/"{ot21-OpenThreads,zlib,libpng16}${SUFFIX}.dll \ + add_runtime_dlls $CONFIGURATION "$(pwd)/OSG/bin/"{ot21-OpenThreads,libpng16}${SUFFIX}.dll \ "$(pwd)/OSG/bin/osg162-osg"{,Animation,DB,FX,GA,Particle,Text,Util,Viewer,Shadow,Sim}${SUFFIX}.dll else - add_runtime_dlls $CONFIGURATION "$(pwd)/OSG/bin/"{OpenThreads,icuuc58,libpng16,zlib}${SUFFIX}.dll \ + add_runtime_dlls $CONFIGURATION "$(pwd)/OSG/bin/"{OpenThreads,icuuc58,libpng16}${SUFFIX}.dll \ "$(pwd)/OSG/bin/libxml2"${SUFFIX_UPCASE}.dll \ "$(pwd)/OSG/bin/osg"{,Animation,DB,FX,GA,Particle,Text,Util,Viewer,Shadow,Sim}${SUFFIX}.dll add_runtime_dlls $CONFIGURATION "$(pwd)/OSG/bin/icudt58.dll" @@ -1028,6 +1032,27 @@ printf "ICU ${ICU_VER/_/.}... " echo Done. } +cd $DEPS +echo +printf "zlib 1.2.11... " +{ + if [ -d zlib-1.2.11-msvc2017-win64 ]; then + printf "Exists. " + elif [ -z $SKIP_EXTRACT ]; then + rm -rf zlib-1.2.11-msvc2017-win64 + eval 7z x -y zlib-1.2.11-msvc2017-win64.7z $STRIP + fi + add_cmake_opts -DZLIB_ROOT="$(real_pwd)/zlib-1.2.11-msvc2017-win64" + for config in ${CONFIGURATIONS[@]}; do + if [ $CONFIGURATION == "Debug" ]; then + add_runtime_dlls $config "$(pwd)/zlib-1.2.11-msvc2017-win64/bin/zlibd.dll" + else + add_runtime_dlls $config "$(pwd)/zlib-1.2.11-msvc2017-win64/bin/zlib.dll" + fi + done + echo Done. +} + echo cd $DEPS_INSTALL/.. echo diff --git a/CMakeLists.txt b/CMakeLists.txt index 0adffa61da..7a559e1cba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -479,6 +479,7 @@ if(OPENMW_USE_SYSTEM_MYGUI) endif() find_package(SDL2 2.0.9 REQUIRED) find_package(OpenAL REQUIRED) +find_package(ZLIB REQUIRED) option(USE_LUAJIT "Switch Lua/LuaJit (TRUE is highly recommended)" TRUE) if(USE_LUAJIT) diff --git a/components/CMakeLists.txt b/components/CMakeLists.txt index dde717d88c..7d96794a98 100644 --- a/components/CMakeLists.txt +++ b/components/CMakeLists.txt @@ -522,6 +522,7 @@ target_link_libraries(components smhasher ${ICU_LIBRARIES} yaml-cpp + ZLIB::ZLIB ) if(Boost_VERSION_STRING VERSION_GREATER_EQUAL 1.77.0) diff --git a/components/esm4/loadclas.cpp b/components/esm4/loadclas.cpp index f5bb0c34ba..14f81abfce 100644 --- a/components/esm4/loadclas.cpp +++ b/components/esm4/loadclas.cpp @@ -55,6 +55,7 @@ void ESM4::Class::load(ESM4::Reader& reader) reader.getZString(mIcon); break; case ESM4::SUB_DATA: + case ESM4::SUB_ATTR: reader.skipSubRecordData(); break; default: diff --git a/components/esm4/reader.cpp b/components/esm4/reader.cpp index 8687f1d9af..70088c9cca 100644 --- a/components/esm4/reader.cpp +++ b/components/esm4/reader.cpp @@ -24,21 +24,15 @@ #undef DEBUG_GROUPSTACK +#include #include #include +#include +#include #include #include -#if defined(_MSC_VER) -#pragma warning(push) -#pragma warning(disable : 4706) -#include -#pragma warning(pop) -#else -#include -#endif -#include -#include +#include #include #include @@ -68,6 +62,92 @@ namespace ESM4 throw std::logic_error("Unsupported LocalizedStringType: " + std::to_string(static_cast(type))); } + + struct InflateEnd + { + void operator()(z_stream* stream) const { inflateEnd(stream); } + }; + + std::optional tryDecompressAll(std::span compressed, std::span decompressed) + { + z_stream stream{}; + + stream.next_in = reinterpret_cast(compressed.data()); + stream.next_out = reinterpret_cast(decompressed.data()); + stream.avail_in = compressed.size(); + stream.avail_out = decompressed.size(); + + if (const int ec = inflateInit(&stream); ec != Z_OK) + return "inflateInit error: " + std::to_string(ec) + " " + std::string(stream.msg); + + const std::unique_ptr streamPtr(&stream); + + if (const int ec = inflate(&stream, Z_NO_FLUSH); ec != Z_STREAM_END) + return "inflate error: " + std::to_string(ec) + " " + std::string(stream.msg); + + return std::nullopt; + } + + std::optional tryDecompressByBlock( + std::span compressed, std::span decompressed, std::size_t blockSize) + { + z_stream stream{}; + + if (const int ec = inflateInit(&stream); ec != Z_OK) + return "inflateInit error: " + std::to_string(ec) + " " + std::string(stream.msg); + + const std::unique_ptr streamPtr(&stream); + + while (!compressed.empty() && !decompressed.empty()) + { + const auto prevTotalIn = stream.total_in; + const auto prevTotalOut = stream.total_out; + stream.next_in = reinterpret_cast(compressed.data()); + stream.avail_in = std::min(blockSize, compressed.size()); + stream.next_out = reinterpret_cast(decompressed.data()); + stream.avail_out = std::min(blockSize, decompressed.size()); + const int ec = inflate(&stream, Z_NO_FLUSH); + if (ec == Z_STREAM_END) + break; + if (ec != Z_OK) + return "inflate error after reading " + std::to_string(stream.total_in) + + " bytes: " + std::to_string(ec) + " " + std::string(stream.msg); + compressed = compressed.subspan(stream.total_in - prevTotalIn); + decompressed = decompressed.subspan(stream.total_out - prevTotalOut); + } + + return std::nullopt; + } + + std::unique_ptr decompress( + std::streamoff position, std::span compressed, std::uint32_t uncompressedSize) + { + auto result = std::make_unique(uncompressedSize); + + const std::span decompressed(result->getRawData(), uncompressedSize); + + const auto allError = tryDecompressAll(compressed, decompressed); + if (!allError.has_value()) + return result; + + Log(Debug::Warning) << "Failed to decompress record data at 0x" << std::hex << position + << std::resetiosflags(std::ios_base::hex) << " compressed size = " << compressed.size() + << " uncompressed size = " << uncompressedSize << ": " << *allError + << ". Trying to decompress by block..."; + + std::memset(result->getRawData(), 0, uncompressedSize); + + constexpr std::size_t blockSize = 4; + const auto blockError = tryDecompressByBlock(compressed, decompressed, blockSize); + if (!blockError.has_value()) + return result; + + std::ostringstream s; + s << "Failed to decompress record data by block of " << blockSize << " bytes at 0x" << std::hex << position + << std::resetiosflags(std::ios_base::hex) << " compressed size = " << compressed.size() + << " uncompressed size = " << uncompressedSize << ": " << *blockError; + throw std::runtime_error(s.str()); + } } ReaderContext::ReaderContext() @@ -423,22 +503,16 @@ namespace ESM4 { mStream->read(reinterpret_cast(&uncompressedSize), sizeof(std::uint32_t)); - std::size_t recordSize = mCtx.recordHeader.record.dataSize - sizeof(std::uint32_t); - Bsa::MemoryInputStream compressedRecord(recordSize); - mStream->read(compressedRecord.getRawData(), recordSize); - std::istream* fileStream = (std::istream*)&compressedRecord; + const std::streamoff position = mStream->tellg(); + + const std::uint32_t recordSize = mCtx.recordHeader.record.dataSize - sizeof(std::uint32_t); + std::vector compressed(recordSize); + mStream->read(compressed.data(), recordSize); mSavedStream = std::move(mStream); mCtx.recordHeader.record.dataSize = uncompressedSize - sizeof(uncompressedSize); - auto memoryStreamPtr = std::make_unique(uncompressedSize); - - boost::iostreams::filtering_streambuf inputStreamBuf; - inputStreamBuf.push(boost::iostreams::zlib_decompressor()); - inputStreamBuf.push(*fileStream); - - boost::iostreams::basic_array_sink sr(memoryStreamPtr->getRawData(), uncompressedSize); - boost::iostreams::copy(inputStreamBuf, sr); + auto memoryStreamPtr = decompress(position, compressed, uncompressedSize); // For debugging only // #if 0