Merge branch 'esmtool_fallout_nv' into 'master'

Support FalloutNV.esm

See merge request OpenMW/openmw!3065

CI/before_script.msvc.sh

@@ -652,6 +652,10 @@ if [ -z $SKIP_DOWNLOAD ]; then
 	download "ICU ${ICU_VER/_/.}"\
 		"https://github.com/unicode-org/icu/releases/download/release-${ICU_VER/_/-}/icu4c-${ICU_VER}-Win${BITS}-MSVC2019.zip" \
 		"icu4c-${ICU_VER}-Win${BITS}-MSVC2019.zip"
+
+	download "zlib 1.2.11"\
+		"https://gitlab.com/OpenMW/openmw-deps/-/raw/main/windows/zlib-1.2.11-msvc2017-win64.7z" \
+		"zlib-1.2.11-msvc2017-win64.7z"
 fi
 
 cd .. #/..
@@ -849,10 +853,10 @@ printf "${OSG_ARCHIVE_NAME}... "
 		fi
 
 		if ! [ -z $OSG_MULTIVIEW_BUILD ]; then
-			add_runtime_dlls $CONFIGURATION "$(pwd)/OSG/bin/"{ot21-OpenThreads,zlib,libpng16}${SUFFIX}.dll \
+			add_runtime_dlls $CONFIGURATION "$(pwd)/OSG/bin/"{ot21-OpenThreads,libpng16}${SUFFIX}.dll \
 				"$(pwd)/OSG/bin/osg162-osg"{,Animation,DB,FX,GA,Particle,Text,Util,Viewer,Shadow,Sim}${SUFFIX}.dll
 		else
-			add_runtime_dlls $CONFIGURATION "$(pwd)/OSG/bin/"{OpenThreads,icuuc58,libpng16,zlib}${SUFFIX}.dll \
+			add_runtime_dlls $CONFIGURATION "$(pwd)/OSG/bin/"{OpenThreads,icuuc58,libpng16}${SUFFIX}.dll \
 				"$(pwd)/OSG/bin/libxml2"${SUFFIX_UPCASE}.dll \
 				"$(pwd)/OSG/bin/osg"{,Animation,DB,FX,GA,Particle,Text,Util,Viewer,Shadow,Sim}${SUFFIX}.dll
 			add_runtime_dlls $CONFIGURATION "$(pwd)/OSG/bin/icudt58.dll"
@@ -1028,6 +1032,27 @@ printf "ICU ${ICU_VER/_/.}... "
 	echo Done.
 }
 
+cd $DEPS
+echo
+printf "zlib 1.2.11... "
+{
+	if [ -d zlib-1.2.11-msvc2017-win64 ]; then
+		printf "Exists. "
+	elif [ -z $SKIP_EXTRACT ]; then
+		rm -rf zlib-1.2.11-msvc2017-win64
+		eval 7z x -y zlib-1.2.11-msvc2017-win64.7z $STRIP
+	fi
+	add_cmake_opts -DZLIB_ROOT="$(real_pwd)/zlib-1.2.11-msvc2017-win64"
+	for config in ${CONFIGURATIONS[@]}; do
+		if [ $CONFIGURATION == "Debug" ]; then
+			add_runtime_dlls $config "$(pwd)/zlib-1.2.11-msvc2017-win64/bin/zlibd.dll"
+		else
+			add_runtime_dlls $config "$(pwd)/zlib-1.2.11-msvc2017-win64/bin/zlib.dll"
+		fi
+	done
+	echo Done.
+}
+
 echo
 cd $DEPS_INSTALL/..
 echo

CMakeLists.txt

@@ -479,6 +479,7 @@ if(OPENMW_USE_SYSTEM_MYGUI)
 endif()
 find_package(SDL2 2.0.9 REQUIRED)
 find_package(OpenAL REQUIRED)
+find_package(ZLIB REQUIRED)
 
 option(USE_LUAJIT "Switch Lua/LuaJit (TRUE is highly recommended)" TRUE)
 if(USE_LUAJIT)

components/CMakeLists.txt

@@ -522,6 +522,7 @@ target_link_libraries(components
     smhasher
     ${ICU_LIBRARIES}
     yaml-cpp
+    ZLIB::ZLIB
     )
 
 if(Boost_VERSION_STRING VERSION_GREATER_EQUAL 1.77.0)

components/esm4/loadclas.cpp

@@ -55,6 +55,7 @@ void ESM4::Class::load(ESM4::Reader& reader)
                 reader.getZString(mIcon);
                 break;
             case ESM4::SUB_DATA:
+            case ESM4::SUB_ATTR:
                 reader.skipSubRecordData();
                 break;
             default:

components/esm4/reader.cpp

@@ -24,21 +24,15 @@
 
 #undef DEBUG_GROUPSTACK
 
+#include <algorithm>
 #include <iomanip>
 #include <iostream>
+#include <optional>
+#include <span>
 #include <sstream>
 #include <stdexcept>
 
-#if defined(_MSC_VER)
+#include <zlib.h>
-#pragma warning(push)
-#pragma warning(disable : 4706)
-#include <boost/iostreams/filter/zlib.hpp>
-#pragma warning(pop)
-#else
-#include <boost/iostreams/filter/zlib.hpp>
-#endif
-#include <boost/iostreams/copy.hpp>
-#include <boost/iostreams/filtering_streambuf.hpp>
 
 #include <components/bsa/memorystream.hpp>
 #include <components/debug/debuglog.hpp>
@@ -68,6 +62,92 @@ namespace ESM4
 
             throw std::logic_error("Unsupported LocalizedStringType: " + std::to_string(static_cast<int>(type)));
         }
+
+        struct InflateEnd
+        {
+            void operator()(z_stream* stream) const { inflateEnd(stream); }
+        };
+
+        std::optional<std::string> tryDecompressAll(std::span<char> compressed, std::span<char> decompressed)
+        {
+            z_stream stream{};
+
+            stream.next_in = reinterpret_cast<Bytef*>(compressed.data());
+            stream.next_out = reinterpret_cast<Bytef*>(decompressed.data());
+            stream.avail_in = compressed.size();
+            stream.avail_out = decompressed.size();
+
+            if (const int ec = inflateInit(&stream); ec != Z_OK)
+                return "inflateInit error: " + std::to_string(ec) + " " + std::string(stream.msg);
+
+            const std::unique_ptr<z_stream, InflateEnd> streamPtr(&stream);
+
+            if (const int ec = inflate(&stream, Z_NO_FLUSH); ec != Z_STREAM_END)
+                return "inflate error: " + std::to_string(ec) + " " + std::string(stream.msg);
+
+            return std::nullopt;
+        }
+
+        std::optional<std::string> tryDecompressByBlock(
+            std::span<char> compressed, std::span<char> decompressed, std::size_t blockSize)
+        {
+            z_stream stream{};
+
+            if (const int ec = inflateInit(&stream); ec != Z_OK)
+                return "inflateInit error: " + std::to_string(ec) + " " + std::string(stream.msg);
+
+            const std::unique_ptr<z_stream, InflateEnd> streamPtr(&stream);
+
+            while (!compressed.empty() && !decompressed.empty())
+            {
+                const auto prevTotalIn = stream.total_in;
+                const auto prevTotalOut = stream.total_out;
+                stream.next_in = reinterpret_cast<Bytef*>(compressed.data());
+                stream.avail_in = std::min(blockSize, compressed.size());
+                stream.next_out = reinterpret_cast<Bytef*>(decompressed.data());
+                stream.avail_out = std::min(blockSize, decompressed.size());
+                const int ec = inflate(&stream, Z_NO_FLUSH);
+                if (ec == Z_STREAM_END)
+                    break;
+                if (ec != Z_OK)
+                    return "inflate error after reading " + std::to_string(stream.total_in)
+                        + " bytes: " + std::to_string(ec) + " " + std::string(stream.msg);
+                compressed = compressed.subspan(stream.total_in - prevTotalIn);
+                decompressed = decompressed.subspan(stream.total_out - prevTotalOut);
+            }
+
+            return std::nullopt;
+        }
+
+        std::unique_ptr<Bsa::MemoryInputStream> decompress(
+            std::streamoff position, std::span<char> compressed, std::uint32_t uncompressedSize)
+        {
+            auto result = std::make_unique<Bsa::MemoryInputStream>(uncompressedSize);
+
+            const std::span decompressed(result->getRawData(), uncompressedSize);
+
+            const auto allError = tryDecompressAll(compressed, decompressed);
+            if (!allError.has_value())
+                return result;
+
+            Log(Debug::Warning) << "Failed to decompress record data at 0x" << std::hex << position
+                                << std::resetiosflags(std::ios_base::hex) << " compressed size = " << compressed.size()
+                                << " uncompressed size = " << uncompressedSize << ": " << *allError
+                                << ". Trying to decompress by block...";
+
+            std::memset(result->getRawData(), 0, uncompressedSize);
+
+            constexpr std::size_t blockSize = 4;
+            const auto blockError = tryDecompressByBlock(compressed, decompressed, blockSize);
+            if (!blockError.has_value())
+                return result;
+
+            std::ostringstream s;
+            s << "Failed to decompress record data by block of " << blockSize << " bytes at 0x" << std::hex << position
+              << std::resetiosflags(std::ios_base::hex) << " compressed size = " << compressed.size()
+              << " uncompressed size = " << uncompressedSize << ": " << *blockError;
+            throw std::runtime_error(s.str());
+        }
     }
 
     ReaderContext::ReaderContext()
@@ -423,22 +503,16 @@ namespace ESM4
         {
             mStream->read(reinterpret_cast<char*>(&uncompressedSize), sizeof(std::uint32_t));
 
-            std::size_t recordSize = mCtx.recordHeader.record.dataSize - sizeof(std::uint32_t);
+            const std::streamoff position = mStream->tellg();
-            Bsa::MemoryInputStream compressedRecord(recordSize);
+
-            mStream->read(compressedRecord.getRawData(), recordSize);
+            const std::uint32_t recordSize = mCtx.recordHeader.record.dataSize - sizeof(std::uint32_t);
-            std::istream* fileStream = (std::istream*)&compressedRecord;
+            std::vector<char> compressed(recordSize);
+            mStream->read(compressed.data(), recordSize);
             mSavedStream = std::move(mStream);
 
             mCtx.recordHeader.record.dataSize = uncompressedSize - sizeof(uncompressedSize);
 
-            auto memoryStreamPtr = std::make_unique<Bsa::MemoryInputStream>(uncompressedSize);
+            auto memoryStreamPtr = decompress(position, compressed, uncompressedSize);
-
-            boost::iostreams::filtering_streambuf<boost::iostreams::input> inputStreamBuf;
-            inputStreamBuf.push(boost::iostreams::zlib_decompressor());
-            inputStreamBuf.push(*fileStream);
-
-            boost::iostreams::basic_array_sink<char> sr(memoryStreamPtr->getRawData(), uncompressedSize);
-            boost::iostreams::copy(inputStreamBuf, sr);
 
             // For debugging only
             // #if 0