From 8a9597e32e98707d5bcdc1a41d8c45e3174d4047 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sat, 4 Apr 2020 20:56:20 +0200 Subject: [PATCH 1/9] DiscIO: Allow converting from formats other than ISO and GCZ The constant DESIRED_BUFFER_SIZE was determined by multiplying the old hardcoded value 32 with the default GCZ block size 16 KiB. Not sure if it actually is the best value, but it seems fine. --- Source/Core/DiscIO/Blob.h | 14 +-- Source/Core/DiscIO/CISOBlob.h | 5 +- Source/Core/DiscIO/CompressedBlob.cpp | 94 ++++++++++----------- Source/Core/DiscIO/CompressedBlob.h | 1 + Source/Core/DiscIO/DiscScrubber.cpp | 35 +------- Source/Core/DiscIO/DiscScrubber.h | 7 +- Source/Core/DiscIO/DriveBlob.h | 5 ++ Source/Core/DiscIO/VolumeVerifier.cpp | 2 +- Source/Core/DiscIO/WbfsBlob.h | 3 + Source/Core/DolphinQt/GameList/GameList.cpp | 10 +-- 10 files changed, 80 insertions(+), 96 deletions(-) diff --git a/Source/Core/DiscIO/Blob.h b/Source/Core/DiscIO/Blob.h index 52b083ab63..9cb00087d2 100644 --- a/Source/Core/DiscIO/Blob.h +++ b/Source/Core/DiscIO/Blob.h @@ -41,11 +41,16 @@ class BlobReader { public: virtual ~BlobReader() {} + virtual BlobType GetBlobType() const = 0; + virtual u64 GetRawSize() const = 0; virtual u64 GetDataSize() const = 0; virtual bool IsDataSizeAccurate() const = 0; + // Returns 0 if the format does not use blocks + virtual u64 GetBlockSize() const { return 0; } + // NOT thread-safe - can't call this from multiple threads. virtual bool Read(u64 offset, u64 size, u8* out_ptr) = 0; template @@ -160,10 +165,9 @@ std::unique_ptr CreateBlobReader(const std::string& filename); typedef bool (*CompressCB)(const std::string& text, float percent, void* arg); -bool CompressFileToBlob(const std::string& infile_path, const std::string& outfile_path, - u32 sub_type = 0, int sector_size = 16384, CompressCB callback = nullptr, - void* arg = nullptr); -bool DecompressBlobToFile(const std::string& infile_path, const std::string& outfile_path, - CompressCB callback = nullptr, void* arg = nullptr); +bool ConvertToGCZ(const std::string& infile_path, const std::string& outfile_path, u32 sub_type = 0, + int sector_size = 16384, CompressCB callback = nullptr, void* arg = nullptr); +bool ConvertToPlain(const std::string& infile_path, const std::string& outfile_path, + CompressCB callback = nullptr, void* arg = nullptr); } // namespace DiscIO diff --git a/Source/Core/DiscIO/CISOBlob.h b/Source/Core/DiscIO/CISOBlob.h index dc606bb4a5..66f950f074 100644 --- a/Source/Core/DiscIO/CISOBlob.h +++ b/Source/Core/DiscIO/CISOBlob.h @@ -37,12 +37,15 @@ public: static std::unique_ptr Create(File::IOFile file); BlobType GetBlobType() const override { return BlobType::CISO; } + + u64 GetRawSize() const override; // The CISO format does not save the original file size. // This function returns an upper bound. u64 GetDataSize() const override; bool IsDataSizeAccurate() const override { return false; } - u64 GetRawSize() const override; + u64 GetBlockSize() const override { return m_block_size; } + bool Read(u64 offset, u64 nbytes, u8* out_ptr) override; private: diff --git a/Source/Core/DiscIO/CompressedBlob.cpp b/Source/Core/DiscIO/CompressedBlob.cpp index 5248170683..33790ad3b0 100644 --- a/Source/Core/DiscIO/CompressedBlob.cpp +++ b/Source/Core/DiscIO/CompressedBlob.cpp @@ -17,6 +17,7 @@ #include #include +#include "Common/Assert.h" #include "Common/CommonTypes.h" #include "Common/File.h" #include "Common/FileUtil.h" @@ -153,24 +154,20 @@ bool CompressedBlobReader::GetBlock(u64 block_num, u8* out_ptr) return true; } -bool CompressFileToBlob(const std::string& infile_path, const std::string& outfile_path, - u32 sub_type, int block_size, CompressCB callback, void* arg) +bool ConvertToGCZ(const std::string& infile_path, const std::string& outfile_path, u32 sub_type, + int block_size, CompressCB callback, void* arg) { bool scrubbing = false; - File::IOFile infile(infile_path, "rb"); - if (IsGCZBlob(infile)) - { - PanicAlertT("\"%s\" is already compressed! Cannot compress it further.", infile_path.c_str()); - return false; - } - + std::unique_ptr infile = CreateDisc(infile_path); if (!infile) { PanicAlertT("Failed to open the input file \"%s\".", infile_path.c_str()); return false; } + ASSERT(infile->IsSizeAccurate()); + File::IOFile outfile(outfile_path, "wb"); if (!outfile) { @@ -182,11 +179,9 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi } DiscScrubber disc_scrubber; - std::unique_ptr volume; if (sub_type == 1) { - volume = CreateDisc(infile_path); - if (!volume || !disc_scrubber.SetupScrub(volume.get(), block_size)) + if (!disc_scrubber.SetupScrub(infile.get())) { PanicAlertT("\"%s\" failed to be scrubbed. Probably the image is corrupt.", infile_path.c_str()); @@ -206,7 +201,7 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi header.magic_cookie = GCZ_MAGIC; header.sub_type = sub_type; header.block_size = block_size; - header.data_size = infile.GetSize(); + header.data_size = infile->GetSize(); // round upwards! header.num_blocks = (u32)((header.data_size + (block_size - 1)) / block_size); @@ -220,10 +215,9 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi outfile.Seek(sizeof(CompressedBlobHeader), SEEK_CUR); // seek past the offset and hash tables (we will write them at the end) outfile.Seek((sizeof(u64) + sizeof(u32)) * header.num_blocks, SEEK_CUR); - // seek to the start of the input file to make sure we get everything - infile.Seek(0, SEEK_SET); // Now we are ready to write compressed data! + u64 inpos = 0; u64 position = 0; int num_compressed = 0; int num_stored = 0; @@ -234,7 +228,6 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi { if (i % progress_monitor == 0) { - const u64 inpos = infile.Tell(); int ratio = 0; if (inpos != 0) ratio = (int)(100 * position / inpos); @@ -252,13 +245,18 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi offsets[i] = position; - size_t read_bytes; - if (scrubbing) - read_bytes = disc_scrubber.GetNextBlock(infile, in_buf.data()); - else - infile.ReadArray(in_buf.data(), header.block_size, &read_bytes); - if (read_bytes < header.block_size) - std::fill(in_buf.begin() + read_bytes, in_buf.begin() + header.block_size, 0); + const u64 bytes_to_read = scrubbing && disc_scrubber.CanBlockBeScrubbed(inpos) ? + 0 : + std::min(block_size, header.data_size - inpos); + + success = infile->Read(inpos, bytes_to_read, in_buf.data(), PARTITION_NONE); + if (!success) + { + PanicAlertT("Failed to read from the input file \"%s\".", infile_path.c_str()); + break; + } + + std::fill(in_buf.begin() + bytes_to_read, in_buf.begin() + header.block_size, 0); int retval = deflateReset(&z); z.next_in = in_buf.data(); @@ -305,6 +303,7 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi break; } + inpos += block_size; position += write_size; hashes[i] = Common::HashAdler32(write_buf, write_size); @@ -337,27 +336,18 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi return success; } -bool DecompressBlobToFile(const std::string& infile_path, const std::string& outfile_path, - CompressCB callback, void* arg) +bool ConvertToPlain(const std::string& infile_path, const std::string& outfile_path, + CompressCB callback, void* arg) { - std::unique_ptr reader; - { - File::IOFile infile(infile_path, "rb"); - if (!IsGCZBlob(infile)) - { - PanicAlertT("File not compressed"); - return false; - } - - reader = CompressedBlobReader::Create(std::move(infile), infile_path); - } - + std::unique_ptr reader = CreateBlobReader(infile_path); if (!reader) { PanicAlertT("Failed to open the input file \"%s\".", infile_path.c_str()); return false; } + ASSERT(reader->IsDataSizeAccurate()); + File::IOFile outfile(outfile_path, "wb"); if (!outfile) { @@ -368,11 +358,20 @@ bool DecompressBlobToFile(const std::string& infile_path, const std::string& out return false; } - const CompressedBlobHeader& header = reader->GetHeader(); - static const size_t BUFFER_BLOCKS = 32; - size_t buffer_size = header.block_size * BUFFER_BLOCKS; + constexpr size_t DESIRED_BUFFER_SIZE = 0x80000; + u64 buffer_size = reader->GetBlockSize(); + if (buffer_size == 0) + { + buffer_size = DESIRED_BUFFER_SIZE; + } + else + { + while (buffer_size < DESIRED_BUFFER_SIZE) + buffer_size *= 2; + } + std::vector buffer(buffer_size); - u32 num_buffers = (header.num_blocks + BUFFER_BLOCKS - 1) / BUFFER_BLOCKS; + const u64 num_buffers = (reader->GetDataSize() + buffer_size - 1) / buffer_size; int progress_monitor = std::max(1, num_buffers / 100); bool success = true; @@ -389,8 +388,13 @@ bool DecompressBlobToFile(const std::string& infile_path, const std::string& out } } const u64 inpos = i * buffer_size; - const u64 sz = std::min(buffer_size, header.data_size - inpos); - reader->Read(inpos, sz, buffer.data()); + const u64 sz = std::min(buffer_size, reader->GetDataSize() - inpos); + if (!reader->Read(inpos, sz, buffer.data())) + { + PanicAlertT("Failed to read from the input file \"%s\".", infile_path.c_str()); + success = false; + break; + } if (!outfile.WriteBytes(buffer.data(), sz)) { PanicAlertT("Failed to write the output file \"%s\".\n" @@ -407,10 +411,6 @@ bool DecompressBlobToFile(const std::string& infile_path, const std::string& out outfile.Close(); File::Delete(outfile_path); } - else - { - outfile.Resize(header.data_size); - } return success; } diff --git a/Source/Core/DiscIO/CompressedBlob.h b/Source/Core/DiscIO/CompressedBlob.h index 0e290542c6..8c63aa68d6 100644 --- a/Source/Core/DiscIO/CompressedBlob.h +++ b/Source/Core/DiscIO/CompressedBlob.h @@ -52,6 +52,7 @@ public: u64 GetRawSize() const override { return m_file_size; } u64 GetDataSize() const override { return m_header.data_size; } bool IsDataSizeAccurate() const override { return true; } + u64 GetBlockSize() const override { return m_header.block_size; } u64 GetBlockCompressedSize(u64 block_num) const; bool GetBlock(u64 block_num, u8* out_ptr) override; diff --git a/Source/Core/DiscIO/DiscScrubber.cpp b/Source/Core/DiscIO/DiscScrubber.cpp index 8a06d9266e..07d504d72a 100644 --- a/Source/Core/DiscIO/DiscScrubber.cpp +++ b/Source/Core/DiscIO/DiscScrubber.cpp @@ -14,6 +14,7 @@ #include #include "Common/Align.h" +#include "Common/Assert.h" #include "Common/CommonTypes.h" #include "Common/File.h" #include "Common/Logging/Log.h" @@ -29,19 +30,11 @@ constexpr size_t CLUSTER_SIZE = 0x8000; DiscScrubber::DiscScrubber() = default; DiscScrubber::~DiscScrubber() = default; -bool DiscScrubber::SetupScrub(const Volume* disc, int block_size) +bool DiscScrubber::SetupScrub(const Volume* disc) { if (!disc) return false; m_disc = disc; - m_block_size = block_size; - - if (CLUSTER_SIZE % m_block_size != 0) - { - ERROR_LOG(DISCIO, "Block size %u is not a factor of 0x8000, scrubbing not possible", - m_block_size); - return false; - } m_file_size = m_disc->GetSize(); @@ -54,34 +47,10 @@ bool DiscScrubber::SetupScrub(const Volume* disc, int block_size) // Fill out table of free blocks const bool success = ParseDisc(); - m_block_count = 0; - m_is_scrubbing = success; return success; } -size_t DiscScrubber::GetNextBlock(File::IOFile& in, u8* buffer) -{ - const u64 current_offset = m_block_count * m_block_size; - - size_t read_bytes = 0; - if (CanBlockBeScrubbed(current_offset)) - { - DEBUG_LOG(DISCIO, "Freeing 0x%016" PRIx64, current_offset); - std::fill(buffer, buffer + m_block_size, 0x00); - in.Seek(m_block_size, SEEK_CUR); - read_bytes = m_block_size; - } - else - { - DEBUG_LOG(DISCIO, "Used 0x%016" PRIx64, current_offset); - in.ReadArray(buffer, m_block_size, &read_bytes); - } - - m_block_count++; - return read_bytes; -} - bool DiscScrubber::CanBlockBeScrubbed(u64 offset) const { return m_is_scrubbing && m_free_table[offset / CLUSTER_SIZE]; diff --git a/Source/Core/DiscIO/DiscScrubber.h b/Source/Core/DiscIO/DiscScrubber.h index 15e0cda7f7..b3e7e7a9a0 100644 --- a/Source/Core/DiscIO/DiscScrubber.h +++ b/Source/Core/DiscIO/DiscScrubber.h @@ -34,8 +34,9 @@ public: DiscScrubber(); ~DiscScrubber(); - bool SetupScrub(const Volume* disc, int block_size); - size_t GetNextBlock(File::IOFile& in, u8* buffer); + bool SetupScrub(const Volume* disc); + + // Returns true if the specified 32 KiB block only contains unused data bool CanBlockBeScrubbed(u64 offset) const; private: @@ -72,8 +73,6 @@ private: std::vector m_free_table; u64 m_file_size = 0; - u64 m_block_count = 0; - u32 m_block_size = 0; bool m_is_scrubbing = false; }; diff --git a/Source/Core/DiscIO/DriveBlob.h b/Source/Core/DiscIO/DriveBlob.h index fddfbee7a2..46c141b1d3 100644 --- a/Source/Core/DiscIO/DriveBlob.h +++ b/Source/Core/DiscIO/DriveBlob.h @@ -23,11 +23,15 @@ class DriveReader : public SectorReader public: static std::unique_ptr Create(const std::string& drive); ~DriveReader(); + BlobType GetBlobType() const override { return BlobType::DRIVE; } + u64 GetRawSize() const override { return m_size; } u64 GetDataSize() const override { return m_size; } bool IsDataSizeAccurate() const override { return true; } + u64 GetBlockSize() const override { return ECC_BLOCK_SIZE; } + private: DriveReader(const std::string& drive); bool GetBlock(u64 block_num, u8* out_ptr) override; @@ -41,6 +45,7 @@ private: File::IOFile m_file; bool IsOK() const { return m_file.IsOpen() && m_file.IsGood(); } #endif + static constexpr u64 ECC_BLOCK_SIZE = 0x8000; u64 m_size = 0; }; diff --git a/Source/Core/DiscIO/VolumeVerifier.cpp b/Source/Core/DiscIO/VolumeVerifier.cpp index 949457cd84..556d40c793 100644 --- a/Source/Core/DiscIO/VolumeVerifier.cpp +++ b/Source/Core/DiscIO/VolumeVerifier.cpp @@ -1031,7 +1031,7 @@ void VolumeVerifier::SetUpHashing() else if (m_volume.GetVolumeType() == Platform::WiiDisc) { // Set up a DiscScrubber for checking whether blocks with errors are unused - m_scrubber.SetupScrub(&m_volume, VolumeWii::BLOCK_TOTAL_SIZE); + m_scrubber.SetupScrub(&m_volume); } std::sort(m_blocks.begin(), m_blocks.end(), diff --git a/Source/Core/DiscIO/WbfsBlob.h b/Source/Core/DiscIO/WbfsBlob.h index 36de02f62d..0283b1c98b 100644 --- a/Source/Core/DiscIO/WbfsBlob.h +++ b/Source/Core/DiscIO/WbfsBlob.h @@ -24,6 +24,7 @@ public: static std::unique_ptr Create(File::IOFile file, const std::string& path); BlobType GetBlobType() const override { return BlobType::WBFS; } + u64 GetRawSize() const override { return m_size; } // The WBFS format does not save the original file size. // This function returns a constant upper bound @@ -31,6 +32,8 @@ public: u64 GetDataSize() const override; bool IsDataSizeAccurate() const override { return false; } + u64 GetBlockSize() const override { return m_wbfs_sector_size; } + bool Read(u64 offset, u64 nbytes, u8* out_ptr) override; private: diff --git a/Source/Core/DolphinQt/GameList/GameList.cpp b/Source/Core/DolphinQt/GameList/GameList.cpp index a635bee291..e4bf1c47bf 100644 --- a/Source/Core/DolphinQt/GameList/GameList.cpp +++ b/Source/Core/DolphinQt/GameList/GameList.cpp @@ -595,8 +595,8 @@ void GameList::CompressISO(bool decompress) } good = std::async(std::launch::async, [&] { - const bool good = DiscIO::DecompressBlobToFile(original_path, dst_path.toStdString(), - &CompressCB, &progress_dialog); + const bool good = DiscIO::ConvertToPlain(original_path, dst_path.toStdString(), &CompressCB, + &progress_dialog); progress_dialog.Reset(); return good; }); @@ -612,9 +612,9 @@ void GameList::CompressISO(bool decompress) good = std::async(std::launch::async, [&] { const bool good = - DiscIO::CompressFileToBlob(original_path, dst_path.toStdString(), - file->GetPlatform() == DiscIO::Platform::WiiDisc ? 1 : 0, - 16384, &CompressCB, &progress_dialog); + DiscIO::ConvertToGCZ(original_path, dst_path.toStdString(), + file->GetPlatform() == DiscIO::Platform::WiiDisc ? 1 : 0, 16384, + &CompressCB, &progress_dialog); progress_dialog.Reset(); return good; }); From 42f6913bccdafcb8a32e5d82bd54947062cfd0a9 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sat, 4 Apr 2020 21:47:28 +0200 Subject: [PATCH 2/9] Move DiscIO::ConvertToPlain to FileBlob.cpp There is no longer anything GCZ specific about it. --- Source/Core/DiscIO/CompressedBlob.cpp | 79 ------------------------- Source/Core/DiscIO/FileBlob.cpp | 84 +++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 79 deletions(-) diff --git a/Source/Core/DiscIO/CompressedBlob.cpp b/Source/Core/DiscIO/CompressedBlob.cpp index 33790ad3b0..5e6723679a 100644 --- a/Source/Core/DiscIO/CompressedBlob.cpp +++ b/Source/Core/DiscIO/CompressedBlob.cpp @@ -336,85 +336,6 @@ bool ConvertToGCZ(const std::string& infile_path, const std::string& outfile_pat return success; } -bool ConvertToPlain(const std::string& infile_path, const std::string& outfile_path, - CompressCB callback, void* arg) -{ - std::unique_ptr reader = CreateBlobReader(infile_path); - if (!reader) - { - PanicAlertT("Failed to open the input file \"%s\".", infile_path.c_str()); - return false; - } - - ASSERT(reader->IsDataSizeAccurate()); - - File::IOFile outfile(outfile_path, "wb"); - if (!outfile) - { - PanicAlertT("Failed to open the output file \"%s\".\n" - "Check that you have permissions to write the target folder and that the media can " - "be written.", - outfile_path.c_str()); - return false; - } - - constexpr size_t DESIRED_BUFFER_SIZE = 0x80000; - u64 buffer_size = reader->GetBlockSize(); - if (buffer_size == 0) - { - buffer_size = DESIRED_BUFFER_SIZE; - } - else - { - while (buffer_size < DESIRED_BUFFER_SIZE) - buffer_size *= 2; - } - - std::vector buffer(buffer_size); - const u64 num_buffers = (reader->GetDataSize() + buffer_size - 1) / buffer_size; - int progress_monitor = std::max(1, num_buffers / 100); - bool success = true; - - for (u64 i = 0; i < num_buffers; i++) - { - if (i % progress_monitor == 0) - { - const bool was_cancelled = - !callback(Common::GetStringT("Unpacking"), (float)i / (float)num_buffers, arg); - if (was_cancelled) - { - success = false; - break; - } - } - const u64 inpos = i * buffer_size; - const u64 sz = std::min(buffer_size, reader->GetDataSize() - inpos); - if (!reader->Read(inpos, sz, buffer.data())) - { - PanicAlertT("Failed to read from the input file \"%s\".", infile_path.c_str()); - success = false; - break; - } - if (!outfile.WriteBytes(buffer.data(), sz)) - { - PanicAlertT("Failed to write the output file \"%s\".\n" - "Check that you have enough space available on the target drive.", - outfile_path.c_str()); - success = false; - break; - } - } - - if (!success) - { - // Remove the incomplete output file. - outfile.Close(); - File::Delete(outfile_path); - } - - return success; -} - bool IsGCZBlob(File::IOFile& file) { const u64 position = file.Tell(); diff --git a/Source/Core/DiscIO/FileBlob.cpp b/Source/Core/DiscIO/FileBlob.cpp index 9e0486d969..95beeb1bc3 100644 --- a/Source/Core/DiscIO/FileBlob.cpp +++ b/Source/Core/DiscIO/FileBlob.cpp @@ -2,10 +2,15 @@ // Licensed under GPLv2+ // Refer to the license.txt file included. +#include #include #include #include +#include +#include "Common/Assert.h" +#include "Common/FileUtil.h" +#include "Common/MsgHandler.h" #include "DiscIO/FileBlob.h" namespace DiscIO @@ -36,4 +41,83 @@ bool PlainFileReader::Read(u64 offset, u64 nbytes, u8* out_ptr) } } +bool ConvertToPlain(const std::string& infile_path, const std::string& outfile_path, + CompressCB callback, void* arg) +{ + std::unique_ptr reader = CreateBlobReader(infile_path); + if (!reader) + { + PanicAlertT("Failed to open the input file \"%s\".", infile_path.c_str()); + return false; + } + + ASSERT(reader->IsDataSizeAccurate()); + + File::IOFile outfile(outfile_path, "wb"); + if (!outfile) + { + PanicAlertT("Failed to open the output file \"%s\".\n" + "Check that you have permissions to write the target folder and that the media can " + "be written.", + outfile_path.c_str()); + return false; + } + + constexpr size_t DESIRED_BUFFER_SIZE = 0x80000; + u64 buffer_size = reader->GetBlockSize(); + if (buffer_size == 0) + { + buffer_size = DESIRED_BUFFER_SIZE; + } + else + { + while (buffer_size < DESIRED_BUFFER_SIZE) + buffer_size *= 2; + } + + std::vector buffer(buffer_size); + const u64 num_buffers = (reader->GetDataSize() + buffer_size - 1) / buffer_size; + int progress_monitor = std::max(1, num_buffers / 100); + bool success = true; + + for (u64 i = 0; i < num_buffers; i++) + { + if (i % progress_monitor == 0) + { + const bool was_cancelled = + !callback(Common::GetStringT("Unpacking"), (float)i / (float)num_buffers, arg); + if (was_cancelled) + { + success = false; + break; + } + } + const u64 inpos = i * buffer_size; + const u64 sz = std::min(buffer_size, reader->GetDataSize() - inpos); + if (!reader->Read(inpos, sz, buffer.data())) + { + PanicAlertT("Failed to read from the input file \"%s\".", infile_path.c_str()); + success = false; + break; + } + if (!outfile.WriteBytes(buffer.data(), sz)) + { + PanicAlertT("Failed to write the output file \"%s\".\n" + "Check that you have enough space available on the target drive.", + outfile_path.c_str()); + success = false; + break; + } + } + + if (!success) + { + // Remove the incomplete output file. + outfile.Close(); + File::Delete(outfile_path); + } + + return success; +} + } // namespace DiscIO From dae2c14f7f215b2da570308a10f533d99fdb80af Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sat, 4 Apr 2020 23:18:15 +0200 Subject: [PATCH 3/9] DolphinQt: Turn the compress/decompress action into a dialog --- Source/Core/DolphinQt/CMakeLists.txt | 2 + Source/Core/DolphinQt/ConvertDialog.cpp | 217 ++++++++++++++++++++ Source/Core/DolphinQt/ConvertDialog.h | 37 ++++ Source/Core/DolphinQt/DolphinQt.vcxproj | 3 + Source/Core/DolphinQt/GameList/GameList.cpp | 210 ++----------------- Source/Core/DolphinQt/GameList/GameList.h | 2 +- Source/Core/UICommon/GameFile.cpp | 3 + Source/Core/UICommon/GameFile.h | 2 + Source/Core/UICommon/GameFileCache.cpp | 2 +- 9 files changed, 283 insertions(+), 195 deletions(-) create mode 100644 Source/Core/DolphinQt/ConvertDialog.cpp create mode 100644 Source/Core/DolphinQt/ConvertDialog.h diff --git a/Source/Core/DolphinQt/CMakeLists.txt b/Source/Core/DolphinQt/CMakeLists.txt index 6a124cd03b..0b468df466 100644 --- a/Source/Core/DolphinQt/CMakeLists.txt +++ b/Source/Core/DolphinQt/CMakeLists.txt @@ -18,6 +18,8 @@ add_executable(dolphin-emu AboutDialog.h CheatsManager.cpp CheatsManager.h + ConvertDialog.cpp + ConvertDialog.h DiscordHandler.cpp DiscordHandler.h DiscordJoinRequestDialog.cpp diff --git a/Source/Core/DolphinQt/ConvertDialog.cpp b/Source/Core/DolphinQt/ConvertDialog.cpp new file mode 100644 index 0000000000..c3e102bd7c --- /dev/null +++ b/Source/Core/DolphinQt/ConvertDialog.cpp @@ -0,0 +1,217 @@ +// Copyright 2020 Dolphin Emulator Project +// Licensed under GPLv2+ +// Refer to the license.txt file included. + +#include "DolphinQt/ConvertDialog.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Common/Assert.h" +#include "DiscIO/Blob.h" +#include "DolphinQt/QtUtils/ModalMessageBox.h" +#include "DolphinQt/QtUtils/ParallelProgressDialog.h" +#include "UICommon/GameFile.h" + +static bool CompressCB(const std::string& text, float percent, void* ptr) +{ + if (ptr == nullptr) + return false; + + auto* progress_dialog = static_cast(ptr); + + progress_dialog->SetValue(percent * 100); + return !progress_dialog->WasCanceled(); +} + +ConvertDialog::ConvertDialog(QList> files, + QWidget* parent) + : QDialog(parent), m_files(std::move(files)) +{ + ASSERT(!m_files.empty()); + + setWindowTitle(tr("Convert")); + setWindowFlags(windowFlags() & ~Qt::WindowContextHelpButtonHint); + + QGridLayout* grid_layout = new QGridLayout; + grid_layout->setColumnStretch(1, 1); + + m_format = new QComboBox; + AddToFormatComboBox(QStringLiteral("ISO"), DiscIO::BlobType::PLAIN); + AddToFormatComboBox(QStringLiteral("GCZ"), DiscIO::BlobType::GCZ); + grid_layout->addWidget(new QLabel(tr("Format:")), 0, 0); + grid_layout->addWidget(m_format, 0, 1); + + QPushButton* convert_button = new QPushButton(tr("Convert")); + + QVBoxLayout* main_layout = new QVBoxLayout; + main_layout->addLayout(grid_layout); + main_layout->addWidget(convert_button); + + setLayout(main_layout); + + connect(convert_button, &QPushButton::clicked, this, &ConvertDialog::Convert); +} + +void ConvertDialog::AddToFormatComboBox(const QString& name, DiscIO::BlobType format) +{ + if (std::all_of(m_files.begin(), m_files.end(), + [format](const auto& file) { return file->GetBlobType() == format; })) + { + return; + } + + m_format->addItem(name, static_cast(format)); +} + +void ConvertDialog::Convert() +{ + const DiscIO::BlobType format = static_cast(m_format->currentData().toInt()); + + const bool scrub_wii = format == DiscIO::BlobType::GCZ; + + if (scrub_wii && std::any_of(m_files.begin(), m_files.end(), [](const auto& file) { + return file->GetPlatform() == DiscIO::Platform::WiiDisc; + })) + { + ModalMessageBox wii_warning(this); + wii_warning.setIcon(QMessageBox::Warning); + wii_warning.setWindowTitle(tr("Confirm")); + wii_warning.setText(tr("Are you sure?")); + wii_warning.setInformativeText( + tr("Compressing a Wii disc image will irreversibly change the compressed copy by removing " + "padding data. Your disc image will still work. Continue?")); + wii_warning.setStandardButtons(QMessageBox::Yes | QMessageBox::No); + + if (wii_warning.exec() == QMessageBox::No) + return; + } + + QString extension; + QString filter; + switch (format) + { + case DiscIO::BlobType::PLAIN: + extension = QStringLiteral(".iso"); + filter = tr("Uncompressed GC/Wii images (*.iso *.gcm)"); + break; + case DiscIO::BlobType::GCZ: + extension = QStringLiteral(".gcz"); + filter = tr("Compressed GC/Wii images (*.gcz)"); + break; + default: + ASSERT(false); + return; + } + + QString dst_dir; + QString dst_path; + + if (m_files.size() > 1) + { + dst_dir = QFileDialog::getExistingDirectory( + this, tr("Select where you want to save the converted images"), + QFileInfo(QString::fromStdString(m_files[0]->GetFilePath())).dir().absolutePath()); + + if (dst_dir.isEmpty()) + return; + } + else + { + dst_path = QFileDialog::getSaveFileName( + this, tr("Select where you want to save the converted image"), + QFileInfo(QString::fromStdString(m_files[0]->GetFilePath())) + .dir() + .absoluteFilePath( + QFileInfo(QString::fromStdString(m_files[0]->GetFilePath())).completeBaseName()) + .append(extension), + filter); + + if (dst_path.isEmpty()) + return; + } + + for (const auto& file : m_files) + { + const auto original_path = file->GetFilePath(); + if (m_files.size() > 1) + { + dst_path = + QDir(dst_dir) + .absoluteFilePath(QFileInfo(QString::fromStdString(original_path)).completeBaseName()) + .append(extension); + QFileInfo dst_info = QFileInfo(dst_path); + if (dst_info.exists()) + { + ModalMessageBox confirm_replace(this); + confirm_replace.setIcon(QMessageBox::Warning); + confirm_replace.setWindowTitle(tr("Confirm")); + confirm_replace.setText(tr("The file %1 already exists.\n" + "Do you wish to replace it?") + .arg(dst_info.fileName())); + confirm_replace.setStandardButtons(QMessageBox::Yes | QMessageBox::No); + + if (confirm_replace.exec() == QMessageBox::No) + continue; + } + } + + ParallelProgressDialog progress_dialog(tr("Converting..."), tr("Abort"), 0, 100, this); + progress_dialog.GetRaw()->setWindowModality(Qt::WindowModal); + progress_dialog.GetRaw()->setWindowTitle(tr("Progress")); + + if (m_files.size() > 1) + { + progress_dialog.GetRaw()->setLabelText( + tr("Converting...") + QLatin1Char{'\n'} + + QFileInfo(QString::fromStdString(original_path)).fileName()); + } + + std::future good; + + if (format == DiscIO::BlobType::PLAIN) + { + good = std::async(std::launch::async, [&] { + const bool good = DiscIO::ConvertToPlain(original_path, dst_path.toStdString(), &CompressCB, + &progress_dialog); + progress_dialog.Reset(); + return good; + }); + } + else if (format == DiscIO::BlobType::GCZ) + { + good = std::async(std::launch::async, [&] { + const bool good = + DiscIO::ConvertToGCZ(original_path, dst_path.toStdString(), + file->GetPlatform() == DiscIO::Platform::WiiDisc ? 1 : 0, 16384, + &CompressCB, &progress_dialog); + progress_dialog.Reset(); + return good; + }); + } + + progress_dialog.GetRaw()->exec(); + if (!good.get()) + { + QErrorMessage(this).showMessage(tr("Dolphin failed to complete the requested action.")); + return; + } + } + + ModalMessageBox::information(this, tr("Success"), + tr("Successfully converted %n image(s).", "", m_files.size())); + + close(); +} diff --git a/Source/Core/DolphinQt/ConvertDialog.h b/Source/Core/DolphinQt/ConvertDialog.h new file mode 100644 index 0000000000..17aeb8f3e7 --- /dev/null +++ b/Source/Core/DolphinQt/ConvertDialog.h @@ -0,0 +1,37 @@ +// Copyright 2020 Dolphin Emulator Project +// Licensed under GPLv2+ +// Refer to the license.txt file included. + +#pragma once + +#include + +#include +#include + +#include "DiscIO/Blob.h" + +class QComboBox; + +namespace UICommon +{ +class GameFile; +} + +class ConvertDialog final : public QDialog +{ + Q_OBJECT + +public: + explicit ConvertDialog(QList> files, + QWidget* parent = nullptr); + +private slots: + void Convert(); + +private: + void AddToFormatComboBox(const QString& name, DiscIO::BlobType format); + + QComboBox* m_format; + QList> m_files; +}; diff --git a/Source/Core/DolphinQt/DolphinQt.vcxproj b/Source/Core/DolphinQt/DolphinQt.vcxproj index 1763c06300..3e9b862ddf 100644 --- a/Source/Core/DolphinQt/DolphinQt.vcxproj +++ b/Source/Core/DolphinQt/DolphinQt.vcxproj @@ -131,6 +131,7 @@ + @@ -211,6 +212,7 @@ + @@ -372,6 +374,7 @@ + diff --git a/Source/Core/DolphinQt/GameList/GameList.cpp b/Source/Core/DolphinQt/GameList/GameList.cpp index e4bf1c47bf..9178980106 100644 --- a/Source/Core/DolphinQt/GameList/GameList.cpp +++ b/Source/Core/DolphinQt/GameList/GameList.cpp @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include @@ -40,6 +40,7 @@ #include "DiscIO/Enums.h" #include "DolphinQt/Config/PropertiesDialog.h" +#include "DolphinQt/ConvertDialog.h" #include "DolphinQt/GameList/GameListModel.h" #include "DolphinQt/GameList/GridProxyModel.h" #include "DolphinQt/GameList/ListProxyModel.h" @@ -53,8 +54,6 @@ #include "UICommon/GameFile.h" -static bool CompressCB(const std::string&, float, void*); - GameList::GameList(QWidget* parent) : QStackedWidget(parent) { m_model = Settings::Instance().GetGameListModel(); @@ -257,35 +256,16 @@ void GameList::ShowContextMenu(const QPoint&) if (HasMultipleSelected()) { - bool wii_saves = true; - bool compress = false; - bool decompress = false; - - for (const auto& game : GetSelectedGames()) + if (std::all_of(GetSelectedGames().begin(), GetSelectedGames().end(), [](const auto& game) { + return DiscIO::IsDisc(game->GetPlatform()) && game->IsVolumeSizeAccurate(); + })) { - DiscIO::Platform platform = game->GetPlatform(); - - if (platform == DiscIO::Platform::GameCubeDisc || platform == DiscIO::Platform::WiiDisc) - { - const auto blob_type = game->GetBlobType(); - if (blob_type == DiscIO::BlobType::GCZ) - decompress = true; - else if (blob_type == DiscIO::BlobType::PLAIN) - compress = true; - } - - if (platform != DiscIO::Platform::WiiWAD && platform != DiscIO::Platform::WiiDisc) - wii_saves = false; + menu->addAction(tr("Convert Selected Files..."), this, &GameList::ConvertFile); + menu->addSeparator(); } - if (compress) - menu->addAction(tr("Compress Selected ISOs..."), this, [this] { CompressISO(false); }); - if (decompress) - menu->addAction(tr("Decompress Selected ISOs..."), this, [this] { CompressISO(true); }); - if (compress || decompress) - menu->addSeparator(); - - if (wii_saves) + if (std::all_of(GetSelectedGames().begin(), GetSelectedGames().end(), + [](const auto& game) { return DiscIO::IsWii(game->GetPlatform()); })) { menu->addAction(tr("Export Wii Saves"), this, &GameList::ExportWiiSave); menu->addSeparator(); @@ -306,15 +286,13 @@ void GameList::ShowContextMenu(const QPoint&) menu->addSeparator(); } - if (platform == DiscIO::Platform::GameCubeDisc || platform == DiscIO::Platform::WiiDisc) + if (DiscIO::IsDisc(platform)) { menu->addAction(tr("Set as &Default ISO"), this, &GameList::SetDefaultISO); const auto blob_type = game->GetBlobType(); - if (blob_type == DiscIO::BlobType::GCZ) - menu->addAction(tr("Decompress ISO..."), this, [this] { CompressISO(true); }); - else if (blob_type == DiscIO::BlobType::PLAIN) - menu->addAction(tr("Compress ISO..."), this, [this] { CompressISO(false); }); + if (game->IsVolumeSizeAccurate()) + menu->addAction(tr("Convert File..."), this, &GameList::ConvertFile); QAction* change_disc = menu->addAction(tr("Change &Disc"), this, &GameList::ChangeDisc); @@ -481,157 +459,14 @@ void GameList::OpenWiki() QDesktopServices::openUrl(QUrl(url)); } -void GameList::CompressISO(bool decompress) +void GameList::ConvertFile() { - auto files = GetSelectedGames(); - const auto game = GetSelectedGame(); - - if (files.empty() || !game) + auto games = GetSelectedGames(); + if (games.empty()) return; - bool wii_warning_given = false; - for (QMutableListIterator> it(files); it.hasNext();) - { - auto file = it.next(); - - if ((file->GetPlatform() != DiscIO::Platform::GameCubeDisc && - file->GetPlatform() != DiscIO::Platform::WiiDisc) || - (decompress && file->GetBlobType() != DiscIO::BlobType::GCZ) || - (!decompress && file->GetBlobType() != DiscIO::BlobType::PLAIN)) - { - it.remove(); - continue; - } - - if (!wii_warning_given && !decompress && file->GetPlatform() == DiscIO::Platform::WiiDisc) - { - ModalMessageBox wii_warning(this); - wii_warning.setIcon(QMessageBox::Warning); - wii_warning.setWindowTitle(tr("Confirm")); - wii_warning.setText(tr("Are you sure?")); - wii_warning.setInformativeText(tr( - "Compressing a Wii disc image will irreversibly change the compressed copy by removing " - "padding data. Your disc image will still work. Continue?")); - wii_warning.setStandardButtons(QMessageBox::Yes | QMessageBox::No); - - if (wii_warning.exec() == QMessageBox::No) - return; - - wii_warning_given = true; - } - } - - QString dst_dir; - QString dst_path; - - if (files.size() > 1) - { - dst_dir = QFileDialog::getExistingDirectory( - this, - decompress ? tr("Select where you want to save the decompressed images") : - tr("Select where you want to save the compressed images"), - QFileInfo(QString::fromStdString(game->GetFilePath())).dir().absolutePath()); - - if (dst_dir.isEmpty()) - return; - } - else - { - dst_path = QFileDialog::getSaveFileName( - this, - decompress ? tr("Select where you want to save the decompressed image") : - tr("Select where you want to save the compressed image"), - QFileInfo(QString::fromStdString(game->GetFilePath())) - .dir() - .absoluteFilePath( - QFileInfo(QString::fromStdString(files[0]->GetFilePath())).completeBaseName()) - .append(decompress ? QStringLiteral(".gcm") : QStringLiteral(".gcz")), - decompress ? tr("Uncompressed GC/Wii images (*.iso *.gcm)") : - tr("Compressed GC/Wii images (*.gcz)")); - - if (dst_path.isEmpty()) - return; - } - - for (const auto& file : files) - { - const auto original_path = file->GetFilePath(); - if (files.size() > 1) - { - dst_path = - QDir(dst_dir) - .absoluteFilePath(QFileInfo(QString::fromStdString(original_path)).completeBaseName()) - .append(decompress ? QStringLiteral(".gcm") : QStringLiteral(".gcz")); - QFileInfo dst_info = QFileInfo(dst_path); - if (dst_info.exists()) - { - ModalMessageBox confirm_replace(this); - confirm_replace.setIcon(QMessageBox::Warning); - confirm_replace.setWindowTitle(tr("Confirm")); - confirm_replace.setText(tr("The file %1 already exists.\n" - "Do you wish to replace it?") - .arg(dst_info.fileName())); - confirm_replace.setStandardButtons(QMessageBox::Yes | QMessageBox::No); - - if (confirm_replace.exec() == QMessageBox::No) - continue; - } - } - - ParallelProgressDialog progress_dialog( - decompress ? tr("Decompressing...") : tr("Compressing..."), tr("Abort"), 0, 100, this); - progress_dialog.GetRaw()->setWindowModality(Qt::WindowModal); - progress_dialog.GetRaw()->setWindowTitle(tr("Progress")); - - std::future good; - - if (decompress) - { - if (files.size() > 1) - { - progress_dialog.GetRaw()->setLabelText( - tr("Decompressing...") + QLatin1Char{'\n'} + - QFileInfo(QString::fromStdString(original_path)).fileName()); - } - - good = std::async(std::launch::async, [&] { - const bool good = DiscIO::ConvertToPlain(original_path, dst_path.toStdString(), &CompressCB, - &progress_dialog); - progress_dialog.Reset(); - return good; - }); - } - else - { - if (files.size() > 1) - { - progress_dialog.GetRaw()->setLabelText( - tr("Compressing...") + QLatin1Char{'\n'} + - QFileInfo(QString::fromStdString(original_path)).fileName()); - } - - good = std::async(std::launch::async, [&] { - const bool good = - DiscIO::ConvertToGCZ(original_path, dst_path.toStdString(), - file->GetPlatform() == DiscIO::Platform::WiiDisc ? 1 : 0, 16384, - &CompressCB, &progress_dialog); - progress_dialog.Reset(); - return good; - }); - } - - progress_dialog.GetRaw()->exec(); - if (!good.get()) - { - QErrorMessage(this).showMessage(tr("Dolphin failed to complete the requested action.")); - return; - } - } - - ModalMessageBox::information(this, tr("Success"), - decompress ? - tr("Successfully decompressed %n image(s).", "", files.size()) : - tr("Successfully compressed %n image(s).", "", files.size())); + ConvertDialog dialog{std::move(games), this}; + dialog.exec(); } void GameList::InstallWAD() @@ -949,17 +784,6 @@ void GameList::OnGameListVisibilityChanged() m_grid_proxy->invalidate(); } -static bool CompressCB(const std::string& text, float percent, void* ptr) -{ - if (ptr == nullptr) - return false; - - auto* progress_dialog = static_cast(ptr); - - progress_dialog->SetValue(percent * 100); - return !progress_dialog->WasCanceled(); -} - void GameList::OnSectionResized(int index, int, int) { auto* hor_header = m_list->horizontalHeader(); diff --git a/Source/Core/DolphinQt/GameList/GameList.h b/Source/Core/DolphinQt/GameList/GameList.h index 596c92f57e..6587653ee4 100644 --- a/Source/Core/DolphinQt/GameList/GameList.h +++ b/Source/Core/DolphinQt/GameList/GameList.h @@ -64,7 +64,7 @@ private: void InstallWAD(); void UninstallWAD(); void ExportWiiSave(); - void CompressISO(bool decompress); + void ConvertFile(); void ChangeDisc(); void NewTag(); void DeleteTag(); diff --git a/Source/Core/UICommon/GameFile.cpp b/Source/Core/UICommon/GameFile.cpp index cc64d9d3e9..139dcce688 100644 --- a/Source/Core/UICommon/GameFile.cpp +++ b/Source/Core/UICommon/GameFile.cpp @@ -116,6 +116,7 @@ GameFile::GameFile(std::string path) : m_file_path(std::move(path)) m_blob_type = volume->GetBlobType(); m_file_size = volume->GetRawSize(); m_volume_size = volume->GetSize(); + m_volume_size_is_accurate = volume->IsSizeAccurate(); m_internal_name = volume->GetInternalName(); m_game_id = volume->GetGameID(); @@ -136,6 +137,7 @@ GameFile::GameFile(std::string path) : m_file_path(std::move(path)) { m_valid = true; m_file_size = m_volume_size = File::GetSize(m_file_path); + m_volume_size_is_accurate = true; m_platform = DiscIO::Platform::ELFOrDOL; m_blob_type = DiscIO::BlobType::DIRECTORY; } @@ -296,6 +298,7 @@ void GameFile::DoState(PointerWrap& p) p.Do(m_file_size); p.Do(m_volume_size); + p.Do(m_volume_size_is_accurate); p.Do(m_short_names); p.Do(m_long_names); diff --git a/Source/Core/UICommon/GameFile.h b/Source/Core/UICommon/GameFile.h index 87513118a9..53401e8756 100644 --- a/Source/Core/UICommon/GameFile.h +++ b/Source/Core/UICommon/GameFile.h @@ -89,6 +89,7 @@ public: const std::string& GetApploaderDate() const { return m_apploader_date; } u64 GetFileSize() const { return m_file_size; } u64 GetVolumeSize() const { return m_volume_size; } + bool IsVolumeSizeAccurate() const { return m_volume_size_is_accurate; } const GameBanner& GetBannerImage() const; const GameCover& GetCoverImage() const; void DoState(PointerWrap& p); @@ -124,6 +125,7 @@ private: u64 m_file_size{}; u64 m_volume_size{}; + bool m_volume_size_is_accurate{}; std::map m_short_names; std::map m_long_names; diff --git a/Source/Core/UICommon/GameFileCache.cpp b/Source/Core/UICommon/GameFileCache.cpp index 07cbf726c4..9b2729407e 100644 --- a/Source/Core/UICommon/GameFileCache.cpp +++ b/Source/Core/UICommon/GameFileCache.cpp @@ -27,7 +27,7 @@ namespace UICommon { -static constexpr u32 CACHE_REVISION = 16; // Last changed in PR 8313 +static constexpr u32 CACHE_REVISION = 17; // Last changed in PR 8738 std::vector FindAllGamePaths(const std::vector& directories_to_scan, bool recursive_scan) From cefc2a7baa9a4dba878352ebc3a5c92b63ed295a Mon Sep 17 00:00:00 2001 From: JosJuice Date: Fri, 10 Apr 2020 16:21:51 +0200 Subject: [PATCH 4/9] DiscIO: Fix edge case where blocks could get scrubbed accidentally If we start 31 KiB into a 32 KiB block and want to mark 2 KiB of data as used, we need to mark 2 blocks as used, not just 1. This problem is avoided when calling MarkAsUsed from MarkAsUsedE, since MarkAsUsedE aligns to 32 KiB on its own. Most calls to MarkAsUsed are from MarkAsUsedE, which is why this hasn't been a noticeable problem in the past. --- Source/Core/DiscIO/DiscScrubber.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/Core/DiscIO/DiscScrubber.cpp b/Source/Core/DiscIO/DiscScrubber.cpp index 07d504d72a..706a2cd00d 100644 --- a/Source/Core/DiscIO/DiscScrubber.cpp +++ b/Source/Core/DiscIO/DiscScrubber.cpp @@ -58,8 +58,8 @@ bool DiscScrubber::CanBlockBeScrubbed(u64 offset) const void DiscScrubber::MarkAsUsed(u64 offset, u64 size) { - u64 current_offset = offset; - const u64 end_offset = current_offset + size; + u64 current_offset = Common::AlignDown(offset, CLUSTER_SIZE); + const u64 end_offset = offset + size; DEBUG_LOG(DISCIO, "Marking 0x%016" PRIx64 " - 0x%016" PRIx64 " as used", offset, end_offset); From 04c7892b93f4ae129609060be5347628fa29eced Mon Sep 17 00:00:00 2001 From: JosJuice Date: Fri, 10 Apr 2020 15:53:14 +0200 Subject: [PATCH 5/9] DiscIO: Add GameCube disc scrubbing support The code was actually already rather well adapted for this. We more or less just have to skip ParseDisc and run ParsePartitionData directly. This required the PartitionHeader struct to be removed (which wasn't that useful anyway). --- Source/Core/DiscIO/DiscScrubber.cpp | 96 +++++++++++++++++------------ Source/Core/DiscIO/DiscScrubber.h | 28 +-------- 2 files changed, 60 insertions(+), 64 deletions(-) diff --git a/Source/Core/DiscIO/DiscScrubber.cpp b/Source/Core/DiscIO/DiscScrubber.cpp index 706a2cd00d..06c444c84d 100644 --- a/Source/Core/DiscIO/DiscScrubber.cpp +++ b/Source/Core/DiscIO/DiscScrubber.cpp @@ -72,20 +72,27 @@ void DiscScrubber::MarkAsUsed(u64 offset, u64 size) void DiscScrubber::MarkAsUsedE(u64 partition_data_offset, u64 offset, u64 size) { - u64 first_cluster_start = ToClusterOffset(offset) + partition_data_offset; - - u64 last_cluster_end; - if (size == 0) + if (partition_data_offset == 0) { - // Without this special case, a size of 0 can be rounded to 1 cluster instead of 0 - last_cluster_end = first_cluster_start; + MarkAsUsed(offset, size); } else { - last_cluster_end = ToClusterOffset(offset + size - 1) + CLUSTER_SIZE + partition_data_offset; - } + u64 first_cluster_start = ToClusterOffset(offset) + partition_data_offset; - MarkAsUsed(first_cluster_start, last_cluster_end - first_cluster_start); + u64 last_cluster_end; + if (size == 0) + { + // Without this special case, a size of 0 can be rounded to 1 cluster instead of 0 + last_cluster_end = first_cluster_start; + } + else + { + last_cluster_end = ToClusterOffset(offset + size - 1) + CLUSTER_SIZE + partition_data_offset; + } + + MarkAsUsed(first_cluster_start, last_cluster_end - first_cluster_start); + } } // Compensate for 0x400 (SHA-1) per 0x8000 (cluster), and round to whole clusters @@ -116,35 +123,38 @@ bool DiscScrubber::ReadFromVolume(u64 offset, u64& buffer, const Partition& part bool DiscScrubber::ParseDisc() { + if (m_disc->GetPartitions().empty()) + return ParsePartitionData(PARTITION_NONE); + // Mark the header as used - it's mostly 0s anyways MarkAsUsed(0, 0x50000); for (const DiscIO::Partition& partition : m_disc->GetPartitions()) { - PartitionHeader header; + u32 tmd_size; + u64 tmd_offset; + u32 cert_chain_size; + u64 cert_chain_offset; + u64 h3_offset; + // The H3 size is always 0x18000 - if (!ReadFromVolume(partition.offset + 0x2a4, header.tmd_size, PARTITION_NONE) || - !ReadFromVolume(partition.offset + 0x2a8, header.tmd_offset, PARTITION_NONE) || - !ReadFromVolume(partition.offset + 0x2ac, header.cert_chain_size, PARTITION_NONE) || - !ReadFromVolume(partition.offset + 0x2b0, header.cert_chain_offset, PARTITION_NONE) || - !ReadFromVolume(partition.offset + 0x2b4, header.h3_offset, PARTITION_NONE) || - !ReadFromVolume(partition.offset + 0x2b8, header.data_offset, PARTITION_NONE) || - !ReadFromVolume(partition.offset + 0x2bc, header.data_size, PARTITION_NONE)) + if (!ReadFromVolume(partition.offset + 0x2a4, tmd_size, PARTITION_NONE) || + !ReadFromVolume(partition.offset + 0x2a8, tmd_offset, PARTITION_NONE) || + !ReadFromVolume(partition.offset + 0x2ac, cert_chain_size, PARTITION_NONE) || + !ReadFromVolume(partition.offset + 0x2b0, cert_chain_offset, PARTITION_NONE) || + !ReadFromVolume(partition.offset + 0x2b4, h3_offset, PARTITION_NONE)) { return false; } MarkAsUsed(partition.offset, 0x2c0); - MarkAsUsed(partition.offset + header.tmd_offset, header.tmd_size); - MarkAsUsed(partition.offset + header.cert_chain_offset, header.cert_chain_size); - MarkAsUsed(partition.offset + header.h3_offset, 0x18000); - // This would mark the whole (encrypted) data area - // we need to parse FST and other crap to find what's free within it! - // MarkAsUsed(partition.offset + header.data_offset, header.data_size); + MarkAsUsed(partition.offset + tmd_offset, tmd_size); + MarkAsUsed(partition.offset + cert_chain_offset, cert_chain_size); + MarkAsUsed(partition.offset + h3_offset, 0x18000); // Parse Data! This is where the big gain is - if (!ParsePartitionData(partition, &header)) + if (!ParsePartitionData(partition)) return false; } @@ -152,7 +162,7 @@ bool DiscScrubber::ParseDisc() } // Operations dealing with encrypted space are done here -bool DiscScrubber::ParsePartitionData(const Partition& partition, PartitionHeader* header) +bool DiscScrubber::ParsePartitionData(const Partition& partition) { const FileSystem* filesystem = m_disc->GetFileSystem(partition); if (!filesystem) @@ -162,17 +172,30 @@ bool DiscScrubber::ParsePartitionData(const Partition& partition, PartitionHeade return false; } - const u64 partition_data_offset = partition.offset + header->data_offset; + u64 partition_data_offset; + if (partition == PARTITION_NONE) + { + partition_data_offset = 0; + } + else + { + u64 data_offset; + if (!ReadFromVolume(partition.offset + 0x2b8, data_offset, PARTITION_NONE)) + return false; + + partition_data_offset = partition.offset + data_offset; + } // Mark things as used which are not in the filesystem // Header, Header Information, Apploader - if (!ReadFromVolume(0x2440 + 0x14, header->apploader_size, partition) || - !ReadFromVolume(0x2440 + 0x18, header->apploader_size, partition)) + u32 apploader_size; + u32 apploader_trailer_size; + if (!ReadFromVolume(0x2440 + 0x14, apploader_size, partition) || + !ReadFromVolume(0x2440 + 0x18, apploader_trailer_size, partition)) { return false; } - MarkAsUsedE(partition_data_offset, 0, - 0x2440 + header->apploader_size + header->apploader_trailer_size); + MarkAsUsedE(partition_data_offset, 0, 0x2440 + apploader_size + apploader_trailer_size); // DOL const std::optional dol_offset = GetBootDOLOffset(*m_disc, partition); @@ -181,17 +204,14 @@ bool DiscScrubber::ParsePartitionData(const Partition& partition, PartitionHeade const std::optional dol_size = GetBootDOLSize(*m_disc, partition, *dol_offset); if (!dol_size) return false; - header->dol_offset = *dol_offset; - header->dol_size = *dol_size; - MarkAsUsedE(partition_data_offset, header->dol_offset, header->dol_size); + MarkAsUsedE(partition_data_offset, *dol_offset, *dol_size); // FST - if (!ReadFromVolume(0x424, header->fst_offset, partition) || - !ReadFromVolume(0x428, header->fst_size, partition)) - { + const std::optional fst_offset = GetFSTOffset(*m_disc, partition); + const std::optional fst_size = GetFSTSize(*m_disc, partition); + if (!fst_offset || !fst_size) return false; - } - MarkAsUsedE(partition_data_offset, header->fst_offset, header->fst_size); + MarkAsUsedE(partition_data_offset, *fst_offset, *fst_size); // Go through the filesystem and mark entries as used ParseFileSystemData(partition_data_offset, filesystem->GetRoot()); diff --git a/Source/Core/DiscIO/DiscScrubber.h b/Source/Core/DiscIO/DiscScrubber.h index b3e7e7a9a0..4b46f6353a 100644 --- a/Source/Core/DiscIO/DiscScrubber.h +++ b/Source/Core/DiscIO/DiscScrubber.h @@ -2,11 +2,7 @@ // Licensed under GPLv2+ // Refer to the license.txt file included. -// DiscScrubber removes the garbage data from discs (currently Wii only) which -// is on the disc due to encryption - -// It could be adapted to GameCube discs, but the gain is most likely negligible, -// and having 1:1 backups of discs is always nice when they are reasonably sized +// DiscScrubber removes the pseudorandom padding data from discs // Note: the technique is inspired by Wiiscrubber, but much simpler - intentionally :) @@ -40,33 +36,13 @@ public: bool CanBlockBeScrubbed(u64 offset) const; private: - struct PartitionHeader final - { - u8* ticket[0x2a4]; - u32 tmd_size; - u64 tmd_offset; - u32 cert_chain_size; - u64 cert_chain_offset; - // H3Size is always 0x18000 - u64 h3_offset; - u64 data_offset; - u64 data_size; - // TMD would be here - u64 dol_offset; - u64 dol_size; - u64 fst_offset; - u64 fst_size; - u32 apploader_size; - u32 apploader_trailer_size; - }; - void MarkAsUsed(u64 offset, u64 size); void MarkAsUsedE(u64 partition_data_offset, u64 offset, u64 size); u64 ToClusterOffset(u64 offset) const; bool ReadFromVolume(u64 offset, u32& buffer, const Partition& partition); bool ReadFromVolume(u64 offset, u64& buffer, const Partition& partition); bool ParseDisc(); - bool ParsePartitionData(const Partition& partition, PartitionHeader* header); + bool ParsePartitionData(const Partition& partition); void ParseFileSystemData(u64 partition_data_offset, const FileInfo& directory); const Volume* m_disc; From 6ffcbcee70c043a0d158d87458f99d20eddbb939 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Fri, 10 Apr 2020 17:40:07 +0200 Subject: [PATCH 6/9] DiscIO: Move scrubbing code out of ConvertToGCZ This way, scrubbing can also be performed when converting to other formats. --- Source/Core/DiscIO/Blob.h | 10 +-- Source/Core/DiscIO/CMakeLists.txt | 2 + Source/Core/DiscIO/CompressedBlob.cpp | 37 ++-------- Source/Core/DiscIO/DiscIO.vcxproj | 2 + Source/Core/DiscIO/DiscIO.vcxproj.filters | 6 ++ Source/Core/DiscIO/DiscScrubber.cpp | 2 - Source/Core/DiscIO/DiscScrubber.h | 2 + Source/Core/DiscIO/FileBlob.cpp | 21 ++---- Source/Core/DiscIO/ScrubbedBlob.cpp | 67 ++++++++++++++++++ Source/Core/DiscIO/ScrubbedBlob.h | 37 ++++++++++ Source/Core/DolphinQt/ConvertDialog.cpp | 82 ++++++++++++++++------- 11 files changed, 195 insertions(+), 73 deletions(-) create mode 100644 Source/Core/DiscIO/ScrubbedBlob.cpp create mode 100644 Source/Core/DiscIO/ScrubbedBlob.h diff --git a/Source/Core/DiscIO/Blob.h b/Source/Core/DiscIO/Blob.h index 9cb00087d2..1d23d7b6b0 100644 --- a/Source/Core/DiscIO/Blob.h +++ b/Source/Core/DiscIO/Blob.h @@ -165,9 +165,11 @@ std::unique_ptr CreateBlobReader(const std::string& filename); typedef bool (*CompressCB)(const std::string& text, float percent, void* arg); -bool ConvertToGCZ(const std::string& infile_path, const std::string& outfile_path, u32 sub_type = 0, - int sector_size = 16384, CompressCB callback = nullptr, void* arg = nullptr); -bool ConvertToPlain(const std::string& infile_path, const std::string& outfile_path, - CompressCB callback = nullptr, void* arg = nullptr); +bool ConvertToGCZ(BlobReader* infile, const std::string& infile_path, + const std::string& outfile_path, u32 sub_type, int sector_size = 16384, + CompressCB callback = nullptr, void* arg = nullptr); +bool ConvertToPlain(BlobReader* infile, const std::string& infile_path, + const std::string& outfile_path, CompressCB callback = nullptr, + void* arg = nullptr); } // namespace DiscIO diff --git a/Source/Core/DiscIO/CMakeLists.txt b/Source/Core/DiscIO/CMakeLists.txt index 279e72030d..4970f135c2 100644 --- a/Source/Core/DiscIO/CMakeLists.txt +++ b/Source/Core/DiscIO/CMakeLists.txt @@ -23,6 +23,8 @@ add_library(discio Filesystem.h NANDImporter.cpp NANDImporter.h + ScrubbedBlob.cpp + ScrubbedBlob.h TGCBlob.cpp TGCBlob.h Volume.cpp diff --git a/Source/Core/DiscIO/CompressedBlob.cpp b/Source/Core/DiscIO/CompressedBlob.cpp index 5e6723679a..cda3cad6d3 100644 --- a/Source/Core/DiscIO/CompressedBlob.cpp +++ b/Source/Core/DiscIO/CompressedBlob.cpp @@ -154,19 +154,11 @@ bool CompressedBlobReader::GetBlock(u64 block_num, u8* out_ptr) return true; } -bool ConvertToGCZ(const std::string& infile_path, const std::string& outfile_path, u32 sub_type, - int block_size, CompressCB callback, void* arg) +bool ConvertToGCZ(BlobReader* infile, const std::string& infile_path, + const std::string& outfile_path, u32 sub_type, int block_size, + CompressCB callback, void* arg) { - bool scrubbing = false; - - std::unique_ptr infile = CreateDisc(infile_path); - if (!infile) - { - PanicAlertT("Failed to open the input file \"%s\".", infile_path.c_str()); - return false; - } - - ASSERT(infile->IsSizeAccurate()); + ASSERT(infile->IsDataSizeAccurate()); File::IOFile outfile(outfile_path, "wb"); if (!outfile) @@ -178,19 +170,6 @@ bool ConvertToGCZ(const std::string& infile_path, const std::string& outfile_pat return false; } - DiscScrubber disc_scrubber; - if (sub_type == 1) - { - if (!disc_scrubber.SetupScrub(infile.get())) - { - PanicAlertT("\"%s\" failed to be scrubbed. Probably the image is corrupt.", - infile_path.c_str()); - return false; - } - - scrubbing = true; - } - z_stream z = {}; if (deflateInit(&z, 9) != Z_OK) return false; @@ -201,7 +180,7 @@ bool ConvertToGCZ(const std::string& infile_path, const std::string& outfile_pat header.magic_cookie = GCZ_MAGIC; header.sub_type = sub_type; header.block_size = block_size; - header.data_size = infile->GetSize(); + header.data_size = infile->GetDataSize(); // round upwards! header.num_blocks = (u32)((header.data_size + (block_size - 1)) / block_size); @@ -245,11 +224,9 @@ bool ConvertToGCZ(const std::string& infile_path, const std::string& outfile_pat offsets[i] = position; - const u64 bytes_to_read = scrubbing && disc_scrubber.CanBlockBeScrubbed(inpos) ? - 0 : - std::min(block_size, header.data_size - inpos); + const u64 bytes_to_read = std::min(block_size, header.data_size - inpos); - success = infile->Read(inpos, bytes_to_read, in_buf.data(), PARTITION_NONE); + success = infile->Read(inpos, bytes_to_read, in_buf.data()); if (!success) { PanicAlertT("Failed to read from the input file \"%s\".", infile_path.c_str()); diff --git a/Source/Core/DiscIO/DiscIO.vcxproj b/Source/Core/DiscIO/DiscIO.vcxproj index dd322dbb62..98bb500452 100644 --- a/Source/Core/DiscIO/DiscIO.vcxproj +++ b/Source/Core/DiscIO/DiscIO.vcxproj @@ -56,6 +56,7 @@ + @@ -80,6 +81,7 @@ + diff --git a/Source/Core/DiscIO/DiscIO.vcxproj.filters b/Source/Core/DiscIO/DiscIO.vcxproj.filters index 5fb5891cfe..9b19fa8471 100644 --- a/Source/Core/DiscIO/DiscIO.vcxproj.filters +++ b/Source/Core/DiscIO/DiscIO.vcxproj.filters @@ -87,6 +87,9 @@ Volume\Blob + + Volume\Blob + @@ -155,6 +158,9 @@ Volume\Blob + + Volume\Blob + diff --git a/Source/Core/DiscIO/DiscScrubber.cpp b/Source/Core/DiscIO/DiscScrubber.cpp index 06c444c84d..e7b098b374 100644 --- a/Source/Core/DiscIO/DiscScrubber.cpp +++ b/Source/Core/DiscIO/DiscScrubber.cpp @@ -25,8 +25,6 @@ namespace DiscIO { -constexpr size_t CLUSTER_SIZE = 0x8000; - DiscScrubber::DiscScrubber() = default; DiscScrubber::~DiscScrubber() = default; diff --git a/Source/Core/DiscIO/DiscScrubber.h b/Source/Core/DiscIO/DiscScrubber.h index 4b46f6353a..5d64fa5376 100644 --- a/Source/Core/DiscIO/DiscScrubber.h +++ b/Source/Core/DiscIO/DiscScrubber.h @@ -35,6 +35,8 @@ public: // Returns true if the specified 32 KiB block only contains unused data bool CanBlockBeScrubbed(u64 offset) const; + static constexpr size_t CLUSTER_SIZE = 0x8000; + private: void MarkAsUsed(u64 offset, u64 size); void MarkAsUsedE(u64 partition_data_offset, u64 offset, u64 size); diff --git a/Source/Core/DiscIO/FileBlob.cpp b/Source/Core/DiscIO/FileBlob.cpp index 95beeb1bc3..ca0aa2cfd0 100644 --- a/Source/Core/DiscIO/FileBlob.cpp +++ b/Source/Core/DiscIO/FileBlob.cpp @@ -41,17 +41,10 @@ bool PlainFileReader::Read(u64 offset, u64 nbytes, u8* out_ptr) } } -bool ConvertToPlain(const std::string& infile_path, const std::string& outfile_path, - CompressCB callback, void* arg) +bool ConvertToPlain(BlobReader* infile, const std::string& infile_path, + const std::string& outfile_path, CompressCB callback, void* arg) { - std::unique_ptr reader = CreateBlobReader(infile_path); - if (!reader) - { - PanicAlertT("Failed to open the input file \"%s\".", infile_path.c_str()); - return false; - } - - ASSERT(reader->IsDataSizeAccurate()); + ASSERT(infile->IsDataSizeAccurate()); File::IOFile outfile(outfile_path, "wb"); if (!outfile) @@ -64,7 +57,7 @@ bool ConvertToPlain(const std::string& infile_path, const std::string& outfile_p } constexpr size_t DESIRED_BUFFER_SIZE = 0x80000; - u64 buffer_size = reader->GetBlockSize(); + u64 buffer_size = infile->GetBlockSize(); if (buffer_size == 0) { buffer_size = DESIRED_BUFFER_SIZE; @@ -76,7 +69,7 @@ bool ConvertToPlain(const std::string& infile_path, const std::string& outfile_p } std::vector buffer(buffer_size); - const u64 num_buffers = (reader->GetDataSize() + buffer_size - 1) / buffer_size; + const u64 num_buffers = (infile->GetDataSize() + buffer_size - 1) / buffer_size; int progress_monitor = std::max(1, num_buffers / 100); bool success = true; @@ -93,8 +86,8 @@ bool ConvertToPlain(const std::string& infile_path, const std::string& outfile_p } } const u64 inpos = i * buffer_size; - const u64 sz = std::min(buffer_size, reader->GetDataSize() - inpos); - if (!reader->Read(inpos, sz, buffer.data())) + const u64 sz = std::min(buffer_size, infile->GetDataSize() - inpos); + if (!infile->Read(inpos, sz, buffer.data())) { PanicAlertT("Failed to read from the input file \"%s\".", infile_path.c_str()); success = false; diff --git a/Source/Core/DiscIO/ScrubbedBlob.cpp b/Source/Core/DiscIO/ScrubbedBlob.cpp new file mode 100644 index 0000000000..255edebf57 --- /dev/null +++ b/Source/Core/DiscIO/ScrubbedBlob.cpp @@ -0,0 +1,67 @@ +// Copyright 2020 Dolphin Emulator Project +// Licensed under GPLv2+ +// Refer to the license.txt file included. + +#include "DiscIO/ScrubbedBlob.h" + +#include +#include +#include +#include + +#include "Common/Align.h" +#include "DiscIO/Blob.h" +#include "DiscIO/DiscScrubber.h" +#include "DiscIO/Volume.h" + +namespace DiscIO +{ +ScrubbedBlob::ScrubbedBlob(std::unique_ptr blob_reader, DiscScrubber scrubber) + : m_blob_reader(std::move(blob_reader)), m_scrubber(std::move(scrubber)) +{ +} + +std::unique_ptr ScrubbedBlob::Create(const std::string& path) +{ + std::unique_ptr disc = CreateDisc(path); + if (!disc) + return nullptr; + + DiscScrubber scrubber; + if (!scrubber.SetupScrub(disc.get())) + return nullptr; + + std::unique_ptr blob = CreateBlobReader(path); + if (!blob) + return nullptr; + + return std::unique_ptr(new ScrubbedBlob(std::move(blob), std::move(scrubber))); +} + +bool ScrubbedBlob::Read(u64 offset, u64 size, u8* out_ptr) +{ + while (size > 0) + { + constexpr size_t CLUSTER_SIZE = DiscScrubber::CLUSTER_SIZE; + const u64 bytes_to_read = + std::min(Common::AlignDown(offset + CLUSTER_SIZE, CLUSTER_SIZE) - offset, size); + + if (m_scrubber.CanBlockBeScrubbed(offset)) + { + std::fill_n(out_ptr, bytes_to_read, 0); + } + else + { + if (!m_blob_reader->Read(offset, bytes_to_read, out_ptr)) + return false; + } + + offset += bytes_to_read; + size -= bytes_to_read; + out_ptr += bytes_to_read; + } + + return true; +} + +} // namespace DiscIO diff --git a/Source/Core/DiscIO/ScrubbedBlob.h b/Source/Core/DiscIO/ScrubbedBlob.h new file mode 100644 index 0000000000..b98b78681e --- /dev/null +++ b/Source/Core/DiscIO/ScrubbedBlob.h @@ -0,0 +1,37 @@ +// Copyright 2020 Dolphin Emulator Project +// Licensed under GPLv2+ +// Refer to the license.txt file included. + +#pragma once + +#include +#include + +#include "DiscIO/Blob.h" +#include "DiscIO/DiscScrubber.h" + +namespace DiscIO +{ +// This class wraps another BlobReader and zeroes out data that has been +// identified by DiscScrubber as unused. +class ScrubbedBlob : public BlobReader +{ +public: + static std::unique_ptr Create(const std::string& path); + + BlobType GetBlobType() const override { return m_blob_reader->GetBlobType(); } + u64 GetRawSize() const override { return m_blob_reader->GetRawSize(); } + u64 GetDataSize() const override { return m_blob_reader->GetDataSize(); } + bool IsDataSizeAccurate() const override { return m_blob_reader->IsDataSizeAccurate(); } + u64 GetBlockSize() const override { return m_blob_reader->GetBlockSize(); } + + bool Read(u64 offset, u64 size, u8* out_ptr) override; + +private: + ScrubbedBlob(std::unique_ptr blob_reader, DiscScrubber scrubber); + + std::unique_ptr m_blob_reader; + DiscScrubber m_scrubber; +}; + +} // namespace DiscIO diff --git a/Source/Core/DolphinQt/ConvertDialog.cpp b/Source/Core/DolphinQt/ConvertDialog.cpp index c3e102bd7c..f35569b96e 100644 --- a/Source/Core/DolphinQt/ConvertDialog.cpp +++ b/Source/Core/DolphinQt/ConvertDialog.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -21,6 +22,7 @@ #include "Common/Assert.h" #include "DiscIO/Blob.h" +#include "DiscIO/ScrubbedBlob.h" #include "DolphinQt/QtUtils/ModalMessageBox.h" #include "DolphinQt/QtUtils/ParallelProgressDialog.h" #include "UICommon/GameFile.h" @@ -179,34 +181,68 @@ void ConvertDialog::Convert() QFileInfo(QString::fromStdString(original_path)).fileName()); } - std::future good; + std::unique_ptr blob_reader; + bool scrub_current_file = scrub_wii && file->GetPlatform() == DiscIO::Platform::WiiDisc; - if (format == DiscIO::BlobType::PLAIN) + if (scrub_current_file) { - good = std::async(std::launch::async, [&] { - const bool good = DiscIO::ConvertToPlain(original_path, dst_path.toStdString(), &CompressCB, - &progress_dialog); - progress_dialog.Reset(); - return good; - }); - } - else if (format == DiscIO::BlobType::GCZ) - { - good = std::async(std::launch::async, [&] { - const bool good = - DiscIO::ConvertToGCZ(original_path, dst_path.toStdString(), - file->GetPlatform() == DiscIO::Platform::WiiDisc ? 1 : 0, 16384, - &CompressCB, &progress_dialog); - progress_dialog.Reset(); - return good; - }); + blob_reader = DiscIO::ScrubbedBlob::Create(original_path); + if (!blob_reader) + { + const int result = + ModalMessageBox::warning(this, tr("Question"), + tr("Failed to remove junk data from file \"%1\".\n\n" + "Would you like to convert it without removing junk data?") + .arg(QString::fromStdString(original_path)), + QMessageBox::Ok | QMessageBox::Abort); + + if (result == QMessageBox::Ok) + scrub_current_file = false; + else + return; + } } - progress_dialog.GetRaw()->exec(); - if (!good.get()) + if (!scrub_current_file) + blob_reader = DiscIO::CreateBlobReader(original_path); + + if (!blob_reader) { - QErrorMessage(this).showMessage(tr("Dolphin failed to complete the requested action.")); - return; + QErrorMessage(this).showMessage( + tr("Failed to open the input file \"%1\".").arg(QString::fromStdString(original_path))); + } + else + { + std::future good; + + if (format == DiscIO::BlobType::PLAIN) + { + good = std::async(std::launch::async, [&] { + const bool good = + DiscIO::ConvertToPlain(blob_reader.get(), original_path, dst_path.toStdString(), + &CompressCB, &progress_dialog); + progress_dialog.Reset(); + return good; + }); + } + else if (format == DiscIO::BlobType::GCZ) + { + good = std::async(std::launch::async, [&] { + const bool good = + DiscIO::ConvertToGCZ(blob_reader.get(), original_path, dst_path.toStdString(), + file->GetPlatform() == DiscIO::Platform::WiiDisc ? 1 : 0, 16384, + &CompressCB, &progress_dialog); + progress_dialog.Reset(); + return good; + }); + } + + progress_dialog.GetRaw()->exec(); + if (!good.get()) + { + QErrorMessage(this).showMessage(tr("Dolphin failed to complete the requested action.")); + return; + } } } From acd00723ad9b6f1e8d178882d2025cfb6aa0d425 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sat, 11 Apr 2020 13:06:45 +0200 Subject: [PATCH 7/9] DolphinQt: Make scrubbing configurable in convert dialog --- Source/Core/DolphinQt/ConvertDialog.cpp | 53 ++++++++++++++++++------- Source/Core/DolphinQt/ConvertDialog.h | 4 ++ Source/Core/UICommon/GameFile.cpp | 5 +++ Source/Core/UICommon/GameFile.h | 2 + 4 files changed, 50 insertions(+), 14 deletions(-) diff --git a/Source/Core/DolphinQt/ConvertDialog.cpp b/Source/Core/DolphinQt/ConvertDialog.cpp index f35569b96e..6d124dcc18 100644 --- a/Source/Core/DolphinQt/ConvertDialog.cpp +++ b/Source/Core/DolphinQt/ConvertDialog.cpp @@ -5,10 +5,12 @@ #include "DolphinQt/ConvertDialog.h" #include +#include #include #include #include +#include #include #include #include @@ -56,6 +58,12 @@ ConvertDialog::ConvertDialog(QList> fi grid_layout->addWidget(new QLabel(tr("Format:")), 0, 0); grid_layout->addWidget(m_format, 0, 1); + m_scrub = new QCheckBox; + grid_layout->addWidget(new QLabel(tr("Remove Junk Data (Irreversible):")), 1, 0); + grid_layout->addWidget(m_scrub, 1, 1); + m_scrub->setEnabled( + std::none_of(m_files.begin(), m_files.end(), std::mem_fn(&UICommon::GameFile::IsDatelDisc))); + QPushButton* convert_button = new QPushButton(tr("Convert")); QVBoxLayout* main_layout = new QVBoxLayout; @@ -78,27 +86,44 @@ void ConvertDialog::AddToFormatComboBox(const QString& name, DiscIO::BlobType fo m_format->addItem(name, static_cast(format)); } +bool ConvertDialog::ShowAreYouSureDialog(const QString& text) +{ + ModalMessageBox warning(this); + warning.setIcon(QMessageBox::Warning); + warning.setWindowTitle(tr("Confirm")); + warning.setText(tr("Are you sure?")); + warning.setInformativeText(text); + warning.setStandardButtons(QMessageBox::Yes | QMessageBox::No); + + return warning.exec() == QMessageBox::Yes; +} + void ConvertDialog::Convert() { const DiscIO::BlobType format = static_cast(m_format->currentData().toInt()); + const bool scrub = m_scrub->isChecked(); - const bool scrub_wii = format == DiscIO::BlobType::GCZ; + if (scrub && format == DiscIO::BlobType::PLAIN) + { + if (!ShowAreYouSureDialog(tr("Removing junk data does not save any space when converting to " + "ISO (unless you package the ISO file in a compressed file format " + "such as ZIP afterwards). Do you want to continue anyway?"))) + { + return; + } + } - if (scrub_wii && std::any_of(m_files.begin(), m_files.end(), [](const auto& file) { - return file->GetPlatform() == DiscIO::Platform::WiiDisc; + if (!scrub && format == DiscIO::BlobType::GCZ && + std::any_of(m_files.begin(), m_files.end(), [](const auto& file) { + return file->GetPlatform() == DiscIO::Platform::WiiDisc && !file->IsDatelDisc(); })) { - ModalMessageBox wii_warning(this); - wii_warning.setIcon(QMessageBox::Warning); - wii_warning.setWindowTitle(tr("Confirm")); - wii_warning.setText(tr("Are you sure?")); - wii_warning.setInformativeText( - tr("Compressing a Wii disc image will irreversibly change the compressed copy by removing " - "padding data. Your disc image will still work. Continue?")); - wii_warning.setStandardButtons(QMessageBox::Yes | QMessageBox::No); - - if (wii_warning.exec() == QMessageBox::No) + if (!ShowAreYouSureDialog(tr("Converting Wii disc images to GCZ without removing junk data " + "does not save any noticeable amount of space compared to " + "converting to ISO. Do you want to continue anyway?"))) + { return; + } } QString extension; @@ -182,7 +207,7 @@ void ConvertDialog::Convert() } std::unique_ptr blob_reader; - bool scrub_current_file = scrub_wii && file->GetPlatform() == DiscIO::Platform::WiiDisc; + bool scrub_current_file = scrub; if (scrub_current_file) { diff --git a/Source/Core/DolphinQt/ConvertDialog.h b/Source/Core/DolphinQt/ConvertDialog.h index 17aeb8f3e7..e4664d3f00 100644 --- a/Source/Core/DolphinQt/ConvertDialog.h +++ b/Source/Core/DolphinQt/ConvertDialog.h @@ -11,6 +11,7 @@ #include "DiscIO/Blob.h" +class QCheckBox; class QComboBox; namespace UICommon @@ -32,6 +33,9 @@ private slots: private: void AddToFormatComboBox(const QString& name, DiscIO::BlobType format); + bool ShowAreYouSureDialog(const QString& text); + QComboBox* m_format; + QCheckBox* m_scrub; QList> m_files; }; diff --git a/Source/Core/UICommon/GameFile.cpp b/Source/Core/UICommon/GameFile.cpp index 139dcce688..ca5c53ae31 100644 --- a/Source/Core/UICommon/GameFile.cpp +++ b/Source/Core/UICommon/GameFile.cpp @@ -38,6 +38,7 @@ #include "Core/TitleDatabase.h" #include "DiscIO/Blob.h" +#include "DiscIO/DiscExtractor.h" #include "DiscIO/Enums.h" #include "DiscIO/Volume.h" #include "DiscIO/WiiSaveBanner.h" @@ -117,6 +118,8 @@ GameFile::GameFile(std::string path) : m_file_path(std::move(path)) m_file_size = volume->GetRawSize(); m_volume_size = volume->GetSize(); m_volume_size_is_accurate = volume->IsSizeAccurate(); + m_is_datel_disc = DiscIO::IsDisc(m_platform) && + !DiscIO::GetBootDOLOffset(*volume, volume->GetGamePartition()); m_internal_name = volume->GetInternalName(); m_game_id = volume->GetGameID(); @@ -138,6 +141,7 @@ GameFile::GameFile(std::string path) : m_file_path(std::move(path)) m_valid = true; m_file_size = m_volume_size = File::GetSize(m_file_path); m_volume_size_is_accurate = true; + m_is_datel_disc = false; m_platform = DiscIO::Platform::ELFOrDOL; m_blob_type = DiscIO::BlobType::DIRECTORY; } @@ -299,6 +303,7 @@ void GameFile::DoState(PointerWrap& p) p.Do(m_file_size); p.Do(m_volume_size); p.Do(m_volume_size_is_accurate); + p.Do(m_is_datel_disc); p.Do(m_short_names); p.Do(m_long_names); diff --git a/Source/Core/UICommon/GameFile.h b/Source/Core/UICommon/GameFile.h index 53401e8756..e3c29860f4 100644 --- a/Source/Core/UICommon/GameFile.h +++ b/Source/Core/UICommon/GameFile.h @@ -90,6 +90,7 @@ public: u64 GetFileSize() const { return m_file_size; } u64 GetVolumeSize() const { return m_volume_size; } bool IsVolumeSizeAccurate() const { return m_volume_size_is_accurate; } + bool IsDatelDisc() const { return m_is_datel_disc; } const GameBanner& GetBannerImage() const; const GameCover& GetCoverImage() const; void DoState(PointerWrap& p); @@ -126,6 +127,7 @@ private: u64 m_file_size{}; u64 m_volume_size{}; bool m_volume_size_is_accurate{}; + bool m_is_datel_disc{}; std::map m_short_names; std::map m_long_names; From 466b2d7202e26ba009277b3d68100b495d5e14f6 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Fri, 10 Apr 2020 15:14:03 +0200 Subject: [PATCH 8/9] DolphinQt: Make block size configurable in convert dialog --- Source/Core/DolphinQt/ConvertDialog.cpp | 84 +++++++++++++++++++++++-- Source/Core/DolphinQt/ConvertDialog.h | 3 + Source/Core/UICommon/UICommon.cpp | 4 +- Source/Core/UICommon/UICommon.h | 2 +- 4 files changed, 86 insertions(+), 7 deletions(-) diff --git a/Source/Core/DolphinQt/ConvertDialog.cpp b/Source/Core/DolphinQt/ConvertDialog.cpp index 6d124dcc18..b257b498f5 100644 --- a/Source/Core/DolphinQt/ConvertDialog.cpp +++ b/Source/Core/DolphinQt/ConvertDialog.cpp @@ -23,11 +23,13 @@ #include #include "Common/Assert.h" +#include "Common/Logging/Log.h" #include "DiscIO/Blob.h" #include "DiscIO/ScrubbedBlob.h" #include "DolphinQt/QtUtils/ModalMessageBox.h" #include "DolphinQt/QtUtils/ParallelProgressDialog.h" #include "UICommon/GameFile.h" +#include "UICommon/UICommon.h" static bool CompressCB(const std::string& text, float percent, void* ptr) { @@ -58,9 +60,13 @@ ConvertDialog::ConvertDialog(QList> fi grid_layout->addWidget(new QLabel(tr("Format:")), 0, 0); grid_layout->addWidget(m_format, 0, 1); + m_block_size = new QComboBox; + grid_layout->addWidget(new QLabel(tr("Block Size:")), 1, 0); + grid_layout->addWidget(m_block_size, 1, 1); + m_scrub = new QCheckBox; - grid_layout->addWidget(new QLabel(tr("Remove Junk Data (Irreversible):")), 1, 0); - grid_layout->addWidget(m_scrub, 1, 1); + grid_layout->addWidget(new QLabel(tr("Remove Junk Data (Irreversible):")), 2, 0); + grid_layout->addWidget(m_scrub, 2, 1); m_scrub->setEnabled( std::none_of(m_files.begin(), m_files.end(), std::mem_fn(&UICommon::GameFile::IsDatelDisc))); @@ -72,7 +78,11 @@ ConvertDialog::ConvertDialog(QList> fi setLayout(main_layout); + connect(m_format, QOverload::of(&QComboBox::currentIndexChanged), this, + &ConvertDialog::OnFormatChanged); connect(convert_button, &QPushButton::clicked, this, &ConvertDialog::Convert); + + OnFormatChanged(); } void ConvertDialog::AddToFormatComboBox(const QString& name, DiscIO::BlobType format) @@ -86,6 +96,71 @@ void ConvertDialog::AddToFormatComboBox(const QString& name, DiscIO::BlobType fo m_format->addItem(name, static_cast(format)); } +void ConvertDialog::AddToBlockSizeComboBox(int size) +{ + m_block_size->addItem(QString::fromStdString(UICommon::FormatSize(size, 0)), size); +} + +void ConvertDialog::OnFormatChanged() +{ + // Because DVD timings are emulated as if we can't read less than an entire ECC block at once + // (32 KiB - 0x8000), there is little reason to use a block size smaller than that. + constexpr int MIN_BLOCK_SIZE = 0x8000; + + // For performance reasons, blocks shouldn't be too large. + // 2 MiB (0x200000) was picked because it is the smallest block size supported by WIA. + constexpr int MAX_BLOCK_SIZE = 0x200000; + + const DiscIO::BlobType format = static_cast(m_format->currentData().toInt()); + + m_block_size->clear(); + switch (format) + { + case DiscIO::BlobType::GCZ: + { + m_block_size->setEnabled(true); + + // In order for versions of Dolphin prior to 5.0-11893 to be able to convert a GCZ file + // to ISO without messing up the final part of the file in some way, the file size + // must be an integer multiple of the block size (fixed in 3aa463c) and must not be + // an integer multiple of the block size multiplied by 32 (fixed in 26b21e3). + + const auto block_size_ok = [this](int block_size) { + return std::all_of(m_files.begin(), m_files.end(), [block_size](const auto& file) { + constexpr u64 BLOCKS_PER_BUFFER = 32; + const u64 file_size = file->GetVolumeSize(); + return file_size % block_size == 0 && file_size % (block_size * BLOCKS_PER_BUFFER) != 0; + }); + }; + + // Add all block sizes in the normal range that do not cause problems + for (int block_size = MIN_BLOCK_SIZE; block_size <= MAX_BLOCK_SIZE; block_size *= 2) + { + if (block_size_ok(block_size)) + AddToBlockSizeComboBox(block_size); + } + + // If we didn't find a good block size, pick the block size which was hardcoded + // in older versions of Dolphin. That way, at least we're not worse than older versions. + if (m_block_size->count() == 0) + { + constexpr int FALLBACK_BLOCK_SIZE = 0x4000; + if (!block_size_ok(FALLBACK_BLOCK_SIZE)) + { + ERROR_LOG(MASTER_LOG, "Failed to find a block size which does not cause problems " + "when decompressing using an old version of Dolphin"); + } + AddToBlockSizeComboBox(FALLBACK_BLOCK_SIZE); + } + + break; + } + default: + m_block_size->setEnabled(false); + break; + } +} + bool ConvertDialog::ShowAreYouSureDialog(const QString& text) { ModalMessageBox warning(this); @@ -101,6 +176,7 @@ bool ConvertDialog::ShowAreYouSureDialog(const QString& text) void ConvertDialog::Convert() { const DiscIO::BlobType format = static_cast(m_format->currentData().toInt()); + const int block_size = m_block_size->currentData().toInt(); const bool scrub = m_scrub->isChecked(); if (scrub && format == DiscIO::BlobType::PLAIN) @@ -255,8 +331,8 @@ void ConvertDialog::Convert() good = std::async(std::launch::async, [&] { const bool good = DiscIO::ConvertToGCZ(blob_reader.get(), original_path, dst_path.toStdString(), - file->GetPlatform() == DiscIO::Platform::WiiDisc ? 1 : 0, 16384, - &CompressCB, &progress_dialog); + file->GetPlatform() == DiscIO::Platform::WiiDisc ? 1 : 0, + block_size, &CompressCB, &progress_dialog); progress_dialog.Reset(); return good; }); diff --git a/Source/Core/DolphinQt/ConvertDialog.h b/Source/Core/DolphinQt/ConvertDialog.h index e4664d3f00..1ff80b4a07 100644 --- a/Source/Core/DolphinQt/ConvertDialog.h +++ b/Source/Core/DolphinQt/ConvertDialog.h @@ -28,14 +28,17 @@ public: QWidget* parent = nullptr); private slots: + void OnFormatChanged(); void Convert(); private: void AddToFormatComboBox(const QString& name, DiscIO::BlobType format); + void AddToBlockSizeComboBox(int size); bool ShowAreYouSureDialog(const QString& text); QComboBox* m_format; + QComboBox* m_block_size; QCheckBox* m_scrub; QList> m_files; }; diff --git a/Source/Core/UICommon/UICommon.cpp b/Source/Core/UICommon/UICommon.cpp index 5af9192711..adff5adca3 100644 --- a/Source/Core/UICommon/UICommon.cpp +++ b/Source/Core/UICommon/UICommon.cpp @@ -453,7 +453,7 @@ void EnableScreenSaver(bool enable) #endif } -std::string FormatSize(u64 bytes) +std::string FormatSize(u64 bytes, int decimals) { // i18n: The symbol for the unit "bytes" const char* const unit_symbols[] = {_trans("B"), _trans("KiB"), _trans("MiB"), _trans("GiB"), @@ -468,7 +468,7 @@ std::string FormatSize(u64 bytes) // Don't need exact values, only 5 most significant digits const double unit_size = std::pow(2, unit * 10); std::ostringstream ss; - ss << std::fixed << std::setprecision(2); + ss << std::fixed << std::setprecision(decimals); ss << bytes / unit_size << ' ' << Common::GetStringT(unit_symbols[unit]); return ss.str(); } diff --git a/Source/Core/UICommon/UICommon.h b/Source/Core/UICommon/UICommon.h index b7169bfb15..2722559a2a 100644 --- a/Source/Core/UICommon/UICommon.h +++ b/Source/Core/UICommon/UICommon.h @@ -32,5 +32,5 @@ void SaveWiimoteSources(); // Return a pretty file size string from byte count. // e.g. 1134278 -> "1.08 MiB" -std::string FormatSize(u64 bytes); +std::string FormatSize(u64 bytes, int decimals = 2); } // namespace UICommon From 99c8ce9d8d1effa2ce67cb2890337b334f285f58 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sat, 11 Apr 2020 14:43:47 +0200 Subject: [PATCH 9/9] DolphinQt: Add info about formats in convert dialog --- Source/Core/DolphinQt/ConvertDialog.cpp | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/Source/Core/DolphinQt/ConvertDialog.cpp b/Source/Core/DolphinQt/ConvertDialog.cpp index b257b498f5..44f41e757e 100644 --- a/Source/Core/DolphinQt/ConvertDialog.cpp +++ b/Source/Core/DolphinQt/ConvertDialog.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -72,9 +73,28 @@ ConvertDialog::ConvertDialog(QList> fi QPushButton* convert_button = new QPushButton(tr("Convert")); + QVBoxLayout* options_layout = new QVBoxLayout; + options_layout->addLayout(grid_layout); + options_layout->addWidget(convert_button); + QGroupBox* options_group = new QGroupBox(tr("Options")); + options_group->setLayout(options_layout); + + QLabel* info_text = + new QLabel(tr("ISO: A simple and robust format which is supported by many programs. " + "It takes up more space than any other format.\n\n" + "GCZ: A basic compressed format which is compatible with most versions of " + "Dolphin and some other programs. It can't efficiently compress junk data " + "(unless removed) or encrypted Wii data.")); + info_text->setWordWrap(true); + + QVBoxLayout* info_layout = new QVBoxLayout; + info_layout->addWidget(info_text); + QGroupBox* info_group = new QGroupBox(tr("Info")); + info_group->setLayout(info_layout); + QVBoxLayout* main_layout = new QVBoxLayout; - main_layout->addLayout(grid_layout); - main_layout->addWidget(convert_button); + main_layout->addWidget(options_group); + main_layout->addWidget(info_group); setLayout(main_layout);