From e2ae2b3b0b11cc869468bed8a850e2012a68c2d9 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Mon, 4 May 2020 12:41:14 +0200 Subject: [PATCH] Add new file format RVZ based on WIA --- .../dolphinemu/utils/FileBrowserHelper.java | 2 +- Source/Core/Core/Boot/Boot.cpp | 2 +- Source/Core/DiscIO/Blob.cpp | 1 + Source/Core/DiscIO/Blob.h | 5 +- Source/Core/DiscIO/WIABlob.cpp | 35 +++++++++---- Source/Core/DiscIO/WIABlob.h | 16 ++++-- Source/Core/DolphinQt/ConvertDialog.cpp | 51 ++++++++++++------- .../Core/DolphinQt/GameList/GameTracker.cpp | 5 +- Source/Core/DolphinQt/Info.plist.in | 1 + Source/Core/DolphinQt/MainWindow.cpp | 4 +- Source/Core/DolphinQt/Settings/PathPane.cpp | 8 +-- Source/Core/UICommon/GameFileCache.cpp | 2 +- 12 files changed, 84 insertions(+), 48 deletions(-) diff --git a/Source/Android/app/src/main/java/org/dolphinemu/dolphinemu/utils/FileBrowserHelper.java b/Source/Android/app/src/main/java/org/dolphinemu/dolphinemu/utils/FileBrowserHelper.java index a27b2f2e05..4c6803d1af 100644 --- a/Source/Android/app/src/main/java/org/dolphinemu/dolphinemu/utils/FileBrowserHelper.java +++ b/Source/Android/app/src/main/java/org/dolphinemu/dolphinemu/utils/FileBrowserHelper.java @@ -22,7 +22,7 @@ import java.util.List; public final class FileBrowserHelper { public static final HashSet GAME_EXTENSIONS = new HashSet<>(Arrays.asList( - "gcm", "tgc", "iso", "ciso", "gcz", "wbfs", "wia", "wad", "dol", "elf", "dff")); + "gcm", "tgc", "iso", "ciso", "gcz", "wbfs", "wia", "rvz", "wad", "dol", "elf", "dff")); public static final HashSet RAW_EXTENSION = new HashSet<>(Collections.singletonList( "raw")); diff --git a/Source/Core/Core/Boot/Boot.cpp b/Source/Core/Core/Boot/Boot.cpp index f06c1077f9..a1ab46ed3c 100644 --- a/Source/Core/Core/Boot/Boot.cpp +++ b/Source/Core/Core/Boot/Boot.cpp @@ -159,7 +159,7 @@ BootParameters::GenerateFromFile(std::vector paths, paths.clear(); static const std::unordered_set disc_image_extensions = { - {".gcm", ".iso", ".tgc", ".wbfs", ".ciso", ".gcz", ".wia", ".dol", ".elf"}}; + {".gcm", ".iso", ".tgc", ".wbfs", ".ciso", ".gcz", ".wia", ".rvz", ".dol", ".elf"}}; if (disc_image_extensions.find(extension) != disc_image_extensions.end() || is_drive) { std::unique_ptr disc = DiscIO::CreateDisc(path); diff --git a/Source/Core/DiscIO/Blob.cpp b/Source/Core/DiscIO/Blob.cpp index 97e48eb047..c2c44caa60 100644 --- a/Source/Core/DiscIO/Blob.cpp +++ b/Source/Core/DiscIO/Blob.cpp @@ -207,6 +207,7 @@ std::unique_ptr CreateBlobReader(const std::string& filename) case WBFS_MAGIC: return WbfsFileReader::Create(std::move(file), filename); case WIA_MAGIC: + case RVZ_MAGIC: return WIAFileReader::Create(std::move(file), filename); default: if (auto directory_blob = DirectoryBlobReader::Create(filename)) diff --git a/Source/Core/DiscIO/Blob.h b/Source/Core/DiscIO/Blob.h index 49e42e6741..149df2afc3 100644 --- a/Source/Core/DiscIO/Blob.h +++ b/Source/Core/DiscIO/Blob.h @@ -37,7 +37,8 @@ enum class BlobType CISO, WBFS, TGC, - WIA + WIA, + RVZ, }; class BlobReader @@ -176,7 +177,7 @@ bool ConvertToPlain(BlobReader* infile, const std::string& infile_path, const std::string& outfile_path, CompressCB callback = nullptr, void* arg = nullptr); bool ConvertToWIA(BlobReader* infile, const std::string& infile_path, - const std::string& outfile_path, WIACompressionType compression_type, + const std::string& outfile_path, bool rvz, WIACompressionType compression_type, int compression_level, int chunk_size, CompressCB callback = nullptr, void* arg = nullptr); diff --git a/Source/Core/DiscIO/WIABlob.cpp b/Source/Core/DiscIO/WIABlob.cpp index 1924c7d08d..7414c48221 100644 --- a/Source/Core/DiscIO/WIABlob.cpp +++ b/Source/Core/DiscIO/WIABlob.cpp @@ -51,14 +51,21 @@ bool WIAFileReader::Initialize(const std::string& path) if (!m_file.Seek(0, SEEK_SET) || !m_file.ReadArray(&m_header_1, 1)) return false; - if (m_header_1.magic != WIA_MAGIC) + if (m_header_1.magic != WIA_MAGIC && m_header_1.magic != RVZ_MAGIC) return false; - const u32 version = Common::swap32(m_header_1.version); - const u32 version_compatible = Common::swap32(m_header_1.version_compatible); - if (WIA_VERSION < version_compatible || WIA_VERSION_READ_COMPATIBLE > version) + m_rvz = m_header_1.magic == RVZ_MAGIC; + + const u32 version = m_rvz ? RVZ_VERSION : WIA_VERSION; + const u32 version_read_compatible = + m_rvz ? RVZ_VERSION_READ_COMPATIBLE : WIA_VERSION_READ_COMPATIBLE; + + const u32 file_version = Common::swap32(m_header_1.version); + const u32 file_version_compatible = Common::swap32(m_header_1.version_compatible); + + if (version < file_version_compatible || version_read_compatible > file_version) { - ERROR_LOG(DISCIO, "Unsupported WIA version %s in %s", VersionToString(version).c_str(), + ERROR_LOG(DISCIO, "Unsupported version %s in %s", VersionToString(file_version).c_str(), path.c_str()); return false; } @@ -229,6 +236,11 @@ std::unique_ptr WIAFileReader::Create(File::IOFile file, const st return blob->m_valid ? std::move(blob) : nullptr; } +BlobType WIAFileReader::GetBlobType() const +{ + return m_rvz ? BlobType::RVZ : BlobType::WIA; +} + bool WIAFileReader::Read(u64 offset, u64 size, u8* out_ptr) { if (offset + size > Common::swap64(m_header_1.iso_file_size)) @@ -1730,7 +1742,7 @@ bool WIAFileReader::WriteHeader(File::IOFile* file, const u8* data, size_t size, ConversionResultCode WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, - File::IOFile* outfile, WIACompressionType compression_type, + File::IOFile* outfile, bool rvz, WIACompressionType compression_type, int compression_level, int chunk_size, CompressCB callback, void* arg) { ASSERT(infile->IsDataSizeAccurate()); @@ -1952,9 +1964,10 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, header_2.group_entries_offset = Common::swap64(group_entries_offset); header_2.group_entries_size = Common::swap32(static_cast(compressed_group_entries->size())); - header_1.magic = WIA_MAGIC; - header_1.version = Common::swap32(WIA_VERSION); - header_1.version_compatible = Common::swap32(WIA_VERSION_WRITE_COMPATIBLE); + header_1.magic = rvz ? RVZ_MAGIC : WIA_MAGIC; + header_1.version = Common::swap32(rvz ? RVZ_VERSION : WIA_VERSION); + header_1.version_compatible = + Common::swap32(rvz ? RVZ_VERSION_WRITE_COMPATIBLE : WIA_VERSION_WRITE_COMPATIBLE); header_1.header_2_size = Common::swap32(sizeof(WIAHeader2)); mbedtls_sha1_ret(reinterpret_cast(&header_2), sizeof(header_2), header_1.header_2_hash.data()); @@ -1975,7 +1988,7 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, } bool ConvertToWIA(BlobReader* infile, const std::string& infile_path, - const std::string& outfile_path, WIACompressionType compression_type, + const std::string& outfile_path, bool rvz, WIACompressionType compression_type, int compression_level, int chunk_size, CompressCB callback, void* arg) { File::IOFile outfile(outfile_path, "wb"); @@ -1991,7 +2004,7 @@ bool ConvertToWIA(BlobReader* infile, const std::string& infile_path, std::unique_ptr infile_volume = CreateDisc(infile_path); const ConversionResultCode result = - WIAFileReader::ConvertToWIA(infile, infile_volume.get(), &outfile, compression_type, + WIAFileReader::ConvertToWIA(infile, infile_volume.get(), &outfile, rvz, compression_type, compression_level, chunk_size, callback, arg); if (result == ConversionResultCode::ReadFailed) diff --git a/Source/Core/DiscIO/WIABlob.h b/Source/Core/DiscIO/WIABlob.h index 653f7312f3..356d8a3edb 100644 --- a/Source/Core/DiscIO/WIABlob.h +++ b/Source/Core/DiscIO/WIABlob.h @@ -37,6 +37,7 @@ enum class WIACompressionType : u32 }; constexpr u32 WIA_MAGIC = 0x01414957; // "WIA\x1" (byteswapped to little endian) +constexpr u32 RVZ_MAGIC = 0x015A5652; // "RVZ\x1" (byteswapped to little endian) class WIAFileReader : public BlobReader { @@ -45,7 +46,7 @@ public: static std::unique_ptr Create(File::IOFile file, const std::string& path); - BlobType GetBlobType() const override { return BlobType::WIA; } + BlobType GetBlobType() const override; u64 GetRawSize() const override { return Common::swap64(m_header_1.wia_file_size); } u64 GetDataSize() const override { return Common::swap64(m_header_1.iso_file_size); } @@ -59,7 +60,7 @@ public: bool ReadWiiDecrypted(u64 offset, u64 size, u8* out_ptr, u64 partition_data_offset) override; static ConversionResultCode ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume, - File::IOFile* outfile, + File::IOFile* outfile, bool rvz, WIACompressionType compression_type, int compression_level, int chunk_size, CompressCB callback, void* arg); @@ -489,6 +490,7 @@ private: } bool m_valid; + bool m_rvz; WIACompressionType m_compression_type; File::IOFile m_file; @@ -504,13 +506,17 @@ private: std::map m_data_entries; + // Perhaps we could set WIA_VERSION_WRITE_COMPATIBLE to 0.9, but WIA version 0.9 was never in + // any official release of wit, and interim versions (either source or binaries) are hard to find. + // Since we've been unable to check if we're write compatible with 0.9, we set it 1.0 to be safe. + static constexpr u32 WIA_VERSION = 0x01000000; static constexpr u32 WIA_VERSION_WRITE_COMPATIBLE = 0x01000000; static constexpr u32 WIA_VERSION_READ_COMPATIBLE = 0x00080000; - // Perhaps we could set WIA_VERSION_WRITE_COMPATIBLE to 0.9, but WIA version 0.9 was never in - // any official release of wit, and interim versions (either source or binaries) are hard to find. - // Since we've been unable to check if we're write compatible with 0.9, we set it 1.0 to be safe. + static constexpr u32 RVZ_VERSION = 0x00010000; + static constexpr u32 RVZ_VERSION_WRITE_COMPATIBLE = 0x00010000; + static constexpr u32 RVZ_VERSION_READ_COMPATIBLE = 0x00010000; }; } // namespace DiscIO diff --git a/Source/Core/DolphinQt/ConvertDialog.cpp b/Source/Core/DolphinQt/ConvertDialog.cpp index e52868039b..120b557c6b 100644 --- a/Source/Core/DolphinQt/ConvertDialog.cpp +++ b/Source/Core/DolphinQt/ConvertDialog.cpp @@ -59,6 +59,7 @@ ConvertDialog::ConvertDialog(QList> fi m_format->addItem(QStringLiteral("ISO"), static_cast(DiscIO::BlobType::PLAIN)); m_format->addItem(QStringLiteral("GCZ"), static_cast(DiscIO::BlobType::GCZ)); m_format->addItem(QStringLiteral("WIA"), static_cast(DiscIO::BlobType::WIA)); + m_format->addItem(QStringLiteral("RVZ"), static_cast(DiscIO::BlobType::RVZ)); if (std::all_of(m_files.begin(), m_files.end(), [](const auto& file) { return file->GetBlobType() == DiscIO::BlobType::PLAIN; })) { @@ -93,15 +94,17 @@ ConvertDialog::ConvertDialog(QList> fi QGroupBox* options_group = new QGroupBox(tr("Options")); options_group->setLayout(options_layout); - QLabel* info_text = - new QLabel(tr("ISO: A simple and robust format which is supported by many programs. " - "It takes up more space than any other format.\n\n" - "GCZ: A basic compressed format which is compatible with most versions of " - "Dolphin and some other programs. It can't efficiently compress junk data " - "(unless removed) or encrypted Wii data.\n\n" - "WIA: An advanced compressed format which is compatible with recent versions " - "of Dolphin and a few other programs. It can efficiently compress encrypted " - "Wii data, but not junk data (unless removed).")); + QLabel* info_text = new QLabel( + tr("ISO: A simple and robust format which is supported by many programs. It takes up more " + "space than any other format.\n\n" + "GCZ: A basic compressed format which is compatible with most versions of Dolphin and " + "some other programs. It can't efficiently compress junk data (unless removed) or " + "encrypted Wii data.\n\n" + "WIA: An advanced compressed format which is compatible with recent versions of Dolphin " + "and a few other programs. It can efficiently compress encrypted Wii data, but not junk " + "data (unless removed).\n\n" + "RVZ: An advanced compressed format which is compatible with recent versions of Dolphin. " + "It can efficiently compress both junk data and encrypted Wii data.")); info_text->setWordWrap(true); QVBoxLayout* info_layout = new QVBoxLayout; @@ -196,6 +199,7 @@ void ConvertDialog::OnFormatChanged() break; } case DiscIO::BlobType::WIA: + case DiscIO::BlobType::RVZ: m_block_size->setEnabled(true); // This is the smallest block size supported by WIA. For performance, larger sizes are avoided. @@ -214,6 +218,7 @@ void ConvertDialog::OnFormatChanged() AddToCompressionComboBox(QStringLiteral("Deflate"), DiscIO::WIACompressionType::None); break; case DiscIO::BlobType::WIA: + case DiscIO::BlobType::RVZ: { m_compression->setEnabled(true); @@ -319,6 +324,10 @@ void ConvertDialog::Convert() extension = QStringLiteral(".wia"); filter = tr("WIA GC/Wii images (*.wia)"); break; + case DiscIO::BlobType::RVZ: + extension = QStringLiteral(".rvz"); + filter = tr("RVZ GC/Wii images (*.rvz)"); + break; default: ASSERT(false); return; @@ -423,8 +432,9 @@ void ConvertDialog::Convert() { std::future good; - if (format == DiscIO::BlobType::PLAIN) + switch (format) { + case DiscIO::BlobType::PLAIN: good = std::async(std::launch::async, [&] { const bool good = DiscIO::ConvertToPlain(blob_reader.get(), original_path, dst_path.toStdString(), @@ -432,9 +442,9 @@ void ConvertDialog::Convert() progress_dialog.Reset(); return good; }); - } - else if (format == DiscIO::BlobType::GCZ) - { + break; + + case DiscIO::BlobType::GCZ: good = std::async(std::launch::async, [&] { const bool good = DiscIO::ConvertToGCZ(blob_reader.get(), original_path, dst_path.toStdString(), @@ -443,16 +453,19 @@ void ConvertDialog::Convert() progress_dialog.Reset(); return good; }); - } - else if (format == DiscIO::BlobType::WIA) - { + break; + + case DiscIO::BlobType::WIA: + case DiscIO::BlobType::RVZ: good = std::async(std::launch::async, [&] { - const bool good = DiscIO::ConvertToWIA( - blob_reader.get(), original_path, dst_path.toStdString(), compression, - compression_level, block_size, &CompressCB, &progress_dialog); + const bool good = + DiscIO::ConvertToWIA(blob_reader.get(), original_path, dst_path.toStdString(), + format == DiscIO::BlobType::RVZ, compression, compression_level, + block_size, &CompressCB, &progress_dialog); progress_dialog.Reset(); return good; }); + break; } progress_dialog.GetRaw()->exec(); diff --git a/Source/Core/DolphinQt/GameList/GameTracker.cpp b/Source/Core/DolphinQt/GameList/GameTracker.cpp index 4aa930140c..1f2b7383dd 100644 --- a/Source/Core/DolphinQt/GameList/GameTracker.cpp +++ b/Source/Core/DolphinQt/GameList/GameTracker.cpp @@ -24,8 +24,9 @@ static const QStringList game_filters{ QStringLiteral("*.[gG][cC][mM]"), QStringLiteral("*.[iI][sS][oO]"), QStringLiteral("*.[tT][gG][cC]"), QStringLiteral("*.[cC][iI][sS][oO]"), QStringLiteral("*.[gG][cC][zZ]"), QStringLiteral("*.[wW][bB][fF][sS]"), - QStringLiteral("*.[wW][iI][aA]"), QStringLiteral("*.[wW][aA][dD]"), - QStringLiteral("*.[eE][lL][fF]"), QStringLiteral("*.[dD][oO][lL]")}; + QStringLiteral("*.[wW][iI][aA]"), QStringLiteral("*.[rR][vV][zZ]"), + QStringLiteral("*.[wW][aA][dD]"), QStringLiteral("*.[eE][lL][fF]"), + QStringLiteral("*.[dD][oO][lL]")}; GameTracker::GameTracker(QObject* parent) : QFileSystemWatcher(parent) { diff --git a/Source/Core/DolphinQt/Info.plist.in b/Source/Core/DolphinQt/Info.plist.in index 979f5f7348..b5f3a3f44b 100644 --- a/Source/Core/DolphinQt/Info.plist.in +++ b/Source/Core/DolphinQt/Info.plist.in @@ -14,6 +14,7 @@ gcz iso m3u + rvz tgc wad wia diff --git a/Source/Core/DolphinQt/MainWindow.cpp b/Source/Core/DolphinQt/MainWindow.cpp index 508b822427..1ecb875986 100644 --- a/Source/Core/DolphinQt/MainWindow.cpp +++ b/Source/Core/DolphinQt/MainWindow.cpp @@ -686,8 +686,8 @@ QStringList MainWindow::PromptFileNames() QStringList paths = QFileDialog::getOpenFileNames( this, tr("Select a File"), settings.value(QStringLiteral("mainwindow/lastdir"), QString{}).toString(), - tr("All GC/Wii files (*.elf *.dol *.gcm *.iso *.tgc *.wbfs *.ciso *.gcz *.wia *.wad *.dff " - "*.m3u);;All Files (*)")); + tr("All GC/Wii files (*.elf *.dol *.gcm *.iso *.tgc *.wbfs *.ciso *.gcz *.wia *.rvz *.wad " + "*.dff *.m3u);;All Files (*)")); if (!paths.isEmpty()) { diff --git a/Source/Core/DolphinQt/Settings/PathPane.cpp b/Source/Core/DolphinQt/Settings/PathPane.cpp index e77f4d09c3..fd85d42a22 100644 --- a/Source/Core/DolphinQt/Settings/PathPane.cpp +++ b/Source/Core/DolphinQt/Settings/PathPane.cpp @@ -42,10 +42,10 @@ void PathPane::Browse() void PathPane::BrowseDefaultGame() { - QString file = QDir::toNativeSeparators(QFileDialog::getOpenFileName( - this, tr("Select a Game"), Settings::Instance().GetDefaultGame(), - tr("All GC/Wii files (*.elf *.dol *.gcm *.iso *.tgc *.wbfs *.ciso *.gcz *.wia *.wad *.m3u);;" - "All Files (*)"))); + QString file = QDir::toNativeSeparators( + QFileDialog::getOpenFileName(this, tr("Select a Game"), Settings::Instance().GetDefaultGame(), + tr("All GC/Wii files (*.elf *.dol *.gcm *.iso *.tgc *.wbfs " + "*.ciso *.gcz *.wia *.rvz *.wad *.m3u);;All Files (*)"))); if (!file.isEmpty()) Settings::Instance().SetDefaultGame(file); diff --git a/Source/Core/UICommon/GameFileCache.cpp b/Source/Core/UICommon/GameFileCache.cpp index bb12eb3ccd..1af33ae44b 100644 --- a/Source/Core/UICommon/GameFileCache.cpp +++ b/Source/Core/UICommon/GameFileCache.cpp @@ -33,7 +33,7 @@ std::vector FindAllGamePaths(const std::vector& direct bool recursive_scan) { static const std::vector search_extensions = { - ".gcm", ".tgc", ".iso", ".ciso", ".gcz", ".wbfs", ".wia", ".wad", ".dol", ".elf"}; + ".gcm", ".tgc", ".iso", ".ciso", ".gcz", ".wbfs", ".wia", ".rvz", ".wad", ".dol", ".elf"}; // TODO: We could process paths iteratively as they are found return Common::DoFileSearch(directories_to_scan, search_extensions, recursive_scan);