From eced9d7c7e2b05df304e800e4cfb7eae3ee9d2bd Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sat, 30 Mar 2019 12:45:00 +0100 Subject: [PATCH] VolumeVerifier: Calculate CRC32/MD5/SHA-1 --- Source/Core/DiscIO/VolumeVerifier.cpp | 83 +++++++++++++++++-- Source/Core/DiscIO/VolumeVerifier.h | 21 ++++- Source/Core/DolphinQt/Config/InfoWidget.cpp | 53 ------------ Source/Core/DolphinQt/Config/InfoWidget.h | 3 - Source/Core/DolphinQt/Config/VerifyWidget.cpp | 42 +++++++++- Source/Core/DolphinQt/Config/VerifyWidget.h | 12 +++ 6 files changed, 151 insertions(+), 63 deletions(-) diff --git a/Source/Core/DiscIO/VolumeVerifier.cpp b/Source/Core/DiscIO/VolumeVerifier.cpp index 2201784d1c..ac2f06cc15 100644 --- a/Source/Core/DiscIO/VolumeVerifier.cpp +++ b/Source/Core/DiscIO/VolumeVerifier.cpp @@ -11,6 +11,10 @@ #include #include +#include +#include +#include + #include "Common/Align.h" #include "Common/Assert.h" #include "Common/CommonTypes.h" @@ -40,9 +44,11 @@ constexpr u64 DL_DVD_R_SIZE = 8543666176; // Wii RVT-R constexpr u64 BLOCK_SIZE = 0x20000; -VolumeVerifier::VolumeVerifier(const Volume& volume) - : m_volume(volume), m_started(false), m_done(false), m_progress(0), - m_max_progress(volume.GetSize()) +VolumeVerifier::VolumeVerifier(const Volume& volume, Hashes hashes_to_calculate) + : m_volume(volume), m_hashes_to_calculate(hashes_to_calculate), + m_calculating_any_hash(hashes_to_calculate.crc32 || hashes_to_calculate.md5 || + hashes_to_calculate.sha1), + m_started(false), m_done(false), m_progress(0), m_max_progress(volume.GetSize()) { } @@ -64,8 +70,7 @@ void VolumeVerifier::Start() CheckDiscSize(); CheckMisc(); - std::sort(m_blocks.begin(), m_blocks.end(), - [](const BlockToVerify& b1, const BlockToVerify& b2) { return b1.offset < b2.offset; }); + SetUpHashing(); } void VolumeVerifier::CheckPartitions() @@ -622,6 +627,27 @@ void VolumeVerifier::CheckMisc() } } +void VolumeVerifier::SetUpHashing() +{ + std::sort(m_blocks.begin(), m_blocks.end(), + [](const BlockToVerify& b1, const BlockToVerify& b2) { return b1.offset < b2.offset; }); + + if (m_hashes_to_calculate.crc32) + m_crc32_context = crc32(0, nullptr, 0); + + if (m_hashes_to_calculate.md5) + { + mbedtls_md5_init(&m_md5_context); + mbedtls_md5_starts(&m_md5_context); + } + + if (m_hashes_to_calculate.sha1) + { + mbedtls_sha1_init(&m_sha1_context); + mbedtls_sha1_starts(&m_sha1_context); + } +} + void VolumeVerifier::Process() { ASSERT(m_started); @@ -641,6 +667,30 @@ void VolumeVerifier::Process() } bytes_to_read = std::min(bytes_to_read, m_max_progress - m_progress); + if (m_calculating_any_hash) + { + std::vector data(bytes_to_read); + if (!m_volume.Read(m_progress, bytes_to_read, data.data(), PARTITION_NONE)) + { + m_calculating_any_hash = false; + } + else + { + if (m_hashes_to_calculate.crc32) + { + // It would be nice to use crc32_z here instead of crc32, but it isn't available on Android + m_crc32_context = + crc32(m_crc32_context, data.data(), static_cast(bytes_to_read)); + } + + if (m_hashes_to_calculate.md5) + mbedtls_md5_update(&m_md5_context, data.data(), bytes_to_read); + + if (m_hashes_to_calculate.sha1) + mbedtls_sha1_update(&m_sha1_context, data.data(), bytes_to_read); + } + } + m_progress += bytes_to_read; while (m_block_index < m_blocks.size() && m_blocks[m_block_index].offset < m_progress) @@ -672,6 +722,29 @@ void VolumeVerifier::Finish() return; m_done = true; + if (m_calculating_any_hash) + { + if (m_hashes_to_calculate.crc32) + { + m_result.hashes.crc32 = std::vector(4); + const u32 crc32_be = Common::swap32(m_crc32_context); + const u8* crc32_be_ptr = reinterpret_cast(&crc32_be); + std::copy(crc32_be_ptr, crc32_be_ptr + 4, m_result.hashes.crc32.begin()); + } + + if (m_hashes_to_calculate.md5) + { + m_result.hashes.md5 = std::vector(16); + mbedtls_md5_finish(&m_md5_context, m_result.hashes.md5.data()); + } + + if (m_hashes_to_calculate.sha1) + { + m_result.hashes.sha1 = std::vector(20); + mbedtls_sha1_finish(&m_sha1_context, m_result.hashes.sha1.data()); + } + } + for (auto pair : m_block_errors) { if (pair.second > 0) diff --git a/Source/Core/DiscIO/VolumeVerifier.h b/Source/Core/DiscIO/VolumeVerifier.h index 22b86107c0..4337fdce46 100644 --- a/Source/Core/DiscIO/VolumeVerifier.h +++ b/Source/Core/DiscIO/VolumeVerifier.h @@ -9,6 +9,9 @@ #include #include +#include +#include + #include "Common/CommonTypes.h" #include "DiscIO/Volume.h" @@ -51,13 +54,22 @@ public: std::string text; }; + template + struct Hashes + { + T crc32; + T md5; + T sha1; + }; + struct Result { + Hashes> hashes; std::string summary_text; std::vector problems; }; - VolumeVerifier(const Volume& volume); + VolumeVerifier(const Volume& volume, Hashes hashes_to_calculate); void Start(); void Process(); u64 GetBytesProcessed() const; @@ -86,6 +98,7 @@ private: u64 GetBiggestUsedOffset(); u64 GetBiggestUsedOffset(const FileInfo& file_info) const; void CheckMisc(); + void SetUpHashing(); void AddProblem(Severity severity, const std::string& text); @@ -95,6 +108,12 @@ private: bool m_is_datel; bool m_is_not_retail; + Hashes m_hashes_to_calculate; + bool m_calculating_any_hash; + unsigned long m_crc32_context; + mbedtls_md5_context m_md5_context; + mbedtls_sha1_context m_sha1_context; + std::vector m_blocks; size_t m_block_index = 0; // Index in m_blocks, not index in a specific partition std::map m_block_errors; diff --git a/Source/Core/DolphinQt/Config/InfoWidget.cpp b/Source/Core/DolphinQt/Config/InfoWidget.cpp index 79b980e52a..9ac8460bb1 100644 --- a/Source/Core/DolphinQt/Config/InfoWidget.cpp +++ b/Source/Core/DolphinQt/Config/InfoWidget.cpp @@ -78,7 +78,6 @@ QGroupBox* InfoWidget::CreateISODetails() QLineEdit* maker = CreateValueDisplay((game_maker.empty() ? UNKNOWN_NAME.toStdString() : game_maker) + " (" + m_game.GetMakerID() + ")"); - QWidget* checksum = CreateChecksumComputer(); layout->addRow(tr("Name:"), internal_name); layout->addRow(tr("File:"), file_path); @@ -89,8 +88,6 @@ QGroupBox* InfoWidget::CreateISODetails() if (!m_game.GetApploaderDate().empty()) layout->addRow(tr("Apploader Date:"), CreateValueDisplay(m_game.GetApploaderDate())); - layout->addRow(tr("MD5 Checksum:"), checksum); - group->setLayout(layout); return group; } @@ -194,53 +191,3 @@ void InfoWidget::ChangeLanguage() m_maker->setText(QString::fromStdString(m_game.GetLongMaker(language))); m_description->setText(QString::fromStdString(m_game.GetDescription(language))); } - -QWidget* InfoWidget::CreateChecksumComputer() -{ - QWidget* widget = new QWidget(); - QHBoxLayout* layout = new QHBoxLayout(); - layout->setContentsMargins(0, 0, 0, 0); - - m_checksum_result = new QLineEdit(); - m_checksum_result->setReadOnly(true); - QPushButton* calculate = new QPushButton(tr("Compute")); - connect(calculate, &QPushButton::clicked, this, &InfoWidget::ComputeChecksum); - layout->addWidget(m_checksum_result); - layout->addWidget(calculate); - - widget->setLayout(layout); - return widget; -} - -void InfoWidget::ComputeChecksum() -{ - QCryptographicHash hash(QCryptographicHash::Md5); - hash.reset(); - std::unique_ptr file(DiscIO::CreateBlobReader(m_game.GetFilePath())); - std::vector file_data(8 * 1080 * 1080); // read 1MB at a time - u64 game_size = file->GetDataSize(); - u64 read_offset = 0; - - // a maximum of 1000 is used instead of game_size because otherwise 8GB games overflow the int - // typed maximum parameter - QProgressDialog* progress = - new QProgressDialog(tr("Computing MD5 Checksum"), tr("Cancel"), 0, 1000, this); - progress->setWindowTitle(tr("Computing MD5 Checksum")); - progress->setWindowFlags(progress->windowFlags() & ~Qt::WindowContextHelpButtonHint); - progress->setMinimumDuration(500); - progress->setWindowModality(Qt::WindowModal); - while (read_offset < game_size) - { - progress->setValue(static_cast(read_offset) / static_cast(game_size) * 1000); - if (progress->wasCanceled()) - return; - - u64 read_size = std::min(file_data.size(), game_size - read_offset); - file->Read(read_offset, read_size, file_data.data()); - hash.addData(reinterpret_cast(file_data.data()), read_size); - read_offset += read_size; - } - m_checksum_result->setText(QString::fromUtf8(hash.result().toHex())); - Q_ASSERT(read_offset == game_size); - progress->setValue(1000); -} diff --git a/Source/Core/DolphinQt/Config/InfoWidget.h b/Source/Core/DolphinQt/Config/InfoWidget.h index b6fd934290..226d8e3c36 100644 --- a/Source/Core/DolphinQt/Config/InfoWidget.h +++ b/Source/Core/DolphinQt/Config/InfoWidget.h @@ -23,7 +23,6 @@ public: explicit InfoWidget(const UICommon::GameFile& game); private: - void ComputeChecksum(); void ChangeLanguage(); void SaveBanner(); @@ -31,12 +30,10 @@ private: QGroupBox* CreateISODetails(); QLineEdit* CreateValueDisplay(const QString& value); QLineEdit* CreateValueDisplay(const std::string& value = ""); - QWidget* CreateChecksumComputer(); void CreateLanguageSelector(); QWidget* CreateBannerGraphic(const QPixmap& image); UICommon::GameFile m_game; - QLineEdit* m_checksum_result; QComboBox* m_language_selector; QLineEdit* m_name; QLineEdit* m_maker; diff --git a/Source/Core/DolphinQt/Config/VerifyWidget.cpp b/Source/Core/DolphinQt/Config/VerifyWidget.cpp index b857d2e88c..f3872df211 100644 --- a/Source/Core/DolphinQt/Config/VerifyWidget.cpp +++ b/Source/Core/DolphinQt/Config/VerifyWidget.cpp @@ -5,12 +5,17 @@ #include "DolphinQt/Config/VerifyWidget.h" #include +#include +#include +#include +#include #include #include #include #include +#include "Common/CommonTypes.h" #include "DiscIO/Volume.h" #include "DiscIO/VolumeVerifier.h" @@ -23,6 +28,7 @@ VerifyWidget::VerifyWidget(std::shared_ptr volume) : m_volume(st layout->addWidget(m_problems); layout->addWidget(m_summary_text); + layout->addLayout(m_hash_layout); layout->addWidget(m_verify_button); layout->setStretchFactor(m_problems, 5); @@ -44,17 +50,47 @@ void VerifyWidget::CreateWidgets() m_summary_text = new QTextEdit(this); m_summary_text->setReadOnly(true); + m_hash_layout = new QFormLayout(this); + std::tie(m_crc32_checkbox, m_crc32_line_edit) = AddHashLine(m_hash_layout, tr("CRC32:")); + std::tie(m_md5_checkbox, m_md5_line_edit) = AddHashLine(m_hash_layout, tr("MD5:")); + std::tie(m_sha1_checkbox, m_sha1_line_edit) = AddHashLine(m_hash_layout, tr("SHA-1:")); + m_verify_button = new QPushButton(tr("Verify Integrity"), this); } +std::pair VerifyWidget::AddHashLine(QFormLayout* layout, QString text) +{ + QLineEdit* line_edit = new QLineEdit(this); + line_edit->setReadOnly(true); + QCheckBox* checkbox = new QCheckBox(tr("Calculate"), this); + checkbox->setChecked(true); + + QHBoxLayout* hbox_layout = new QHBoxLayout(this); + hbox_layout->addWidget(line_edit); + hbox_layout->addWidget(checkbox); + + layout->addRow(text, hbox_layout); + + return std::pair(checkbox, line_edit); +} + void VerifyWidget::ConnectWidgets() { connect(m_verify_button, &QPushButton::clicked, this, &VerifyWidget::Verify); } +static void SetHash(QLineEdit* line_edit, const std::vector& hash) +{ + const QByteArray byte_array = QByteArray::fromRawData(reinterpret_cast(hash.data()), + static_cast(hash.size())); + line_edit->setText(QString::fromLatin1(byte_array.toHex())); +} + void VerifyWidget::Verify() { - DiscIO::VolumeVerifier verifier(*m_volume); + DiscIO::VolumeVerifier verifier( + *m_volume, + {m_crc32_checkbox->isChecked(), m_md5_checkbox->isChecked(), m_sha1_checkbox->isChecked()}); // We have to divide the number of processed bytes with something so it won't make ints overflow constexpr int DIVISOR = 0x100; @@ -104,6 +140,10 @@ void VerifyWidget::Verify() SetProblemCellText(i, 0, QString::fromStdString(problem.text)); SetProblemCellText(i, 1, severity); } + + SetHash(m_crc32_line_edit, result.hashes.crc32); + SetHash(m_md5_line_edit, result.hashes.md5); + SetHash(m_sha1_line_edit, result.hashes.sha1); } void VerifyWidget::SetProblemCellText(int row, int column, QString text) diff --git a/Source/Core/DolphinQt/Config/VerifyWidget.h b/Source/Core/DolphinQt/Config/VerifyWidget.h index 4a895c651c..c2433cbd49 100644 --- a/Source/Core/DolphinQt/Config/VerifyWidget.h +++ b/Source/Core/DolphinQt/Config/VerifyWidget.h @@ -6,7 +6,11 @@ #include #include +#include +#include +#include +#include #include #include #include @@ -25,6 +29,7 @@ public: private: void CreateWidgets(); + std::pair AddHashLine(QFormLayout* layout, QString text); void ConnectWidgets(); void Verify(); @@ -33,5 +38,12 @@ private: std::shared_ptr m_volume; QTableWidget* m_problems; QTextEdit* m_summary_text; + QFormLayout* m_hash_layout; + QCheckBox* m_crc32_checkbox; + QCheckBox* m_md5_checkbox; + QCheckBox* m_sha1_checkbox; + QLineEdit* m_crc32_line_edit; + QLineEdit* m_md5_line_edit; + QLineEdit* m_sha1_line_edit; QPushButton* m_verify_button; };