Merge pull request #5059 from arthomnix/feature/mrpack_export

NOISSUE Modrinth exporter: lookup all hashes in one API request

launcher/CMakeLists.txt

@@ -529,6 +529,8 @@ set(MODRINTH_SOURCES
     modplatform/modrinth/ModrinthPackManifest.h
     modplatform/modrinth/ModrinthInstanceExportTask.h
     modplatform/modrinth/ModrinthInstanceExportTask.cpp
+    modplatform/modrinth/ModrinthHashLookupRequest.h
+    modplatform/modrinth/ModrinthHashLookupRequest.cpp
 )
 
 add_unit_test(Index

launcher/modplatform/modrinth/ModrinthHashLookupRequest.cpp

@@ -0,0 +1,124 @@
+/*
+ * Copyright 2023 arthomnix
+ *
+ * This source is subject to the Microsoft Public License (MS-PL).
+ * Please see the COPYING.md file for more information.
+ */
+
+#include <QJsonArray>
+#include <QJsonDocument>
+#include "ModrinthHashLookupRequest.h"
+#include "BuildConfig.h"
+#include "Json.h"
+
+namespace Modrinth
+{
+
+HashLookupRequest::HashLookupRequest(QList<HashLookupData> hashes, QList<HashLookupResponseData> *output) : NetAction(), m_hashes(hashes), m_output(output)
+{
+    m_url = "https://api.modrinth.com/v2/version_files";
+    m_status = Job_NotStarted;
+}
+
+void HashLookupRequest::startImpl()
+{
+    finished = false;
+    m_status = Job_InProgress;
+
+    QNetworkRequest request(m_url);
+    request.setHeader(QNetworkRequest::UserAgentHeader, BuildConfig.USER_AGENT_UNCACHED);
+    request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
+
+    QJsonObject requestObject;
+    QJsonArray hashes;
+
+    for (const auto &data : m_hashes) {
+        hashes.append(data.hash);
+    }
+
+    requestObject.insert("hashes", hashes);
+    requestObject.insert("algorithm", QJsonValue("sha512"));
+
+    QNetworkReply *rep = m_network->post(request, QJsonDocument(requestObject).toJson());
+    m_reply.reset(rep);
+    connect(rep, &QNetworkReply::uploadProgress, this, &HashLookupRequest::downloadProgress);
+    connect(rep, &QNetworkReply::finished, this, &HashLookupRequest::downloadFinished);
+    connect(rep, &QNetworkReply::errorOccurred, this, &HashLookupRequest::downloadError);
+}
+
+void HashLookupRequest::downloadError(QNetworkReply::NetworkError error)
+{
+    qCritical() << "Modrinth hash lookup request failed with error" << m_reply->errorString() << "Server reply:\n" << m_reply->readAll();
+    if (finished) {
+        qCritical() << "Double finished ModrinthHashLookupRequest!";
+        return;
+    }
+    m_status = Job_Failed;
+    finished = true;
+    m_reply.reset();
+    emit failed(m_index_within_job);
+}
+
+void HashLookupRequest::downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
+{
+    m_total_progress = bytesTotal;
+    m_progress = bytesReceived;
+    emit netActionProgress(m_index_within_job, bytesReceived, bytesTotal);
+}
+
+void HashLookupRequest::downloadFinished()
+{
+    if (finished) {
+        qCritical() << "Double finished ModrinthHashLookupRequest!";
+        return;
+    }
+
+    QByteArray data = m_reply->readAll();
+    m_reply.reset();
+
+    try {
+        auto document = Json::requireDocument(data);
+        auto rootObject = Json::requireObject(document);
+
+        for (const auto &hashData : m_hashes) {
+            if (rootObject.contains(hashData.hash)) {
+                auto versionObject = Json::requireObject(rootObject, hashData.hash);
+
+                auto files = Json::requireIsArrayOf<QJsonObject>(versionObject, "files");
+
+                QJsonObject file;
+
+                for (const auto &fileJson : files) {
+                    auto hashes = Json::requireObject(fileJson, "hashes");
+                    QString sha512 = Json::requireString(hashes, "sha512");
+
+                    if (sha512 == hashData.hash) {
+                        file = fileJson;
+                    }
+                }
+
+                m_output->append(HashLookupResponseData {
+                    hashData.fileInfo,
+                    true,
+                    file
+                });
+            } else {
+                m_output->append(HashLookupResponseData {
+                    hashData.fileInfo,
+                    false,
+                    QJsonObject()
+                });
+            }
+        }
+
+        m_status = Job_Finished;
+        finished = true;
+        emit succeeded(m_index_within_job);
+    } catch (const Json::JsonException &e) {
+        qCritical() << "Failed to parse Modrinth hash lookup response: " << e.cause();
+        m_status = Job_Failed;
+        finished = true;
+        emit failed(m_index_within_job);
+    }
+}
+}

launcher/modplatform/modrinth/ModrinthHashLookupRequest.h

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2023 arthomnix
+ *
+ * This source is subject to the Microsoft Public License (MS-PL).
+ * Please see the COPYING.md file for more information.
+ */
+
+#pragma once
+
+#include <QFileInfo>
+#include <QJsonObject>
+#include "net/NetAction.h"
+
+namespace Modrinth
+{
+
+struct HashLookupData
+{
+    QFileInfo fileInfo;
+    QString hash;
+};
+
+struct HashLookupResponseData
+{
+    QFileInfo fileInfo;
+    bool found;
+    QJsonObject fileJson;
+};
+
+class HashLookupRequest : public NetAction
+{
+public:
+    using Ptr = shared_qobject_ptr<HashLookupRequest>;
+
+    explicit HashLookupRequest(QList<HashLookupData> hashes, QList<HashLookupResponseData> *output);
+    static Ptr make(QList<HashLookupData> hashes, QList<HashLookupResponseData> *output) {
+        return Ptr(new HashLookupRequest(hashes, output));
+    }
+
+protected slots:
+    void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) override;
+    void downloadError(QNetworkReply::NetworkError error) override;
+    void downloadFinished() override;
+    void downloadReadyRead() override {}
+
+public slots:
+    void startImpl() override;
+
+private:
+    QList<HashLookupData> m_hashes;
+    std::shared_ptr<QList<HashLookupResponseData>> m_output;
+    bool finished = true;
+};
+
+}

launcher/modplatform/modrinth/ModrinthInstanceExportTask.cpp

@@ -16,6 +16,7 @@
 #include "ui/dialogs/ModrinthExportDialog.h"
 #include "JlCompress.h"
 #include "FileSystem.h"
+#include "ModrinthHashLookupRequest.h"
 
 namespace Modrinth
 {
@@ -76,6 +77,10 @@ void InstanceExportTask::executeTask()
 
     m_netJob = new NetJob(tr("Modrinth pack export"), APPLICATION->network());
 
+    QList<HashLookupData> hashes;
+
+    qint64 progress = 0;
+    setProgress(progress, filesToResolve.length());
     for (const QString &filePath: filesToResolve) {
         qDebug() << "Attempting to resolve file hash from Modrinth API: " << filePath;
         QFile file(filePath);
@@ -86,20 +91,20 @@ void InstanceExportTask::executeTask()
             hasher.addData(contents);
             QString hash = hasher.result().toHex();
 
-            m_responses.append(HashLookupData{
-                    QFileInfo(file),
-                    hash,
-                    QByteArray()
+            hashes.append(HashLookupData {
+                QFileInfo(file),
+                hash
             });
 
-            m_netJob->addNetAction(Net::Download::makeByteArray(
-                    QString("https://api.modrinth.com/v2/version_file/%1?algorithm=sha512").arg(hash),
-                    &m_responses.last().response,
-                    Net::Download::Options(Net::Download::Option::AllowNotFound)
-            ));
+            progress++;
+            setProgress(progress, filesToResolve.length());
         }
     }
 
+    m_response.reset(new QList<HashLookupResponseData>);
+
+    m_netJob->addNetAction(HashLookupRequest::make(hashes, m_response.get()));
+
     connect(m_netJob.get(), &NetJob::succeeded, this, &InstanceExportTask::lookupSucceeded);
     connect(m_netJob.get(), &NetJob::failed, this, &InstanceExportTask::lookupFailed);
     connect(m_netJob.get(), &NetJob::progress, this, &InstanceExportTask::lookupProgress);
@@ -114,43 +119,32 @@ void InstanceExportTask::lookupSucceeded()
     QList<ExportFile> resolvedFiles;
     QFileInfoList failedFiles;
 
-    for (const auto &data : m_responses) {
-        try {
-            auto document = Json::requireDocument(data.response);
-            auto object = Json::requireObject(document);
-            auto files = Json::requireIsArrayOf<QJsonObject>(object, "files");
+    for (const auto &file : *m_response) {
+        if (file.found) {
+            try {
+                auto url = Json::requireString(file.fileJson, "url");
+                auto hashes = Json::requireObject(file.fileJson, "hashes");
 
-            QJsonObject file;
+                QString sha512Hash = Json::requireString(hashes, "sha512");
+                QString sha1Hash = Json::requireString(hashes, "sha1");
 
-            for (const auto &fileJson : files) {
-                auto hashes = Json::requireObject(fileJson, "hashes");
-                QString sha512 = Json::requireString(hashes, "sha512");
+                ExportFile fileData;
 
-                if (sha512 == data.sha512) {
-                    file = fileJson;
-                }
+                QDir gameDir(m_instance->gameRoot());
+
+                fileData.path = gameDir.relativeFilePath(file.fileInfo.absoluteFilePath());
+                fileData.download = url;
+                fileData.sha512 = sha512Hash;
+                fileData.sha1 = sha1Hash;
+                fileData.fileSize = file.fileInfo.size();
+
+                resolvedFiles << fileData;
+            } catch (const Json::JsonException &e) {
+                qDebug() << "File " << file.fileInfo.absoluteFilePath() << " failed to process for reason " << e.cause() << ", adding to overrides";
+                failedFiles << file.fileInfo;
             }
-
-            auto url = Json::requireString(file, "url");
-            auto hashes = Json::requireObject(file, "hashes");
-
-            QString sha512Hash = Json::requireString(hashes, "sha512");
-            QString sha1Hash = Json::requireString(hashes, "sha1");
-
-            ExportFile fileData;
-
-            QDir gameDir(m_instance->gameRoot());
-
-            fileData.path = gameDir.relativeFilePath(data.fileInfo.absoluteFilePath());
-            fileData.download = url;
-            fileData.sha512 = sha512Hash;
-            fileData.sha1 = sha1Hash;
-            fileData.fileSize = data.fileInfo.size();
-
-            resolvedFiles << fileData;
-        } catch (const Json::JsonException &e) {
-            qDebug() << "File " << data.fileInfo.absoluteFilePath() << " failed to process for reason " << e.cause() << ", adding to overrides";
-            failedFiles << data.fileInfo;
+        } else {
+            failedFiles << file.fileInfo;
         }
     }
 

launcher/modplatform/modrinth/ModrinthInstanceExportTask.h

@@ -11,6 +11,7 @@
 #include "BaseInstance.h"
 #include "net/NetJob.h"
 #include "ui/dialogs/ModrinthExportDialog.h"
+#include "ModrinthHashLookupRequest.h"
 
 namespace Modrinth
 {
@@ -35,13 +36,6 @@ struct ExportSettings
     QString exportPath;
 };
 
-struct HashLookupData
-{
-    QFileInfo fileInfo;
-    QString sha512;
-    QByteArray response;
-};
-
 // Using the existing Modrinth::File struct from the importer doesn't actually make much sense here (doesn't support multiple hashes, hash is a byte array rather than a string, no file size, etc)
 struct ExportFile
 {
@@ -71,7 +65,7 @@ private slots:
 private:
     InstancePtr m_instance;
     ExportSettings m_settings;
-    QList<HashLookupData> m_responses;
+    std::shared_ptr<QList<HashLookupResponseData>> m_response;
     NetJob::Ptr m_netJob;
 };