mirror of
https://github.com/MultiMC/MultiMC5.git
synced 2025-03-14 04:18:29 +00:00
Merge pull request #5059 from arthomnix/feature/mrpack_export
NOISSUE Modrinth exporter: lookup all hashes in one API request
This commit is contained in:
commit
34f8de4682
@ -529,6 +529,8 @@ set(MODRINTH_SOURCES
|
||||
modplatform/modrinth/ModrinthPackManifest.h
|
||||
modplatform/modrinth/ModrinthInstanceExportTask.h
|
||||
modplatform/modrinth/ModrinthInstanceExportTask.cpp
|
||||
modplatform/modrinth/ModrinthHashLookupRequest.h
|
||||
modplatform/modrinth/ModrinthHashLookupRequest.cpp
|
||||
)
|
||||
|
||||
add_unit_test(Index
|
||||
|
124
launcher/modplatform/modrinth/ModrinthHashLookupRequest.cpp
Normal file
124
launcher/modplatform/modrinth/ModrinthHashLookupRequest.cpp
Normal file
@ -0,0 +1,124 @@
|
||||
/*
|
||||
* Copyright 2023 arthomnix
|
||||
*
|
||||
* This source is subject to the Microsoft Public License (MS-PL).
|
||||
* Please see the COPYING.md file for more information.
|
||||
*/
|
||||
|
||||
#include <QJsonArray>
|
||||
#include <QJsonDocument>
|
||||
#include "ModrinthHashLookupRequest.h"
|
||||
#include "BuildConfig.h"
|
||||
#include "Json.h"
|
||||
|
||||
namespace Modrinth
|
||||
{
|
||||
|
||||
HashLookupRequest::HashLookupRequest(QList<HashLookupData> hashes, QList<HashLookupResponseData> *output) : NetAction(), m_hashes(hashes), m_output(output)
|
||||
{
|
||||
m_url = "https://api.modrinth.com/v2/version_files";
|
||||
m_status = Job_NotStarted;
|
||||
}
|
||||
|
||||
void HashLookupRequest::startImpl()
|
||||
{
|
||||
finished = false;
|
||||
m_status = Job_InProgress;
|
||||
|
||||
QNetworkRequest request(m_url);
|
||||
request.setHeader(QNetworkRequest::UserAgentHeader, BuildConfig.USER_AGENT_UNCACHED);
|
||||
request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
|
||||
|
||||
QJsonObject requestObject;
|
||||
QJsonArray hashes;
|
||||
|
||||
for (const auto &data : m_hashes) {
|
||||
hashes.append(data.hash);
|
||||
}
|
||||
|
||||
requestObject.insert("hashes", hashes);
|
||||
requestObject.insert("algorithm", QJsonValue("sha512"));
|
||||
|
||||
QNetworkReply *rep = m_network->post(request, QJsonDocument(requestObject).toJson());
|
||||
m_reply.reset(rep);
|
||||
connect(rep, &QNetworkReply::uploadProgress, this, &HashLookupRequest::downloadProgress);
|
||||
connect(rep, &QNetworkReply::finished, this, &HashLookupRequest::downloadFinished);
|
||||
connect(rep, &QNetworkReply::errorOccurred, this, &HashLookupRequest::downloadError);
|
||||
}
|
||||
|
||||
void HashLookupRequest::downloadError(QNetworkReply::NetworkError error)
|
||||
{
|
||||
qCritical() << "Modrinth hash lookup request failed with error" << m_reply->errorString() << "Server reply:\n" << m_reply->readAll();
|
||||
if (finished) {
|
||||
qCritical() << "Double finished ModrinthHashLookupRequest!";
|
||||
return;
|
||||
}
|
||||
m_status = Job_Failed;
|
||||
finished = true;
|
||||
m_reply.reset();
|
||||
emit failed(m_index_within_job);
|
||||
}
|
||||
|
||||
void HashLookupRequest::downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
|
||||
{
|
||||
m_total_progress = bytesTotal;
|
||||
m_progress = bytesReceived;
|
||||
emit netActionProgress(m_index_within_job, bytesReceived, bytesTotal);
|
||||
}
|
||||
|
||||
void HashLookupRequest::downloadFinished()
|
||||
{
|
||||
if (finished) {
|
||||
qCritical() << "Double finished ModrinthHashLookupRequest!";
|
||||
return;
|
||||
}
|
||||
|
||||
QByteArray data = m_reply->readAll();
|
||||
m_reply.reset();
|
||||
|
||||
try {
|
||||
auto document = Json::requireDocument(data);
|
||||
auto rootObject = Json::requireObject(document);
|
||||
|
||||
for (const auto &hashData : m_hashes) {
|
||||
if (rootObject.contains(hashData.hash)) {
|
||||
auto versionObject = Json::requireObject(rootObject, hashData.hash);
|
||||
|
||||
auto files = Json::requireIsArrayOf<QJsonObject>(versionObject, "files");
|
||||
|
||||
QJsonObject file;
|
||||
|
||||
for (const auto &fileJson : files) {
|
||||
auto hashes = Json::requireObject(fileJson, "hashes");
|
||||
QString sha512 = Json::requireString(hashes, "sha512");
|
||||
|
||||
if (sha512 == hashData.hash) {
|
||||
file = fileJson;
|
||||
}
|
||||
}
|
||||
|
||||
m_output->append(HashLookupResponseData {
|
||||
hashData.fileInfo,
|
||||
true,
|
||||
file
|
||||
});
|
||||
} else {
|
||||
m_output->append(HashLookupResponseData {
|
||||
hashData.fileInfo,
|
||||
false,
|
||||
QJsonObject()
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
m_status = Job_Finished;
|
||||
finished = true;
|
||||
emit succeeded(m_index_within_job);
|
||||
} catch (const Json::JsonException &e) {
|
||||
qCritical() << "Failed to parse Modrinth hash lookup response: " << e.cause();
|
||||
m_status = Job_Failed;
|
||||
finished = true;
|
||||
emit failed(m_index_within_job);
|
||||
}
|
||||
}
|
||||
}
|
55
launcher/modplatform/modrinth/ModrinthHashLookupRequest.h
Normal file
55
launcher/modplatform/modrinth/ModrinthHashLookupRequest.h
Normal file
@ -0,0 +1,55 @@
|
||||
/*
|
||||
* Copyright 2023 arthomnix
|
||||
*
|
||||
* This source is subject to the Microsoft Public License (MS-PL).
|
||||
* Please see the COPYING.md file for more information.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <QFileInfo>
|
||||
#include <QJsonObject>
|
||||
#include "net/NetAction.h"
|
||||
|
||||
namespace Modrinth
|
||||
{
|
||||
|
||||
struct HashLookupData
|
||||
{
|
||||
QFileInfo fileInfo;
|
||||
QString hash;
|
||||
};
|
||||
|
||||
struct HashLookupResponseData
|
||||
{
|
||||
QFileInfo fileInfo;
|
||||
bool found;
|
||||
QJsonObject fileJson;
|
||||
};
|
||||
|
||||
class HashLookupRequest : public NetAction
|
||||
{
|
||||
public:
|
||||
using Ptr = shared_qobject_ptr<HashLookupRequest>;
|
||||
|
||||
explicit HashLookupRequest(QList<HashLookupData> hashes, QList<HashLookupResponseData> *output);
|
||||
static Ptr make(QList<HashLookupData> hashes, QList<HashLookupResponseData> *output) {
|
||||
return Ptr(new HashLookupRequest(hashes, output));
|
||||
}
|
||||
|
||||
protected slots:
|
||||
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) override;
|
||||
void downloadError(QNetworkReply::NetworkError error) override;
|
||||
void downloadFinished() override;
|
||||
void downloadReadyRead() override {}
|
||||
|
||||
public slots:
|
||||
void startImpl() override;
|
||||
|
||||
private:
|
||||
QList<HashLookupData> m_hashes;
|
||||
std::shared_ptr<QList<HashLookupResponseData>> m_output;
|
||||
bool finished = true;
|
||||
};
|
||||
|
||||
}
|
@ -16,6 +16,7 @@
|
||||
#include "ui/dialogs/ModrinthExportDialog.h"
|
||||
#include "JlCompress.h"
|
||||
#include "FileSystem.h"
|
||||
#include "ModrinthHashLookupRequest.h"
|
||||
|
||||
namespace Modrinth
|
||||
{
|
||||
@ -76,6 +77,10 @@ void InstanceExportTask::executeTask()
|
||||
|
||||
m_netJob = new NetJob(tr("Modrinth pack export"), APPLICATION->network());
|
||||
|
||||
QList<HashLookupData> hashes;
|
||||
|
||||
qint64 progress = 0;
|
||||
setProgress(progress, filesToResolve.length());
|
||||
for (const QString &filePath: filesToResolve) {
|
||||
qDebug() << "Attempting to resolve file hash from Modrinth API: " << filePath;
|
||||
QFile file(filePath);
|
||||
@ -86,20 +91,20 @@ void InstanceExportTask::executeTask()
|
||||
hasher.addData(contents);
|
||||
QString hash = hasher.result().toHex();
|
||||
|
||||
m_responses.append(HashLookupData{
|
||||
QFileInfo(file),
|
||||
hash,
|
||||
QByteArray()
|
||||
hashes.append(HashLookupData {
|
||||
QFileInfo(file),
|
||||
hash
|
||||
});
|
||||
|
||||
m_netJob->addNetAction(Net::Download::makeByteArray(
|
||||
QString("https://api.modrinth.com/v2/version_file/%1?algorithm=sha512").arg(hash),
|
||||
&m_responses.last().response,
|
||||
Net::Download::Options(Net::Download::Option::AllowNotFound)
|
||||
));
|
||||
progress++;
|
||||
setProgress(progress, filesToResolve.length());
|
||||
}
|
||||
}
|
||||
|
||||
m_response.reset(new QList<HashLookupResponseData>);
|
||||
|
||||
m_netJob->addNetAction(HashLookupRequest::make(hashes, m_response.get()));
|
||||
|
||||
connect(m_netJob.get(), &NetJob::succeeded, this, &InstanceExportTask::lookupSucceeded);
|
||||
connect(m_netJob.get(), &NetJob::failed, this, &InstanceExportTask::lookupFailed);
|
||||
connect(m_netJob.get(), &NetJob::progress, this, &InstanceExportTask::lookupProgress);
|
||||
@ -114,43 +119,32 @@ void InstanceExportTask::lookupSucceeded()
|
||||
QList<ExportFile> resolvedFiles;
|
||||
QFileInfoList failedFiles;
|
||||
|
||||
for (const auto &data : m_responses) {
|
||||
try {
|
||||
auto document = Json::requireDocument(data.response);
|
||||
auto object = Json::requireObject(document);
|
||||
auto files = Json::requireIsArrayOf<QJsonObject>(object, "files");
|
||||
for (const auto &file : *m_response) {
|
||||
if (file.found) {
|
||||
try {
|
||||
auto url = Json::requireString(file.fileJson, "url");
|
||||
auto hashes = Json::requireObject(file.fileJson, "hashes");
|
||||
|
||||
QJsonObject file;
|
||||
QString sha512Hash = Json::requireString(hashes, "sha512");
|
||||
QString sha1Hash = Json::requireString(hashes, "sha1");
|
||||
|
||||
for (const auto &fileJson : files) {
|
||||
auto hashes = Json::requireObject(fileJson, "hashes");
|
||||
QString sha512 = Json::requireString(hashes, "sha512");
|
||||
ExportFile fileData;
|
||||
|
||||
if (sha512 == data.sha512) {
|
||||
file = fileJson;
|
||||
}
|
||||
QDir gameDir(m_instance->gameRoot());
|
||||
|
||||
fileData.path = gameDir.relativeFilePath(file.fileInfo.absoluteFilePath());
|
||||
fileData.download = url;
|
||||
fileData.sha512 = sha512Hash;
|
||||
fileData.sha1 = sha1Hash;
|
||||
fileData.fileSize = file.fileInfo.size();
|
||||
|
||||
resolvedFiles << fileData;
|
||||
} catch (const Json::JsonException &e) {
|
||||
qDebug() << "File " << file.fileInfo.absoluteFilePath() << " failed to process for reason " << e.cause() << ", adding to overrides";
|
||||
failedFiles << file.fileInfo;
|
||||
}
|
||||
|
||||
auto url = Json::requireString(file, "url");
|
||||
auto hashes = Json::requireObject(file, "hashes");
|
||||
|
||||
QString sha512Hash = Json::requireString(hashes, "sha512");
|
||||
QString sha1Hash = Json::requireString(hashes, "sha1");
|
||||
|
||||
ExportFile fileData;
|
||||
|
||||
QDir gameDir(m_instance->gameRoot());
|
||||
|
||||
fileData.path = gameDir.relativeFilePath(data.fileInfo.absoluteFilePath());
|
||||
fileData.download = url;
|
||||
fileData.sha512 = sha512Hash;
|
||||
fileData.sha1 = sha1Hash;
|
||||
fileData.fileSize = data.fileInfo.size();
|
||||
|
||||
resolvedFiles << fileData;
|
||||
} catch (const Json::JsonException &e) {
|
||||
qDebug() << "File " << data.fileInfo.absoluteFilePath() << " failed to process for reason " << e.cause() << ", adding to overrides";
|
||||
failedFiles << data.fileInfo;
|
||||
} else {
|
||||
failedFiles << file.fileInfo;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -11,6 +11,7 @@
|
||||
#include "BaseInstance.h"
|
||||
#include "net/NetJob.h"
|
||||
#include "ui/dialogs/ModrinthExportDialog.h"
|
||||
#include "ModrinthHashLookupRequest.h"
|
||||
|
||||
namespace Modrinth
|
||||
{
|
||||
@ -35,13 +36,6 @@ struct ExportSettings
|
||||
QString exportPath;
|
||||
};
|
||||
|
||||
struct HashLookupData
|
||||
{
|
||||
QFileInfo fileInfo;
|
||||
QString sha512;
|
||||
QByteArray response;
|
||||
};
|
||||
|
||||
// Using the existing Modrinth::File struct from the importer doesn't actually make much sense here (doesn't support multiple hashes, hash is a byte array rather than a string, no file size, etc)
|
||||
struct ExportFile
|
||||
{
|
||||
@ -71,7 +65,7 @@ private slots:
|
||||
private:
|
||||
InstancePtr m_instance;
|
||||
ExportSettings m_settings;
|
||||
QList<HashLookupData> m_responses;
|
||||
std::shared_ptr<QList<HashLookupResponseData>> m_response;
|
||||
NetJob::Ptr m_netJob;
|
||||
};
|
||||
|
||||
|
Loading…
x
Reference in New Issue
Block a user