From f6f3565a9e3cc9b80ad4b90964ad650344ff89b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20Mr=C3=A1zek?= Date: Tue, 13 Oct 2020 21:46:09 +0200 Subject: [PATCH] NOISSUE Some work on JRE downloading --- CMakeLists.txt | 6 + api/logic/CMakeLists.txt | 23 ++ api/logic/java/JavaInstall.cpp | 12 +- api/logic/java/JavaInstall.h | 4 + api/logic/java/JavaInstallList.cpp | 71 +++++- api/logic/mojang/ComponentsManifest.cpp | 106 ++++++++ api/logic/mojang/ComponentsManifest.h | 50 ++++ api/logic/mojang/ComponentsManifest_test.cpp | 103 ++++++++ api/logic/mojang/PackageInstallTask.cpp | 251 +++++++++++++++++++ api/logic/mojang/PackageInstallTask.h | 47 ++++ api/logic/mojang/PackageManifest.cpp | 32 ++- api/logic/mojang/PackageManifest.h | 103 ++------ api/logic/mojang/PackageManifest_test.cpp | 1 + api/logic/mojang/Path.cpp | 1 + api/logic/mojang/Path.h | 93 +++++++ api/logic/mojang/testdata/all.json | Bin 0 -> 6346 bytes api/logic/net/ChecksumValidator.h | 4 +- api/logic/net/Download.cpp | 2 +- api/logic/net/Download.h | 3 +- api/logic/net/FileSink.cpp | 8 +- api/logic/net/FileSink.h | 3 +- tools/CMakeLists.txt | 2 + tools/GrabJRE.cpp | 56 +++++ 23 files changed, 870 insertions(+), 111 deletions(-) create mode 100644 api/logic/mojang/ComponentsManifest.cpp create mode 100644 api/logic/mojang/ComponentsManifest.h create mode 100644 api/logic/mojang/ComponentsManifest_test.cpp create mode 100644 api/logic/mojang/PackageInstallTask.cpp create mode 100644 api/logic/mojang/PackageInstallTask.h create mode 100644 api/logic/mojang/Path.cpp create mode 100644 api/logic/mojang/Path.h create mode 100644 api/logic/mojang/testdata/all.json create mode 100644 tools/CMakeLists.txt create mode 100644 tools/GrabJRE.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 4ea92f68..a5f3c73d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -275,3 +275,9 @@ add_subdirectory(api/gui) # NOTE: this must always be last to appease the CMake deity of quirky install command evaluation order. add_subdirectory(application) + +option(BUILD_TOOLS "Build miscellaneous tools" OFF) + +if(BUILD_TOOLS) + add_subdirectory(tools) +endif() diff --git a/api/logic/CMakeLists.txt b/api/logic/CMakeLists.txt index c3322955..056de04e 100644 --- a/api/logic/CMakeLists.txt +++ b/api/logic/CMakeLists.txt @@ -315,6 +315,12 @@ set(MINECRAFT_SOURCES mojang/PackageManifest.h mojang/PackageManifest.cpp + mojang/ComponentsManifest.h + mojang/ComponentsManifest.cpp + mojang/Path.h + mojang/Path.cpp + mojang/PackageInstallTask.h + mojang/PackageInstallTask.cpp ) add_unit_test(GradleSpecifier @@ -338,6 +344,23 @@ add_test( WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) + +add_executable(ComponentsManifest + mojang/ComponentsManifest_test.cpp +) +target_link_libraries(ComponentsManifest + MultiMC_logic + Qt5::Test +) +target_include_directories(ComponentsManifest + PRIVATE ../../cmake/UnitTest/ +) +add_test( + NAME ComponentsManifest + COMMAND ComponentsManifest + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} +) + add_unit_test(MojangVersionFormat SOURCES minecraft/MojangVersionFormat_test.cpp LIBS MultiMC_logic diff --git a/api/logic/java/JavaInstall.cpp b/api/logic/java/JavaInstall.cpp index 5bcf7bcb..4c489163 100644 --- a/api/logic/java/JavaInstall.cpp +++ b/api/logic/java/JavaInstall.cpp @@ -3,9 +3,19 @@ bool JavaInstall::operator<(const JavaInstall &rhs) { + // prefer remote + if(remote < rhs.remote) { + return true; + } + if(remote > rhs.remote) { + return false; + } + // FIXME: make this prefer native arch auto archCompare = Strings::naturalCompare(arch, rhs.arch, Qt::CaseInsensitive); if(archCompare != 0) return archCompare < 0; + + // FIXME: make this a version compare if(id < rhs.id) { return true; @@ -19,7 +29,7 @@ bool JavaInstall::operator<(const JavaInstall &rhs) bool JavaInstall::operator==(const JavaInstall &rhs) { - return arch == rhs.arch && id == rhs.id && path == rhs.path; + return arch == rhs.arch && id == rhs.id && path == rhs.path && remote == rhs.remote; } bool JavaInstall::operator>(const JavaInstall &rhs) diff --git a/api/logic/java/JavaInstall.h b/api/logic/java/JavaInstall.h index 64be40d1..14dbfa6a 100644 --- a/api/logic/java/JavaInstall.h +++ b/api/logic/java/JavaInstall.h @@ -33,6 +33,10 @@ struct JavaInstall : public BaseVersion QString arch; QString path; bool recommended = false; + + bool remote = false; + QString url; + QString installRoot; }; typedef std::shared_ptr JavaInstallPtr; diff --git a/api/logic/java/JavaInstallList.cpp b/api/logic/java/JavaInstallList.cpp index 0bded03c..379f5da9 100644 --- a/api/logic/java/JavaInstallList.cpp +++ b/api/logic/java/JavaInstallList.cpp @@ -24,6 +24,7 @@ #include "java/JavaUtils.h" #include "MMCStrings.h" #include "minecraft/VersionFilterData.h" +#include "sys.h" JavaInstallList::JavaInstallList(QObject *parent) : BaseVersionList(parent) { @@ -191,16 +192,72 @@ void JavaListLoadTask::javaCheckerFinished() } } + auto kernelInfo = Sys::getKernelInfo(); + + // FIXME: data-drive + // FIXME: architecture being '32' or '64' is dumb + // FIXME: limited. + JavaInstallPtr remoteJava; + if(kernelInfo.kernelName == "Windows") { + remoteJava.reset(new JavaInstall()); + remoteJava->id = "1.8.0_51"; + if(Sys::isSystem64bit()) { + remoteJava->arch = "64"; + remoteJava->url = "https://launchermeta.mojang.com/v1/packages/ddc568a50326d2cf85765abb61e752aab191c366/manifest.json"; + } + else { + remoteJava->arch = "32"; + remoteJava->url = "https://launchermeta.mojang.com/v1/packages/baa62193c2785f54d877d871d9859c67d65f08ba/manifest.json"; + } + } + + if(kernelInfo.kernelName == "Linux") { + remoteJava.reset(new JavaInstall()); + remoteJava->id = "1.8.0_202"; + if(Sys::isSystem64bit()) { + remoteJava->arch = "64"; + remoteJava->url = "https://launchermeta.mojang.com/v1/packages/a1c15cc788f8893fba7e988eb27404772f699a84/manifest.json"; + } + else { + remoteJava->arch = "32"; + remoteJava->url = "https://launchermeta.mojang.com/v1/packages/64c6a0b8e3427c6c3f3ce82729aada8b2634a955/manifest.json"; + } + } + + if(kernelInfo.kernelName == "Darwin") { + if(Sys::isSystem64bit()) { + remoteJava.reset(new JavaInstall()); + remoteJava->id = "1.8.0_74"; + remoteJava->arch = "64"; + remoteJava->url = "https://launchermeta.mojang.com/v1/packages/341663b48a0d4e1c448dc789463fced6ba0962e1/manifest.json"; + } + } + + if(remoteJava) { + remoteJava->remote = true; + remoteJava->recommended = true; + + // Example: "runtimes/Windows/64/1.8.0_51" + auto rootPath = QString("runtimes/%1/%2/%3").arg(kernelInfo.kernelName, remoteJava->arch, remoteJava->id.toString()); + remoteJava->installRoot = rootPath; + + if(kernelInfo.kernelName == "Windows") { + remoteJava->path = rootPath + "/data/bin/javaw.exe"; + } + else if (kernelInfo.kernelName == "Darwin") { + remoteJava->path = rootPath + "/data/jre.bundle/Contents/Home/bin/java"; + } + else if (kernelInfo.kernelName == "Linux") { + remoteJava->path = rootPath + "/data/bin/java"; + } + candidates.append(remoteJava); + } + + QList javas_bvp; for (auto java : candidates) { - //qDebug() << java->id << java->arch << " at " << java->path; - BaseVersionPtr bp_java = std::dynamic_pointer_cast(java); - - if (bp_java) - { - javas_bvp.append(java); - } + javas_bvp.append(java); } m_list->updateListData(javas_bvp); diff --git a/api/logic/mojang/ComponentsManifest.cpp b/api/logic/mojang/ComponentsManifest.cpp new file mode 100644 index 00000000..87aeaf82 --- /dev/null +++ b/api/logic/mojang/ComponentsManifest.cpp @@ -0,0 +1,106 @@ +#include "ComponentsManifest.h" + +#include +#include + +// https://launchermeta.mojang.com/v1/products/java-runtime/2ec0cc96c44e5a76b9c8b7c39df7210883d12871/all.json + +namespace mojang_files { + +namespace { +void fromJson(QJsonDocument & doc, AllPlatformsManifest & out) { + if (!doc.isObject()) + { + throw JSONValidationError("file manifest is not an object"); + } + QJsonObject root = doc.object(); + + auto platformIter = root.begin(); + while (platformIter != root.end()) { + QString platformName = platformIter.key(); + auto platformValue = platformIter.value(); + platformIter++; + if (!platformValue.isObject()) + { + throw JSONValidationError("platform entry inside manifest is not an object: " + platformName); + } + auto platformObject = platformValue.toObject(); + auto componentIter = platformObject.begin(); + ComponentsPlatform outPlatform; + while (componentIter != platformObject.end()) { + QString componentName = componentIter.key(); + auto componentValue = componentIter.value(); + componentIter++; + if (!componentValue.isArray()) + { + throw JSONValidationError("component version list inside manifest is not an array: " + componentName); + } + auto versionArray = componentValue.toArray(); + VersionList outVersionList; + int i = 0; + for (auto versionValue: versionArray) { + if (!versionValue.isObject()) + { + throw JSONValidationError("version is not an object: " + componentName + "[" + i + "]"); + } + i++; + auto versionObject = versionValue.toObject(); + ComponentVersion outVersion; + auto availaibility = Json::requireObject(versionObject, "availability"); + outVersion.availability_group = Json::requireInteger(availaibility, "group"); + outVersion.availability_progress = Json::requireInteger(availaibility, "progress"); + auto manifest = Json::requireObject(versionObject, "manifest"); + outVersion.manifest_sha1 = Json::requireString(manifest, "sha1"); + outVersion.manifest_size = Json::requireInteger(manifest, "size"); + outVersion.manifest_url = Json::requireUrl(manifest, "url"); + auto version = Json::requireObject(versionObject, "version"); + outVersion.version_name = Json::requireString(version, "name"); + outVersion.version_released = Json::requireDateTime(version, "released"); + outVersionList.versions.push_back(outVersion); + } + if(outVersionList.versions.size()) { + outPlatform.components[componentName] = std::move(outVersionList); + } + } + if(outPlatform.components.size()) { + out.platforms[platformName] = outPlatform; + } + } + out.valid = true; +} +} + +AllPlatformsManifest AllPlatformsManifest::fromManifestContents(const QByteArray& contents) +{ + AllPlatformsManifest out; + try + { + auto doc = Json::requireDocument(contents, "AllPlatformsManifest"); + fromJson(doc, out); + return out; + } + catch (const Exception &e) + { + qDebug() << QString("Unable to parse manifest: %1").arg(e.cause()); + out.valid = false; + return out; + } +} + +AllPlatformsManifest AllPlatformsManifest::fromManifestFile(const QString & filename) { + AllPlatformsManifest out; + try + { + auto doc = Json::requireDocument(filename, filename); + fromJson(doc, out); + return out; + } + catch (const Exception &e) + { + qDebug() << QString("Unable to parse manifest file %1: %2").arg(filename, e.cause()); + out.valid = false; + return out; + } +} + +} diff --git a/api/logic/mojang/ComponentsManifest.h b/api/logic/mojang/ComponentsManifest.h new file mode 100644 index 00000000..fa8524d2 --- /dev/null +++ b/api/logic/mojang/ComponentsManifest.h @@ -0,0 +1,50 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include "tasks/Task.h" + + +#include "Path.h" + +#include "multimc_logic_export.h" + +namespace mojang_files { + +struct MULTIMC_LOGIC_EXPORT ComponentVersion { + int availability_group = 0; + int availability_progress = 0; + + QString manifest_sha1; + size_t manifest_size = 0; + QUrl manifest_url; + + QString version_name; + QDateTime version_released; +}; + +struct MULTIMC_LOGIC_EXPORT VersionList { + std::vector versions; +}; + +struct MULTIMC_LOGIC_EXPORT ComponentsPlatform { + std::map components; +}; + +struct MULTIMC_LOGIC_EXPORT AllPlatformsManifest { + static AllPlatformsManifest fromManifestFile(const QString &path); + static AllPlatformsManifest fromManifestContents(const QByteArray& contents); + + explicit operator bool() const + { + return valid; + } + + std::map platforms; + bool valid = false; +}; + +} diff --git a/api/logic/mojang/ComponentsManifest_test.cpp b/api/logic/mojang/ComponentsManifest_test.cpp new file mode 100644 index 00000000..6bd0b565 --- /dev/null +++ b/api/logic/mojang/ComponentsManifest_test.cpp @@ -0,0 +1,103 @@ +#include +#include +#include "TestUtil.h" + +#include "mojang/ComponentsManifest.h" + +using namespace mojang_files; + +class ComponentsManifestTest : public QObject +{ + Q_OBJECT + +private slots: + void test_parse(); + void test_parse_file(); +}; + +namespace { +QByteArray basic_manifest = R"END( +{ + "gamecore": { + "java-runtime-alpha": [], + "jre-legacy": [], + "minecraft-java-exe": [] + }, + "linux": { + "java-runtime-alpha": [{ + "availability": { + "group": 5851, + "progress": 100 + }, + "manifest": { + "sha1": "e968e71afd3360e5032deac19e1c14d7aa32f5bb", + "size": 81882, + "url": "https://launchermeta.mojang.com/v1/packages/e968e71afd3360e5032deac19e1c14d7aa32f5bb/manifest.json" + }, + "version": { + "name": "16.0.1.9.1", + "released": "2021-05-10T16:43:02+00:00" + } + }], + "jre-legacy": [{ + "availability": { + "group": 6513, + "progress": 100 + }, + "manifest": { + "sha1": "a1c15cc788f8893fba7e988eb27404772f699a84", + "size": 125581, + "url": "https://launchermeta.mojang.com/v1/packages/a1c15cc788f8893fba7e988eb27404772f699a84/manifest.json" + }, + "version": { + "name": "8u202", + "released": "2020-11-17T19:26:25+00:00" + } + }], + "minecraft-java-exe": [] + } +} +)END"; +} + +void ComponentsManifestTest::test_parse() +{ + auto manifest = AllPlatformsManifest::fromManifestContents(basic_manifest); + QVERIFY(manifest.valid == true); + QVERIFY(manifest.platforms.count("gamecore") == 0); + QVERIFY(manifest.platforms.count("linux") == 1); + /* + QVERIFY(manifest.files.size() == 1); + QVERIFY(manifest.files.count(Path("a/b.txt"))); + auto &file = manifest.files[Path("a/b.txt")]; + QVERIFY(file.executable == true); + QVERIFY(file.hash == "da39a3ee5e6b4b0d3255bfef95601890afd80709"); + QVERIFY(file.size == 0); + QVERIFY(manifest.folders.size() == 4); + QVERIFY(manifest.folders.count(Path("."))); + QVERIFY(manifest.folders.count(Path("a"))); + QVERIFY(manifest.folders.count(Path("a/b"))); + QVERIFY(manifest.folders.count(Path("a/b/c"))); + QVERIFY(manifest.symlinks.size() == 1); + auto symlinkPath = Path("a/b/c.txt"); + QVERIFY(manifest.symlinks.count(symlinkPath)); + auto &symlink = manifest.symlinks[symlinkPath]; + QVERIFY(symlink == Path("../b.txt")); + QVERIFY(manifest.sources.size() == 1); + */ +} + +void ComponentsManifestTest::test_parse_file() { + auto path = QFINDTESTDATA("testdata/all.json"); + auto manifest = AllPlatformsManifest::fromManifestFile(path); + QVERIFY(manifest.valid == true); + QVERIFY(manifest.platforms.count("gamecore") == 0); + QVERIFY(manifest.platforms.count("linux") == 1); + /* + QVERIFY(manifest.sources.count("c725183c757011e7ba96c83c1e86ee7e8b516a2b") == 1); + */ +} + +QTEST_GUILESS_MAIN(ComponentsManifestTest) + +#include "ComponentsManifest_test.moc" diff --git a/api/logic/mojang/PackageInstallTask.cpp b/api/logic/mojang/PackageInstallTask.cpp new file mode 100644 index 00000000..e315cad8 --- /dev/null +++ b/api/logic/mojang/PackageInstallTask.cpp @@ -0,0 +1,251 @@ +#include "PackageInstallTask.h" + +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "PackageManifest.h" +#include + +using Package = mojang_files::Package; +using UpdateOperations = mojang_files::UpdateOperations; + +struct PackageInstallTaskData +{ + QString root; + QString version; + QString packageURL; + Net::Mode netmode; + QFuture inspectionFuture; + QFutureWatcher inspectionWatcher; + Package inspectedPackage; + + NetJobPtr manifestDownloadJob; + bool manifestDone = false; + Package downloadedPackage; + + UpdateOperations updateOps; + NetJobPtr mainDownloadJob; +}; + +namespace { +class InspectFolder +{ +public: + InspectFolder(const QString & folder) : folder(folder) {} + Package operator()() + { + return Package::fromInspectedFolder(folder); + } +private: + QString folder; +}; + +class ParsingValidator : public Net::Validator +{ +public: /* con/des */ + ParsingValidator(Package &package) : m_package(package) + { + } + virtual ~ParsingValidator() = default; + +public: /* methods */ + bool init(QNetworkRequest &) override + { + m_package.valid = false; + return true; + } + bool write(QByteArray & data) override + { + this->data.append(data); + return true; + } + bool abort() override + { + return true; + } + bool validate(QNetworkReply &) override + { + m_package = Package::fromManifestContents(data); + return m_package.valid; + } + +private: /* data */ + QByteArray data; + Package &m_package; +}; + +} + +PackageInstallTask::PackageInstallTask( + Net::Mode netmode, + QString version, + QString packageURL, + QString targetPath, + QObject* parent +) : Task(parent) { + d.reset(new PackageInstallTaskData); + d->netmode = netmode; + d->root = targetPath; + d->packageURL = packageURL; + d->version = version; +} + +PackageInstallTask::~PackageInstallTask() {} + +void PackageInstallTask::executeTask() { + // inspect the data folder in a thread + d->inspectionFuture = QtConcurrent::run(QThreadPool::globalInstance(), InspectFolder(FS::PathCombine(d->root, "data"))); + connect(&d->inspectionWatcher, &QFutureWatcher::finished, this, &PackageInstallTask::inspectionFinished); + d->inspectionWatcher.setFuture(d->inspectionFuture); + + // while inspecting, grab the manifest from remote + d->manifestDownloadJob.reset(new NetJob(QObject::tr("Download of package manifest %1").arg(d->packageURL))); + auto url = d->packageURL; + auto dl = Net::Download::makeFile(url, FS::PathCombine(d->root, "manifest.json")); + /* + * The validator parses the file and loads it into the object. + * If that fails, the file is not written to storage. + */ + dl->addValidator(new ParsingValidator(d->downloadedPackage)); + d->manifestDownloadJob->addNetAction(dl); + auto job = d->manifestDownloadJob.get(); + connect(job, &NetJob::finished, this, &PackageInstallTask::manifestFetchFinished); + d->manifestDownloadJob->start(); +} + +void PackageInstallTask::inspectionFinished() { + d->inspectedPackage = d->inspectionWatcher.result(); + processInputs(); +} + +void PackageInstallTask::manifestFetchFinished() +{ + d->manifestDone = true; + d->manifestDownloadJob.reset(); + processInputs(); +} + +void PackageInstallTask::processInputs() +{ + if(!d->manifestDone) { + return; + } + if(!d->inspectionFuture.isFinished()) { + return; + } + + if(!d->downloadedPackage.valid) { + emitFailed("Downloading package manifest failed..."); + return; + } + + if(!d->inspectedPackage.valid) { + emitFailed("Inspecting local data folder failed..."); + return; + } + + d->updateOps = UpdateOperations::resolve(d->inspectedPackage, d->downloadedPackage); + + if(!d->updateOps.valid) { + emitFailed("Unable to determine update actions..."); + return; + } + + if(d->updateOps.empty()) { + emitSucceeded(); + } + + auto dataRoot = FS::PathCombine(d->root, "data"); + + // first, ensure data path exists + QDir temp; + temp.mkpath(dataRoot); + + for(auto & rm: d->updateOps.deletes) { + auto filePath = FS::PathCombine(dataRoot, rm.toString()); + qDebug() << "RM" << filePath; + QFile::remove(filePath); + } + + for(auto & rmdir: d->updateOps.rmdirs) { + auto folderPath = FS::PathCombine(dataRoot, rmdir.toString()); + qDebug() << "RMDIR" << folderPath; + QDir dir; + dir.rmdir(folderPath); + } + + for(auto & mkdir: d->updateOps.mkdirs) { + auto folderPath = FS::PathCombine(dataRoot, mkdir.toString()); + qDebug() << "MKDIR" << folderPath; + QDir dir; + dir.mkdir(folderPath); + } + + for(auto & mklink: d->updateOps.mklinks) { + auto linkPath = FS::PathCombine(dataRoot, mklink.first.toString()); + auto linkTarget = mklink.second.toString(); + qDebug() << "MKLINK" << linkPath << "->" << linkTarget; + QFile::link(linkTarget, linkPath); + } + + for(auto & fix: d->updateOps.executable_fixes) { + const auto &path = fix.first; + bool executable = fix.second; + auto targetPath = FS::PathCombine(dataRoot, path.toString()); + qDebug() << "FIX_EXEC" << targetPath << "->" << (executable ? "EXECUTABLE" : "REGULAR"); + + auto perms = QFile::permissions(targetPath); + if(executable) { + perms |= QFileDevice::ExeUser | QFileDevice::ExeGroup | QFileDevice::ExeOther; + } + else { + perms &= ~(QFileDevice::ExeUser | QFileDevice::ExeGroup | QFileDevice::ExeOther); + } + QFile::setPermissions(targetPath, perms); + } + + if(!d->updateOps.downloads.size()) { + emitSucceeded(); + return; + } + + // we download. + d->manifestDownloadJob.reset(new NetJob(QObject::tr("Download of files for %1").arg(d->packageURL))); + connect(d->manifestDownloadJob.get(), &NetJob::succeeded, this, &PackageInstallTask::downloadsSucceeded); + connect(d->manifestDownloadJob.get(), &NetJob::failed, this, &PackageInstallTask::downloadsFailed); + connect(d->manifestDownloadJob.get(), &NetJob::progress, [&](qint64 current, qint64 total) { + setProgress(current, total); + }); + + using Option = Net::Download::Option; + for(auto & download: d->updateOps.downloads) { + const auto &path = download.first; + const auto &object = download.second; + auto targetPath = FS::PathCombine(dataRoot, path.toString()); + auto dl = Net::Download::makeFile(object.url, targetPath, object.executable ? Option::SetExecutable : Option::NoOptions); + dl->addValidator(new Net::ChecksumValidator(QCryptographicHash::Sha1, QByteArray::fromHex(object.hash.toLatin1()))); + dl->m_total_progress = object.size; + d->manifestDownloadJob->addNetAction(dl); + qDebug() << "DOWNLOAD" << object.url << "to" << targetPath << (object.executable ? "(EXECUTABLE)" : "(REGULAR)"); + } + d->manifestDownloadJob->start(); +} + +void PackageInstallTask::downloadsFailed(QString reason) +{ + d->manifestDownloadJob.reset(); + emitFailed(reason); +} + +void PackageInstallTask::downloadsSucceeded() +{ + d->manifestDownloadJob.reset(); + emitSucceeded(); +} diff --git a/api/logic/mojang/PackageInstallTask.h b/api/logic/mojang/PackageInstallTask.h new file mode 100644 index 00000000..45323407 --- /dev/null +++ b/api/logic/mojang/PackageInstallTask.h @@ -0,0 +1,47 @@ +#pragma once + +#include "tasks/Task.h" +#include "net/Mode.h" +#include "multimc_logic_export.h" + +#include + +struct PackageInstallTaskData; + +class MULTIMC_LOGIC_EXPORT PackageInstallTask : public Task +{ + Q_OBJECT +public: + enum class Mode + { + Launch, + Resolution + }; + +public: + explicit PackageInstallTask( + Net::Mode netmode, + QString version, + QString packageURL, + QString targetPath, + QObject *parent = 0 + ); + virtual ~PackageInstallTask(); + +protected: + void executeTask() override; + +private slots: + void inspectionFinished(); + void manifestFetchFinished(); + + void processInputs(); + + void downloadsSucceeded(); + void downloadsFailed(QString reason); + +private: + std::unique_ptr d; +}; + + diff --git a/api/logic/mojang/PackageManifest.cpp b/api/logic/mojang/PackageManifest.cpp index b3dfd7fc..b85a4d01 100644 --- a/api/logic/mojang/PackageManifest.cpp +++ b/api/logic/mojang/PackageManifest.cpp @@ -67,8 +67,8 @@ void Package::addLink(const Path& path, const Path& target) { symlinks[path] = target; } -void Package::addSource(const FileSource& source) { - sources[source.hash] = source; +void Package::addSource(const Hash & rawHash, const FileSource& source) { + sources[rawHash] = source; } @@ -124,6 +124,8 @@ void fromJson(QJsonDocument & doc, Package & out) { } else if (compression == "lzma") { source.compression = Compression::Lzma; + // FIXME: remove this line when we implement LZMA filter for downloads again + continue; } else { continue; @@ -134,11 +136,10 @@ void fromJson(QJsonDocument & doc, Package & out) { throw JSONValidationError("No valid compression method for file " + iter.key()); } out.addFile(objectPath, file); - out.addSource(bestSource); + out.addSource(file.hash, bestSource); } else if(type == "link") { auto target = Json::requireString(fileObject, "target"); - out.symlinks[objectPath] = target; out.addLink(objectPath, target); } else { @@ -238,7 +239,8 @@ Package Package::fromInspectedFolder(const QString& folderPath) iterator.next(); auto fileInfo = iterator.fileInfo(); - auto relPath = root.relativeFilePath(fileInfo.filePath()); + auto itemPath = fileInfo.absoluteFilePath(); + auto relPath = root.relativeFilePath(itemPath); // FIXME: this is probably completely busted on Windows anyway, so just disable it. // Qt makes shit up and doesn't understand the platform details // TODO: Actually use a filesystem library that isn't terrible and has decen license. @@ -246,7 +248,7 @@ Package Package::fromInspectedFolder(const QString& folderPath) #ifndef Q_OS_WIN32 if(fileInfo.isSymLink()) { Path targetPath; - if(!actually_read_symlink_target(fileInfo.filePath(), targetPath)) { + if(!actually_read_symlink_target(fileInfo.absoluteFilePath(), targetPath)) { qCritical() << "Folder inspection: Unknown filesystem object:" << fileInfo.absoluteFilePath(); out.valid = false; } @@ -364,12 +366,21 @@ UpdateOperations UpdateOperations::resolve(const Package& from, const Package& t } } for(auto iter = to.files.begin(); iter != to.files.end(); iter++) { - auto path = iter->first; + auto & path = iter->first; + auto & file = iter->second; + auto & fileHash = file.hash; + auto executable = file.executable; + if(!to.sources.count(fileHash)) { + out.valid = false; + return out; + } + auto & source = to.sources.at(fileHash); + // it wasn't there before, it is there now... therefore we fill it in if(!from.files.count(path)) { out.downloads.emplace( std::pair{ path, - FileDownload(to.sources.at(iter->second.hash), iter->second.executable) + FileDownload(source, executable) } ); } @@ -411,13 +422,14 @@ UpdateOperations UpdateOperations::resolve(const Package& from, const Package& t const auto &new_target = iter2->second; if (current_target != new_target) { out.deletes.push_back(path); - out.mklinks[path] = iter2->second; + out.mklinks[path] = new_target; } } for(auto iter = to.symlinks.begin(); iter != to.symlinks.end(); iter++) { auto path = iter->first; + const auto &new_target = iter->second; if(!from.symlinks.count(path)) { - out.mklinks[path] = iter->second; + out.mklinks[path] = new_target; } } out.valid = true; diff --git a/api/logic/mojang/PackageManifest.h b/api/logic/mojang/PackageManifest.h index d01a0554..fbc44845 100644 --- a/api/logic/mojang/PackageManifest.h +++ b/api/logic/mojang/PackageManifest.h @@ -6,6 +6,9 @@ #include #include "tasks/Task.h" + +#include "Path.h" + #include "multimc_logic_export.h" namespace mojang_files { @@ -13,91 +16,6 @@ namespace mojang_files { using Hash = QString; extern const Hash empty_hash; -// simple-ish path implementation. assumes always relative and does not allow '..' entries -class MULTIMC_LOGIC_EXPORT Path -{ -public: - using parts_type = QStringList; - - Path() = default; - Path(QString string) { - auto parts_in = string.split('/'); - for(auto & part: parts_in) { - if(part.isEmpty() || part == ".") { - continue; - } - if(part == "..") { - if(parts.size()) { - parts.pop_back(); - } - continue; - } - parts.push_back(part); - } - } - - bool has_parent_path() const - { - return parts.size() > 0; - } - - Path parent_path() const - { - if (parts.empty()) - return Path(); - return Path(parts.begin(), std::prev(parts.end())); - } - - bool empty() const - { - return parts.empty(); - } - - int length() const - { - return parts.length(); - } - - bool operator==(const Path & rhs) const { - return parts == rhs.parts; - } - - bool operator!=(const Path & rhs) const { - return parts != rhs.parts; - } - - inline bool operator<(const Path& rhs) const - { - return compare(rhs) < 0; - } - - parts_type::const_iterator begin() const - { - return parts.begin(); - } - - parts_type::const_iterator end() const - { - return parts.end(); - } - - QString toString() const { - return parts.join("/"); - } - -private: - Path(const parts_type::const_iterator & start, const parts_type::const_iterator & end) { - auto cursor = start; - while(cursor != end) { - parts.push_back(*cursor); - cursor++; - } - } - int compare(const Path& p) const; - - parts_type parts; -}; - enum class Compression { Raw, @@ -141,7 +59,7 @@ struct MULTIMC_LOGIC_EXPORT Package { void addFolder(Path folder); void addFile(const Path & path, const File & file); void addLink(const Path & path, const Path & target); - void addSource(const FileSource & source); + void addSource(const Hash & rawHash, const FileSource & source); std::map sources; bool valid = true; @@ -168,6 +86,19 @@ struct MULTIMC_LOGIC_EXPORT UpdateOperations { std::map downloads; std::map mklinks; std::map executable_fixes; + + bool empty() const { + if(!valid) { + return true; + } + return + deletes.empty() && + rmdirs.empty() && + mkdirs.empty() && + downloads.empty() && + mklinks.empty() && + executable_fixes.empty(); + } }; } diff --git a/api/logic/mojang/PackageManifest_test.cpp b/api/logic/mojang/PackageManifest_test.cpp index d4c55c5a..634cbe12 100644 --- a/api/logic/mojang/PackageManifest_test.cpp +++ b/api/logic/mojang/PackageManifest_test.cpp @@ -86,6 +86,7 @@ void PackageManifestTest::test_parse_file() { auto path = QFINDTESTDATA("testdata/1.8.0_202-x64.json"); auto manifest = Package::fromManifestFile(path); QVERIFY(manifest.valid == true); + QVERIFY(manifest.sources.count("c725183c757011e7ba96c83c1e86ee7e8b516a2b") == 1); } diff --git a/api/logic/mojang/Path.cpp b/api/logic/mojang/Path.cpp new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/api/logic/mojang/Path.cpp @@ -0,0 +1 @@ + diff --git a/api/logic/mojang/Path.h b/api/logic/mojang/Path.h new file mode 100644 index 00000000..cdfa0db7 --- /dev/null +++ b/api/logic/mojang/Path.h @@ -0,0 +1,93 @@ +#pragma once + +#include +#include + +namespace mojang_files { + +// simple-ish path implementation. assumes always relative and does not allow '..' entries +class MULTIMC_LOGIC_EXPORT Path +{ +public: + using parts_type = QStringList; + + Path() = default; + Path(QString string) { + auto parts_in = string.split('/'); + for(auto & part: parts_in) { + if(part.isEmpty() || part == ".") { + continue; + } + if(part == "..") { + if(parts.size()) { + parts.pop_back(); + continue; + } + } + parts.push_back(part); + } + } + + bool has_parent_path() const + { + return parts.size() > 0; + } + + Path parent_path() const + { + if (parts.empty()) + return Path(); + return Path(parts.begin(), std::prev(parts.end())); + } + + bool empty() const + { + return parts.empty(); + } + + int length() const + { + return parts.length(); + } + + bool operator==(const Path & rhs) const { + return parts == rhs.parts; + } + + bool operator!=(const Path & rhs) const { + return parts != rhs.parts; + } + + inline bool operator<(const Path& rhs) const + { + return compare(rhs) < 0; + } + + parts_type::const_iterator begin() const + { + return parts.begin(); + } + + parts_type::const_iterator end() const + { + return parts.end(); + } + + QString toString() const { + return parts.join("/"); + } + +private: + Path(const parts_type::const_iterator & start, const parts_type::const_iterator & end) { + auto cursor = start; + while(cursor != end) { + parts.push_back(*cursor); + cursor++; + } + } + int compare(const Path& p) const; + + parts_type parts; +}; + +} diff --git a/api/logic/mojang/testdata/all.json b/api/logic/mojang/testdata/all.json new file mode 100644 index 0000000000000000000000000000000000000000..84f1f1300a41b30fac865470bfe07786d0a692bf GIT binary patch literal 6346 zcmd6r+iu%96h+_j6@^}BWb=A>sPD*IQ49_*N|gE%NOqc;BL6;=lg5d`IW0?QZ~Is^Uw$5bIsAG@4^;R2&8}O=npXF>l-j?3`}0kYKGv&v zL;mm6mUXv@zNdCqAMO$#$9sSMcKGYl^Bt;Hx7j~VdE`;kL06L1t;lt&ZZ|&nee3gh z`tANcHEGoRD0gfcuj;u|=l7j5I$@v7Fe>DoB#^3NG=iKmt-TEJ*H zwLwZ?Q8Ot*B+qT+o~vMpq-eEsRYuIv{WH!Dazv47-uJ8V3Ael5{m?9zE81`TZS2=^ zM~n6Ljy5+7zg;gM`0}3oce;tga;lo;ufZ+uhV7<0I}Z=BAG*{jJKdXv&nS|^f-U&M zE_iOBeOyHvViFM{_yY^9GVjIcz3&g-u)$WLp|}e=fvaJTsKT!=4*HF z3a#0Epb;jsnTzuw6$g+`A+exR4Wtx?lz&!%Y<&#ih*=OZTPD0Xk1?`TJv3EKaXxTw zvY~#$#hbiisTfj)edkBF3ELk-{RrwmH6JBDC)Suo5MUfBCL}~|zzQ9R8=q(sHWl7M z2FXHhNB)X6rIUeWcESwznJT9^T}SuHjr4@Ex79W_%v``&A@7l(Rc6MEdrt^n1Jd{o z20{21wE<1e0gpxtBF8QF5+FBezq|*nDOB`KRa2Y}-0Q5po^bKTd$?%0ynu^*tvx%N zveiIwI;@5O8Vsa5STS-Sg&`hMj+9`));NUZWM}9|1x|+J8O||POK~>%lvGNNA#0(p zg=myX14iG87{#$c?J)|_GS#`(IOkaFQ|cHig#*JpNS4YR zW6og1l{5$$u`|e-DyKX-b5D45BQB|H3T2K$$jx|hJRA|E>PLEEP`6qIG$ws`u&AvE zlPsrY=+3m_url = url; dl->m_options = options; - dl->m_sink.reset(new FileSink(path)); + dl->m_sink.reset(new FileSink(path, options & Option::SetExecutable)); return std::shared_ptr(dl); } diff --git a/api/logic/net/Download.h b/api/logic/net/Download.h index 2c436032..51855aff 100644 --- a/api/logic/net/Download.h +++ b/api/logic/net/Download.h @@ -31,7 +31,8 @@ public: /* types */ enum class Option { NoOptions = 0, - AcceptLocalFiles = 1 + AcceptLocalFiles = 1, + SetExecutable = 2 }; Q_DECLARE_FLAGS(Options, Option) diff --git a/api/logic/net/FileSink.cpp b/api/logic/net/FileSink.cpp index 8b3e917d..2504ecca 100644 --- a/api/logic/net/FileSink.cpp +++ b/api/logic/net/FileSink.cpp @@ -6,8 +6,8 @@ namespace Net { -FileSink::FileSink(QString filename) - :m_filename(filename) +FileSink::FileSink(QString filename, bool setExecutable) + :m_filename(filename), setExecutable(setExecutable) { // nil } @@ -95,6 +95,10 @@ JobStatus FileSink::finalize(QNetworkReply& reply) m_output_file->cancelWriting(); return Job_Failed; } + if(setExecutable) { + auto permissions = QFile::permissions(m_filename); + QFile::setPermissions(m_filename, permissions | QFileDevice::ExeUser | QFileDevice::ExeGroup | QFileDevice::ExeOther); + } } // then get rid of the save file m_output_file.reset(); diff --git a/api/logic/net/FileSink.h b/api/logic/net/FileSink.h index 875fe511..ca94a0b4 100644 --- a/api/logic/net/FileSink.h +++ b/api/logic/net/FileSink.h @@ -6,7 +6,7 @@ namespace Net { class FileSink : public Sink { public: /* con/des */ - FileSink(QString filename); + FileSink(QString filename, bool setExecutable = false); virtual ~FileSink(); public: /* methods */ @@ -24,5 +24,6 @@ protected: /* data */ QString m_filename; bool wroteAnyData = false; std::unique_ptr m_output_file; + bool setExecutable = false; }; } diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt new file mode 100644 index 00000000..96d70da0 --- /dev/null +++ b/tools/CMakeLists.txt @@ -0,0 +1,2 @@ +add_executable(GrabJRE WIN32 GrabJRE.cpp) +target_link_libraries(GrabJRE MultiMC_logic) diff --git a/tools/GrabJRE.cpp b/tools/GrabJRE.cpp new file mode 100644 index 00000000..ead9563f --- /dev/null +++ b/tools/GrabJRE.cpp @@ -0,0 +1,56 @@ +#include +#include + +#include "mojang/PackageManifest.h" +#include "mojang/PackageInstallTask.h" +#include + +int main(int argc, char ** argv) { + QCoreApplication app(argc, argv); + QCoreApplication::setApplicationName("GrabJRE"); + QCoreApplication::setApplicationVersion("1.0"); + + QCommandLineParser parser; + parser.setApplicationDescription("Stupid thing that grabs a piston package and updates a local folder with it"); + parser.addHelpOption(); + parser.addVersionOption(); + + parser.addPositionalArgument("url", "Source URL"); + parser.addPositionalArgument("version", "Source version"); + parser.addPositionalArgument("destination", "Destination folder to update"); + + parser.process(app); + + const QStringList args = parser.positionalArguments(); + if (args.size() != 3) { + parser.showHelp(1); + } + + // Run like ./GrabJRE "https://launchermeta.mojang.com/v1/packages/a1c15cc788f8893fba7e988eb27404772f699a84/manifest.json" "1.8.0_202" "test" + + auto url = args[0]; + auto version = args[1]; + auto destination = args[2]; + PackageInstallTask installTask( + Net::Mode::Online, + version, + url, + destination + ); + installTask.start(); + QCoreApplication::connect(&installTask, &PackageInstallTask::progress, [&](qint64 now, qint64 total) { + static int percentage = 0; + if(total > 0) { + int newPercentage = (now * 100.0f) / double(total); + if(newPercentage != percentage) { + percentage = newPercentage; + QTextStream(stdout) << "Downloading: " << percentage << "% done\n"; + } + } + }); + QCoreApplication::connect(&installTask, &PackageInstallTask::finished, [&]() { + app.exit(installTask.wasSuccessful() ? 0 : 1); + }); + app.exec(); + return 0; +}