From a6c2e995af2c5f61477bf7e3d03e4f3c8bae6cdf Mon Sep 17 00:00:00 2001 From: Eladash <18193363+elad335@users.noreply.github.com> Date: Fri, 2 Feb 2024 22:28:44 +0200 Subject: [PATCH] Crypto/PKG installer: Fix potential RAM shortage when extracing EDAT files --- rpcs3/Crypto/decrypt_binaries.cpp | 11 +- rpcs3/Crypto/unedat.cpp | 94 +++++++++----- rpcs3/Crypto/unedat.h | 21 ++- rpcs3/Crypto/unpkg.cpp | 208 +++++++++++++++++++++++++++--- rpcs3/Emu/Cell/lv2/sys_fs.cpp | 2 +- 5 files changed, 286 insertions(+), 50 deletions(-) diff --git a/rpcs3/Crypto/decrypt_binaries.cpp b/rpcs3/Crypto/decrypt_binaries.cpp index 7ae85c59ed..c0ee5a4abe 100644 --- a/rpcs3/Crypto/decrypt_binaries.cpp +++ b/rpcs3/Crypto/decrypt_binaries.cpp @@ -139,7 +139,16 @@ usz decrypt_binaries_t::decrypt(std::string klic_input) if (fs::file new_file{new_path, fs::rewrite}) { - new_file.write(elf_file.to_string()); + // 16MB buffer + std::vector buffer(std::min(elf_file.size(), 1u << 24)); + + elf_file.seek(0); + + while (usz read_size = elf_file.read(buffer.data(), buffer.size())) + { + new_file.write(buffer.data(), read_size); + } + dec_log.success("Decrypted %s -> %s", old_path, new_path); std::cout << "Decrypted " << old_path << " -> " << new_path << std::endl; // For CLI m_index++; diff --git a/rpcs3/Crypto/unedat.cpp b/rpcs3/Crypto/unedat.cpp index 1fab69e798..470fab2f0b 100644 --- a/rpcs3/Crypto/unedat.cpp +++ b/rpcs3/Crypto/unedat.cpp @@ -610,21 +610,25 @@ bool validate_dev_klic(const u8* klicensee, NPD_HEADER *npd) return cmac_hash_compare(reinterpret_cast(&key), 0x10, dev, 0x60, npd->dev_hash, 0x10); } -bool validate_npd_hashes(const char* file_name, const u8* klicensee, NPD_HEADER *npd, EDAT_HEADER* edat, bool verbose) +bool validate_npd_hashes(std::string_view file_name, const u8* klicensee, NPD_HEADER* npd, EDAT_HEADER* edat, bool verbose) { - if (!file_name) - { - fmt::throw_exception("Empty filename"); - } - // Ignore header validation in DEBUG data. if (edat->flags & EDAT_DEBUG_DATA_FLAG) { return true; } - const usz file_name_length = std::strlen(file_name); - const usz buf_len = 0x30 + file_name_length; + if (!validate_dev_klic(klicensee, npd)) + { + return false; + } + + if (file_name.empty()) + { + return true; + } + + const usz buf_len = 0x30 + file_name.size(); std::unique_ptr buf(new u8[buf_len]); std::unique_ptr buf_lower(new u8[buf_len]); @@ -632,12 +636,12 @@ bool validate_npd_hashes(const char* file_name, const u8* klicensee, NPD_HEADER // Build the title buffer (content_id + file_name). std::memcpy(buf.get(), npd->content_id, 0x30); - std::memcpy(buf.get() + 0x30, file_name, file_name_length); + std::memcpy(buf.get() + 0x30, file_name.data(), file_name.size()); std::memcpy(buf_lower.get(), buf.get(), buf_len); std::memcpy(buf_upper.get(), buf.get(), buf_len); - for (usz i = std::basic_string_view(buf.get() + 0x30, file_name_length).find_last_of('.'); i < buf_len; i++) + for (usz i = std::basic_string_view(buf.get() + 0x30, file_name.size()).find_last_of('.'); i < buf_len; i++) { const u8 c = static_cast(buf[i]); buf_upper[i] = std::toupper(c); @@ -659,17 +663,17 @@ bool validate_npd_hashes(const char* file_name, const u8* klicensee, NPD_HEADER edat_log.warning("NPD title hash is invalid!"); } - const bool dev_hash_result = validate_dev_klic(klicensee, npd); - - return title_hash_result && dev_hash_result; + return title_hash_result; } void read_npd_edat_header(const fs::file* input, NPD_HEADER& NPD, EDAT_HEADER& EDAT) { - char npd_header[0x80]; - char edat_header[0x10]; - input->read(npd_header, sizeof(npd_header)); - input->read(edat_header, sizeof(edat_header)); + char npd_header[0x80]{}; + char edat_header[0x10]{}; + + usz pos = 0; + pos = input->read_at(pos, npd_header, sizeof(npd_header)); + input->read_at(pos, edat_header, sizeof(edat_header)); std::memcpy(&NPD.magic, npd_header, 4); NPD.version = read_from_ptr>(npd_header, 4); @@ -739,7 +743,7 @@ bool extract_all_data(const fs::file* input, const fs::file* output, const char* } // Perform header validation (EDAT only). - char real_file_name[CRYPTO_MAX_PATH]; + char real_file_name[CRYPTO_MAX_PATH]{}; extract_file_name(input_file_name, real_file_name); if (!validate_npd_hashes(real_file_name, devklic, &NPD, &EDAT, verbose)) { @@ -839,7 +843,7 @@ bool VerifyEDATHeaderWithKLicense(const fs::file& input, const std::string& inpu } // Perform header validation (EDAT only). - char real_file_name[CRYPTO_MAX_PATH]; + char real_file_name[CRYPTO_MAX_PATH]{}; extract_file_name(input_file_name.c_str(), real_file_name); if (!validate_npd_hashes(real_file_name, custom_klic, &NPD, &EDAT, false)) { @@ -915,20 +919,21 @@ fs::file DecryptEDAT(const fs::file& input, const std::string& input_file_name, } // Delete the bad output file if any errors arise. - fs::file output = fs::make_stream>(); - if (extract_all_data(&input, &output, input_file_name.c_str(), reinterpret_cast(&devklic), verbose)) + auto data = std::make_unique(input, devklic, input_file_name, false); + + if (!data->ReadHeader()) { - output.release(); return fs::file{}; } - output.seek(0); + fs::file output; + output.reset(std::move(data)); + return output; } bool EDATADecrypter::ReadHeader() { - edata_file.seek(0); // Read in the NPD and EDAT/SDAT headers. read_npd_edat_header(&edata_file, npdHeader, edatHeader); @@ -945,14 +950,45 @@ bool EDATADecrypter::ReadHeader() } else { - // verify key - if (!validate_dev_klic(reinterpret_cast(&dec_key), &npdHeader)) + // extract key from RIF + char real_file_name[CRYPTO_MAX_PATH]{}; + extract_file_name(m_file_name.c_str(), real_file_name); + + if (!validate_npd_hashes(real_file_name, reinterpret_cast(&dec_key), &npdHeader, &edatHeader, false)) { - edat_log.error("Failed validating klic"); - return false; + edat_log.error("NPD hash validation failed!"); + return true; } - // Use provided dec_key + // Select EDAT key. + if (m_is_key_final) + { + // Already provided + } + // Type 3: Use supplied dec_key. + else if ((npdHeader.license & 0x3) == 0x3) + { + // + } + // Type 2: Use key from RAP file (RIF key). (also used for type 1 at the moment) + else + { + const std::string rap_path = rpcs3::utils::get_rap_file_path(npdHeader.content_id); + + if (fs::file rap{rap_path}; rap && rap.size() >= sizeof(dec_key)) + { + dec_key = GetEdatRifKeyFromRapFile(rap); + } + + // Make sure we don't have an empty RIF key. + if (!dec_key) + { + edat_log.error("A valid RAP file is needed for this EDAT file! (license=%d)", npdHeader.license); + return true; + } + + edat_log.trace("RIFKEY: %s", std::bit_cast>(dec_key)); + } } edata_file.seek(0); diff --git a/rpcs3/Crypto/unedat.h b/rpcs3/Crypto/unedat.h index 50ea211c20..6a8977ede9 100644 --- a/rpcs3/Crypto/unedat.h +++ b/rpcs3/Crypto/unedat.h @@ -71,7 +71,10 @@ u128 GetEdatRifKeyFromRapFile(const fs::file& rap_file); struct EDATADecrypter final : fs::file_base { // file stream - fs::file edata_file; + fs::file m_edata_file; + const fs::file& edata_file; + std::string m_file_name; + bool m_is_key_final = true; u64 file_size{0}; u32 total_blocks{0}; u64 pos{0}; @@ -82,8 +85,20 @@ struct EDATADecrypter final : fs::file_base u128 dec_key{}; public: - EDATADecrypter(fs::file&& input, u128 dec_key = {}) - : edata_file(std::move(input)) + EDATADecrypter(fs::file&& input, u128 dec_key = {}, std::string file_name = {}, bool is_key_final = true) noexcept + : m_edata_file(std::move(input)) + , edata_file(m_edata_file) + , m_file_name(std::move(file_name)) + , m_is_key_final(is_key_final) + , dec_key(dec_key) + { + } + + EDATADecrypter(const fs::file& input, u128 dec_key = {}, std::string file_name = {}, bool is_key_final = true) noexcept + : m_edata_file(fs::file{}) + , edata_file(input) + , m_file_name(std::move(file_name)) + , m_is_key_final(is_key_final) , dec_key(dec_key) { } diff --git a/rpcs3/Crypto/unpkg.cpp b/rpcs3/Crypto/unpkg.cpp index 0ee2f4bdc1..e92c00449b 100644 --- a/rpcs3/Crypto/unpkg.cpp +++ b/rpcs3/Crypto/unpkg.cpp @@ -866,6 +866,8 @@ fs::file DecryptEDAT(const fs::file& input, const std::string& input_file_name, void package_reader::extract_worker(thread_key thread_data_key) { + std::vector read_cache; + while (m_num_failures == 0 && !m_aborted) { // Make sure m_entry_indexer does not exceed m_install_entries @@ -941,11 +943,18 @@ void package_reader::extract_worker(thread_key thread_data_key) pkg_log.warning("NPDRM EDAT!"); } - if (fs::file out = is_buffered ? fs::make_stream>() : fs::file{ path, did_overwrite ? fs::rewrite : fs::write_new }) + if (fs::file out{ path, did_overwrite ? fs::rewrite : fs::write_new }) { bool extract_success = true; - for (u64 pos = 0; pos < entry.file_size; pos += BUF_SIZE) + + auto read_op = [&](usz pos, usz size) -> std::span { + if (pos >= entry.file_size) + { + return {}; + } + + // Because that is the length of the buffer at the moment const u64 block_size = std::min(BUF_SIZE, entry.file_size - pos); const std::span data_span = decrypt(entry.file_offset + pos, block_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data(), thread_data_key); @@ -954,29 +963,196 @@ void package_reader::extract_worker(thread_key thread_data_key) { extract_success = false; pkg_log.error("Failed to extract file %s (data_span.size=%d, block_size=%d)", path, data_span.size(), block_size); - break; } - if (out.write(data_span.data(), block_size) != block_size) + return data_span; + }; + + struct pkg_file_reader : fs::file_base + { + const std::function m_read_func; + const install_entry& m_entry; + usz m_pos; + + explicit pkg_file_reader(std::function read_func, const install_entry& entry) noexcept + : m_read_func(std::move(read_func)) + , m_entry(entry) + , m_pos(0) { - extract_success = false; - pkg_log.error("Failed to write file %s (error=%s)", path, fs::g_tls_error); - break; } - m_written_bytes += block_size; - } + fs::stat_t get_stat() override + { + fs::stat_t stat{}; + stat.size = m_entry.file_size; + return stat; + } + + bool trunc(u64) override + { + return false; + } + + u64 read(void* buffer, u64 size) override + { + const u64 result = pkg_file_reader::read_at(m_pos, buffer, size); + m_pos += result; + return result; + } + + u64 read_at(u64 offset, void* buffer, u64 size) override + { + return m_read_func(offset, buffer, size); + } + + u64 write(const void*, u64) override + { + return 0; + } + + u64 seek(s64 offset, fs::seek_mode whence) override + { + const s64 new_pos = + whence == fs::seek_set ? offset : + whence == fs::seek_cur ? offset + m_pos : + whence == fs::seek_end ? offset + size() : -1; + + if (new_pos < 0) + { + fs::g_tls_error = fs::error::inval; + return -1; + } + + m_pos = new_pos; + return m_pos; + } + + u64 size() override + { + return m_entry.file_size; + } + + fs::file_id get_id() override + { + fs::file_id id{}; + + id.type.insert(0, "pkg_file_reader: "sv); + return id; + } + }; + + read_cache.clear(); + + auto reader = std::make_unique([&, cache_off = u64{umax}](usz pos, void* ptr, usz size) mutable -> u64 + { + if (pos >= entry.file_size || !size) + { + return 0; + } + + size = std::min(entry.file_size - pos, size); + + u64 size_cache_end = 0; + u64 read_size = 0; + + // Check if exists in cache + if (!read_cache.empty() && cache_off <= pos && pos < cache_off + read_cache.size()) + { + read_size = std::min(pos + size, cache_off + read_cache.size()) - pos; + + std::memcpy(ptr, read_cache.data() + (pos - cache_off), read_size); + pos += read_size; + } + else if (!read_cache.empty() && cache_off < pos + size && cache_off + read_cache.size() >= pos + size) + { + size_cache_end = size - (std::max(cache_off, pos) - pos); + + std::memcpy(static_cast(ptr) + (cache_off - pos), read_cache.data(), size_cache_end); + size -= size_cache_end; + } + + if (pos >= entry.file_size || !size) + { + return read_size + size_cache_end; + } + + // Try to cache for later + if (size <= BUF_SIZE && !size_cache_end && !read_size) + { + const u64 block_size = std::min({BUF_SIZE, std::max(size * 5 / 3, 65536), entry.file_size - pos}); + + read_cache.resize(block_size); + cache_off = pos; + + const std::span data_span = decrypt(entry.file_offset + pos, block_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data(), thread_data_key); + + if (data_span.empty()) + { + cache_off = umax; + read_cache.clear(); + return 0; + } + + read_cache.resize(data_span.size()); + std::memcpy(read_cache.data(), data_span.data(), data_span.size()); + + size = std::min(data_span.size(), size); + std::memcpy(ptr, data_span.data(), size); + return size; + } + + while (read_size < size) + { + const u64 block_size = std::min(BUF_SIZE, size - read_size); + + const std::span data_span = decrypt(entry.file_offset + pos, block_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data(), thread_data_key); + + if (data_span.empty()) + { + break; + } + + std::memcpy(static_cast(ptr) + read_size, data_span.data(), data_span.size()); + + read_size += data_span.size(); + pos += data_span.size(); + } + + return read_size + size_cache_end; + }, entry); + + fs::file in_data; + in_data.reset(std::move(reader)); + + fs::file final_data; if (is_buffered) { - out = DecryptEDAT(out, name, 1, reinterpret_cast(&m_header.klicensee), true); - if (!out || !fs::write_file(path, fs::rewrite, static_cast>*>(out.release().get())->obj)) - { - m_num_failures++; - pkg_log.error("Failed to create file %s (error=%s)", path, fs::g_tls_error); - break; - } + final_data = DecryptEDAT(in_data, name, 1, reinterpret_cast(&m_header.klicensee), true); } + else + { + final_data = std::move(in_data); + } + + if (!final_data) + { + m_num_failures++; + pkg_log.error("Failed to decrypt EDAT file %s (error=%s)", path, fs::g_tls_error); + break; + } + + // 16MB buffer + std::vector buffer(std::min(entry.file_size, 1u << 24)); + + while (usz read_size = final_data.read(buffer.data(), buffer.size())) + { + out.write(buffer.data(), read_size); + m_written_bytes += read_size; + } + + final_data.close(); + out.close(); if (extract_success) { diff --git a/rpcs3/Emu/Cell/lv2/sys_fs.cpp b/rpcs3/Emu/Cell/lv2/sys_fs.cpp index fec2e1615e..13547c3799 100644 --- a/rpcs3/Emu/Cell/lv2/sys_fs.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_fs.cpp @@ -915,7 +915,7 @@ lv2_file::open_raw_result_t lv2_file::open_raw(const std::string& local_path, s3 if (!edata_file->ReadHeader()) { // Prepare file for the next iteration - file = std::move(edata_file->edata_file); + file = std::move(edata_file->m_edata_file); continue; }