mirror/rpcs3
1
0
Fork 0
mirror of https://github.com/RPCS3/rpcs3.git synced 2025-02-06 18:40:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Crypto/PKG installer: Fix potential RAM shortage when extracing EDAT files

Browse Source
This commit is contained in:
Eladash 2024-02-02 22:28:44 +02:00 committed by Elad.Ash
parent 596e671973
commit a6c2e995af
5 changed files with 286 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
rpcs3/Crypto/decrypt_binaries.cpp
View File

@ -139,7 +139,16 @@ usz decrypt_binaries_t::decrypt(std::string klic_input)
if (fs::file new_file{new_path, fs::rewrite})
{
new_file.write(elf_file.to_string());
// 16MB buffer
std::vector<u8> buffer(std::min<usz>(elf_file.size(), 1u << 24));
elf_file.seek(0);
while (usz read_size = elf_file.read(buffer.data(), buffer.size()))
{
new_file.write(buffer.data(), read_size);
}
dec_log.success("Decrypted %s -> %s", old_path, new_path);
std::cout << "Decrypted " << old_path << " -> " << new_path << std::endl; // For CLI
m_index++;

94
rpcs3/Crypto/unedat.cpp
View File

@ -610,21 +610,25 @@ bool validate_dev_klic(const u8* klicensee, NPD_HEADER *npd)
return cmac_hash_compare(reinterpret_cast<uchar*>(&key), 0x10, dev, 0x60, npd->dev_hash, 0x10);
}
bool validate_npd_hashes(const char* file_name, const u8* klicensee, NPD_HEADER *npd, EDAT_HEADER* edat, bool verbose)
bool validate_npd_hashes(std::string_view file_name, const u8* klicensee, NPD_HEADER* npd, EDAT_HEADER* edat, bool verbose)
{
if (!file_name)
{
fmt::throw_exception("Empty filename");
}
// Ignore header validation in DEBUG data.
if (edat->flags & EDAT_DEBUG_DATA_FLAG)
{
return true;
}
const usz file_name_length = std::strlen(file_name);
const usz buf_len = 0x30 + file_name_length;
if (!validate_dev_klic(klicensee, npd))
{
return false;
}
if (file_name.empty())
{
return true;
}
const usz buf_len = 0x30 + file_name.size();
std::unique_ptr<u8[]> buf(new u8[buf_len]);
std::unique_ptr<u8[]> buf_lower(new u8[buf_len]);
@ -632,12 +636,12 @@ bool validate_npd_hashes(const char* file_name, const u8* klicensee, NPD_HEADER
// Build the title buffer (content_id + file_name).
std::memcpy(buf.get(), npd->content_id, 0x30);
std::memcpy(buf.get() + 0x30, file_name, file_name_length);
std::memcpy(buf.get() + 0x30, file_name.data(), file_name.size());
std::memcpy(buf_lower.get(), buf.get(), buf_len);
std::memcpy(buf_upper.get(), buf.get(), buf_len);
for (usz i = std::basic_string_view<u8>(buf.get() + 0x30, file_name_length).find_last_of('.'); i < buf_len; i++)
for (usz i = std::basic_string_view<u8>(buf.get() + 0x30, file_name.size()).find_last_of('.'); i < buf_len; i++)
{
const u8 c = static_cast<u8>(buf[i]);
buf_upper[i] = std::toupper(c);
@ -659,17 +663,17 @@ bool validate_npd_hashes(const char* file_name, const u8* klicensee, NPD_HEADER
edat_log.warning("NPD title hash is invalid!");
}
const bool dev_hash_result = validate_dev_klic(klicensee, npd);
return title_hash_result && dev_hash_result;
return title_hash_result;
}
void read_npd_edat_header(const fs::file* input, NPD_HEADER& NPD, EDAT_HEADER& EDAT)
{
char npd_header[0x80];
char edat_header[0x10];
input->read(npd_header, sizeof(npd_header));
input->read(edat_header, sizeof(edat_header));
char npd_header[0x80]{};
char edat_header[0x10]{};
usz pos = 0;
pos = input->read_at(pos, npd_header, sizeof(npd_header));
input->read_at(pos, edat_header, sizeof(edat_header));
std::memcpy(&NPD.magic, npd_header, 4);
NPD.version = read_from_ptr<be_t<s32>>(npd_header, 4);
@ -739,7 +743,7 @@ bool extract_all_data(const fs::file* input, const fs::file* output, const char*
}
// Perform header validation (EDAT only).
char real_file_name[CRYPTO_MAX_PATH];
char real_file_name[CRYPTO_MAX_PATH]{};
extract_file_name(input_file_name, real_file_name);
if (!validate_npd_hashes(real_file_name, devklic, &NPD, &EDAT, verbose))
{
@ -839,7 +843,7 @@ bool VerifyEDATHeaderWithKLicense(const fs::file& input, const std::string& inpu
}
// Perform header validation (EDAT only).
char real_file_name[CRYPTO_MAX_PATH];
char real_file_name[CRYPTO_MAX_PATH]{};
extract_file_name(input_file_name.c_str(), real_file_name);
if (!validate_npd_hashes(real_file_name, custom_klic, &NPD, &EDAT, false))
{
@ -915,20 +919,21 @@ fs::file DecryptEDAT(const fs::file& input, const std::string& input_file_name,
}
// Delete the bad output file if any errors arise.
fs::file output = fs::make_stream<std::vector<u8>>();
if (extract_all_data(&input, &output, input_file_name.c_str(), reinterpret_cast<uchar*>(&devklic), verbose))
auto data = std::make_unique<EDATADecrypter>(input, devklic, input_file_name, false);
if (!data->ReadHeader())
{
output.release();
return fs::file{};
}
output.seek(0);
fs::file output;
output.reset(std::move(data));
return output;
}
bool EDATADecrypter::ReadHeader()
{
edata_file.seek(0);
// Read in the NPD and EDAT/SDAT headers.
read_npd_edat_header(&edata_file, npdHeader, edatHeader);
@ -945,14 +950,45 @@ bool EDATADecrypter::ReadHeader()
}
else
{
// verify key
if (!validate_dev_klic(reinterpret_cast<uchar*>(&dec_key), &npdHeader))
// extract key from RIF
char real_file_name[CRYPTO_MAX_PATH]{};
extract_file_name(m_file_name.c_str(), real_file_name);
if (!validate_npd_hashes(real_file_name, reinterpret_cast<const u8*>(&dec_key), &npdHeader, &edatHeader, false))
{
edat_log.error("Failed validating klic");
return false;
edat_log.error("NPD hash validation failed!");
return true;
}
// Use provided dec_key
// Select EDAT key.
if (m_is_key_final)
{
// Already provided
}
// Type 3: Use supplied dec_key.
else if ((npdHeader.license & 0x3) == 0x3)
{
//
}
// Type 2: Use key from RAP file (RIF key). (also used for type 1 at the moment)
else
{
const std::string rap_path = rpcs3::utils::get_rap_file_path(npdHeader.content_id);
if (fs::file rap{rap_path}; rap && rap.size() >= sizeof(dec_key))
{
dec_key = GetEdatRifKeyFromRapFile(rap);
}
// Make sure we don't have an empty RIF key.
if (!dec_key)
{
edat_log.error("A valid RAP file is needed for this EDAT file! (license=%d)", npdHeader.license);
return true;
}
edat_log.trace("RIFKEY: %s", std::bit_cast<be_t<u128>>(dec_key));
}
}
edata_file.seek(0);

21
rpcs3/Crypto/unedat.h
View File

@ -71,7 +71,10 @@ u128 GetEdatRifKeyFromRapFile(const fs::file& rap_file);
struct EDATADecrypter final : fs::file_base
{
// file stream
fs::file edata_file;
fs::file m_edata_file;
const fs::file& edata_file;
std::string m_file_name;
bool m_is_key_final = true;
u64 file_size{0};
u32 total_blocks{0};
u64 pos{0};
@ -82,8 +85,20 @@ struct EDATADecrypter final : fs::file_base
u128 dec_key{};
public:
EDATADecrypter(fs::file&& input, u128 dec_key = {})
: edata_file(std::move(input))
EDATADecrypter(fs::file&& input, u128 dec_key = {}, std::string file_name = {}, bool is_key_final = true) noexcept
: m_edata_file(std::move(input))
, edata_file(m_edata_file)
, m_file_name(std::move(file_name))
, m_is_key_final(is_key_final)
, dec_key(dec_key)
{
}
EDATADecrypter(const fs::file& input, u128 dec_key = {}, std::string file_name = {}, bool is_key_final = true) noexcept
: m_edata_file(fs::file{})
, edata_file(input)
, m_file_name(std::move(file_name))
, m_is_key_final(is_key_final)
, dec_key(dec_key)
{
}

208
rpcs3/Crypto/unpkg.cpp
View File

@ -866,6 +866,8 @@ fs::file DecryptEDAT(const fs::file& input, const std::string& input_file_name,
void package_reader::extract_worker(thread_key thread_data_key)
{
std::vector<u8> read_cache;
while (m_num_failures == 0 && !m_aborted)
{
// Make sure m_entry_indexer does not exceed m_install_entries
@ -941,11 +943,18 @@ void package_reader::extract_worker(thread_key thread_data_key)
pkg_log.warning("NPDRM EDAT!");
}
if (fs::file out = is_buffered ? fs::make_stream<std::vector<u8>>() : fs::file{ path, did_overwrite ? fs::rewrite : fs::write_new })
if (fs::file out{ path, did_overwrite ? fs::rewrite : fs::write_new })
{
bool extract_success = true;
for (u64 pos = 0; pos < entry.file_size; pos += BUF_SIZE)
auto read_op = [&](usz pos, usz size) -> std::span<const char>
{
if (pos >= entry.file_size)
{
return {};
}
// Because that is the length of the buffer at the moment
const u64 block_size = std::min<u64>(BUF_SIZE, entry.file_size - pos);
const std::span<const char> data_span = decrypt(entry.file_offset + pos, block_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data(), thread_data_key);
@ -954,29 +963,196 @@ void package_reader::extract_worker(thread_key thread_data_key)
{
extract_success = false;
pkg_log.error("Failed to extract file %s (data_span.size=%d, block_size=%d)", path, data_span.size(), block_size);
break;
}
if (out.write(data_span.data(), block_size) != block_size)
return data_span;
};
struct pkg_file_reader : fs::file_base
{
const std::function<u64(u64, void*, u64)> m_read_func;
const install_entry& m_entry;
usz m_pos;
explicit pkg_file_reader(std::function<u64(u64, void* buffer, u64)> read_func, const install_entry& entry) noexcept
: m_read_func(std::move(read_func))
, m_entry(entry)
, m_pos(0)
{
extract_success = false;
pkg_log.error("Failed to write file %s (error=%s)", path, fs::g_tls_error);
break;
}
m_written_bytes += block_size;
}
fs::stat_t get_stat() override
{
fs::stat_t stat{};
stat.size = m_entry.file_size;
return stat;
}
bool trunc(u64) override
{
return false;
}
u64 read(void* buffer, u64 size) override
{
const u64 result = pkg_file_reader::read_at(m_pos, buffer, size);
m_pos += result;
return result;
}
u64 read_at(u64 offset, void* buffer, u64 size) override
{
return m_read_func(offset, buffer, size);
}
u64 write(const void*, u64) override
{
return 0;
}
u64 seek(s64 offset, fs::seek_mode whence) override
{
const s64 new_pos =
whence == fs::seek_set ? offset :
whence == fs::seek_cur ? offset + m_pos :
whence == fs::seek_end ? offset + size() : -1;
if (new_pos < 0)
{
fs::g_tls_error = fs::error::inval;
return -1;
}
m_pos = new_pos;
return m_pos;
}
u64 size() override
{
return m_entry.file_size;
}
fs::file_id get_id() override
{
fs::file_id id{};
id.type.insert(0, "pkg_file_reader: "sv);
return id;
}
};
read_cache.clear();
auto reader = std::make_unique<pkg_file_reader>([&, cache_off = u64{umax}](usz pos, void* ptr, usz size) mutable -> u64
{
if (pos >= entry.file_size || !size)
{
return 0;
}
size = std::min<u64>(entry.file_size - pos, size);
u64 size_cache_end = 0;
u64 read_size = 0;
// Check if exists in cache
if (!read_cache.empty() && cache_off <= pos && pos < cache_off + read_cache.size())
{
read_size = std::min<u64>(pos + size, cache_off + read_cache.size()) - pos;
std::memcpy(ptr, read_cache.data() + (pos - cache_off), read_size);
pos += read_size;
}
else if (!read_cache.empty() && cache_off < pos + size && cache_off + read_cache.size() >= pos + size)
{
size_cache_end = size - (std::max<u64>(cache_off, pos) - pos);
std::memcpy(static_cast<u8*>(ptr) + (cache_off - pos), read_cache.data(), size_cache_end);
size -= size_cache_end;
}
if (pos >= entry.file_size || !size)
{
return read_size + size_cache_end;
}
// Try to cache for later
if (size <= BUF_SIZE && !size_cache_end && !read_size)
{
const u64 block_size = std::min<u64>({BUF_SIZE, std::max<u64>(size * 5 / 3, 65536), entry.file_size - pos});
read_cache.resize(block_size);
cache_off = pos;
const std::span<const char> data_span = decrypt(entry.file_offset + pos, block_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data(), thread_data_key);
if (data_span.empty())
{
cache_off = umax;
read_cache.clear();
return 0;
}
read_cache.resize(data_span.size());
std::memcpy(read_cache.data(), data_span.data(), data_span.size());
size = std::min<usz>(data_span.size(), size);
std::memcpy(ptr, data_span.data(), size);
return size;
}
while (read_size < size)
{
const u64 block_size = std::min<u64>(BUF_SIZE, size - read_size);
const std::span<const char> data_span = decrypt(entry.file_offset + pos, block_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data(), thread_data_key);
if (data_span.empty())
{
break;
}
std::memcpy(static_cast<u8*>(ptr) + read_size, data_span.data(), data_span.size());
read_size += data_span.size();
pos += data_span.size();
}
return read_size + size_cache_end;
}, entry);
fs::file in_data;
in_data.reset(std::move(reader));
fs::file final_data;
if (is_buffered)
{
out = DecryptEDAT(out, name, 1, reinterpret_cast<u8*>(&m_header.klicensee), true);
if (!out || !fs::write_file(path, fs::rewrite, static_cast<fs::container_stream<std::vector<u8>>*>(out.release().get())->obj))
{
m_num_failures++;
pkg_log.error("Failed to create file %s (error=%s)", path, fs::g_tls_error);
break;
}
final_data = DecryptEDAT(in_data, name, 1, reinterpret_cast<u8*>(&m_header.klicensee), true);
}
else
{
final_data = std::move(in_data);
}
if (!final_data)
{
m_num_failures++;
pkg_log.error("Failed to decrypt EDAT file %s (error=%s)", path, fs::g_tls_error);
break;
}
// 16MB buffer
std::vector<u8> buffer(std::min<usz>(entry.file_size, 1u << 24));
while (usz read_size = final_data.read(buffer.data(), buffer.size()))
{
out.write(buffer.data(), read_size);
m_written_bytes += read_size;
}
final_data.close();
out.close();
if (extract_success)
{

2
rpcs3/Emu/Cell/lv2/sys_fs.cpp
View File

@ -915,7 +915,7 @@ lv2_file::open_raw_result_t lv2_file::open_raw(const std::string& local_path, s3
if (!edata_file->ReadHeader())
{
// Prepare file for the next iteration
file = std::move(edata_file->edata_file);
file = std::move(edata_file->m_edata_file);
continue;
}