mirror/rpcs3
1
0
Fork 0
mirror of https://github.com/RPCS3/rpcs3.git synced 2024-09-29 07:21:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move PPU and shader cache

Browse Source 
New hash-based location (already used for SPU)
Bump PPU cache version, improve naming and decrease size

Remove fs::get_data_dir
Disable boot.elf cache
This commit is contained in:
Nekotekina 2019-01-13 20:06:30 +03:00
parent aefee04c4a
commit a419e98acb
7 changed files with 91 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
Utilities/File.cpp
View File

@ -1463,75 +1463,6 @@ const std::string& fs::get_cache_dir()
return s_dir; return s_dir;
} }
std::string fs::get_data_dir(const std::string& prefix, const std::string& location, const std::string& suffix)
{
static const std::string s_dir = []
{
const std::string dir = get_cache_dir() + "data/";
if (!create_path(dir))
{
return get_cache_dir();
}
return dir;
}();
std::vector<u8> buf;
buf.reserve(location.size() + 1);
// Normalize location
for (char c : location)
{
#ifdef _WIN32
if (c == '/' || c == '\\')
#else
if (c == '/')
#endif
{
if (buf.empty() || buf.back() != '/')
{
buf.push_back('/');
}
continue;
}
buf.push_back(c);
}
// Calculate hash
u8 hash[20];
sha1(buf.data(), buf.size(), hash);
// Concatenate
std::string result = fmt::format("%s%s/%016llx%08x-%s/", s_dir, prefix, reinterpret_cast<be_t<u64>&>(hash[0]), reinterpret_cast<be_t<u32>&>(hash[8]), suffix);
// Create dir if necessary
if (create_path(result))
{
// Acknowledge original location
file(result + ".location", rewrite).write(buf);
}
return result;
}
std::string fs::get_data_dir(const std::string& prefix, const std::string& path)
{
#ifdef _WIN32
const auto& delim = "/\\";
#else
const auto& delim = "/";
#endif
// Extract file name and location
const std::string& location = fs::get_parent_dir(path);
const std::size_t name_pos = path.find_first_not_of(delim, location.size());
return fs::get_data_dir(prefix, location, name_pos == -1 ? std::string{} : path.substr(name_pos));
}
void fs::remove_all(const std::string& path, bool remove_root) void fs::remove_all(const std::string& path, bool remove_root)
{ {
for (const auto& entry : dir(path)) for (const auto& entry : dir(path))

6
Utilities/File.h
View File

@ -487,12 +487,6 @@ namespace fs
// Get common cache directory // Get common cache directory
const std::string& get_cache_dir(); const std::string& get_cache_dir();
// Get data/cache directory for specified prefix and suffix
std::string get_data_dir(const std::string& prefix, const std::string& location, const std::string& suffix);
// Get data/cache directory for specified prefix and path (suffix will be filename)
std::string get_data_dir(const std::string& prefix, const std::string& path);
// Delete directory and all its contents recursively // Delete directory and all its contents recursively
void remove_all(const std::string& path, bool remove_root = true); void remove_all(const std::string& path, bool remove_root = true);

84
rpcs3/Emu/Cell/PPUThread.cpp
View File

@ -1224,23 +1224,6 @@ extern void ppu_initialize()
return; return;
} }
// New PPU cache location
_main->cache = fs::get_cache_dir() + "data/";
if (!Emu.GetTitleID().empty() && Emu.GetCat() != "1P")
{
// TODO
_main->cache += Emu.GetTitleID();
_main->cache += '/';
}
fmt::append(_main->cache, "ppu-%s-%s/", fmt::base57(_main->sha1), _main->path.substr(_main->path.find_last_of('/') + 1));
if (!fs::create_path(_main->cache))
{
fmt::throw_exception("Failed to create cache directory: %s (%s)", _main->cache, fs::g_tls_error);
}
if (Emu.IsStopped()) if (Emu.IsStopped())
{ {
return; return;
@ -1336,23 +1319,29 @@ extern void ppu_initialize(const ppu_module& info)
if (info.name.empty()) if (info.name.empty())
{ {
cache_path = Emu.GetCachePath(); cache_path = info.cache;
} }
else else
{ {
cache_path = vfs::get("/dev_flash/"); // New PPU cache location
cache_path = fs::get_cache_dir() + "cache/";
if (info.path.compare(0, cache_path.size(), cache_path) == 0) const std::string dev_flash = vfs::get("/dev_flash/");
if (info.path.compare(0, dev_flash.size(), dev_flash) != 0 && !Emu.GetTitleID().empty() && Emu.GetCat() != "1P")
{ {
// Remove prefix for dev_flash files // Add prefix for anything except dev_flash files, standalone elfs or PS1 classics
cache_path.clear(); cache_path += Emu.GetTitleID();
} cache_path += '/';
else
{
cache_path = Emu.GetTitleID();
} }
cache_path = fs::get_data_dir(cache_path, info.path); // Add PPU hash and filename
fmt::append(cache_path, "ppu-%s-%s/", fmt::base57(info.sha1), info.path.substr(info.path.find_last_of('/') + 1));
if (!fs::create_path(cache_path))
{
fmt::throw_exception("Failed to create cache directory: %s (%s)", cache_path, fs::g_tls_error);
}
} }
#ifdef LLVM_AVAILABLE #ifdef LLVM_AVAILABLE
@ -1428,7 +1417,7 @@ extern void ppu_initialize(const ppu_module& info)
{ {
auto& func = info.funcs[fpos]; auto& func = info.funcs[fpos];
if (bsize + func.size > 256 * 1024 && bsize) if (bsize + func.size > 100 * 1024 && bsize)
{ {
break; break;
} }
@ -1449,21 +1438,8 @@ extern void ppu_initialize(const ppu_module& info)
fpos++; fpos++;
} }
// Version, module name and hash: vX-liblv2.sprx-0123456789ABCDEF.obj // Compute module hash to generate (hopefully) unique object name
std::string obj_name = "v2"; std::string obj_name;
if (info.name.size())
{
obj_name += '-';
obj_name += info.name;
}
if (fstart || fpos < info.funcs.size())
{
fmt::append(obj_name, "+%06X", suffix);
}
// Compute module hash
{ {
sha1_context ctx; sha1_context ctx;
u8 output[20]; u8 output[20];
@ -1524,14 +1500,30 @@ extern void ppu_initialize(const ppu_module& info)
sha1_update(&ctx, vm::_ptr<const u8>(func.addr), func.size); sha1_update(&ctx, vm::_ptr<const u8>(func.addr), func.size);
} }
if (info.name == "liblv2.sprx" || info.name == "libsysmodule.sprx" || info.name == "libnet.sprx") if (false)
{ {
const be_t<u64> forced_upd = 3; const be_t<u64> forced_upd = 3;
sha1_update(&ctx, reinterpret_cast<const u8*>(&forced_upd), sizeof(forced_upd)); sha1_update(&ctx, reinterpret_cast<const u8*>(&forced_upd), sizeof(forced_upd));
} }
sha1_finish(&ctx, output); sha1_finish(&ctx, output);
fmt::append(obj_name, "-%016X-%s.obj", reinterpret_cast<be_t<u64>&>(output), jit_compiler::cpu(g_cfg.core.llvm_cpu));
// Settings: should be populated by settings which affect codegen (TODO)
enum class ppu_settings : u32
{
non_win32,
__bitset_enum_max
};
be_t<bs_t<ppu_settings>> settings{};
#ifndef _WIN32
settings += ppu_settings::non_win32;
#endif
// Write version, hash, CPU, settings
fmt::append(obj_name, "v1-tane-%s-%s-%s.obj", fmt::base57(output, 16), fmt::base57(settings), jit_compiler::cpu(g_cfg.core.llvm_cpu));
} }
if (Emu.IsStopped()) if (Emu.IsStopped())
@ -1579,7 +1571,7 @@ extern void ppu_initialize(const ppu_module& info)
if (!Emu.IsStopped()) if (!Emu.IsStopped())
{ {
LOG_WARNING(PPU, "LLVM: Compiling module %s", obj_name); LOG_WARNING(PPU, "LLVM: Compiling module %s%s", cache_path, obj_name);
// Use another JIT instance // Use another JIT instance
jit_compiler jit2({}, g_cfg.core.llvm_cpu); jit_compiler jit2({}, g_cfg.core.llvm_cpu);

9
rpcs3/Emu/Cell/SPURecompiler.cpp
View File

@ -10,7 +10,6 @@
#include "SPUInterpreter.h" #include "SPUInterpreter.h"
#include "SPUDisAsm.h" #include "SPUDisAsm.h"
#include "SPURecompiler.h" #include "SPURecompiler.h"
#include "PPUAnalyser.h"
#include <algorithm> #include <algorithm>
#include <mutex> #include <mutex>
#include <thread> #include <thread>
@ -86,15 +85,15 @@ void spu_cache::add(const std::vector<u32>& func)
void spu_cache::initialize() void spu_cache::initialize()
{ {
const auto _main = fxm::get<ppu_module>(); const std::string ppu_cache = Emu.PPUCache();
if (!_main || !g_cfg.core.spu_shared_runtime) if (ppu_cache.empty() || !g_cfg.core.spu_shared_runtime)
{ {
return; return;
} }
// SPU cache file (version + block size type) // SPU cache file (version + block size type)
const std::string loc = _main->cache + "spu-" + fmt::to_lower(g_cfg.core.spu_block_size.to_string()) + "-v1-tane.dat"; const std::string loc = ppu_cache + "spu-" + fmt::to_lower(g_cfg.core.spu_block_size.to_string()) + "-v1-tane.dat";
auto cache = std::make_shared<spu_cache>(loc); auto cache = std::make_shared<spu_cache>(loc);
@ -1525,7 +1524,7 @@ public:
m_map[std::vector<u32>()] = &spu_recompiler_base::dispatch; m_map[std::vector<u32>()] = &spu_recompiler_base::dispatch;
// Clear LLVM output // Clear LLVM output
m_cache_path = fxm::check_unlocked<ppu_module>()->cache; m_cache_path = Emu.PPUCache();
fs::create_dir(m_cache_path + "llvm/"); fs::create_dir(m_cache_path + "llvm/");
fs::remove_all(m_cache_path + "llvm/", false); fs::remove_all(m_cache_path + "llvm/", false);

9
rpcs3/Emu/RSX/rsx_cache.h
View File

@ -505,13 +505,16 @@ namespace rsx
, pipeline_class_name(pipeline_class) , pipeline_class_name(pipeline_class)
, m_storage(storage) , m_storage(storage)
{ {
root_path = Emu.GetCachePath() + "/shaders_cache"; if (!g_cfg.video.disable_on_disk_shader_cache)
{
root_path = Emu.PPUCache() + "shaders_cache";
}
} }
template <typename... Args> template <typename... Args>
void load(progress_dialog_helper* dlg, Args&& ...args) void load(progress_dialog_helper* dlg, Args&& ...args)
{ {
if (g_cfg.video.disable_on_disk_shader_cache || Emu.GetCachePath() == "") if (g_cfg.video.disable_on_disk_shader_cache)
{ {
return; return;
} }
@ -682,7 +685,7 @@ namespace rsx
void store(pipeline_storage_type &pipeline, RSXVertexProgram &vp, RSXFragmentProgram &fp) void store(pipeline_storage_type &pipeline, RSXVertexProgram &vp, RSXFragmentProgram &fp)
{ {
if (g_cfg.video.disable_on_disk_shader_cache || Emu.GetCachePath() == "") if (g_cfg.video.disable_on_disk_shader_cache)
{ {
return; return;
} }

66
rpcs3/Emu/System.cpp
View File

@ -451,6 +451,18 @@ const bool Emulator::SetUsr(const std::string& user)
return true; return true;
} }
std::string Emulator::PPUCache() const
{
const auto _main = fxm::check_unlocked<ppu_module>();
if (!_main || _main->cache.empty())
{
fmt::throw_exception("PPU Cache location not initialized.");
}
return _main->cache;
}
bool Emulator::BootRsxCapture(const std::string& path) bool Emulator::BootRsxCapture(const std::string& path)
{ {
if (!fs::is_file(path)) if (!fs::is_file(path))
@ -475,6 +487,7 @@ bool Emulator::BootRsxCapture(const std::string& path)
} }
Init(); Init();
g_cfg.video.disable_on_disk_shader_cache.set(true);
vm::init(); vm::init();
@ -768,25 +781,6 @@ void Emulator::Load(bool add_only)
LOG_NOTICE(LOADER, "Serial: %s", GetTitleID()); LOG_NOTICE(LOADER, "Serial: %s", GetTitleID());
LOG_NOTICE(LOADER, "Category: %s", GetCat()); LOG_NOTICE(LOADER, "Category: %s", GetCat());
// Initialize data/cache directory
if (fs::is_dir(m_path))
{
m_cache_path = fs::get_cache_dir() + "data/" + GetTitleID() + '/';
LOG_NOTICE(LOADER, "Cache: %s", GetCachePath());
}
else
{
m_cache_path = fs::get_data_dir(m_title_id, m_path);
LOG_NOTICE(LOADER, "Cache: %s", GetCachePath());
}
// Load custom config-0
if (fs::file cfg_file{m_cache_path + "/config.yml"})
{
LOG_NOTICE(LOADER, "Applying custom config: %s/config.yml", m_cache_path);
g_cfg.from_string(cfg_file.to_string());
}
// Load custom config-1 // Load custom config-1
if (fs::file cfg_file{fs::get_config_dir() + "data/" + m_title_id + "/config.yml"}) if (fs::file cfg_file{fs::get_config_dir() + "data/" + m_title_id + "/config.yml"})
{ {
@ -819,7 +813,6 @@ void Emulator::Load(bool add_only)
// Load patches from different locations // Load patches from different locations
fxm::check_unlocked<patch_engine>()->append(fs::get_config_dir() + "data/" + m_title_id + "/patch.yml"); fxm::check_unlocked<patch_engine>()->append(fs::get_config_dir() + "data/" + m_title_id + "/patch.yml");
fxm::check_unlocked<patch_engine>()->append(m_cache_path + "/patch.yml");
// Mount all devices // Mount all devices
const std::string emu_dir = GetEmuDir(); const std::string emu_dir = GetEmuDir();
@ -1064,8 +1057,6 @@ void Emulator::Load(bool add_only)
card_1_file.trunc(128 * 1024); card_1_file.trunc(128 * 1024);
fs::file card_2_file(vfs::get("/dev_hdd0/savedata/vmc/" + argv[2]), fs::write + fs::create); fs::file card_2_file(vfs::get("/dev_hdd0/savedata/vmc/" + argv[2]), fs::write + fs::create);
card_2_file.trunc(128 * 1024); card_2_file.trunc(128 * 1024);
m_cache_path = fs::get_data_dir("", vfs::get(argv[0]));
} }
else if (m_cat != "DG" && m_cat != "GD") else if (m_cat != "DG" && m_cat != "GD")
{ {
@ -1197,20 +1188,23 @@ void Emulator::Load(bool add_only)
// Check SELF header // Check SELF header
if (elf_file.size() >= 4 && elf_file.read<u32>() == "SCE\0"_u32) if (elf_file.size() >= 4 && elf_file.read<u32>() == "SCE\0"_u32)
{ {
const std::string decrypted_path = m_cache_path + "boot.elf"; const std::string decrypted_path = "boot.elf";
fs::stat_t encrypted_stat = elf_file.stat(); fs::stat_t encrypted_stat = elf_file.stat();
fs::stat_t decrypted_stat; fs::stat_t decrypted_stat;
// Check modification time and try to load decrypted ELF // Check modification time and try to load decrypted ELF
if (fs::stat(decrypted_path, decrypted_stat) && decrypted_stat.mtime == encrypted_stat.mtime) if (false && fs::stat(decrypted_path, decrypted_stat) && decrypted_stat.mtime == encrypted_stat.mtime)
{ {
elf_file.open(decrypted_path); elf_file.open(decrypted_path);
} }
// Decrypt SELF // Decrypt SELF
else if (elf_file = decrypt_self(std::move(elf_file), klic.empty() ? nullptr : klic.data())) else if (elf_file = decrypt_self(std::move(elf_file), klic.empty() ? nullptr : klic.data()))
{ {
if (fs::file elf_out{decrypted_path, fs::rewrite}) if (true)
{
}
else if (fs::file elf_out{decrypted_path, fs::rewrite})
{ {
elf_out.write(elf_file.to_vector<u8>()); elf_out.write(elf_file.to_vector<u8>());
elf_out.close(); elf_out.close();
@ -1279,6 +1273,28 @@ void Emulator::Load(bool add_only)
ppu_load_exec(ppu_exec); ppu_load_exec(ppu_exec);
const auto _main = fxm::get<ppu_module>();
_main->cache = fs::get_cache_dir() + "cache/";
if (!m_title_id.empty() && m_cat != "1P")
{
// TODO
_main->cache += Emu.GetTitleID();
_main->cache += '/';
}
fmt::append(_main->cache, "ppu-%s-%s/", fmt::base57(_main->sha1), _main->path.substr(_main->path.find_last_of('/') + 1));
if (!fs::create_path(_main->cache))
{
fmt::throw_exception("Failed to create cache directory: %s (%s)", _main->cache, fs::g_tls_error);
}
else
{
LOG_NOTICE(LOADER, "Cache: %s", _main->cache);
}
fxm::import<GSRender>(Emu.GetCallbacks().get_gs_render); // TODO: must be created in appropriate sys_rsx syscall fxm::import<GSRender>(Emu.GetCallbacks().get_gs_render); // TODO: must be created in appropriate sys_rsx syscall
fxm::import<pad_thread>(Emu.GetCallbacks().get_pad_handler); fxm::import<pad_thread>(Emu.GetCallbacks().get_pad_handler);
network_thread_init(); network_thread_init();

8
rpcs3/Emu/System.h
View File

@ -213,7 +213,6 @@ class Emulator final
atomic_t<u64> m_pause_amend_time; // increased when resumed atomic_t<u64> m_pause_amend_time; // increased when resumed
std::string m_path; std::string m_path;
std::string m_cache_path;
std::string m_title_id; std::string m_title_id;
std::string m_title; std::string m_title;
std::string m_cat; std::string m_cat;
@ -279,11 +278,6 @@ public:
return m_cat; return m_cat;
} }
const std::string& GetCachePath() const
{
return m_cache_path;
}
const std::string& GetDir() const const std::string& GetDir() const
{ {
return m_dir; return m_dir;
@ -313,6 +307,8 @@ public:
return m_pause_amend_time; return m_pause_amend_time;
} }
std::string PPUCache() const;
bool BootGame(const std::string& path, bool direct = false, bool add_only = false); bool BootGame(const std::string& path, bool direct = false, bool add_only = false);
bool BootRsxCapture(const std::string& path); bool BootRsxCapture(const std::string& path);
bool InstallPkg(const std::string& path); bool InstallPkg(const std::string& path);