From a419e98acb74d3650bdbb946025351d51a23345b Mon Sep 17 00:00:00 2001 From: Nekotekina Date: Sun, 13 Jan 2019 20:06:30 +0300 Subject: [PATCH] Move PPU and shader cache New hash-based location (already used for SPU) Bump PPU cache version, improve naming and decrease size Remove fs::get_data_dir Disable boot.elf cache --- Utilities/File.cpp | 69 -------------------------- Utilities/File.h | 6 --- rpcs3/Emu/Cell/PPUThread.cpp | 84 +++++++++++++++----------------- rpcs3/Emu/Cell/SPURecompiler.cpp | 9 ++-- rpcs3/Emu/RSX/rsx_cache.h | 9 ++-- rpcs3/Emu/System.cpp | 66 +++++++++++++++---------- rpcs3/Emu/System.h | 8 +-- 7 files changed, 91 insertions(+), 160 deletions(-) diff --git a/Utilities/File.cpp b/Utilities/File.cpp index e2ea1fb2de..745fb35ac1 100644 --- a/Utilities/File.cpp +++ b/Utilities/File.cpp @@ -1463,75 +1463,6 @@ const std::string& fs::get_cache_dir() return s_dir; } -std::string fs::get_data_dir(const std::string& prefix, const std::string& location, const std::string& suffix) -{ - static const std::string s_dir = [] - { - const std::string dir = get_cache_dir() + "data/"; - - if (!create_path(dir)) - { - return get_cache_dir(); - } - - return dir; - }(); - - std::vector buf; - buf.reserve(location.size() + 1); - - // Normalize location - for (char c : location) - { -#ifdef _WIN32 - if (c == '/' || c == '\\') -#else - if (c == '/') -#endif - { - if (buf.empty() || buf.back() != '/') - { - buf.push_back('/'); - } - - continue; - } - - buf.push_back(c); - } - - // Calculate hash - u8 hash[20]; - sha1(buf.data(), buf.size(), hash); - - // Concatenate - std::string result = fmt::format("%s%s/%016llx%08x-%s/", s_dir, prefix, reinterpret_cast&>(hash[0]), reinterpret_cast&>(hash[8]), suffix); - - // Create dir if necessary - if (create_path(result)) - { - // Acknowledge original location - file(result + ".location", rewrite).write(buf); - } - - return result; -} - -std::string fs::get_data_dir(const std::string& prefix, const std::string& path) -{ -#ifdef _WIN32 - const auto& delim = "/\\"; -#else - const auto& delim = "/"; -#endif - - // Extract file name and location - const std::string& location = fs::get_parent_dir(path); - const std::size_t name_pos = path.find_first_not_of(delim, location.size()); - - return fs::get_data_dir(prefix, location, name_pos == -1 ? std::string{} : path.substr(name_pos)); -} - void fs::remove_all(const std::string& path, bool remove_root) { for (const auto& entry : dir(path)) diff --git a/Utilities/File.h b/Utilities/File.h index 792b7ad22b..652b2cf4d8 100644 --- a/Utilities/File.h +++ b/Utilities/File.h @@ -487,12 +487,6 @@ namespace fs // Get common cache directory const std::string& get_cache_dir(); - // Get data/cache directory for specified prefix and suffix - std::string get_data_dir(const std::string& prefix, const std::string& location, const std::string& suffix); - - // Get data/cache directory for specified prefix and path (suffix will be filename) - std::string get_data_dir(const std::string& prefix, const std::string& path); - // Delete directory and all its contents recursively void remove_all(const std::string& path, bool remove_root = true); diff --git a/rpcs3/Emu/Cell/PPUThread.cpp b/rpcs3/Emu/Cell/PPUThread.cpp index d6883151e7..30d1a27ef5 100644 --- a/rpcs3/Emu/Cell/PPUThread.cpp +++ b/rpcs3/Emu/Cell/PPUThread.cpp @@ -1224,23 +1224,6 @@ extern void ppu_initialize() return; } - // New PPU cache location - _main->cache = fs::get_cache_dir() + "data/"; - - if (!Emu.GetTitleID().empty() && Emu.GetCat() != "1P") - { - // TODO - _main->cache += Emu.GetTitleID(); - _main->cache += '/'; - } - - fmt::append(_main->cache, "ppu-%s-%s/", fmt::base57(_main->sha1), _main->path.substr(_main->path.find_last_of('/') + 1)); - - if (!fs::create_path(_main->cache)) - { - fmt::throw_exception("Failed to create cache directory: %s (%s)", _main->cache, fs::g_tls_error); - } - if (Emu.IsStopped()) { return; @@ -1336,23 +1319,29 @@ extern void ppu_initialize(const ppu_module& info) if (info.name.empty()) { - cache_path = Emu.GetCachePath(); + cache_path = info.cache; } else { - cache_path = vfs::get("/dev_flash/"); + // New PPU cache location + cache_path = fs::get_cache_dir() + "cache/"; - if (info.path.compare(0, cache_path.size(), cache_path) == 0) + const std::string dev_flash = vfs::get("/dev_flash/"); + + if (info.path.compare(0, dev_flash.size(), dev_flash) != 0 && !Emu.GetTitleID().empty() && Emu.GetCat() != "1P") { - // Remove prefix for dev_flash files - cache_path.clear(); - } - else - { - cache_path = Emu.GetTitleID(); + // Add prefix for anything except dev_flash files, standalone elfs or PS1 classics + cache_path += Emu.GetTitleID(); + cache_path += '/'; } - cache_path = fs::get_data_dir(cache_path, info.path); + // Add PPU hash and filename + fmt::append(cache_path, "ppu-%s-%s/", fmt::base57(info.sha1), info.path.substr(info.path.find_last_of('/') + 1)); + + if (!fs::create_path(cache_path)) + { + fmt::throw_exception("Failed to create cache directory: %s (%s)", cache_path, fs::g_tls_error); + } } #ifdef LLVM_AVAILABLE @@ -1428,7 +1417,7 @@ extern void ppu_initialize(const ppu_module& info) { auto& func = info.funcs[fpos]; - if (bsize + func.size > 256 * 1024 && bsize) + if (bsize + func.size > 100 * 1024 && bsize) { break; } @@ -1449,21 +1438,8 @@ extern void ppu_initialize(const ppu_module& info) fpos++; } - // Version, module name and hash: vX-liblv2.sprx-0123456789ABCDEF.obj - std::string obj_name = "v2"; - - if (info.name.size()) - { - obj_name += '-'; - obj_name += info.name; - } - - if (fstart || fpos < info.funcs.size()) - { - fmt::append(obj_name, "+%06X", suffix); - } - - // Compute module hash + // Compute module hash to generate (hopefully) unique object name + std::string obj_name; { sha1_context ctx; u8 output[20]; @@ -1524,14 +1500,30 @@ extern void ppu_initialize(const ppu_module& info) sha1_update(&ctx, vm::_ptr(func.addr), func.size); } - if (info.name == "liblv2.sprx" || info.name == "libsysmodule.sprx" || info.name == "libnet.sprx") + if (false) { const be_t forced_upd = 3; sha1_update(&ctx, reinterpret_cast(&forced_upd), sizeof(forced_upd)); } sha1_finish(&ctx, output); - fmt::append(obj_name, "-%016X-%s.obj", reinterpret_cast&>(output), jit_compiler::cpu(g_cfg.core.llvm_cpu)); + + // Settings: should be populated by settings which affect codegen (TODO) + enum class ppu_settings : u32 + { + non_win32, + + __bitset_enum_max + }; + + be_t> settings{}; + +#ifndef _WIN32 + settings += ppu_settings::non_win32; +#endif + + // Write version, hash, CPU, settings + fmt::append(obj_name, "v1-tane-%s-%s-%s.obj", fmt::base57(output, 16), fmt::base57(settings), jit_compiler::cpu(g_cfg.core.llvm_cpu)); } if (Emu.IsStopped()) @@ -1579,7 +1571,7 @@ extern void ppu_initialize(const ppu_module& info) if (!Emu.IsStopped()) { - LOG_WARNING(PPU, "LLVM: Compiling module %s", obj_name); + LOG_WARNING(PPU, "LLVM: Compiling module %s%s", cache_path, obj_name); // Use another JIT instance jit_compiler jit2({}, g_cfg.core.llvm_cpu); diff --git a/rpcs3/Emu/Cell/SPURecompiler.cpp b/rpcs3/Emu/Cell/SPURecompiler.cpp index b6445c7552..2e1ccb7ee5 100644 --- a/rpcs3/Emu/Cell/SPURecompiler.cpp +++ b/rpcs3/Emu/Cell/SPURecompiler.cpp @@ -10,7 +10,6 @@ #include "SPUInterpreter.h" #include "SPUDisAsm.h" #include "SPURecompiler.h" -#include "PPUAnalyser.h" #include #include #include @@ -86,15 +85,15 @@ void spu_cache::add(const std::vector& func) void spu_cache::initialize() { - const auto _main = fxm::get(); + const std::string ppu_cache = Emu.PPUCache(); - if (!_main || !g_cfg.core.spu_shared_runtime) + if (ppu_cache.empty() || !g_cfg.core.spu_shared_runtime) { return; } // SPU cache file (version + block size type) - const std::string loc = _main->cache + "spu-" + fmt::to_lower(g_cfg.core.spu_block_size.to_string()) + "-v1-tane.dat"; + const std::string loc = ppu_cache + "spu-" + fmt::to_lower(g_cfg.core.spu_block_size.to_string()) + "-v1-tane.dat"; auto cache = std::make_shared(loc); @@ -1525,7 +1524,7 @@ public: m_map[std::vector()] = &spu_recompiler_base::dispatch; // Clear LLVM output - m_cache_path = fxm::check_unlocked()->cache; + m_cache_path = Emu.PPUCache(); fs::create_dir(m_cache_path + "llvm/"); fs::remove_all(m_cache_path + "llvm/", false); diff --git a/rpcs3/Emu/RSX/rsx_cache.h b/rpcs3/Emu/RSX/rsx_cache.h index 52509f40b5..c94ba4a54b 100644 --- a/rpcs3/Emu/RSX/rsx_cache.h +++ b/rpcs3/Emu/RSX/rsx_cache.h @@ -505,13 +505,16 @@ namespace rsx , pipeline_class_name(pipeline_class) , m_storage(storage) { - root_path = Emu.GetCachePath() + "/shaders_cache"; + if (!g_cfg.video.disable_on_disk_shader_cache) + { + root_path = Emu.PPUCache() + "shaders_cache"; + } } template void load(progress_dialog_helper* dlg, Args&& ...args) { - if (g_cfg.video.disable_on_disk_shader_cache || Emu.GetCachePath() == "") + if (g_cfg.video.disable_on_disk_shader_cache) { return; } @@ -682,7 +685,7 @@ namespace rsx void store(pipeline_storage_type &pipeline, RSXVertexProgram &vp, RSXFragmentProgram &fp) { - if (g_cfg.video.disable_on_disk_shader_cache || Emu.GetCachePath() == "") + if (g_cfg.video.disable_on_disk_shader_cache) { return; } diff --git a/rpcs3/Emu/System.cpp b/rpcs3/Emu/System.cpp index 917cc0e8dc..b1a4aa7d73 100644 --- a/rpcs3/Emu/System.cpp +++ b/rpcs3/Emu/System.cpp @@ -451,6 +451,18 @@ const bool Emulator::SetUsr(const std::string& user) return true; } +std::string Emulator::PPUCache() const +{ + const auto _main = fxm::check_unlocked(); + + if (!_main || _main->cache.empty()) + { + fmt::throw_exception("PPU Cache location not initialized."); + } + + return _main->cache; +} + bool Emulator::BootRsxCapture(const std::string& path) { if (!fs::is_file(path)) @@ -475,6 +487,7 @@ bool Emulator::BootRsxCapture(const std::string& path) } Init(); + g_cfg.video.disable_on_disk_shader_cache.set(true); vm::init(); @@ -768,25 +781,6 @@ void Emulator::Load(bool add_only) LOG_NOTICE(LOADER, "Serial: %s", GetTitleID()); LOG_NOTICE(LOADER, "Category: %s", GetCat()); - // Initialize data/cache directory - if (fs::is_dir(m_path)) - { - m_cache_path = fs::get_cache_dir() + "data/" + GetTitleID() + '/'; - LOG_NOTICE(LOADER, "Cache: %s", GetCachePath()); - } - else - { - m_cache_path = fs::get_data_dir(m_title_id, m_path); - LOG_NOTICE(LOADER, "Cache: %s", GetCachePath()); - } - - // Load custom config-0 - if (fs::file cfg_file{m_cache_path + "/config.yml"}) - { - LOG_NOTICE(LOADER, "Applying custom config: %s/config.yml", m_cache_path); - g_cfg.from_string(cfg_file.to_string()); - } - // Load custom config-1 if (fs::file cfg_file{fs::get_config_dir() + "data/" + m_title_id + "/config.yml"}) { @@ -819,7 +813,6 @@ void Emulator::Load(bool add_only) // Load patches from different locations fxm::check_unlocked()->append(fs::get_config_dir() + "data/" + m_title_id + "/patch.yml"); - fxm::check_unlocked()->append(m_cache_path + "/patch.yml"); // Mount all devices const std::string emu_dir = GetEmuDir(); @@ -1064,8 +1057,6 @@ void Emulator::Load(bool add_only) card_1_file.trunc(128 * 1024); fs::file card_2_file(vfs::get("/dev_hdd0/savedata/vmc/" + argv[2]), fs::write + fs::create); card_2_file.trunc(128 * 1024); - - m_cache_path = fs::get_data_dir("", vfs::get(argv[0])); } else if (m_cat != "DG" && m_cat != "GD") { @@ -1197,20 +1188,23 @@ void Emulator::Load(bool add_only) // Check SELF header if (elf_file.size() >= 4 && elf_file.read() == "SCE\0"_u32) { - const std::string decrypted_path = m_cache_path + "boot.elf"; + const std::string decrypted_path = "boot.elf"; fs::stat_t encrypted_stat = elf_file.stat(); fs::stat_t decrypted_stat; // Check modification time and try to load decrypted ELF - if (fs::stat(decrypted_path, decrypted_stat) && decrypted_stat.mtime == encrypted_stat.mtime) + if (false && fs::stat(decrypted_path, decrypted_stat) && decrypted_stat.mtime == encrypted_stat.mtime) { elf_file.open(decrypted_path); } // Decrypt SELF else if (elf_file = decrypt_self(std::move(elf_file), klic.empty() ? nullptr : klic.data())) { - if (fs::file elf_out{decrypted_path, fs::rewrite}) + if (true) + { + } + else if (fs::file elf_out{decrypted_path, fs::rewrite}) { elf_out.write(elf_file.to_vector()); elf_out.close(); @@ -1279,6 +1273,28 @@ void Emulator::Load(bool add_only) ppu_load_exec(ppu_exec); + const auto _main = fxm::get(); + + _main->cache = fs::get_cache_dir() + "cache/"; + + if (!m_title_id.empty() && m_cat != "1P") + { + // TODO + _main->cache += Emu.GetTitleID(); + _main->cache += '/'; + } + + fmt::append(_main->cache, "ppu-%s-%s/", fmt::base57(_main->sha1), _main->path.substr(_main->path.find_last_of('/') + 1)); + + if (!fs::create_path(_main->cache)) + { + fmt::throw_exception("Failed to create cache directory: %s (%s)", _main->cache, fs::g_tls_error); + } + else + { + LOG_NOTICE(LOADER, "Cache: %s", _main->cache); + } + fxm::import(Emu.GetCallbacks().get_gs_render); // TODO: must be created in appropriate sys_rsx syscall fxm::import(Emu.GetCallbacks().get_pad_handler); network_thread_init(); diff --git a/rpcs3/Emu/System.h b/rpcs3/Emu/System.h index 8bf517728a..aa8872e868 100644 --- a/rpcs3/Emu/System.h +++ b/rpcs3/Emu/System.h @@ -213,7 +213,6 @@ class Emulator final atomic_t m_pause_amend_time; // increased when resumed std::string m_path; - std::string m_cache_path; std::string m_title_id; std::string m_title; std::string m_cat; @@ -279,11 +278,6 @@ public: return m_cat; } - const std::string& GetCachePath() const - { - return m_cache_path; - } - const std::string& GetDir() const { return m_dir; @@ -313,6 +307,8 @@ public: return m_pause_amend_time; } + std::string PPUCache() const; + bool BootGame(const std::string& path, bool direct = false, bool add_only = false); bool BootRsxCapture(const std::string& path); bool InstallPkg(const std::string& path);