mirror/rpcs3
1
0
Fork 0
mirror of https://github.com/RPCS3/rpcs3.git synced 2025-02-10 21:40:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move PPU and shader cache

Browse Source 
New hash-based location (already used for SPU)
Bump PPU cache version, improve naming and decrease size

Remove fs::get_data_dir
Disable boot.elf cache
This commit is contained in:
Nekotekina 2019-01-13 20:06:30 +03:00
parent aefee04c4a
commit a419e98acb
7 changed files with 91 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
Utilities/File.cpp
View File

@ -1463,75 +1463,6 @@ const std::string& fs::get_cache_dir()
return s_dir;
}
std::string fs::get_data_dir(const std::string& prefix, const std::string& location, const std::string& suffix)
{
static const std::string s_dir = []
{
const std::string dir = get_cache_dir() + "data/";
if (!create_path(dir))
{
return get_cache_dir();
}
return dir;
}();
std::vector<u8> buf;
buf.reserve(location.size() + 1);
// Normalize location
for (char c : location)
{
#ifdef _WIN32
if (c == '/' || c == '\\')
#else
if (c == '/')
#endif
{
if (buf.empty() || buf.back() != '/')
{
buf.push_back('/');
}
continue;
}
buf.push_back(c);
}
// Calculate hash
u8 hash[20];
sha1(buf.data(), buf.size(), hash);
// Concatenate
std::string result = fmt::format("%s%s/%016llx%08x-%s/", s_dir, prefix, reinterpret_cast<be_t<u64>&>(hash[0]), reinterpret_cast<be_t<u32>&>(hash[8]), suffix);
// Create dir if necessary
if (create_path(result))
{
// Acknowledge original location
file(result + ".location", rewrite).write(buf);
}
return result;
}
std::string fs::get_data_dir(const std::string& prefix, const std::string& path)
{
#ifdef _WIN32
const auto& delim = "/\\";
#else
const auto& delim = "/";
#endif
// Extract file name and location
const std::string& location = fs::get_parent_dir(path);
const std::size_t name_pos = path.find_first_not_of(delim, location.size());
return fs::get_data_dir(prefix, location, name_pos == -1 ? std::string{} : path.substr(name_pos));
}
void fs::remove_all(const std::string& path, bool remove_root)
{
for (const auto& entry : dir(path))

6
Utilities/File.h
View File

@ -487,12 +487,6 @@ namespace fs
// Get common cache directory
const std::string& get_cache_dir();
// Get data/cache directory for specified prefix and suffix
std::string get_data_dir(const std::string& prefix, const std::string& location, const std::string& suffix);
// Get data/cache directory for specified prefix and path (suffix will be filename)
std::string get_data_dir(const std::string& prefix, const std::string& path);
// Delete directory and all its contents recursively
void remove_all(const std::string& path, bool remove_root = true);

84
rpcs3/Emu/Cell/PPUThread.cpp
View File

@ -1224,23 +1224,6 @@ extern void ppu_initialize()
return;
}
// New PPU cache location
_main->cache = fs::get_cache_dir() + "data/";
if (!Emu.GetTitleID().empty() && Emu.GetCat() != "1P")
{
// TODO
_main->cache += Emu.GetTitleID();
_main->cache += '/';
}
fmt::append(_main->cache, "ppu-%s-%s/", fmt::base57(_main->sha1), _main->path.substr(_main->path.find_last_of('/') + 1));
if (!fs::create_path(_main->cache))
{
fmt::throw_exception("Failed to create cache directory: %s (%s)", _main->cache, fs::g_tls_error);
}
if (Emu.IsStopped())
{
return;
@ -1336,23 +1319,29 @@ extern void ppu_initialize(const ppu_module& info)
if (info.name.empty())
{
cache_path = Emu.GetCachePath();
cache_path = info.cache;
}
else
{
cache_path = vfs::get("/dev_flash/");
// New PPU cache location
cache_path = fs::get_cache_dir() + "cache/";
if (info.path.compare(0, cache_path.size(), cache_path) == 0)
const std::string dev_flash = vfs::get("/dev_flash/");
if (info.path.compare(0, dev_flash.size(), dev_flash) != 0 && !Emu.GetTitleID().empty() && Emu.GetCat() != "1P")
{
// Remove prefix for dev_flash files
cache_path.clear();
}
else
{
cache_path = Emu.GetTitleID();
// Add prefix for anything except dev_flash files, standalone elfs or PS1 classics
cache_path += Emu.GetTitleID();
cache_path += '/';
}
cache_path = fs::get_data_dir(cache_path, info.path);
// Add PPU hash and filename
fmt::append(cache_path, "ppu-%s-%s/", fmt::base57(info.sha1), info.path.substr(info.path.find_last_of('/') + 1));
if (!fs::create_path(cache_path))
{
fmt::throw_exception("Failed to create cache directory: %s (%s)", cache_path, fs::g_tls_error);
}
}
#ifdef LLVM_AVAILABLE
@ -1428,7 +1417,7 @@ extern void ppu_initialize(const ppu_module& info)
{
auto& func = info.funcs[fpos];
if (bsize + func.size > 256 * 1024 && bsize)
if (bsize + func.size > 100 * 1024 && bsize)
{
break;
}
@ -1449,21 +1438,8 @@ extern void ppu_initialize(const ppu_module& info)
fpos++;
}
// Version, module name and hash: vX-liblv2.sprx-0123456789ABCDEF.obj
std::string obj_name = "v2";
if (info.name.size())
{
obj_name += '-';
obj_name += info.name;
}
if (fstart || fpos < info.funcs.size())
{
fmt::append(obj_name, "+%06X", suffix);
}
// Compute module hash
// Compute module hash to generate (hopefully) unique object name
std::string obj_name;
{
sha1_context ctx;
u8 output[20];
@ -1524,14 +1500,30 @@ extern void ppu_initialize(const ppu_module& info)
sha1_update(&ctx, vm::_ptr<const u8>(func.addr), func.size);
}
if (info.name == "liblv2.sprx" || info.name == "libsysmodule.sprx" || info.name == "libnet.sprx")
if (false)
{
const be_t<u64> forced_upd = 3;
sha1_update(&ctx, reinterpret_cast<const u8*>(&forced_upd), sizeof(forced_upd));
}
sha1_finish(&ctx, output);
fmt::append(obj_name, "-%016X-%s.obj", reinterpret_cast<be_t<u64>&>(output), jit_compiler::cpu(g_cfg.core.llvm_cpu));
// Settings: should be populated by settings which affect codegen (TODO)
enum class ppu_settings : u32
{
non_win32,
__bitset_enum_max
};
be_t<bs_t<ppu_settings>> settings{};
#ifndef _WIN32
settings += ppu_settings::non_win32;
#endif
// Write version, hash, CPU, settings
fmt::append(obj_name, "v1-tane-%s-%s-%s.obj", fmt::base57(output, 16), fmt::base57(settings), jit_compiler::cpu(g_cfg.core.llvm_cpu));
}
if (Emu.IsStopped())
@ -1579,7 +1571,7 @@ extern void ppu_initialize(const ppu_module& info)
if (!Emu.IsStopped())
{
LOG_WARNING(PPU, "LLVM: Compiling module %s", obj_name);
LOG_WARNING(PPU, "LLVM: Compiling module %s%s", cache_path, obj_name);
// Use another JIT instance
jit_compiler jit2({}, g_cfg.core.llvm_cpu);

9
rpcs3/Emu/Cell/SPURecompiler.cpp
View File

@ -10,7 +10,6 @@
#include "SPUInterpreter.h"
#include "SPUDisAsm.h"
#include "SPURecompiler.h"
#include "PPUAnalyser.h"
#include <algorithm>
#include <mutex>
#include <thread>
@ -86,15 +85,15 @@ void spu_cache::add(const std::vector<u32>& func)
void spu_cache::initialize()
{
const auto _main = fxm::get<ppu_module>();
const std::string ppu_cache = Emu.PPUCache();
if (!_main || !g_cfg.core.spu_shared_runtime)
if (ppu_cache.empty() || !g_cfg.core.spu_shared_runtime)
{
return;
}
// SPU cache file (version + block size type)
const std::string loc = _main->cache + "spu-" + fmt::to_lower(g_cfg.core.spu_block_size.to_string()) + "-v1-tane.dat";
const std::string loc = ppu_cache + "spu-" + fmt::to_lower(g_cfg.core.spu_block_size.to_string()) + "-v1-tane.dat";
auto cache = std::make_shared<spu_cache>(loc);
@ -1525,7 +1524,7 @@ public:
m_map[std::vector<u32>()] = &spu_recompiler_base::dispatch;
// Clear LLVM output
m_cache_path = fxm::check_unlocked<ppu_module>()->cache;
m_cache_path = Emu.PPUCache();
fs::create_dir(m_cache_path + "llvm/");
fs::remove_all(m_cache_path + "llvm/", false);

9
rpcs3/Emu/RSX/rsx_cache.h
View File

@ -505,13 +505,16 @@ namespace rsx
, pipeline_class_name(pipeline_class)
, m_storage(storage)
{
root_path = Emu.GetCachePath() + "/shaders_cache";
if (!g_cfg.video.disable_on_disk_shader_cache)
{
root_path = Emu.PPUCache() + "shaders_cache";
}
}
template <typename... Args>
void load(progress_dialog_helper* dlg, Args&& ...args)
{
if (g_cfg.video.disable_on_disk_shader_cache || Emu.GetCachePath() == "")
if (g_cfg.video.disable_on_disk_shader_cache)
{
return;
}
@ -682,7 +685,7 @@ namespace rsx
void store(pipeline_storage_type &pipeline, RSXVertexProgram &vp, RSXFragmentProgram &fp)
{
if (g_cfg.video.disable_on_disk_shader_cache || Emu.GetCachePath() == "")
if (g_cfg.video.disable_on_disk_shader_cache)
{
return;
}

66
rpcs3/Emu/System.cpp
View File

@ -451,6 +451,18 @@ const bool Emulator::SetUsr(const std::string& user)
return true;
}
std::string Emulator::PPUCache() const
{
const auto _main = fxm::check_unlocked<ppu_module>();
if (!_main || _main->cache.empty())
{
fmt::throw_exception("PPU Cache location not initialized.");
}
return _main->cache;
}
bool Emulator::BootRsxCapture(const std::string& path)
{
if (!fs::is_file(path))
@ -475,6 +487,7 @@ bool Emulator::BootRsxCapture(const std::string& path)
}
Init();
g_cfg.video.disable_on_disk_shader_cache.set(true);
vm::init();
@ -768,25 +781,6 @@ void Emulator::Load(bool add_only)
LOG_NOTICE(LOADER, "Serial: %s", GetTitleID());
LOG_NOTICE(LOADER, "Category: %s", GetCat());
// Initialize data/cache directory
if (fs::is_dir(m_path))
{
m_cache_path = fs::get_cache_dir() + "data/" + GetTitleID() + '/';
LOG_NOTICE(LOADER, "Cache: %s", GetCachePath());
}
else
{
m_cache_path = fs::get_data_dir(m_title_id, m_path);
LOG_NOTICE(LOADER, "Cache: %s", GetCachePath());
}
// Load custom config-0
if (fs::file cfg_file{m_cache_path + "/config.yml"})
{
LOG_NOTICE(LOADER, "Applying custom config: %s/config.yml", m_cache_path);
g_cfg.from_string(cfg_file.to_string());
}
// Load custom config-1
if (fs::file cfg_file{fs::get_config_dir() + "data/" + m_title_id + "/config.yml"})
{
@ -819,7 +813,6 @@ void Emulator::Load(bool add_only)
// Load patches from different locations
fxm::check_unlocked<patch_engine>()->append(fs::get_config_dir() + "data/" + m_title_id + "/patch.yml");
fxm::check_unlocked<patch_engine>()->append(m_cache_path + "/patch.yml");
// Mount all devices
const std::string emu_dir = GetEmuDir();
@ -1064,8 +1057,6 @@ void Emulator::Load(bool add_only)
card_1_file.trunc(128 * 1024);
fs::file card_2_file(vfs::get("/dev_hdd0/savedata/vmc/" + argv[2]), fs::write + fs::create);
card_2_file.trunc(128 * 1024);
m_cache_path = fs::get_data_dir("", vfs::get(argv[0]));
}
else if (m_cat != "DG" && m_cat != "GD")
{
@ -1197,20 +1188,23 @@ void Emulator::Load(bool add_only)
// Check SELF header
if (elf_file.size() >= 4 && elf_file.read<u32>() == "SCE\0"_u32)
{
const std::string decrypted_path = m_cache_path + "boot.elf";
const std::string decrypted_path = "boot.elf";
fs::stat_t encrypted_stat = elf_file.stat();
fs::stat_t decrypted_stat;
// Check modification time and try to load decrypted ELF
if (fs::stat(decrypted_path, decrypted_stat) && decrypted_stat.mtime == encrypted_stat.mtime)
if (false && fs::stat(decrypted_path, decrypted_stat) && decrypted_stat.mtime == encrypted_stat.mtime)
{
elf_file.open(decrypted_path);
}
// Decrypt SELF
else if (elf_file = decrypt_self(std::move(elf_file), klic.empty() ? nullptr : klic.data()))
{
if (fs::file elf_out{decrypted_path, fs::rewrite})
if (true)
{
}
else if (fs::file elf_out{decrypted_path, fs::rewrite})
{
elf_out.write(elf_file.to_vector<u8>());
elf_out.close();
@ -1279,6 +1273,28 @@ void Emulator::Load(bool add_only)
ppu_load_exec(ppu_exec);
const auto _main = fxm::get<ppu_module>();
_main->cache = fs::get_cache_dir() + "cache/";
if (!m_title_id.empty() && m_cat != "1P")
{
// TODO
_main->cache += Emu.GetTitleID();
_main->cache += '/';
}
fmt::append(_main->cache, "ppu-%s-%s/", fmt::base57(_main->sha1), _main->path.substr(_main->path.find_last_of('/') + 1));
if (!fs::create_path(_main->cache))
{
fmt::throw_exception("Failed to create cache directory: %s (%s)", _main->cache, fs::g_tls_error);
}
else
{
LOG_NOTICE(LOADER, "Cache: %s", _main->cache);
}
fxm::import<GSRender>(Emu.GetCallbacks().get_gs_render); // TODO: must be created in appropriate sys_rsx syscall
fxm::import<pad_thread>(Emu.GetCallbacks().get_pad_handler);
network_thread_init();

8
rpcs3/Emu/System.h
View File

@ -213,7 +213,6 @@ class Emulator final
atomic_t<u64> m_pause_amend_time; // increased when resumed
std::string m_path;
std::string m_cache_path;
std::string m_title_id;
std::string m_title;
std::string m_cat;
@ -279,11 +278,6 @@ public:
return m_cat;
}
const std::string& GetCachePath() const
{
return m_cache_path;
}
const std::string& GetDir() const
{
return m_dir;
@ -313,6 +307,8 @@ public:
return m_pause_amend_time;
}
std::string PPUCache() const;
bool BootGame(const std::string& path, bool direct = false, bool add_only = false);
bool BootRsxCapture(const std::string& path);
bool InstallPkg(const std::string& path);