vm: add "hook" memory (32G)

Implement overcommit emulation for shm.

rpcs3/Emu/Memory/vm.cpp

@@ -44,8 +44,11 @@ namespace vm
 	// Auxiliary virtual memory for executable areas
 	u8* const g_exec_addr = memory_reserve_4GiB(g_sudo_addr, 0x200000000);
 
+	// Hooks for memory R/W interception (default: zero offset to some function with only ret instructions)
+	u8* const g_hook_addr = memory_reserve_4GiB(g_exec_addr, 0x800000000);
+
 	// Stats for debugging
-	u8* const g_stat_addr = memory_reserve_4GiB(g_exec_addr);
+	u8* const g_stat_addr = memory_reserve_4GiB(g_hook_addr);
 
 	// For SPU
 	u8* const g_free_addr = g_stat_addr + 0x1'0000'0000;
@@ -1125,7 +1128,7 @@ namespace vm
 		if (flags & page_size_4k || flags & preallocated)
 		{
 			// Special path for whole-allocated areas allowing 4k granularity
-			m_common = std::make_shared<utils::shm>(size);
+			m_common = std::make_shared<utils::shm>(size, fs::get_cache_dir() + std::to_string(utils::get_unique_tsc()));
 			m_common->map_critical(vm::base(addr), utils::protection::no);
 			m_common->map_critical(vm::get_super_ptr(addr));
 			lock_sudo(addr, size);
@@ -1630,17 +1633,21 @@ namespace vm
 
 	inline namespace ps3_
 	{
+		static utils::shm s_hook{0x800000000, fs::get_cache_dir() + "hook.dat"};
+
 		void init()
 		{
 			vm_log.notice("Guest memory bases address ranges:\n"
 			"vm::g_base_addr = %p - %p\n"
 			"vm::g_sudo_addr = %p - %p\n"
 			"vm::g_exec_addr = %p - %p\n"
+			"vm::g_hook_addr = %p - %p\n"
 			"vm::g_stat_addr = %p - %p\n"
 			"vm::g_reservations = %p - %p\n",
 			g_base_addr, g_base_addr + UINT32_MAX,
 			g_sudo_addr, g_sudo_addr + UINT32_MAX,
 			g_exec_addr, g_exec_addr + 0x200000000 - 1,
+			g_hook_addr, g_hook_addr + 0x800000000 - 1,
 			g_stat_addr, g_stat_addr + UINT32_MAX,
 			g_reservations, g_reservations + sizeof(g_reservations) - 1);
 
@@ -1661,6 +1668,11 @@ namespace vm
 			std::memset(g_shmem, 0, sizeof(g_shmem));
 			std::memset(g_range_lock_set, 0, sizeof(g_range_lock_set));
 			g_range_lock_bits = 0;
+
+#ifdef _WIN32
+			utils::memory_release(g_hook_addr, 0x800000000);
+#endif
+			ensure(s_hook.map(g_hook_addr, utils::protection::rw, true));
 		}
 	}
 
@@ -1672,6 +1684,13 @@ namespace vm
 		utils::memory_decommit(g_exec_addr, 0x200000000);
 		utils::memory_decommit(g_stat_addr, 0x100000000);
 
+#ifdef _WIN32
+		s_hook.unmap(g_hook_addr);
+		ensure(utils::memory_reserve(0x800000000, g_hook_addr));
+#else
+		utils::memory_decommit(g_hook_addr, 0x800000000);
+#endif
+
 		std::memset(g_range_lock_set, 0, sizeof(g_range_lock_set));
 		g_range_lock_bits = 0;
 	}

rpcs3/util/vm.hpp

@@ -51,13 +51,16 @@ namespace utils
 #else
 		int m_file{};
 #endif
-		u32 m_size{};
 		u32 m_flags{};
+		u64 m_size{};
 		atomic_t<void*> m_ptr{nullptr};
 
 	public:
 		explicit shm(u32 size, u32 flags = 0);
 
+		// Construct with specified path as sparse file storage
+		shm(u64 size, const std::string& storage);
+
 		shm(const shm&) = delete;
 
 		shm& operator=(const shm&) = delete;
@@ -91,7 +94,7 @@ namespace utils
 			return static_cast<u8*>(+m_ptr);
 		}
 
-		u32 size() const
+		u64 size() const
 		{
 			return m_size;
 		}

rpcs3/util/vm_native.cpp

@@ -3,6 +3,7 @@
 #include "util/vm.hpp"
 #include "util/asm.hpp"
 #ifdef _WIN32
+#include "Utilities/File.h"
 #include "util/dyn_lib.hpp"
 #include <Windows.h>
 #else
@@ -283,12 +284,11 @@ namespace utils
 	}
 
 	shm::shm(u32 size, u32 flags)
-		: m_size(utils::align(size, 0x10000))
+		: m_flags(flags)
-		, m_flags(flags)
+		, m_size(utils::align(size, 0x10000))
 	{
 #ifdef _WIN32
-		m_handle = ::CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_EXECUTE_READWRITE, 0, m_size, nullptr);
+		m_handle = ensure(::CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_EXECUTE_READWRITE, 0, m_size, nullptr));
-		ensure(m_handle != INVALID_HANDLE_VALUE);
 #elif __linux__
 		m_file = -1;
 
@@ -326,6 +326,24 @@ namespace utils
 #endif
 	}
 
+	shm::shm(u64 size, const std::string& storage)
+		: m_size(utils::align(size, 0x10000))
+	{
+#ifdef _WIN32
+		fs::file f = ensure(fs::file(storage, fs::read + fs::rewrite));
+		FILE_DISPOSITION_INFO disp{ .DeleteFileW = true };
+		ensure(SetFileInformationByHandle(f.get_handle(), FileDispositionInfo, &disp, sizeof(disp)));
+		ensure(DeviceIoControl(f.get_handle(), FSCTL_SET_SPARSE, nullptr, 0, nullptr, 0, nullptr, nullptr));
+		ensure(f.trunc(m_size));
+		m_handle = ensure(::CreateFileMappingW(f.get_handle(), nullptr, PAGE_READWRITE, 0, 0, nullptr));
+#else
+		m_file = ::open(storage.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IWUSR | S_IRUSR);
+		ensure(m_file >= 0);
+		ensure(::ftruncate(m_file, m_size) >= 0);
+		::unlink(storage.c_str());
+#endif
+	}
+
 	shm::~shm()
 	{
 		this->unmap_self();