From d1ee7c651f423285340a4db75e5fd2fc3d427f96 Mon Sep 17 00:00:00 2001
From: Nekotekina <nekotekina@gmail.com>
Date: Tue, 10 Nov 2020 05:56:29 +0300
Subject: [PATCH] Make memory locking optional (mlock, VirtualLock).

Fix desired locking operation (to fix "sudo" memory).
It was discovered that some systems have outdated configuration.
With too tight limit, it's almost impossible to lock anything in memory.
---
 rpcs3/Emu/Memory/vm.cpp  |  6 +++++-
 rpcs3/main.cpp           | 13 +++++++++++--
 rpcs3/util/vm.hpp        |  2 +-
 rpcs3/util/vm_native.cpp |  6 +++---
 4 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/rpcs3/Emu/Memory/vm.cpp b/rpcs3/Emu/Memory/vm.cpp
index e00b4837b0..196fe08794 100644
--- a/rpcs3/Emu/Memory/vm.cpp
+++ b/rpcs3/Emu/Memory/vm.cpp
@@ -773,7 +773,11 @@ namespace vm
 
 		perf_meter<"PAGE_LCK"_u64> perf1;
 
-		utils::memory_lock(g_base_addr + addr, size);
+		if (!utils::memory_lock(g_sudo_addr + addr, size))
+		{
+			vm_log.error("Failed to lock memory. Consider increasing your system limits.\n"
+				"addr=0x%x, size=0x%x, shm=%d, shm:[f=%d,l=%u]", addr, size, +!!shm, shm ? shm->flags() : 0, shm ? shm->info : 0);
+		}
 	}
 
 	bool page_protect(u32 addr, u32 size, u8 flags_test, u8 flags_set, u8 flags_clear)
diff --git a/rpcs3/main.cpp b/rpcs3/main.cpp
index daf63e3c69..08b973bd7d 100644
--- a/rpcs3/main.cpp
+++ b/rpcs3/main.cpp
@@ -292,7 +292,7 @@ int main(int argc, char** argv)
 #ifdef _WIN32
 	ULONG64 intro_cycles{};
 	QueryThreadCycleTime(GetCurrentThread(), &intro_cycles);
-	verify("SetProcessWorkingSetSize" HERE), SetProcessWorkingSetSize(GetCurrentProcess(), 0x60000000, 0x80000000); // 2GiB
+	SetProcessWorkingSetSize(GetCurrentProcess(), 0x60000000, 0x80000000); // 2GiB
 #elif defined(RUSAGE_THREAD)
 	struct ::rusage intro_stats{};
 	::getrusage(RUSAGE_THREAD, &intro_stats);
@@ -413,8 +413,17 @@ int main(int argc, char** argv)
 	struct ::rlimit rlim;
 	rlim.rlim_cur = 4096;
 	rlim.rlim_max = 4096;
+#ifdef RLIMIT_NOFILE
 	if (::setrlimit(RLIMIT_NOFILE, &rlim) != 0)
-		std::fprintf(stderr, "Failed to set max open file limit (4096).");
+		std::fprintf(stderr, "Failed to set max open file limit (4096).\n");
+#endif
+
+	rlim.rlim_cur = 0x80000000;
+	rlim.rlim_max = 0x80000000;
+#ifdef RLIMIT_MEMLOCK
+	if (::setrlimit(RLIMIT_MEMLOCK, &rlim) != 0)
+		std::fprintf(stderr, "Failed to set RLIMIT_MEMLOCK size to 2 GiB. Try to update your system configuration.\n");
+#endif
 	// Work around crash on startup on KDE: https://bugs.kde.org/show_bug.cgi?id=401637
 	setenv( "KDE_DEBUG", "1", 0 );
 #endif
diff --git a/rpcs3/util/vm.hpp b/rpcs3/util/vm.hpp
index 689eded128..3941a0600d 100644
--- a/rpcs3/util/vm.hpp
+++ b/rpcs3/util/vm.hpp
@@ -40,7 +40,7 @@ namespace utils
 	void memory_protect(void* pointer, std::size_t size, protection prot);
 
 	// Lock pages in memory
-	void memory_lock(void* pointer, std::size_t size);
+	bool memory_lock(void* pointer, std::size_t size);
 
 	// Shared memory handle
 	class shm
diff --git a/rpcs3/util/vm_native.cpp b/rpcs3/util/vm_native.cpp
index e2117d0613..4fe925545c 100644
--- a/rpcs3/util/vm_native.cpp
+++ b/rpcs3/util/vm_native.cpp
@@ -186,12 +186,12 @@ namespace utils
 #endif
 	}
 
-	void memory_lock(void* pointer, std::size_t size)
+	bool memory_lock(void* pointer, std::size_t size)
 	{
 #ifdef _WIN32
-		verify("VirtualLock" HERE), ::VirtualLock(pointer, size);
+		return ::VirtualLock(pointer, size);
 #else
-		verify("mlock" HERE), !::mlock(pointer, size);
+		return !::mlock(pointer, size);
 #endif
 	}