Make memory locking optional (mlock, VirtualLock).

Fix desired locking operation (to fix "sudo" memory).
It was discovered that some systems have outdated configuration.
With too tight limit, it's almost impossible to lock anything in memory.

rpcs3/Emu/Memory/vm.cpp

@@ -773,7 +773,11 @@ namespace vm
 
 		perf_meter<"PAGE_LCK"_u64> perf1;
 
-		utils::memory_lock(g_base_addr + addr, size);
+		if (!utils::memory_lock(g_sudo_addr + addr, size))
+		{
+			vm_log.error("Failed to lock memory. Consider increasing your system limits.\n"
+				"addr=0x%x, size=0x%x, shm=%d, shm:[f=%d,l=%u]", addr, size, +!!shm, shm ? shm->flags() : 0, shm ? shm->info : 0);
+		}
 	}
 
 	bool page_protect(u32 addr, u32 size, u8 flags_test, u8 flags_set, u8 flags_clear)

rpcs3/main.cpp

@@ -292,7 +292,7 @@ int main(int argc, char** argv)
 #ifdef _WIN32
 	ULONG64 intro_cycles{};
 	QueryThreadCycleTime(GetCurrentThread(), &intro_cycles);
-	verify("SetProcessWorkingSetSize" HERE), SetProcessWorkingSetSize(GetCurrentProcess(), 0x60000000, 0x80000000); // 2GiB
+	SetProcessWorkingSetSize(GetCurrentProcess(), 0x60000000, 0x80000000); // 2GiB
 #elif defined(RUSAGE_THREAD)
 	struct ::rusage intro_stats{};
 	::getrusage(RUSAGE_THREAD, &intro_stats);
@@ -413,8 +413,17 @@ int main(int argc, char** argv)
 	struct ::rlimit rlim;
 	rlim.rlim_cur = 4096;
 	rlim.rlim_max = 4096;
+#ifdef RLIMIT_NOFILE
 	if (::setrlimit(RLIMIT_NOFILE, &rlim) != 0)
-		std::fprintf(stderr, "Failed to set max open file limit (4096).");
+		std::fprintf(stderr, "Failed to set max open file limit (4096).\n");
+#endif
+
+	rlim.rlim_cur = 0x80000000;
+	rlim.rlim_max = 0x80000000;
+#ifdef RLIMIT_MEMLOCK
+	if (::setrlimit(RLIMIT_MEMLOCK, &rlim) != 0)
+		std::fprintf(stderr, "Failed to set RLIMIT_MEMLOCK size to 2 GiB. Try to update your system configuration.\n");
+#endif
 	// Work around crash on startup on KDE: https://bugs.kde.org/show_bug.cgi?id=401637
 	setenv( "KDE_DEBUG", "1", 0 );
 #endif

rpcs3/util/vm.hpp

@@ -40,7 +40,7 @@ namespace utils
 	void memory_protect(void* pointer, std::size_t size, protection prot);
 
 	// Lock pages in memory
-	void memory_lock(void* pointer, std::size_t size);
+	bool memory_lock(void* pointer, std::size_t size);
 
 	// Shared memory handle
 	class shm

rpcs3/util/vm_native.cpp

@@ -186,12 +186,12 @@ namespace utils
 #endif
 	}
 
-	void memory_lock(void* pointer, std::size_t size)
+	bool memory_lock(void* pointer, std::size_t size)
 	{
 #ifdef _WIN32
-		verify("VirtualLock" HERE), ::VirtualLock(pointer, size);
+		return ::VirtualLock(pointer, size);
 #else
-		verify("mlock" HERE), !::mlock(pointer, size);
+		return !::mlock(pointer, size);
 #endif
 	}