From 2d9929059f1d9524946ee2feb53b912bbaeda38c Mon Sep 17 00:00:00 2001
From: Eladash <elad3356p@gmail.com>
Date: Thu, 26 Aug 2021 18:14:08 +0300
Subject: [PATCH] vm: Fix an overflow at vm::alloc, fix vm::find_map (#10760)

* The statement addr += align could have overflowed resulting in either infinite loop or allocating memory outside of the region (illegal).
Add a check checking if it's the last iteration of the loop, then break without adding.

* vm::find_map condition didn't consider the size of the map to be allocated, allowing illegal occupation of [<=0xB000'0000]-0xCFFF'FFFF. (0xC000'0000-0xCFFF'FFFF is reserved for RSX)
---
 rpcs3/Emu/Memory/vm.cpp | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/rpcs3/Emu/Memory/vm.cpp b/rpcs3/Emu/Memory/vm.cpp
index fd9d31fe56..f6f9dcf0db 100644
--- a/rpcs3/Emu/Memory/vm.cpp
+++ b/rpcs3/Emu/Memory/vm.cpp
@@ -1205,15 +1205,29 @@ namespace vm
 			shm = std::make_shared<utils::shm>(size);
 		}
 
+		const u32 max = (this->addr + this->size - size) & (0 - align);
+
+		u32 addr = utils::align(this->addr, align);
+
+		if (this->addr > std::min(max, addr))
+		{
+			return 0;
+		}
+
 		vm::writer_lock lock(0);
 
 		// Search for an appropriate place (unoptimized)
-		for (u32 addr = utils::align(this->addr, align); u64{addr} + size <= u64{this->addr} + this->size; addr += align)
+		for (;; addr += align)
 		{
 			if (try_alloc(addr, pflags, size, std::move(shm)))
 			{
 				return addr + (flags & stack_guarded ? 0x1000 : 0);
 			}
+
+			if (addr == max)
+			{
+				break;
+			}
 		}
 
 		return 0;
@@ -1417,12 +1431,24 @@ namespace vm
 
 	static std::shared_ptr<block_t> _find_map(u32 size, u32 align, u64 flags)
 	{
-		for (u32 addr = utils::align<u32>(0x20000000, align); addr - 1 < 0xC0000000 - 1; addr += align)
+		const u32 max = (0xC0000000 - size) & (0 - align);
+
+		if (size > 0xC0000000 - 0x20000000 || max < 0x20000000)
+		{
+			return nullptr;
+		}
+
+		for (u32 addr = utils::align<u32>(0x20000000, align);; addr += align)
 		{
 			if (_test_map(addr, size))
 			{
 				return std::make_shared<block_t>(addr, size, flags);
 			}
+
+			if (addr == max)
+			{
+				break;
+			}
 		}
 
 		return nullptr;