Channel<> rewritten

rpcs3/Emu/Cell/SPUThread.h

@@ -1,4 +1,5 @@
 #pragma once
+#include "Emu/Memory/atomic_type.h"
 #include "PPCThread.h"
 #include "Emu/Event.h"
 #include "MFC.h"
@@ -246,181 +247,127 @@ public:
 		}
 	} m_intrtag[3];
 
-	template<size_t _max_count>
+	// limited lock-free queue, most functions are barrier-free
+	template<size_t max_count>
 	class Channel
 	{
-	public:
+		static_assert(max_count >= 1, "Invalid channel count");
-		static const size_t max_count = _max_count;
 
-	private:
+		struct ChannelData
-		union _CRT_ALIGN(8) {
+		{
-			struct {
+			u32 value;
-				volatile u32 m_index;
+			u32 is_set;
-				u32 m_value[max_count];
-			};
-			volatile u64 m_indval;
 		};
-		std::mutex m_lock;
+
+		atomic_t<ChannelData> m_data[max_count];
+		size_t m_push;
+		size_t m_pop;
 
 	public:
-		Channel()
+		__noinline Channel()
 		{
-			Init();
+			for (size_t i = 0; i < max_count; i++)
-		}
-
-		void Init()
-		{
-			m_indval = 0;
-		}
-
-		__forceinline bool Pop(u32& res)
-		{
-			if (max_count > 1)
 			{
-				std::lock_guard<std::mutex> lock(m_lock);
+				m_data[i].write_relaxed({});
-				if(!m_index) 
-				{
-					return false;
-				}
-				res = m_value[0];
-				if (max_count > 1) for (u32 i = 1; i < max_count; i++) // FIFO
-				{
-					m_value[i-1] = m_value[i];
-				}
-				m_value[max_count-1] = 0;
-				m_index--;
-				return true;
-			}
-			else
-			{ //lock-free
-				if ((m_indval & 0xffffffff) == 0)
-					return false;
-				else
-				{
-					res = (m_indval >> 32);
-					m_indval = 0;
-					return true;
-				}				
-			}
-		}
-
-		__forceinline bool Push(u32 value)
-		{
-			if (max_count > 1)
-			{
-				std::lock_guard<std::mutex> lock(m_lock);
-				if(m_index >= max_count) 
-				{
-					return false;
-				}
-				m_value[m_index++] = value;
-				return true;
-			}
-			else
-			{ //lock-free
-				if (m_indval & 0xffffffff)
-					return false;
-				else
-				{
-					const u64 new_value = ((u64)value << 32) | 1;
-					m_indval = new_value;
-					return true;
-				}
-			}
-		}
-
-		__forceinline void PushUncond(u32 value)
-		{
-			if (max_count > 1)
-			{
-				std::lock_guard<std::mutex> lock(m_lock);
-				if(m_index >= max_count) 
-					m_value[max_count-1] = value; //last message is overwritten
-				else
-					m_value[m_index++] = value;
-			}
-			else
-			{ //lock-free
-				const u64 new_value = ((u64)value << 32) | 1;
-				m_indval = new_value;
-			}
-		}
-
-		__forceinline void PushUncond_OR(u32 value)
-		{
-			if (max_count > 1)
-			{
-				std::lock_guard<std::mutex> lock(m_lock);
-				if(m_index >= max_count) 
-					m_value[max_count-1] |= value; //last message is logically ORed
-				else
-					m_value[m_index++] = value;
-			}
-			else
-			{
-				InterlockedOr(&m_indval, ((u64)value << 32) | 1);
 			}
+			m_push = 0;
+			m_pop = 0;
 		}
 
 		__forceinline void PopUncond(u32& res)
 		{
-			if (max_count > 1)
+			res = m_data[m_pop].read_relaxed().value;
-			{
+			m_data[m_pop].write_relaxed({});
-				std::lock_guard<std::mutex> lock(m_lock);
+			m_pop = (m_pop + 1) % max_count;
-				if(!m_index) 
-					res = 0; //result is undefined
-				else
-				{
-					res = m_value[--m_index];
-					m_value[m_index] = 0;
-				}
-			}
-			else
-			{ //lock-free
-				if(!m_index)
-					res = 0;
-				else
-				{
-					res = (m_indval >> 32);
-					m_indval = 0;
-				}
-			}
 		}
 
-		__forceinline u32 GetCount()
+		__forceinline bool Pop(u32& res)
 		{
-			if (max_count > 1)
+			const auto data = m_data[m_pop].read_relaxed();
+			if (data.is_set)
 			{
-				std::lock_guard<std::mutex> lock(m_lock);
+				res = data.value;
-				return m_index;
+				m_data[m_pop].write_relaxed({});
+				m_pop = (m_pop + 1) % max_count;
+				return true;
 			}
 			else
 			{
-				return m_index;
+				return false;
 			}
 		}
 
-		__forceinline u32 GetFreeCount()
+		__forceinline bool Pop_XCHG(u32& res) // not barrier-free, not tested
 		{
-			if (max_count > 1)
+			const auto data = m_data[m_pop].exchange({});
+			if (data.is_set)
 			{
-				std::lock_guard<std::mutex> lock(m_lock);
+				res = data.value;
-				return max_count - m_index;
+				m_pop = (m_pop + 1) % max_count;
+				return true;
 			}
 			else
 			{
-				return max_count - m_index;
+				return false;
 			}
 		}
 
-		void SetValue(u32 value)
+		__forceinline void PushUncond_OR(const u32 value) // not barrier-free, not tested
 		{
-			m_value[0] = value;
+			m_data[m_push]._or({ value, 1 });
+			m_push = (m_push + 1) % max_count;
 		}
 
-		u32 GetValue() const
+		__forceinline void PushUncond(const u32 value)
 		{
-			return m_value[0];
+			m_data[m_push].write_relaxed({ value, 1 });
+			m_push = (m_push + 1) % max_count;
+		}
+
+		__forceinline bool Push(const u32 value)
+		{
+			if (m_data[m_push].read_relaxed().is_set)
+			{
+				return false;
+			}
+			else
+			{
+				PushUncond(value);
+				return true;
+			}
+		}
+
+		__forceinline u32 GetCount() const
+		{
+			u32 res = 0;
+			for (size_t i = 0; i < max_count; i++)
+			{
+				res += m_data[i].read_relaxed().is_set ? 1 : 0;
+			}
+			return res;
+		}
+
+		__forceinline u32 GetFreeCount() const
+		{
+			u32 res = 0;
+			for (size_t i = 0; i < max_count; i++)
+			{
+				res += m_data[i].read_relaxed().is_set ? 0 : 1;
+			}
+			return res;
+		}
+
+		__forceinline void SetValue(const u32 value)
+		{
+			m_data[m_push].direct_op([value](ChannelData& v)
+			{
+				v.value = value;
+			});
+		}
+
+		__forceinline u32 GetValue() const
+		{
+			return m_data[m_pop].read_relaxed().value;
 		}
 	};
 

rpcs3/Emu/Memory/vm.h

@@ -205,4 +205,3 @@ namespace vm
 #include "vm_ref.h"
 #include "vm_ptr.h"
 #include "vm_var.h"
-#include "atomic_type.h"

rpcs3/Emu/SysCalls/Modules/cellSpurs.cpp

@@ -241,7 +241,7 @@ s64 spursInit(
 						{
 							if (spurs->m.wklStat1[i].read_relaxed() == 2 &&
 								spurs->m.wklG1[i].wklPriority.ToBE() != 0 &&
-								spurs->_u8[0x50 + i] & 0xf // check wklMaxCnt
+								spurs->m.wklMaxCnt[i].read_relaxed() & 0xf
 								)
 							{
 								if (spurs->m.wklReadyCount[i].read_relaxed() ||
@@ -259,7 +259,7 @@ s64 spursInit(
 						{
 							if (spurs->m.wklStat2[i].read_relaxed() == 2 &&
 								spurs->m.wklG2[i].wklPriority.ToBE() != 0 &&
-								spurs->_u8[0x50 + i] & 0xf0 // check wklMaxCnt
+								spurs->m.wklMaxCnt[i].read_relaxed() & 0xf0
 								)
 							{
 								if (spurs->m.wklReadyCount[i + 0x10].read_relaxed() ||
@@ -995,19 +995,22 @@ s32 spursAddWorkload(
 	}
 	spurs->m.wklReadyCount[wnum].write_relaxed(0);
 
-	u32 pos = ((~wnum * 8) | (wnum / 4)) & 0x1c;
-	spurs->m.wklMaxCnt[index / 4].atomic_op([pos, maxContention](be_t<u32>& v)
-	{
-		v &= ~(0xf << pos);
-		v |= (maxContention > 8 ? 8 : maxContention) << pos;
-	});
-
 	if (wnum <= 15)
 	{
+		spurs->m.wklMaxCnt[wnum].atomic_op([maxContention](u8& v)
+		{
+			v &= ~0xf;
+			v |= (maxContention > 8 ? 8 : maxContention);
+		});
 		spurs->m.wklSet1._and_not({ be_t<u16>::make(0x8000 >> index) }); // clear bit in wklFlag1
 	}
 	else
 	{
+		spurs->m.wklMaxCnt[index].atomic_op([maxContention](u8& v)
+		{
+			v &= ~0xf0;
+			v |= (maxContention > 8 ? 8 : maxContention) << 4;
+		});
 		spurs->m.wklSet2._and_not({ be_t<u16>::make(0x8000 >> index) }); // clear bit in wklFlag2
 	}
 

rpcs3/Emu/SysCalls/Modules/cellSpurs.h

@@ -275,7 +275,7 @@ struct CellSpurs
 			u8 wklA[0x10];        // 0x20
 			u8 wklB[0x10];        // 0x30
 			u8 wklMinCnt[0x10];   // 0x40
-			atomic_t<u32> wklMaxCnt[4]; // 0x50
+			atomic_t<u8> wklMaxCnt[0x10]; // 0x50
 			CellSpursWorkloadFlag wklFlag; // 0x60
 			atomic_t<u16> wklSet1; // 0x70
 			atomic_t<u8> x72;     // 0x72

rpcs3/Emu/SysCalls/lv2/sys_ppu_thread.cpp

@@ -3,6 +3,7 @@
 #include "Emu/System.h"
 #include "Emu/SysCalls/SysCalls.h"
 #include "Emu/SysCalls/Callback.h"
+#include "Emu/Memory/atomic_type.h"
 
 #include "Emu/CPU/CPUThreadManager.h"
 #include "Emu/Cell/PPUThread.h"

rpcs3/Emu/SysCalls/lv2/sys_spinlock.cpp

@@ -2,6 +2,7 @@
 #include "Emu/Memory/Memory.h"
 #include "Emu/System.h"
 #include "Emu/SysCalls/SysCalls.h"
+#include "Emu/Memory/atomic_type.h"
 
 #include "sys_spinlock.h"
 

rpcs3/Emu/SysCalls/lv2/sys_spu.cpp

@@ -589,7 +589,7 @@ s32 sys_spu_thread_read_ls(u32 id, u32 address, vm::ptr<be_t<u64>> value, u32 ty
 
 s32 sys_spu_thread_write_spu_mb(u32 id, u32 value)
 {
-	sys_spu.Log("sys_spu_thread_write_spu_mb(id=%d, value=0x%x)", id, value);
+	sys_spu.Warning("sys_spu_thread_write_spu_mb(id=%d, value=0x%x)", id, value);
 
 	CPUThread* thr = Emu.GetCPU().GetThread(id);