Implement atomic_t<>::release

More relaxed store with release memory order

Utilities/Atomic.h

@@ -4,7 +4,7 @@
 #include <functional>
 
 // Helper class, provides access to compiler-specific atomic intrinsics
-template<typename T, std::size_t Size = sizeof(T)>
+template <typename T, std::size_t Size = sizeof(T)>
 struct atomic_storage
 {
 	static_assert(sizeof(T) <= 16 && sizeof(T) == alignof(T), "atomic_storage<> error: invalid type");
@@ -29,6 +29,11 @@ struct atomic_storage
 		__atomic_store(&dest, &value, __ATOMIC_SEQ_CST);
 	}
 
+	static inline void release(T& dest, T value)
+	{
+		__atomic_store(&dest, &value, __ATOMIC_RELEASE);
+	}
+
 	static inline T exchange(T& dest, T value)
 	{
 		T result;
@@ -176,7 +181,7 @@ struct atomic_storage
 
 /* The rest: ugly MSVC intrinsics + inline asm implementations */
 
-template<typename T>
+template <typename T>
 struct atomic_storage<T, 1> : atomic_storage<T, 0>
 {
 #ifdef _MSC_VER
@@ -200,6 +205,12 @@ struct atomic_storage<T, 1> : atomic_storage<T, 0>
 		_InterlockedExchange8((volatile char*)&dest, (char&)value);
 	}
 
+	static inline void release(T& dest, T value)
+	{
+		_ReadWriteBarrier();
+		*(volatile char*)&dest = (char&)value;
+	}
+
 	static inline T exchange(T& dest, T value)
 	{
 		char r = _InterlockedExchange8((volatile char*)&dest, (char&)value);
@@ -232,7 +243,7 @@ struct atomic_storage<T, 1> : atomic_storage<T, 0>
 #endif
 };
 
-template<typename T>
+template <typename T>
 struct atomic_storage<T, 2> : atomic_storage<T, 0>
 {
 #ifdef _MSC_VER
@@ -256,6 +267,12 @@ struct atomic_storage<T, 2> : atomic_storage<T, 0>
 		_InterlockedExchange16((volatile short*)&dest, (short&)value);
 	}
 
+	static inline void release(T& dest, T value)
+	{
+		_ReadWriteBarrier();
+		*(volatile short*)&dest = (short&)value;
+	}
+
 	static inline T exchange(T& dest, T value)
 	{
 		short r = _InterlockedExchange16((volatile short*)&dest, (short&)value);
@@ -324,7 +341,7 @@ struct atomic_storage<T, 2> : atomic_storage<T, 0>
 #endif
 };
 
-template<typename T>
+template <typename T>
 struct atomic_storage<T, 4> : atomic_storage<T, 0>
 {
 #ifdef _MSC_VER
@@ -348,6 +365,12 @@ struct atomic_storage<T, 4> : atomic_storage<T, 0>
 		_InterlockedExchange((volatile long*)&dest, (long&)value);
 	}
 
+	static inline void release(T& dest, T value)
+	{
+		_ReadWriteBarrier();
+		*(volatile long*)&dest = (long&)value;
+	}
+
 	static inline T exchange(T& dest, T value)
 	{
 		long r = _InterlockedExchange((volatile long*)&dest, (long&)value);
@@ -423,7 +446,7 @@ struct atomic_storage<T, 4> : atomic_storage<T, 0>
 #endif
 };
 
-template<typename T>
+template <typename T>
 struct atomic_storage<T, 8> : atomic_storage<T, 0>
 {
 #ifdef _MSC_VER
@@ -447,6 +470,12 @@ struct atomic_storage<T, 8> : atomic_storage<T, 0>
 		_InterlockedExchange64((volatile llong*)&dest, (llong&)value);
 	}
 
+	static inline void release(T& dest, T value)
+	{
+		_ReadWriteBarrier();
+		*(volatile llong*)&dest = (llong&)value;
+	}
+
 	static inline T exchange(T& dest, T value)
 	{
 		llong r = _InterlockedExchange64((volatile llong*)&dest, (llong&)value);
@@ -525,7 +554,7 @@ struct atomic_storage<T, 8> : atomic_storage<T, 0>
 #endif
 };
 
-template<typename T>
+template <typename T>
 struct atomic_storage<T, 16> : atomic_storage<T, 0>
 {
 #ifdef _MSC_VER
@@ -550,6 +579,14 @@ struct atomic_storage<T, 16> : atomic_storage<T, 0>
 		while (!_InterlockedCompareExchange128((volatile llong*)&dest, hi, lo, cmp));
 	}
 
+	static inline void release(T& dest, T value)
+	{
+		llong lo = *(llong*)&value;
+		llong hi = *((llong*)&value + 1);
+		llong cmp[2]{ *(volatile llong*)&dest, *((volatile llong*)&dest + 1) };
+		while (!_InterlockedCompareExchange128((volatile llong*)&dest, hi, lo, cmp));
+	}
+
 	static inline T exchange(T& dest, T value)
 	{
 		llong lo = *(llong*)&value;
@@ -764,6 +801,12 @@ public:
 		return rhs;
 	}
 
+	// Atomically write data with release memory order (faster on x86)
+	void release(const type& rhs)
+	{
+		atomic_storage<type>::release(m_data, rhs);
+	}
+
 	// Atomically replace data with value, return previous data value
 	type exchange(const type& rhs)
 	{

rpcs3/Emu/Cell/SPURecompiler.cpp

@@ -543,11 +543,7 @@ void spu_recompiler_base::dispatch(spu_thread& spu, void*, u8* rip)
 	// If code verification failed from a patched patchpoint, clear it with a single NOP
 	if (rip)
 	{
-#ifdef _MSC_VER
+		atomic_storage<u64>::release(*reinterpret_cast<u64*>(rip), 0x841f0f);
-		*(volatile u64*)(rip) = 0x841f0f;
-#else
-		__atomic_store_n(reinterpret_cast<u64*>(rip), 0x841f0f, __ATOMIC_RELAXED);
-#endif
 	}
 
 	// Second attempt (recover from the recursion after repeated unsuccessful trampoline call)
@@ -580,7 +576,11 @@ void spu_recompiler_base::branch(spu_thread& spu, void*, u8* rip)
 	// Overwrite jump to this function with jump to the compiled function
 	const s64 rel = reinterpret_cast<u64>(func) - reinterpret_cast<u64>(rip) - 5;
 
-	alignas(8) u8 bytes[8];
+	union
+	{
+		u8 bytes[8];
+		u64 result;
+	};
 
 	if (rel >= INT32_MIN && rel <= INT32_MAX)
 	{
@@ -609,11 +609,7 @@ void spu_recompiler_base::branch(spu_thread& spu, void*, u8* rip)
 		std::memset(bytes + 3, 0x00, 5);
 	}
 
-#ifdef _MSC_VER
+	atomic_storage<u64>::release(*reinterpret_cast<u64*>(rip), result);
-	*(volatile u64*)(rip) = *reinterpret_cast<u64*>(+bytes);
-#else
-	__atomic_store_n(reinterpret_cast<u64*>(rip), *reinterpret_cast<u64*>(+bytes), __ATOMIC_RELAXED);
-#endif
 }
 
 std::vector<u32> spu_recompiler_base::block(const be_t<u32>* ls, u32 entry_point)

rpcs3/Emu/Cell/SPUThread.h

@@ -245,12 +245,7 @@ public:
 
 	void set_value(u32 value, bool count = true)
 	{
-		const u64 new_data = u64{count} << off_count | value;
+		data.release(u64{count} << off_count | value);
-#ifdef _MSC_VER
-		const_cast<volatile u64&>(data.raw()) = new_data;
-#else
-		__atomic_store_n(&data.raw(), new_data, __ATOMIC_RELAXED);
-#endif
 	}
 
 	u32 get_value()

rpcs3/Emu/Memory/vm.cpp

@@ -295,7 +295,7 @@ namespace vm
 
 	writer_lock::~writer_lock()
 	{
-		g_addr_lock.raw() = 0;
+		g_addr_lock.release(0);
 		g_mutex.unlock();
 	}