diff --git a/Utilities/Atomic.h b/Utilities/Atomic.h index 1ea3e4f597..e172793672 100644 --- a/Utilities/Atomic.h +++ b/Utilities/Atomic.h @@ -4,7 +4,7 @@ #include // Helper class, provides access to compiler-specific atomic intrinsics -template +template struct atomic_storage { static_assert(sizeof(T) <= 16 && sizeof(T) == alignof(T), "atomic_storage<> error: invalid type"); @@ -29,6 +29,11 @@ struct atomic_storage __atomic_store(&dest, &value, __ATOMIC_SEQ_CST); } + static inline void release(T& dest, T value) + { + __atomic_store(&dest, &value, __ATOMIC_RELEASE); + } + static inline T exchange(T& dest, T value) { T result; @@ -176,7 +181,7 @@ struct atomic_storage /* The rest: ugly MSVC intrinsics + inline asm implementations */ -template +template struct atomic_storage : atomic_storage { #ifdef _MSC_VER @@ -200,6 +205,12 @@ struct atomic_storage : atomic_storage _InterlockedExchange8((volatile char*)&dest, (char&)value); } + static inline void release(T& dest, T value) + { + _ReadWriteBarrier(); + *(volatile char*)&dest = (char&)value; + } + static inline T exchange(T& dest, T value) { char r = _InterlockedExchange8((volatile char*)&dest, (char&)value); @@ -232,7 +243,7 @@ struct atomic_storage : atomic_storage #endif }; -template +template struct atomic_storage : atomic_storage { #ifdef _MSC_VER @@ -256,6 +267,12 @@ struct atomic_storage : atomic_storage _InterlockedExchange16((volatile short*)&dest, (short&)value); } + static inline void release(T& dest, T value) + { + _ReadWriteBarrier(); + *(volatile short*)&dest = (short&)value; + } + static inline T exchange(T& dest, T value) { short r = _InterlockedExchange16((volatile short*)&dest, (short&)value); @@ -324,7 +341,7 @@ struct atomic_storage : atomic_storage #endif }; -template +template struct atomic_storage : atomic_storage { #ifdef _MSC_VER @@ -348,6 +365,12 @@ struct atomic_storage : atomic_storage _InterlockedExchange((volatile long*)&dest, (long&)value); } + static inline void release(T& dest, T value) + { + _ReadWriteBarrier(); + *(volatile long*)&dest = (long&)value; + } + static inline T exchange(T& dest, T value) { long r = _InterlockedExchange((volatile long*)&dest, (long&)value); @@ -423,7 +446,7 @@ struct atomic_storage : atomic_storage #endif }; -template +template struct atomic_storage : atomic_storage { #ifdef _MSC_VER @@ -447,6 +470,12 @@ struct atomic_storage : atomic_storage _InterlockedExchange64((volatile llong*)&dest, (llong&)value); } + static inline void release(T& dest, T value) + { + _ReadWriteBarrier(); + *(volatile llong*)&dest = (llong&)value; + } + static inline T exchange(T& dest, T value) { llong r = _InterlockedExchange64((volatile llong*)&dest, (llong&)value); @@ -525,7 +554,7 @@ struct atomic_storage : atomic_storage #endif }; -template +template struct atomic_storage : atomic_storage { #ifdef _MSC_VER @@ -550,6 +579,14 @@ struct atomic_storage : atomic_storage while (!_InterlockedCompareExchange128((volatile llong*)&dest, hi, lo, cmp)); } + static inline void release(T& dest, T value) + { + llong lo = *(llong*)&value; + llong hi = *((llong*)&value + 1); + llong cmp[2]{ *(volatile llong*)&dest, *((volatile llong*)&dest + 1) }; + while (!_InterlockedCompareExchange128((volatile llong*)&dest, hi, lo, cmp)); + } + static inline T exchange(T& dest, T value) { llong lo = *(llong*)&value; @@ -764,6 +801,12 @@ public: return rhs; } + // Atomically write data with release memory order (faster on x86) + void release(const type& rhs) + { + atomic_storage::release(m_data, rhs); + } + // Atomically replace data with value, return previous data value type exchange(const type& rhs) { diff --git a/rpcs3/Emu/Cell/SPURecompiler.cpp b/rpcs3/Emu/Cell/SPURecompiler.cpp index 7db917ef9f..261ef759ac 100644 --- a/rpcs3/Emu/Cell/SPURecompiler.cpp +++ b/rpcs3/Emu/Cell/SPURecompiler.cpp @@ -543,11 +543,7 @@ void spu_recompiler_base::dispatch(spu_thread& spu, void*, u8* rip) // If code verification failed from a patched patchpoint, clear it with a single NOP if (rip) { -#ifdef _MSC_VER - *(volatile u64*)(rip) = 0x841f0f; -#else - __atomic_store_n(reinterpret_cast(rip), 0x841f0f, __ATOMIC_RELAXED); -#endif + atomic_storage::release(*reinterpret_cast(rip), 0x841f0f); } // Second attempt (recover from the recursion after repeated unsuccessful trampoline call) @@ -580,7 +576,11 @@ void spu_recompiler_base::branch(spu_thread& spu, void*, u8* rip) // Overwrite jump to this function with jump to the compiled function const s64 rel = reinterpret_cast(func) - reinterpret_cast(rip) - 5; - alignas(8) u8 bytes[8]; + union + { + u8 bytes[8]; + u64 result; + }; if (rel >= INT32_MIN && rel <= INT32_MAX) { @@ -609,11 +609,7 @@ void spu_recompiler_base::branch(spu_thread& spu, void*, u8* rip) std::memset(bytes + 3, 0x00, 5); } -#ifdef _MSC_VER - *(volatile u64*)(rip) = *reinterpret_cast(+bytes); -#else - __atomic_store_n(reinterpret_cast(rip), *reinterpret_cast(+bytes), __ATOMIC_RELAXED); -#endif + atomic_storage::release(*reinterpret_cast(rip), result); } std::vector spu_recompiler_base::block(const be_t* ls, u32 entry_point) diff --git a/rpcs3/Emu/Cell/SPUThread.h b/rpcs3/Emu/Cell/SPUThread.h index 47dc7cbe04..21bd74d22c 100644 --- a/rpcs3/Emu/Cell/SPUThread.h +++ b/rpcs3/Emu/Cell/SPUThread.h @@ -245,12 +245,7 @@ public: void set_value(u32 value, bool count = true) { - const u64 new_data = u64{count} << off_count | value; -#ifdef _MSC_VER - const_cast(data.raw()) = new_data; -#else - __atomic_store_n(&data.raw(), new_data, __ATOMIC_RELAXED); -#endif + data.release(u64{count} << off_count | value); } u32 get_value() diff --git a/rpcs3/Emu/Memory/vm.cpp b/rpcs3/Emu/Memory/vm.cpp index 9202cc03fa..457f311f8a 100644 --- a/rpcs3/Emu/Memory/vm.cpp +++ b/rpcs3/Emu/Memory/vm.cpp @@ -85,7 +85,7 @@ namespace vm if (!lock && lock.compare_and_swap_test(0, lock_info)) { return &lock; - } + } } } } @@ -256,7 +256,7 @@ namespace vm { const u64 value = lock; - // Test beginning address + // Test beginning address if (static_cast(value) > addr) { break; @@ -295,7 +295,7 @@ namespace vm writer_lock::~writer_lock() { - g_addr_lock.raw() = 0; + g_addr_lock.release(0); g_mutex.unlock(); }