vm::atomic update (or, and, xor)

CellSyncMutex, CellSyncBarrier, CellSyncRwm refactoring

Utilities/GNU.h

@@ -46,18 +46,21 @@ void strcpy_trunc(char(&dst)[size], const char(&src)[rsize])
 #define _CRT_ALIGN(x) __attribute__((aligned(x)))
 #define InterlockedCompareExchange(ptr,new_val,old_val)  __sync_val_compare_and_swap(ptr,old_val,new_val)
 #define InterlockedExchange(ptr, value) __sync_lock_test_and_set(ptr, value)
+#define InterlockedOr(ptr, value) __sync_fetch_and_or(ptr, value)
+#define InterlockedAnd(ptr, value) __sync_fetch_and_and(ptr, value)
+#define InterlockedXor(ptr, value) __sync_fetch_and_xor(ptr, value)
 
-inline int64_t InterlockedOr64(volatile int64_t *dest, int64_t val)
+//inline int64_t InterlockedOr64(volatile int64_t *dest, int64_t val)
-{
+//{
-	int64_t olderval;
+//	int64_t olderval;
-	int64_t oldval = *dest;
+//	int64_t oldval = *dest;
-	do
+//	do
-	{
+//	{
-		olderval = oldval;
+//		olderval = oldval;
-		oldval = __sync_val_compare_and_swap(dest, olderval | val, olderval);
+//		oldval = __sync_val_compare_and_swap(dest, olderval | val, olderval);
-	} while (olderval != oldval);
+//	} while (olderval != oldval);
-	return oldval;
+//	return oldval;
-}
+//}
 
 inline uint64_t __umulh(uint64_t a, uint64_t b)
 {
@@ -112,3 +115,36 @@ static __forceinline uint64_t InterlockedExchange(volatile uint64_t* dest, uint6
 	return _InterlockedExchange64((volatile long long*)dest, value);
 }
 #endif
+
+#ifndef InterlockedOr
+static __forceinline uint32_t InterlockedOr(volatile uint32_t* dest, uint32_t value)
+{
+	return _InterlockedOr((volatile long*)dest, value);
+}
+static __forceinline uint64_t InterlockedOr(volatile uint64_t* dest, uint64_t value)
+{
+	return _InterlockedOr64((volatile long long*)dest, value);
+}
+#endif
+
+#ifndef InterlockedAnd
+static __forceinline uint32_t InterlockedAnd(volatile uint32_t* dest, uint32_t value)
+{
+	return _InterlockedAnd((volatile long*)dest, value);
+}
+static __forceinline uint64_t InterlockedAnd(volatile uint64_t* dest, uint64_t value)
+{
+	return _InterlockedAnd64((volatile long long*)dest, value);
+}
+#endif
+
+#ifndef InterlockedXor
+static __forceinline uint32_t InterlockedXor(volatile uint32_t* dest, uint32_t value)
+{
+	return _InterlockedXor((volatile long*)dest, value);
+}
+static __forceinline uint64_t InterlockedXor(volatile uint64_t* dest, uint64_t value)
+{
+	return _InterlockedXor64((volatile long long*)dest, value);
+}
+#endif

rpcs3/Emu/Cell/SPUThread.h

@@ -358,7 +358,7 @@ public:
 			}
 			else
 			{
-				InterlockedOr64((volatile s64*)m_indval, ((u64)value << 32) | 1);
+				InterlockedOr((volatile u64*)m_indval, ((u64)value << 32) | 1);
 			}
 		}
 

rpcs3/Emu/Memory/vm_atomic.h

@@ -93,6 +93,42 @@ namespace vm
 			}
 		}
 
+		__forceinline const T _or(const T right) volatile
+		{
+			const atomic_type res = InterlockedOr((volatile atomic_type*)&data, (atomic_type&)right);
+			return (T&)res;
+		}
+
+		__forceinline const T _and(const T right) volatile
+		{
+			const atomic_type res = InterlockedAnd((volatile atomic_type*)&data, (atomic_type&)right);
+			return (T&)res;
+		}
+
+		__forceinline const T _xor(const T right) volatile
+		{
+			const atomic_type res = InterlockedXor((volatile atomic_type*)&data, (atomic_type&)right);
+			return (T&)res;
+		}
+
+		__forceinline const T operator |= (const T right) volatile
+		{
+			const atomic_type res = InterlockedOr((volatile atomic_type*)&data, (atomic_type&)right) | (atomic_type&)right;
+			return (T&)res;
+		}
+
+		__forceinline const T operator &= (const T right) volatile
+		{
+			const atomic_type res = InterlockedAnd((volatile atomic_type*)&data, (atomic_type&)right) & (atomic_type&)right;
+			return (T&)res;
+		}
+
+		__forceinline const T operator ^= (const T right) volatile
+		{
+			const atomic_type res = InterlockedXor((volatile atomic_type*)&data, (atomic_type&)right) ^ (atomic_type&)right;
+			return (T&)res;
+		}
+
 	};
 
 	template<typename T> struct atomic_le : public _atomic_base<T>

rpcs3/Emu/SysCalls/Modules/cellSync.cpp

@@ -54,9 +54,9 @@ s32 cellSyncMutexLock(vm::ptr<CellSyncMutex> mutex)
 
 	// prx: increase m_acq and remember its old value
 	be_t<u16> order;
-	mutex->data.atomic_op([&order](CellSyncMutex::data_t& _mutex)
+	mutex->data.atomic_op([&order](CellSyncMutex::data_t& mutex)
 	{
-		order = _mutex.m_acq++;
+		order = mutex.m_acq++;
 	});
 
 	// prx: wait until this old value is equal to m_rel
@@ -89,9 +89,9 @@ s32 cellSyncMutexTryLock(vm::ptr<CellSyncMutex> mutex)
 	}
 
 	// prx: exit if m_acq and m_rel are not equal, increase m_acq
-	return mutex->data.atomic_op<s32>(CELL_OK, [](CellSyncMutex::data_t& _mutex) -> s32
+	return mutex->data.atomic_op(CELL_OK, [](CellSyncMutex::data_t& mutex) -> s32
 	{
-		if (_mutex.m_acq++ != _mutex.m_rel)
+		if (mutex.m_acq++ != mutex.m_rel)
 		{
 			return CELL_SYNC_ERROR_BUSY;
 		}
@@ -113,9 +113,9 @@ s32 cellSyncMutexUnlock(vm::ptr<CellSyncMutex> mutex)
 	}
 
 	mutex->data.read_sync();
-	mutex->data.atomic_op([](CellSyncMutex::data_t& _mutex)
+	mutex->data.atomic_op([](CellSyncMutex::data_t& mutex)
 	{
-		_mutex.m_rel++;
+		mutex.m_rel++;
 	});
 	return CELL_OK;
 }
@@ -147,6 +147,24 @@ s32 cellSyncBarrierInitialize(vm::ptr<CellSyncBarrier> barrier, u16 total_count)
 	return syncBarrierInitialize(barrier, total_count);
 }
 
+s32 syncBarrierTryNotifyOp(CellSyncBarrier::data_t& barrier)
+{
+	// prx: extract m_value (repeat if < 0), increase, compare with second s16, set sign bit if equal, insert it back
+	s16 value = (s16)barrier.m_value;
+	if (value < 0)
+	{
+		return CELL_SYNC_ERROR_BUSY;
+	}
+
+	value++;
+	if (value == (s16)barrier.m_count)
+	{
+		value |= 0x8000;
+	}
+	barrier.m_value = value;
+	return CELL_OK;
+};
+
 s32 cellSyncBarrierNotify(vm::ptr<CellSyncBarrier> barrier)
 {
 	cellSync->Log("cellSyncBarrierNotify(barrier_addr=0x%x)", barrier.addr());
@@ -160,16 +178,8 @@ s32 cellSyncBarrierNotify(vm::ptr<CellSyncBarrier> barrier)
 		return CELL_SYNC_ERROR_ALIGN;
 	}
 
-	// prx: sync, extract m_value, repeat if < 0, increase, compare with second s16, set sign bit if equal, insert it back
 	barrier->data.read_sync();
-
+	while (barrier->data.atomic_op(CELL_OK, syncBarrierTryNotifyOp))
-	while (true)
-	{
-		const auto old = barrier->data.read_relaxed();
-		auto _barrier = old;
-
-		s16 value = (s16)_barrier.m_value;
-		if (value < 0)
 	{
 		std::this_thread::sleep_for(std::chrono::milliseconds(1)); // hack
 		if (Emu.IsStopped())
@@ -177,18 +187,7 @@ s32 cellSyncBarrierNotify(vm::ptr<CellSyncBarrier> barrier)
 			cellSync->Warning("cellSyncBarrierNotify(barrier_addr=0x%x) aborted", barrier.addr());
 			return CELL_OK;
 		}
-			continue;
 	}
-
-		value++;
-		if (value == (s16)_barrier.m_count)
-		{
-			value |= 0x8000;
-		}
-		_barrier.m_value = value;
-		if (barrier->data.compare_and_swap_test(old, _barrier)) break;
-	}
-
 	return CELL_OK;
 }
 
@@ -206,29 +205,24 @@ s32 cellSyncBarrierTryNotify(vm::ptr<CellSyncBarrier> barrier)
 	}
 
 	barrier->data.read_sync();
+	return barrier->data.atomic_op(CELL_OK, syncBarrierTryNotifyOp);
+}
 
-	while (true)
+s32 syncBarrierTryWaitOp(CellSyncBarrier::data_t& barrier)
-	{
+{
-		const auto old = barrier->data.read_relaxed();
+	// prx: extract m_value (repeat if >= 0), decrease it, set 0 if == 0x8000, insert it back
-		auto _barrier = old;
+	s16 value = (s16)barrier.m_value;
-
-		s16 value = (s16)_barrier.m_value;
 	if (value >= 0)
 	{
-			value++;
+		return CELL_SYNC_ERROR_BUSY;
-			if (value == (s16)_barrier.m_count)
-			{
-				value |= 0x8000;
-			}
-			_barrier.m_value = value;
-			if (barrier->data.compare_and_swap_test(old, _barrier)) break;
-		}		
-		else
-		{
-			if (barrier->data.compare_and_swap_test(old, _barrier)) return CELL_SYNC_ERROR_BUSY;
-		}
 	}
 
+	value--;
+	if (value == (s16)0x8000)
+	{
+		value = 0;
+	}
+	barrier.m_value = value;
 	return CELL_OK;
 }
 
@@ -245,16 +239,8 @@ s32 cellSyncBarrierWait(vm::ptr<CellSyncBarrier> barrier)
 		return CELL_SYNC_ERROR_ALIGN;
 	}
 
-	// prx: sync, extract m_value (repeat if >= 0), decrease it, set 0 if == 0x8000, insert it back
 	barrier->data.read_sync();
-
+	while (barrier->data.atomic_op(CELL_OK, syncBarrierTryWaitOp))
-	while (true)
-	{
-		const auto old = barrier->data.read_relaxed();
-		auto _barrier = old;
-
-		s16 value = (s16)_barrier.m_value;
-		if (value >= 0)
 	{
 		std::this_thread::sleep_for(std::chrono::milliseconds(1)); // hack
 		if (Emu.IsStopped())
@@ -262,18 +248,7 @@ s32 cellSyncBarrierWait(vm::ptr<CellSyncBarrier> barrier)
 			cellSync->Warning("cellSyncBarrierWait(barrier_addr=0x%x) aborted", barrier.addr());
 			return CELL_OK;
 		}
-			continue;
 	}
-
-		value--;
-		if (value == (s16)0x8000)
-		{
-			value = 0;
-		}
-		_barrier.m_value = value;
-		if (barrier->data.compare_and_swap_test(old, _barrier)) break;
-	}
-
 	return CELL_OK;
 }
 
@@ -291,28 +266,7 @@ s32 cellSyncBarrierTryWait(vm::ptr<CellSyncBarrier> barrier)
 	}
 
 	barrier->data.read_sync();
-
+	return barrier->data.atomic_op(CELL_OK, syncBarrierTryWaitOp);
-	while (true)
-	{
-		const auto old = barrier->data.read_relaxed();
-		auto _barrier = old;
-
-		s16 value = (s16)_barrier.m_value;
-		if (value >= 0)
-		{
-			return CELL_SYNC_ERROR_BUSY;
-		}
-
-		value--;
-		if (value == (s16)0x8000)
-		{
-			value = 0;
-		}
-		_barrier.m_value = value;
-		if (barrier->data.compare_and_swap_test(old, _barrier)) break;
-	}
-
-	return CELL_OK;
 }
 
 s32 syncRwmInitialize(vm::ptr<CellSyncRwm> rwm, vm::ptr<void> buffer, u32 buffer_size)
@@ -331,10 +285,9 @@ s32 syncRwmInitialize(vm::ptr<CellSyncRwm> rwm, vm::ptr<void> buffer, u32 buffer
 	}
 
 	// prx: zeroize first u16 and second u16, write buffer_size in second u32, write buffer_addr in second u64 and sync
-	rwm->m_data() = 0;
 	rwm->m_size = buffer_size;
 	rwm->m_buffer = buffer;
-	InterlockedCompareExchange(&rwm->m_data(), 0, 0);
+	rwm->data.exchange({});
 	return CELL_OK;
 }
 
@@ -345,6 +298,27 @@ s32 cellSyncRwmInitialize(vm::ptr<CellSyncRwm> rwm, vm::ptr<void> buffer, u32 bu
 	return syncRwmInitialize(rwm, buffer, buffer_size);
 }
 
+s32 syncRwmTryReadBeginOp(CellSyncRwm::data_t& rwm)
+{
+	if (rwm.m_writers.ToBE())
+	{
+		return CELL_SYNC_ERROR_BUSY;
+	}
+	rwm.m_readers++;
+	return CELL_OK;
+}
+
+s32 syncRwmReadEndOp(CellSyncRwm::data_t& rwm)
+{
+	if (!rwm.m_readers.ToBE())
+	{
+		cellSync->Error("syncRwmReadEndOp(rwm_addr=0x%x): m_readers == 0 (m_writers=%d)", Memory.RealToVirtualAddr(&rwm), (u16)rwm.m_writers);
+		return CELL_SYNC_ERROR_ABORT;
+	}
+	rwm.m_readers--;
+	return CELL_OK;
+}
+
 s32 cellSyncRwmRead(vm::ptr<CellSyncRwm> rwm, vm::ptr<void> buffer)
 {
 	cellSync->Log("cellSyncRwmRead(rwm_addr=0x%x, buffer_addr=0x%x)", rwm.addr(), buffer.addr());
@@ -358,14 +332,8 @@ s32 cellSyncRwmRead(vm::ptr<CellSyncRwm> rwm, vm::ptr<void> buffer)
 		return CELL_SYNC_ERROR_ALIGN;
 	}
 
-	// prx: atomically load first u32, repeat until second u16 == 0, increase first u16 and sync
+	// prx: increase m_readers, wait until m_writers is zero
-	while (true)
+	while (rwm->data.atomic_op(CELL_OK, syncRwmTryReadBeginOp))
-	{
-		const u32 old_data = rwm->m_data();
-		CellSyncRwm new_rwm;
-		new_rwm.m_data() = old_data;
-
-		if (new_rwm.m_writers.ToBE())
 	{
 		std::this_thread::sleep_for(std::chrono::milliseconds(1)); // hack
 		if (Emu.IsStopped())
@@ -373,33 +341,13 @@ s32 cellSyncRwmRead(vm::ptr<CellSyncRwm> rwm, vm::ptr<void> buffer)
 			cellSync->Warning("cellSyncRwmRead(rwm_addr=0x%x) aborted", rwm.addr());
 			return CELL_OK;
 		}
-			continue;
-		}
-		
-		new_rwm.m_readers++;
-		if (InterlockedCompareExchange(&rwm->m_data(), new_rwm.m_data(), old_data) == old_data) break;
 	}
 
 	// copy data to buffer_addr
 	memcpy(buffer.get_ptr(), rwm->m_buffer.get_ptr(), (u32)rwm->m_size);
 
-	// prx: load first u32, return 0x8041010C if first u16 == 0, atomically decrease it
+	// prx: decrease m_readers (return 0x8041010C if already zero)
-	while (true)
+	return rwm->data.atomic_op(CELL_OK, syncRwmReadEndOp);
-	{
-		const u32 old_data = rwm->m_data();
-		CellSyncRwm new_rwm;
-		new_rwm.m_data() = old_data;
-
-		if (!new_rwm.m_readers.ToBE())
-		{
-			cellSync->Error("cellSyncRwmRead(rwm_addr=0x%x): m_readers == 0 (m_writers=%d)", rwm.addr(), (u16)new_rwm.m_writers);
-			return CELL_SYNC_ERROR_ABORT;
-		}
-
-		new_rwm.m_readers--;
-		if (InterlockedCompareExchange(&rwm->m_data(), new_rwm.m_data(), old_data) == old_data) break;
-	}
-	return CELL_OK;
 }
 
 s32 cellSyncRwmTryRead(vm::ptr<CellSyncRwm> rwm, vm::ptr<void> buffer)
@@ -415,38 +363,22 @@ s32 cellSyncRwmTryRead(vm::ptr<CellSyncRwm> rwm, vm::ptr<void> buffer)
 		return CELL_SYNC_ERROR_ALIGN;
 	}
 
-	while (true)
+	if (s32 res = rwm->data.atomic_op(CELL_OK, syncRwmTryReadBeginOp))
 	{
-		const u32 old_data = rwm->m_data();
+		return res;
-		CellSyncRwm new_rwm;
+	}
-		new_rwm.m_data() = old_data;
+	memcpy(buffer.get_ptr(), rwm->m_buffer.get_ptr(), (u32)rwm->m_size);
 
-		if (new_rwm.m_writers.ToBE())
+	return rwm->data.atomic_op(CELL_OK, syncRwmReadEndOp);
+}
+
+s32 syncRwmTryWriteBeginOp(CellSyncRwm::data_t& rwm)
+{
+	if (rwm.m_writers.ToBE())
 	{
 		return CELL_SYNC_ERROR_BUSY;
 	}
-
+	rwm.m_writers = 1;
-		new_rwm.m_readers++;
-		if (InterlockedCompareExchange(&rwm->m_data(), new_rwm.m_data(), old_data) == old_data) break;
-	}
-
-	memcpy(buffer.get_ptr(), rwm->m_buffer.get_ptr(), (u32)rwm->m_size);
-
-	while (true)
-	{
-		const u32 old_data = rwm->m_data();
-		CellSyncRwm new_rwm;
-		new_rwm.m_data() = old_data;
-
-		if (!new_rwm.m_readers.ToBE())
-		{
-			cellSync->Error("cellSyncRwmRead(rwm_addr=0x%x): m_readers == 0 (m_writers=%d)", rwm.addr(), (u16)new_rwm.m_writers);
-			return CELL_SYNC_ERROR_ABORT;
-		}
-
-		new_rwm.m_readers--;
-		if (InterlockedCompareExchange(&rwm->m_data(), new_rwm.m_data(), old_data) == old_data) break;
-	}
 	return CELL_OK;
 }
 
@@ -463,14 +395,7 @@ s32 cellSyncRwmWrite(vm::ptr<CellSyncRwm> rwm, vm::ptr<const void> buffer)
 		return CELL_SYNC_ERROR_ALIGN;
 	}
 
-	// prx: atomically compare second u16 (m_writers) with 0, repeat if not 0, set 1, sync
+	while (rwm->data.atomic_op(CELL_OK, syncRwmTryWriteBeginOp))
-	while (true)
-	{
-		const u32 old_data = rwm->m_data();
-		CellSyncRwm new_rwm;
-		new_rwm.m_data() = old_data;
-
-		if (new_rwm.m_writers.ToBE())
 	{
 		std::this_thread::sleep_for(std::chrono::milliseconds(1)); // hack
 		if (Emu.IsStopped())
@@ -478,15 +403,10 @@ s32 cellSyncRwmWrite(vm::ptr<CellSyncRwm> rwm, vm::ptr<const void> buffer)
 			cellSync->Warning("cellSyncRwmWrite(rwm_addr=0x%x) aborted (I)", rwm.addr());
 			return CELL_OK;
 		}
-			continue;
-		}
-
-		new_rwm.m_writers = 1;
-		if (InterlockedCompareExchange(&rwm->m_data(), new_rwm.m_data(), old_data) == old_data) break;
 	}
 
 	// prx: wait until m_readers == 0
-	while (rwm->m_readers.ToBE())
+	while (rwm->data.read_relaxed().m_readers.ToBE())
 	{
 		std::this_thread::sleep_for(std::chrono::milliseconds(1)); // hack
 		if (Emu.IsStopped())
@@ -500,8 +420,7 @@ s32 cellSyncRwmWrite(vm::ptr<CellSyncRwm> rwm, vm::ptr<const void> buffer)
 	memcpy(rwm->m_buffer.get_ptr(), buffer.get_ptr(), (u32)rwm->m_size);
 
 	// prx: sync and zeroize m_readers and m_writers
-	InterlockedCompareExchange(&rwm->m_data(), 0, 0);
+	rwm->data.exchange({});
-	rwm->m_data() = 0;
 	return CELL_OK;
 }
 
@@ -518,15 +437,17 @@ s32 cellSyncRwmTryWrite(vm::ptr<CellSyncRwm> rwm, vm::ptr<const void> buffer)
 		return CELL_SYNC_ERROR_ALIGN;
 	}
 
-	// prx: compare m_readers | m_writers with 0, return busy if not zero, set m_writers to 1
+	// prx: compare m_readers | m_writers with 0, return if not zero, set m_writers to 1
-	if (InterlockedCompareExchange(&rwm->m_data(), se32(1), 0) != 0) return CELL_SYNC_ERROR_BUSY;
+	if (!rwm->data.compare_and_swap_test({}, {be_t<u16>::make(0), be_t<u16>::make(1)}))
+	{
+		return CELL_SYNC_ERROR_BUSY;
+	}
 
 	// prx: copy data from buffer_addr
 	memcpy(rwm->m_buffer.get_ptr(), buffer.get_ptr(), (u32)rwm->m_size);
 
 	// prx: sync and zeroize m_readers and m_writers
-	InterlockedCompareExchange(&rwm->m_data(), 0, 0);
+	rwm->data.exchange({});
-	rwm->m_data() = 0;
 	return CELL_OK;
 }
 

rpcs3/Emu/SysCalls/Modules/cellSync.h

@@ -57,23 +57,30 @@ static_assert(sizeof(CellSyncBarrier) == 4, "CellSyncBarrier: wrong size");
 
 struct CellSyncRwm
 {
+	struct data_t
+	{
 		be_t<u16> m_readers;
 		be_t<u16> m_writers;
+	};
+
+	vm::atomic<data_t> data;
 	be_t<u32> m_size;
 	vm::bptr<void, 1, u64> m_buffer;
-
-	volatile u32& m_data()
-	{
-		return *reinterpret_cast<u32*>(this);
-	};
 };
 
 static_assert(sizeof(CellSyncRwm) == 16, "CellSyncRwm: wrong size");
 
 struct CellSyncQueue
 {
+	struct data_t
+	{
 		be_t<u32> m_v1;
 		be_t<u32> m_v2;
+	};
+
+	be_t<u32> m_v1;
+	be_t<u32> m_v2;
+	//vm::atomic<data_t> data;
 	be_t<u32> m_size;
 	be_t<u32> m_depth;
 	vm::bptr<u8, 1, u64> m_buffer;