vm::atomic update

rpcs3/Emu/Memory/vm_atomic.h

@@ -29,23 +29,70 @@ namespace vm
 		typedef typename _to_atomic<T, sizeof(T)>::type atomic_type;
 
 	public:
+		// atomically compare data with cmp, replace with exch if equal, return previous data value anyway
 		__forceinline const T compare_and_swap(const T cmp, const T exch) volatile
 		{
 			const atomic_type res = InterlockedCompareExchange((volatile atomic_type*)&data, (atomic_type&)exch, (atomic_type&)cmp);
 			return (T&)res;
 		}
 
-		__forceinline const T exchange(const T value) volatile
+		// atomically compare data with cmp, replace with exch if equal, return true if data was replaced
+		__forceinline bool compare_and_swap_test(const T cmp, const T exch) volatile
 		{
-			const atomic_type res = InterlockedExchange((volatile atomic_type*)&data, (atomic_type&)value);
+			return InterlockedCompareExchange((volatile atomic_type*)&data, (atomic_type&)exch, (atomic_type&)cmp) == (atomic_type&)cmp;
+		}
+
+		// read data with memory barrier
+		__forceinline const T read_sync() const volatile 
+		{
+			const atomic_type res = InterlockedCompareExchange((volatile atomic_type*)&data, 0, 0);
 			return (T&)res;
 		}
 
+		// atomically replace data with exch, return previous data value
+		__forceinline const T exchange(const T exch) volatile
+		{
+			const atomic_type res = InterlockedExchange((volatile atomic_type*)&data, (atomic_type&)exch);
+			return (T&)res;
+		}
+
+		// read data without memory barrier
 		__forceinline const T read_relaxed() const volatile
 		{
 			return (T&)data;
 		}
 
+		// write data without memory barrier
+		__forceinline void write_relaxed(const T value) volatile
+		{
+			(T&)data = value;
+		}
+
+		// perform atomic operation on data
+		template<typename FT> __forceinline void atomic_op(const FT atomic_proc) volatile
+		{
+			while (true)
+			{
+				const T old = read_relaxed();
+				T _new = old;
+				atomic_proc(_new); // function should accept reference to T type
+				if (compare_and_swap_test(old, _new)) return;
+			}
+		}
+
+		// perform atomic operation on data with special exit condition (if intermediate result != proceed_value)
+		template<typename RT, typename FT> __forceinline RT atomic_op(const RT proceed_value, const FT atomic_proc) volatile
+		{
+			while (true)
+			{
+				const T old = read_relaxed();
+				T _new = old;
+				RT res = (RT)atomic_proc(_new); // function should accept reference to T type and return some value
+				if (res != proceed_value) return res;
+				if (compare_and_swap_test(old, _new)) return proceed_value;
+			}
+		}
+
 	};
 
 	template<typename T> struct atomic_le : public _atomic_base<T>

rpcs3/Emu/SysCalls/Modules/cellSpurs.cpp

@@ -219,7 +219,7 @@ s64 spursInit(
 	// some unknown subroutine
 	spurs->m.sub3.unk1 = spurs.addr() + 0xc9;
 	spurs->m.sub3.unk2 = 3; // unknown const
-	spurs->m.sub3.port = spurs->m.port;
+	spurs->m.sub3.port = (u64)spurs->m.port;
 
 	if (flags & SAF_SYSTEM_WORKLOAD_ENABLED) // initialize system workload
 	{
@@ -247,7 +247,7 @@ s64 cellSpursInitialize(vm::ptr<CellSpurs> spurs, s32 nSpus, s32 spuPriority, s3
 	cellSpurs->Warning("cellSpursInitialize(spurs_addr=0x%x, nSpus=%d, spuPriority=%d, ppuPriority=%d, exitIfNoWork=%d)",
 		spurs.addr(), nSpus, spuPriority, ppuPriority, exitIfNoWork ? 1 : 0);
 
-#ifdef PRX_DEBUG_XXX
+#ifdef PRX_DEBUG
 	return GetCurrentPPUThread().FastCall2(libsre + 0x8480, libsre_rtoc);
 #else
 	return spursInit(

rpcs3/Emu/SysCalls/Modules/cellSync.cpp

@@ -16,7 +16,7 @@ u32 libsre;
 u32 libsre_rtoc;
 #endif
 
-s32 syncMutexInitialize(vm::ptr<vm::atomic<CellSyncMutex>> mutex)
+s32 syncMutexInitialize(vm::ptr<CellSyncMutex> mutex)
 {
 	if (!mutex)
 	{
@@ -28,11 +28,11 @@ s32 syncMutexInitialize(vm::ptr<vm::atomic<CellSyncMutex>> mutex)
 	}
 
 	// prx: set zero and sync
-	mutex->exchange({});
+	mutex->data.exchange({});
 	return CELL_OK;
 }
 
-s32 cellSyncMutexInitialize(vm::ptr<vm::atomic<CellSyncMutex>> mutex)
+s32 cellSyncMutexInitialize(vm::ptr<CellSyncMutex> mutex)
 {
 	cellSync->Log("cellSyncMutexInitialize(mutex_addr=0x%x)", mutex.addr());
 
@@ -52,21 +52,15 @@ s32 cellSyncMutexLock(vm::ptr<CellSyncMutex> mutex)
 		return CELL_SYNC_ERROR_ALIGN;
 	}
 
-	// prx: increase u16 and remember its old value
+	// prx: increase m_acq and remember its old value
-	be_t<u16> old_order;
+	be_t<u16> order;
-	while (true)
+	mutex->data.atomic_op([&order](CellSyncMutex::data_t& _mutex)
 	{
-		const u32 old_data = mutex->m_data();
+		order = _mutex.m_acq++;
-		CellSyncMutex new_mutex;
+	});
-		new_mutex.m_data() = old_data;
 
-		old_order = new_mutex.m_order;
+	// prx: wait until this old value is equal to m_rel
-		new_mutex.m_order++; // increase m_order
+	while (order != mutex->data.read_relaxed().m_rel)
-		if (InterlockedCompareExchange(&mutex->m_data(), new_mutex.m_data(), old_data) == old_data) break;
-	}
-
-	// prx: wait until another u16 value == old value
-	while (old_order != mutex->m_freed)
 	{
 		std::this_thread::sleep_for(std::chrono::milliseconds(1)); // hack
 		if (Emu.IsStopped())
@@ -77,7 +71,7 @@ s32 cellSyncMutexLock(vm::ptr<CellSyncMutex> mutex)
 	}
 
 	// prx: sync
-	InterlockedCompareExchange(&mutex->m_data(), 0, 0);
+	mutex->data.read_sync();
 	return CELL_OK;
 }
 
@@ -94,25 +88,15 @@ s32 cellSyncMutexTryLock(vm::ptr<CellSyncMutex> mutex)
 		return CELL_SYNC_ERROR_ALIGN;
 	}
 
-	while (true)
+	// prx: exit if m_acq and m_rel are not equal, increase m_acq
+	return mutex->data.atomic_op<s32>(CELL_OK, [](CellSyncMutex::data_t& _mutex) -> s32
 	{
-		const u32 old_data = mutex->m_data();
+		if (_mutex.m_acq++ != _mutex.m_rel)
-		CellSyncMutex new_mutex;
-		new_mutex.m_data() = old_data;
-
-		// prx: compare two u16 values and exit if not equal
-		if (new_mutex.m_order != new_mutex.m_freed)
 		{
 			return CELL_SYNC_ERROR_BUSY;
 		}
-		else
+		return CELL_OK;
-		{
+	});
-			new_mutex.m_order++;
-		}
-		if (InterlockedCompareExchange(&mutex->m_data(), new_mutex.m_data(), old_data) == old_data) break;
-	}
-
-	return CELL_OK;
 }
 
 s32 cellSyncMutexUnlock(vm::ptr<CellSyncMutex> mutex)
@@ -128,18 +112,11 @@ s32 cellSyncMutexUnlock(vm::ptr<CellSyncMutex> mutex)
 		return CELL_SYNC_ERROR_ALIGN;
 	}
 
-	InterlockedCompareExchange(&mutex->m_data(), 0, 0);
+	mutex->data.read_sync();
-
+	mutex->data.atomic_op([](CellSyncMutex::data_t& _mutex)
-	while (true)
 	{
-		const u32 old_data = mutex->m_data();
+		_mutex.m_rel++;
-		CellSyncMutex new_mutex;
+	});
-		new_mutex.m_data() = old_data;
-
-		new_mutex.m_freed++;
-		if (InterlockedCompareExchange(&mutex->m_data(), new_mutex.m_data(), old_data) == old_data) break;
-	}
-
 	return CELL_OK;
 }
 
@@ -159,9 +136,7 @@ s32 syncBarrierInitialize(vm::ptr<CellSyncBarrier> barrier, u16 total_count)
 	}
 
 	// prx: zeroize first u16, write total_count in second u16 and sync
-	barrier->m_value = 0;
+	barrier->data.exchange({ be_t<s16>::make(0), be_t<s16>::make(total_count) });
-	barrier->m_count = total_count;
-	InterlockedCompareExchange(&barrier->m_data(), 0, 0);
 	return CELL_OK;
 }
 
@@ -186,15 +161,14 @@ s32 cellSyncBarrierNotify(vm::ptr<CellSyncBarrier> barrier)
 	}
 
 	// prx: sync, extract m_value, repeat if < 0, increase, compare with second s16, set sign bit if equal, insert it back
-	InterlockedCompareExchange(&barrier->m_data(), 0, 0);
+	barrier->data.read_sync();
 
 	while (true)
 	{
-		const u32 old_data = barrier->m_data();
+		const auto old = barrier->data.read_relaxed();
-		CellSyncBarrier new_barrier;
+		auto _barrier = old;
-		new_barrier.m_data() = old_data;
 
-		s16 value = (s16)new_barrier.m_value;
+		s16 value = (s16)_barrier.m_value;
 		if (value < 0)
 		{
 			std::this_thread::sleep_for(std::chrono::milliseconds(1)); // hack
@@ -207,12 +181,12 @@ s32 cellSyncBarrierNotify(vm::ptr<CellSyncBarrier> barrier)
 		}
 
 		value++;
-		if (value == (s16)new_barrier.m_count)
+		if (value == (s16)_barrier.m_count)
 		{
 			value |= 0x8000;
 		}
-		new_barrier.m_value = value;
+		_barrier.m_value = value;
-		if (InterlockedCompareExchange(&barrier->m_data(), new_barrier.m_data(), old_data) == old_data) break;
+		if (barrier->data.compare_and_swap_test(old, _barrier)) break;
 	}
 
 	return CELL_OK;
@@ -231,28 +205,27 @@ s32 cellSyncBarrierTryNotify(vm::ptr<CellSyncBarrier> barrier)
 		return CELL_SYNC_ERROR_ALIGN;
 	}
 
-	InterlockedCompareExchange(&barrier->m_data(), 0, 0);
+	barrier->data.read_sync();
 
 	while (true)
 	{
-		const u32 old_data = barrier->m_data();
+		const auto old = barrier->data.read_relaxed();
-		CellSyncBarrier new_barrier;
+		auto _barrier = old;
-		new_barrier.m_data() = old_data;
 
-		s16 value = (s16)new_barrier.m_value;
+		s16 value = (s16)_barrier.m_value;
 		if (value >= 0)
 		{
 			value++;
-			if (value == (s16)new_barrier.m_count)
+			if (value == (s16)_barrier.m_count)
 			{
 				value |= 0x8000;
 			}
-			new_barrier.m_value = value;
+			_barrier.m_value = value;
-			if (InterlockedCompareExchange(&barrier->m_data(), new_barrier.m_data(), old_data) == old_data) break;
+			if (barrier->data.compare_and_swap_test(old, _barrier)) break;
 		}		
 		else
 		{
-			if (InterlockedCompareExchange(&barrier->m_data(), new_barrier.m_data(), old_data) == old_data) return CELL_SYNC_ERROR_BUSY;
+			if (barrier->data.compare_and_swap_test(old, _barrier)) return CELL_SYNC_ERROR_BUSY;
 		}
 	}
 
@@ -273,15 +246,14 @@ s32 cellSyncBarrierWait(vm::ptr<CellSyncBarrier> barrier)
 	}
 
 	// prx: sync, extract m_value (repeat if >= 0), decrease it, set 0 if == 0x8000, insert it back
-	InterlockedCompareExchange(&barrier->m_data(), 0, 0);
+	barrier->data.read_sync();
 
 	while (true)
 	{
-		const u32 old_data = barrier->m_data();
+		const auto old = barrier->data.read_relaxed();
-		CellSyncBarrier new_barrier;
+		auto _barrier = old;
-		new_barrier.m_data() = old_data;
 
-		s16 value = (s16)new_barrier.m_value;
+		s16 value = (s16)_barrier.m_value;
 		if (value >= 0)
 		{
 			std::this_thread::sleep_for(std::chrono::milliseconds(1)); // hack
@@ -298,8 +270,8 @@ s32 cellSyncBarrierWait(vm::ptr<CellSyncBarrier> barrier)
 		{
 			value = 0;
 		}
-		new_barrier.m_value = value;
+		_barrier.m_value = value;
-		if (InterlockedCompareExchange(&barrier->m_data(), new_barrier.m_data(), old_data) == old_data) break;
+		if (barrier->data.compare_and_swap_test(old, _barrier)) break;
 	}
 
 	return CELL_OK;
@@ -318,15 +290,14 @@ s32 cellSyncBarrierTryWait(vm::ptr<CellSyncBarrier> barrier)
 		return CELL_SYNC_ERROR_ALIGN;
 	}
 
-	InterlockedCompareExchange(&barrier->m_data(), 0, 0);
+	barrier->data.read_sync();
 
 	while (true)
 	{
-		const u32 old_data = barrier->m_data();
+		const auto old = barrier->data.read_relaxed();
-		CellSyncBarrier new_barrier;
+		auto _barrier = old;
-		new_barrier.m_data() = old_data;
 
-		s16 value = (s16)new_barrier.m_value;
+		s16 value = (s16)_barrier.m_value;
 		if (value >= 0)
 		{
 			return CELL_SYNC_ERROR_BUSY;
@@ -337,8 +308,8 @@ s32 cellSyncBarrierTryWait(vm::ptr<CellSyncBarrier> barrier)
 		{
 			value = 0;
 		}
-		new_barrier.m_value = value;
+		_barrier.m_value = value;
-		if (InterlockedCompareExchange(&barrier->m_data(), new_barrier.m_data(), old_data) == old_data) break;
+		if (barrier->data.compare_and_swap_test(old, _barrier)) break;
 	}
 
 	return CELL_OK;

rpcs3/Emu/SysCalls/Modules/cellSync.h

@@ -31,26 +31,26 @@ enum
 
 struct CellSyncMutex
 {
-	be_t<u16> m_freed;
+	struct data_t
-	be_t<u16> m_order;
-
-	volatile u32& m_data()
 	{
-		return *reinterpret_cast<u32*>(this);
+		be_t<u16> m_rel; // release order (increased when mutex is unlocked)
+		be_t<u16> m_acq; // acquire order (increased when mutex is locked)
 	};
+
+	vm::atomic<data_t> data;
 };
 
 static_assert(sizeof(CellSyncMutex) == 4, "CellSyncMutex: wrong size");
 
 struct CellSyncBarrier
 {
-	be_t<s16> m_value;
+	struct data_t
-	be_t<s16> m_count;
-
-	volatile u32& m_data()
 	{
-		return *reinterpret_cast<u32*>(this);
+		be_t<s16> m_value;
+		be_t<s16> m_count;
 	};
+
+	vm::atomic<data_t> data;
 };
 
 static_assert(sizeof(CellSyncBarrier) == 4, "CellSyncBarrier: wrong size");