Implement cpu_thread::suspend_all

Remove Accurate PUTLLC option.
Implement fallback path for SPU transactions.

Utilities/JIT.cpp

@@ -190,18 +190,34 @@ asmjit::JitRuntime& asmjit::get_global_runtime()
 	return g_rt;
 }
 
-asmjit::Label asmjit::build_transaction_enter(asmjit::X86Assembler& c, asmjit::Label fallback)
+void asmjit::build_transaction_enter(asmjit::X86Assembler& c, asmjit::Label fallback, const asmjit::X86Gp& ctr, uint less_than)
 {
 	Label fall = c.newLabel();
 	Label begin = c.newLabel();
 	c.jmp(begin);
 	c.bind(fall);
-	c.test(x86::eax, _XABORT_RETRY);
-	c.jz(fallback);
+
+	if (less_than < 65)
+	{
+		c.add(ctr, 1);
+		c.test(x86::eax, _XABORT_RETRY);
+		c.jz(fallback);
+	}
+	else
+	{
+		// Count an attempt without RETRY flag as 65 normal attempts and continue
+		c.not_(x86::eax);
+		c.and_(x86::eax, _XABORT_RETRY);
+		c.shl(x86::eax, 5);
+		c.add(x86::eax, 1); // eax = RETRY ? 1 : 65
+		c.add(ctr, x86::rax);
+	}
+
+	c.cmp(ctr, less_than);
+	c.jae(fallback);
 	c.align(kAlignCode, 16);
 	c.bind(begin);
 	c.xbegin(fall);
-	return begin;
 }
 
 void asmjit::build_transaction_abort(asmjit::X86Assembler& c, unsigned char code)

Utilities/JIT.h

@@ -43,7 +43,7 @@ namespace asmjit
 	asmjit::JitRuntime& get_global_runtime();
 
 	// Emit xbegin and adjacent loop, return label at xbegin
-	Label build_transaction_enter(X86Assembler& c, Label fallback);
+	void build_transaction_enter(X86Assembler& c, Label fallback, const X86Gp& ctr, uint less_than);
 
 	// Emit xabort
 	void build_transaction_abort(X86Assembler& c, unsigned char code);

Utilities/Thread.cpp

@@ -3,6 +3,7 @@
 #include "Emu/System.h"
 #include "Emu/IdManager.h"
 #include "Emu/Cell/SPUThread.h"
+#include "Emu/Cell/PPUThread.h"
 #include "Emu/Cell/RawSPUThread.h"
 #include "Emu/Cell/lv2/sys_mmapper.h"
 #include "Emu/Cell/lv2/sys_event.h"
@@ -1101,6 +1102,11 @@ bool handle_access_violation(u32 addr, bool is_writing, x64_context* context)
 
 		try
 		{
+			if (cpu)
+			{
+				vm::temporary_unlock(*cpu);
+			}
+
 			handled = rsx::g_access_violation_handler(addr, is_writing);
 		}
 		catch (const std::exception& e)
@@ -1109,7 +1115,6 @@ bool handle_access_violation(u32 addr, bool is_writing, x64_context* context)
 
 			if (cpu)
 			{
-				vm::temporary_unlock(*cpu);
 				cpu->state += cpu_flag::dbg_pause;
 
 				if (cpu->test_stopped())
@@ -1131,6 +1136,10 @@ bool handle_access_violation(u32 addr, bool is_writing, x64_context* context)
 
 			return true;
 		}
+
+		if (cpu && cpu->test_stopped())
+		{
+		}
 	}
 
 	auto code = (const u8*)RIP(context);

Utilities/cond.cpp

@@ -273,6 +273,177 @@ void shared_cond::imp_notify() noexcept
 	balanced_awaken<true>(m_cvx32, utils::popcnt32(wait_mask));
 }
 
+void shared_cond::wait_all() noexcept
+{
+	// Try to acquire waiting state without locking but only if there are other locks
+	const auto [old_, result] = m_cvx32.fetch_op([](u64& cvx32) -> u64
+	{
+		// Check waiting alone
+		if ((cvx32 & 0xffffffff) == 0)
+		{
+			return 0;
+		}
+
+		// Combine used bits and invert to find least significant bit unused
+		const u32 slot = utils::cnttz64(~((cvx32 & 0xffffffff) | (cvx32 >> 32)), true);
+
+		// Set waiting bit (does nothing if all slots are used)
+		cvx32 |= (1ull << slot) & 0xffffffff;
+		return 1ull << slot;
+	});
+
+	if (!result)
+	{
+		return;
+	}
+
+	if (result > 0xffffffffu)
+	{
+		// All slots are used, fallback to spin wait
+		while (m_cvx32 & 0xffffffff)
+		{
+			busy_wait();
+		}
+
+		return;
+	}
+
+	const u64 wait_bit = result;
+	const u64 lock_bit = wait_bit | (wait_bit << 32);
+
+	balanced_wait_until(m_cvx32, -1, [&](u64& cvx32, auto... ret) -> int
+	{
+		if ((cvx32 & wait_bit) == 0)
+		{
+			// Remove signal and unlock at once
+			cvx32 &= ~lock_bit;
+			return +1;
+		}
+
+		if constexpr (sizeof...(ret))
+		{
+			cvx32 &= ~lock_bit;
+			return -1;
+		}
+
+		return 0;
+	});
+}
+
+bool shared_cond::wait_all(shared_cond::shared_lock& lock) noexcept
+{
+	AUDIT(lock.m_this == this);
+
+	if (lock.m_slot >= 32)
+	{
+		// Invalid argument, assume notified
+		return true;
+	}
+
+	const u64 wait_bit = c_wait << lock.m_slot;
+	const u64 lock_bit = c_lock << lock.m_slot;
+
+	// Try to acquire waiting state only if there are other locks
+	const auto [old_, not_alone] = m_cvx32.fetch_op([&](u64& cvx32)
+	{
+		// Check locking alone
+		if (((cvx32 >> 32) & cvx32) == (lock_bit >> 32))
+		{
+			return false;
+		}
+
+		// c_lock -> c_wait, c_sig -> unlock
+		cvx32 &= ~(lock_bit & ~wait_bit);
+		return true;
+	});
+
+	if (!not_alone)
+	{
+		return false;
+	}
+	else
+	{
+		// Set invalid slot to acknowledge unlocking
+		lock.m_slot = 33;
+	}
+
+	if ((old_ & wait_bit) == 0)
+	{
+		// Already signaled, return without waiting
+		return true;
+	}
+
+	balanced_wait_until(m_cvx32, -1, [&](u64& cvx32, auto... ret) -> int
+	{
+		if ((cvx32 & wait_bit) == 0)
+		{
+			// Remove signal and unlock at once
+			cvx32 &= ~lock_bit;
+			return +1;
+		}
+
+		if constexpr (sizeof...(ret))
+		{
+			cvx32 &= ~lock_bit;
+			return -1;
+		}
+
+		return 0;
+	});
+
+	return true;
+}
+
+bool shared_cond::notify_all(shared_cond::shared_lock& lock) noexcept
+{
+	AUDIT(lock.m_this == this);
+
+	if (lock.m_slot >= 32)
+	{
+		// Invalid argument
+		return false;
+	}
+
+	const u64 slot_mask = c_sig << lock.m_slot;
+
+	auto [old, ok] = m_cvx32.fetch_op([&](u64& cvx32)
+	{
+		if (((cvx32 << 32) & cvx32) != slot_mask)
+		{
+			return false;
+		}
+
+		if (const u64 sig_mask = cvx32 & 0xffffffff)
+		{
+			cvx32 &= (0xffffffffull << 32) & ~slot_mask;
+			cvx32 |= (sig_mask << 32) & ~slot_mask;
+			return true;
+		}
+
+		return false;
+	});
+
+	if (!ok)
+	{
+		// Not an exclusive reader
+		return false;
+	}
+
+	// Set invalid slot to acknowledge unlocking
+	lock.m_slot = 34;
+
+	// Determine if some waiters need a syscall notification
+	const u64 wait_mask = old & (~old >> 32);
+
+	if (UNLIKELY(!wait_mask))
+	{
+		return true;
+	}
+
+	balanced_awaken<true>(m_cvx32, utils::popcnt32(wait_mask));
+	return true;
+}
+
 bool lf_queue_base::wait(u64 _timeout)
 {
 	auto _old = m_head.compare_and_swap(0, 1);

Utilities/cond.h

@@ -206,7 +206,7 @@ class shared_cond
 			m_slot = m_this->m_cvx32.atomic_op([](u64& cvx32)
 			{
 				// Combine used bits and invert to find least significant bit unused
-				const u32 slot = utils::cnttz32(~((cvx32 & 0xffffffff) | (cvx32 >> 32)), true);
+				const u32 slot = utils::cnttz64(~((cvx32 & 0xffffffff) | (cvx32 >> 32)), true);
 
 				// Set lock bits (does nothing if all slots are used)
 				const u64 bit = (1ull << slot) & 0xffffffff;
@@ -217,6 +217,13 @@ class shared_cond
 
 		shared_lock(const shared_lock&) = delete;
 
+		shared_lock(shared_lock&& rhs)
+			: m_this(rhs.m_this)
+			, m_slot(rhs.m_slot)
+		{
+			rhs.m_slot = 32;
+		}
+
 		shared_lock& operator=(const shared_lock&) = delete;
 
 		~shared_lock()
@@ -261,6 +268,10 @@ public:
 		return imp_wait(lock.m_slot, usec_timeout);
 	}
 
+	void wait_all() noexcept;
+
+	bool wait_all(shared_lock& lock) noexcept;
+
 	void notify_all() noexcept
 	{
 		if (LIKELY(!m_cvx32))
@@ -268,4 +279,6 @@ public:
 
 		imp_notify();
 	}
+
+	bool notify_all(shared_lock& lock) noexcept;
 };