Build transactions at runtime

Drop _xbegin family intrinsics due to bad codegen
Implemented `notifier` class, replacing vm::notify
Minor optimization: detach transactions from global mutex on TSX path
Minor optimization: don't acquire vm::passive_lock on PPU on TSX path

Utilities/JIT.cpp

@@ -7,14 +7,14 @@ asmjit::JitRuntime& asmjit::get_global_runtime()
 	return g_rt;
 }
 
-void asmjit::build_transaction_enter(asmjit::X86Assembler& c, asmjit::Label abort)
+void asmjit::build_transaction_enter(asmjit::X86Assembler& c, asmjit::Label fallback)
 {
 	Label fall = c.newLabel();
 	Label begin = c.newLabel();
 	c.jmp(begin);
 	c.bind(fall);
 	c.test(x86::eax, _XABORT_RETRY);
-	c.jz(abort);
+	c.jz(fallback);
 	c.align(kAlignCode, 16);
 	c.bind(begin);
 	c.xbegin(fall);
@@ -25,8 +25,6 @@ void asmjit::build_transaction_abort(asmjit::X86Assembler& c, unsigned char code
 	c.db(0xc6);
 	c.db(0xf8);
 	c.db(code);
-	c.xor_(x86::eax, x86::eax);
-	c.ret();
 }
 
 #ifdef LLVM_AVAILABLE

Utilities/JIT.h

@@ -12,9 +12,9 @@ namespace asmjit
 	JitRuntime& get_global_runtime();
 
 	// Emit xbegin and adjacent loop
-	void build_transaction_enter(X86Assembler& c, Label abort);
+	void build_transaction_enter(X86Assembler& c, Label fallback);
 
-	// Emit xabort and return zero
+	// Emit xabort
 	void build_transaction_abort(X86Assembler& c, unsigned char code);
 }
 

Utilities/cond.cpp

@@ -16,7 +16,7 @@ bool cond_variable::imp_wait(u32 _old, u64 _timeout) noexcept
 	LARGE_INTEGER timeout;
 	timeout.QuadPart = _timeout * -10;
 
-	if (HRESULT rc = NtWaitForKeyedEvent(nullptr, &m_value, false, is_inf ? nullptr : &timeout))
+	if (HRESULT rc = _timeout ? NtWaitForKeyedEvent(nullptr, &m_value, false, is_inf ? nullptr : &timeout) : WAIT_TIMEOUT)
 	{
 		verify(HERE), rc == WAIT_TIMEOUT;
 
@@ -32,6 +32,12 @@ bool cond_variable::imp_wait(u32 _old, u64 _timeout) noexcept
 
 	return true;
 #else
+	if (!_timeout)
+	{
+		verify(HERE), m_value--;
+		return false;
+	}
+
 	timespec timeout;
 	timeout.tv_sec  = _timeout / 1000000;
 	timeout.tv_nsec = (_timeout % 1000000) * 1000;

Utilities/cond.h

@@ -9,6 +9,8 @@ class cond_variable
 	// Internal waiter counter
 	atomic_t<u32> m_value{0};
 
+	friend class notifier;
+
 protected:
 	// Internal waiting function
 	bool imp_wait(u32 _old, u64 _timeout) noexcept;
@@ -50,3 +52,94 @@ public:
 
 	static constexpr u64 max_timeout = u64{UINT32_MAX} / 1000 * 1000000;
 };
+
+// Pair of a fake shared mutex (only limited shared locking) and a condition variable
+class notifier
+{
+	atomic_t<u32> m_counter{0};
+	cond_variable m_cond;
+
+public:
+	constexpr notifier() = default;
+
+	void lock_shared()
+	{
+		m_counter++;
+	}
+
+	void unlock_shared()
+	{
+		const u32 counter = --m_counter;
+
+		if (counter & 0x7f)
+		{
+			return;
+		}
+
+		if (counter >= 0x80)
+		{
+			const u32 _old = m_counter.atomic_op([](u32& value) -> u32
+			{
+				if (value & 0x7f)
+				{
+					return 0;
+				}
+
+				return std::exchange(value, 0) >> 7;
+			});
+
+			if (_old && m_cond.m_value)
+			{
+				m_cond.imp_wake(_old);
+			}
+		}
+	}
+
+	explicit_bool_t wait(u64 usec_timeout = -1)
+	{
+		const u32 _old = m_cond.m_value.fetch_add(1);
+
+		if (0x80 <= m_counter.fetch_op([](u32& value)
+		{
+			value--;
+
+			if (value >= 0x80)
+			{
+				value -= 0x80;
+			}
+		}))
+		{
+			// Return without waiting
+			m_cond.imp_wait(_old, 0);
+			m_counter++;
+			return true;
+		}
+
+		const bool res = m_cond.imp_wait(_old, usec_timeout);
+		m_counter++;
+		return res;
+	}
+
+	void notify_all()
+	{
+		if (m_counter)
+		{
+			m_counter.atomic_op([](u32& value)
+			{
+				if (const u32 add = value & 0x7f)
+				{
+					// Mutex is locked in shared mode
+					value += add << 7;
+				}
+				else
+				{
+					// Mutex is unlocked
+					value = 0;
+				}
+			});
+		}
+
+		// Notify after imaginary "exclusive" lock+unlock
+		m_cond.notify_all();
+	}
+};

Utilities/sysinfo.h

@@ -41,28 +41,5 @@ namespace utils
 
 	bool has_xop();
 
-	FORCE_INLINE bool transaction_enter(uint* out = nullptr)
-	{
-		while (true)
-		{
-			const uint status = _xbegin();
-
-			if (status == _XBEGIN_STARTED)
-			{
-				return true;
-			}
-
-			if (!(status & _XABORT_RETRY))
-			{
-				if (out)
-				{
-					*out = status;
-				}
-
-				return false;
-			}
-		}
-	}
-
 	std::string get_system_info();
 }