Implement class cond_x16

Use as reservation notifier
Limited to 16 threads but allows more precise control of contention

Utilities/cond.cpp

@@ -293,3 +293,106 @@ void cond_one::imp_notify() noexcept
 	futex(&m_value, FUTEX_WAKE_PRIVATE, 1);
 #endif
 }
+
+bool cond_x16::imp_wait(u32 _new, u32 slot, u64 _timeout) noexcept
+{
+	const u32 wait_bit = c_wait << slot;
+	const u32 lock_bit = c_lock << slot;
+
+	const bool is_inf = _timeout > cond_variable::max_timeout;
+
+#ifdef _WIN32
+	LARGE_INTEGER timeout;
+	timeout.QuadPart = _timeout * -10;
+
+	if (HRESULT rc = _timeout ? NtWaitForKeyedEvent(nullptr, &m_cvx16, false, is_inf ? nullptr : &timeout) : WAIT_TIMEOUT)
+	{
+		verify(HERE), rc == WAIT_TIMEOUT;
+
+		// Retire
+		const bool signaled = this->retire(slot);
+
+		while (signaled)
+		{
+			timeout.QuadPart = 0;
+
+			if (HRESULT rc2 = NtWaitForKeyedEvent(nullptr, &m_cvx16, false, &timeout))
+			{
+				verify(HERE), rc2 == WAIT_TIMEOUT;
+				SwitchToThread();
+				continue;
+			}
+
+			return true;
+		}
+
+		return false;
+	}
+
+	if (!this->retire(slot))
+	{
+		// Stolen notification: restore balance
+		NtReleaseKeyedEvent(nullptr, &m_cvx16, false, nullptr);
+	}
+#else
+	timespec timeout;
+	timeout.tv_sec  = _timeout / 1000000;
+	timeout.tv_nsec = (_timeout % 1000000) * 1000;
+
+	for (u32 value = _new; ((value >> slot) & c_sig) != c_sig; value = m_cvx16)
+	{
+		const int err = futex(&m_cvx16, FUTEX_WAIT_PRIVATE, value, is_inf ? nullptr : &timeout) == 0
+			? 0
+			: errno;
+
+		// Normal or timeout wakeup
+		if (!err || (!is_inf && err == ETIMEDOUT))
+		{
+			return this->retire(slot);
+		}
+
+		// Not a wakeup
+		verify(HERE), err == EAGAIN;
+	}
+
+	// Convert c_sig to c_lock
+	m_cvx16 &= ~wait_bit;
+#endif
+
+	return true;
+}
+
+void cond_x16::imp_notify() noexcept
+{
+	auto [old, ok] = m_cvx16.fetch_op([](u32& v)
+	{
+		const u32 lock_mask = v >> 16;
+		const u32 wait_mask = v & 0xffff;
+
+		if (const u32 sig_mask = lock_mask ^ wait_mask)
+		{
+			v |= sig_mask | sig_mask << 16;
+			return true;
+		}
+
+		return false;
+	});
+
+	// Determine if some waiters need a syscall notification
+	const u32 wait_mask = old & (~old >> 16);
+
+	if (UNLIKELY(!ok || !wait_mask))
+	{
+		return;
+	}
+
+#ifdef _WIN32
+	for (u32 i = 0; i < 16; i++)
+	{
+		if ((wait_mask >> i) & 1)
+			NtReleaseKeyedEvent(nullptr, &m_cvx16, false, nullptr);
+	}
+#else
+	futex(&m_cvx16, FUTEX_WAKE_PRIVATE, INT_MAX);
+#endif
+}