archive/rpcs3
1
0
Fork 0
mirror of https://github.com/RPCS3/rpcs3.git synced 2025-07-10 00:41:26 +12:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improve vm::reservation_op

Browse source 
Remove XABORT, sync status handling with SPU/PPU transaction.
Limit max number of transaction attempts in loop.
Add Ack template parameter, as in vm::reservation_light_op.
Remove utils::tx_abort, improve utils::tx_start as well.
This commit is contained in:
Nekotekina 2020-10-20 08:41:10 +03:00
parent dc8252bb9f
commit 4384ae15b4
3 changed files with 55 additions and 80 deletions
Download patch file Download diff file Expand all files Collapse all files

17
Utilities/asm.h
View file

@ -4,7 +4,7 @@
namespace utils namespace utils
{ {
// Transaction helper (Max = max attempts) (result = pair of success and op result, which is only meaningful on success) // Transaction helper (Max = max attempts) (result = pair of success and op result)
template <uint Max = 10, typename F, typename R = std::invoke_result_t<F>> template <uint Max = 10, typename F, typename R = std::invoke_result_t<F>>
inline auto tx_start(F op) inline auto tx_start(F op)
{ {
@ -48,9 +48,8 @@ namespace utils
#ifndef _MSC_VER #ifndef _MSC_VER
__asm__ volatile ("movl %%eax, %0;" : "=r" (status) :: "memory"); __asm__ volatile ("movl %%eax, %0;" : "=r" (status) :: "memory");
#endif #endif
if (!(status & _XABORT_RETRY)) [[unlikely]] if (!status) [[unlikely]]
{ {
// In order to abort transaction, tx_abort() can be used, so there is no need for "special" return value
break; break;
} }
} }
@ -65,18 +64,6 @@ namespace utils
} }
}; };
// Special function to abort transaction
[[noreturn]] FORCE_INLINE void tx_abort()
{
#ifndef _MSC_VER
__asm__ volatile ("xabort $0;" ::: "memory");
__builtin_unreachable();
#else
_xabort(0);
__assume(0);
#endif
}
// Rotate helpers // Rotate helpers
#if defined(__GNUG__) #if defined(__GNUG__)

9
rpcs3/Emu/Memory/vm.cpp
View file

@ -509,25 +509,18 @@ namespace vm
void reservation_op_internal(u32 addr, std::function<bool()> func) void reservation_op_internal(u32 addr, std::function<bool()> func)
{ {
const bool ok = cpu_thread::suspend_all(get_current_cpu_thread(), [&] cpu_thread::suspend_all(get_current_cpu_thread(), [&]
{ {
if (func()) if (func())
{ {
// Success, release all locks if necessary // Success, release all locks if necessary
vm::reservation_acquire(addr, 128) += 127; vm::reservation_acquire(addr, 128) += 127;
return true;
} }
else else
{ {
vm::reservation_acquire(addr, 128) -= 1; vm::reservation_acquire(addr, 128) -= 1;
return false;
} }
}); });
if (ok)
{
vm::reservation_notifier(addr, 128).notify_all();
}
} }
void reservation_escape_internal() void reservation_escape_internal()

109
rpcs3/Emu/Memory/vm_reservation.h
View file

@ -67,9 +67,10 @@ namespace vm
return {*res, rtime}; return {*res, rtime};
} }
// TODO: remove and make it external
void reservation_op_internal(u32 addr, std::function<bool()> func); void reservation_op_internal(u32 addr, std::function<bool()> func);
template <typename T, typename AT = u32, typename F> template <bool Ack = false, typename T, typename AT = u32, typename F>
SAFE_BUFFERS inline auto reservation_op(_ptr_base<T, AT> ptr, F op) SAFE_BUFFERS inline auto reservation_op(_ptr_base<T, AT> ptr, F op)
{ {
// Atomic operation will be performed on aligned 128 bytes of data, so the data size and alignment must comply // Atomic operation will be performed on aligned 128 bytes of data, so the data size and alignment must comply
@ -79,15 +80,21 @@ namespace vm
// Use "super" pointer to prevent access violation handling during atomic op // Use "super" pointer to prevent access violation handling during atomic op
const auto sptr = vm::get_super_ptr<T>(static_cast<u32>(ptr.addr())); const auto sptr = vm::get_super_ptr<T>(static_cast<u32>(ptr.addr()));
// Prefetch some data
_m_prefetchw(sptr);
_m_prefetchw(reinterpret_cast<char*>(sptr) + 64);
// Use 128-byte aligned addr // Use 128-byte aligned addr
const u32 addr = static_cast<u32>(ptr.addr()) & -128; const u32 addr = static_cast<u32>(ptr.addr()) & -128;
if (g_use_rtm) if (g_use_rtm)
{ {
auto& res = vm::reservation_acquire(addr, 128); auto& res = vm::reservation_acquire(addr, 128);
_m_prefetchw(&res);
// Stage 1: single optimistic transaction attempt // Stage 1: single optimistic transaction attempt
unsigned status = _XBEGIN_STARTED; unsigned status = _XBEGIN_STARTED;
unsigned count = 0;
u64 _old = 0; u64 _old = 0;
#ifndef _MSC_VER #ifndef _MSC_VER
@ -100,22 +107,24 @@ namespace vm
if (res & rsrv_unique_lock) if (res & rsrv_unique_lock)
{ {
#ifndef _MSC_VER #ifndef _MSC_VER
__asm__ volatile ("xabort $0;" ::: "memory"); __asm__ volatile ("xend; mov $-1, %%eax;" ::: "memory");
#else #else
_xabort(0); _xend();
#endif #endif
goto stage2;
} }
if constexpr (std::is_void_v<std::invoke_result_t<F, T&>>) if constexpr (std::is_void_v<std::invoke_result_t<F, T&>>)
{ {
res += 128;
std::invoke(op, *sptr); std::invoke(op, *sptr);
res += 128;
#ifndef _MSC_VER #ifndef _MSC_VER
__asm__ volatile ("xend;" ::: "memory"); __asm__ volatile ("xend;" ::: "memory");
#else #else
_xend(); _xend();
#endif #endif
res.notify_all(); if constexpr (Ack)
res.notify_all();
return; return;
} }
else else
@ -128,37 +137,29 @@ namespace vm
#else #else
_xend(); _xend();
#endif #endif
res.notify_all(); if constexpr (Ack)
res.notify_all();
return result; return result;
} }
else else
{ {
#ifndef _MSC_VER #ifndef _MSC_VER
__asm__ volatile ("xabort $1;" ::: "memory"); __asm__ volatile ("xend;" ::: "memory");
#else #else
_xabort(1); _xend();
#endif #endif
// Unreachable code return result;
return std::invoke_result_t<F, T&>();
} }
} }
} }
stage2: stage2:
#ifndef _MSC_VER #ifndef _MSC_VER
__asm__ volatile ("movl %%eax, %0;" : "=r" (status) :: "memory"); __asm__ volatile ("mov %%eax, %0;" : "=r" (status) :: "memory");
#endif #endif
if constexpr (!std::is_void_v<std::invoke_result_t<F, T&>>)
{
if (_XABORT_CODE(status))
{
// Unfortunately, actual function result is not recoverable in this case
return std::invoke_result_t<F, T&>();
}
}
// Touch memory if transaction failed without RETRY flag on the first attempt (TODO) // Touch memory if transaction failed with status 0
if (!(status & _XABORT_RETRY)) if (!status)
{ {
reinterpret_cast<atomic_t<u8>*>(sptr)->fetch_add(0); reinterpret_cast<atomic_t<u8>*>(sptr)->fetch_add(0);
} }
@ -166,8 +167,11 @@ namespace vm
// Stage 2: try to lock reservation first // Stage 2: try to lock reservation first
_old = res.fetch_add(1); _old = res.fetch_add(1);
// Also identify atomic op
count = 1;
// Start lightened transaction (TODO: tweaking) // Start lightened transaction (TODO: tweaking)
while (!(_old & rsrv_unique_lock)) for (; !(_old & rsrv_unique_lock) && count < 60; count++)
{ {
#ifndef _MSC_VER #ifndef _MSC_VER
__asm__ goto ("xbegin %l[retry];" ::: "memory" : retry); __asm__ goto ("xbegin %l[retry];" ::: "memory" : retry);
@ -188,7 +192,8 @@ namespace vm
_xend(); _xend();
#endif #endif
res += 127; res += 127;
res.notify_all(); if (Ack)
res.notify_all();
return; return;
} }
else else
@ -201,35 +206,28 @@ namespace vm
_xend(); _xend();
#endif #endif
res += 127; res += 127;
res.notify_all(); if (Ack)
res.notify_all();
return result; return result;
} }
else else
{ {
#ifndef _MSC_VER #ifndef _MSC_VER
__asm__ volatile ("xabort $1;" ::: "memory"); __asm__ volatile ("xend;" ::: "memory");
#else #else
_xabort(1); _xend();
#endif #endif
return std::invoke_result_t<F, T&>(); return result;
} }
} }
retry: retry:
#ifndef _MSC_VER #ifndef _MSC_VER
__asm__ volatile ("movl %%eax, %0;" : "=r" (status) :: "memory"); __asm__ volatile ("mov %%eax, %0;" : "=r" (status) :: "memory");
#endif #endif
if (!(status & _XABORT_RETRY)) [[unlikely]]
{
if constexpr (!std::is_void_v<std::invoke_result_t<F, T&>>)
{
if (_XABORT_CODE(status))
{
res -= 1;
return std::invoke_result_t<F, T&>();
}
}
if (!status)
{
break; break;
} }
} }
@ -237,11 +235,15 @@ namespace vm
// Stage 3: all failed, heavyweight fallback (see comments at the bottom) // Stage 3: all failed, heavyweight fallback (see comments at the bottom)
if constexpr (std::is_void_v<std::invoke_result_t<F, T&>>) if constexpr (std::is_void_v<std::invoke_result_t<F, T&>>)
{ {
return vm::reservation_op_internal(addr, [&] vm::reservation_op_internal(addr, [&]
{ {
std::invoke(op, *sptr); std::invoke(op, *sptr);
return true; return true;
}); });
if constexpr (Ack)
res.notify_all();
return;
} }
else else
{ {
@ -249,11 +251,8 @@ namespace vm
vm::reservation_op_internal(addr, [&] vm::reservation_op_internal(addr, [&]
{ {
T buf = *sptr; if ((result = std::invoke(op, *sptr)))
if ((result = std::invoke(op, buf)))
{ {
*sptr = buf;
return true; return true;
} }
else else
@ -262,6 +261,8 @@ namespace vm
} }
}); });
if (Ack && result)
res.notify_all();
return result; return result;
} }
} }
@ -269,42 +270,36 @@ namespace vm
// Perform heavyweight lock // Perform heavyweight lock
auto [res, rtime] = vm::reservation_lock(addr); auto [res, rtime] = vm::reservation_lock(addr);
// Write directly if the op cannot fail
if constexpr (std::is_void_v<std::invoke_result_t<F, T&>>) if constexpr (std::is_void_v<std::invoke_result_t<F, T&>>)
{ {
{ {
vm::writer_lock lock(addr); vm::writer_lock lock(addr);
std::invoke(op, *sptr); std::invoke(op, *sptr);
res += 127; res += 64;
} }
res.notify_all(); if constexpr (Ack)
res.notify_all();
return; return;
} }
else else
{ {
// Make an operational copy of data (TODO: volatile storage?)
auto result = std::invoke_result_t<F, T&>(); auto result = std::invoke_result_t<F, T&>();
{ {
vm::writer_lock lock(addr); vm::writer_lock lock(addr);
T buf = *sptr;
if ((result = std::invoke(op, buf))) if ((result = std::invoke(op, *sptr)))
{ {
// If operation succeeds, write the data back res += 64;
*sptr = buf;
res.release(rtime + 128);
} }
else else
{ {
// Operation failed, no memory has been modified res -= 64;
res.release(rtime);
return std::invoke_result_t<F, T&>();
} }
} }
res.notify_all(); if (Ack && result)
res.notify_all();
return result; return result;
} }
} }