archive/rpcs3
1
0
Fork 0
mirror of https://github.com/RPCS3/rpcs3.git synced 2025-07-13 10:18:40 +12:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

atomic.cpp: use some actual hash for hashtable (std::minstd_rand)

Browse source 
Should only take about 5ns for initialization.
Should also make collisions almost impossible.

I also suspect I fixed some bad bug in process...
This commit is contained in:
Nekotekina 2020-11-20 04:59:05 +03:00
parent 5bcaa25c02
commit 72ac1aa537
1 changed files with 65 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

127
rpcs3/util/atomic.cpp
View file

@ -734,12 +734,14 @@ namespace atomic_wait
// For collision statistics (bit difference stick flags) // For collision statistics (bit difference stick flags)
atomic_t<u32> diff_lz{}, diff_tz{}, diff_pop{}; atomic_t<u32> diff_lz{}, diff_tz{}, diff_pop{};
atomic_t<u16>* slot_alloc(std::uintptr_t ptr) noexcept; static atomic_t<u16>* slot_alloc(std::uintptr_t ptr) noexcept;
root_info* slot_free(std::uintptr_t ptr, atomic_t<u16>* slot) noexcept; static void slot_free(std::uintptr_t ptr, atomic_t<u16>* slot) noexcept;
template <typename F> template <typename F>
auto slot_search(std::uintptr_t iptr, u32 size, u64 thread_id, __m128i mask, F func) noexcept; static auto slot_search(std::uintptr_t iptr, u32 size, u64 thread_id, __m128i mask, F func) noexcept;
void register_collisions(std::uintptr_t ptr);
}; };
static_assert(sizeof(root_info) == 256); static_assert(sizeof(root_info) == 256);
@ -748,6 +750,37 @@ namespace atomic_wait
// Main hashtable for atomic wait. // Main hashtable for atomic wait.
static atomic_wait::root_info s_hashtable[s_hashtable_size]{}; static atomic_wait::root_info s_hashtable[s_hashtable_size]{};
namespace
{
struct hash_engine
{
// Must be very lightweight
std::minstd_rand rnd;
// Pointer to the current hashtable slot
atomic_wait::root_info* current;
// Initialize
hash_engine(std::uintptr_t iptr)
: rnd(static_cast<u32>(iptr >> 15))
, current(&s_hashtable[((rnd() >> 1) + iptr) % s_hashtable_size])
{
}
// Advance
void operator++(int) noexcept
{
current = &s_hashtable[(rnd() >> 1) % s_hashtable_size];
}
// Access current
atomic_wait::root_info* operator ->() const noexcept
{
return current;
}
};
}
u64 atomic_wait::get_unique_tsc() u64 atomic_wait::get_unique_tsc()
{ {
const u64 stamp0 = __rdtsc(); const u64 stamp0 = __rdtsc();
@ -773,9 +806,9 @@ atomic_t<u16>* atomic_wait::root_info::slot_alloc(std::uintptr_t ptr) noexcept
u32 limit = 0; u32 limit = 0;
for (auto* _this = this;;) for (hash_engine _this(ptr);; _this++)
{ {
slot = _this->bits.atomic_op([_this](slot_allocator& bits) -> atomic_t<u16>* slot = _this->bits.atomic_op([&](slot_allocator& bits) -> atomic_t<u16>*
{ {
// Increment reference counter on every hashtable slot we attempt to allocate on // Increment reference counter on every hashtable slot we attempt to allocate on
if (bits.ref == UINT16_MAX) if (bits.ref == UINT16_MAX)
@ -803,13 +836,14 @@ atomic_t<u16>* atomic_wait::root_info::slot_alloc(std::uintptr_t ptr) noexcept
return nullptr; return nullptr;
}); });
_this->register_collisions(ptr);
if (slot) if (slot)
{ {
break; break;
} }
// Keep trying adjacent slots in the hashtable, they are often free due to alignment. // Keep trying adjacent slots in the hashtable, they are often free due to alignment.
_this++;
limit++; limit++;
if (limit == max_distance) [[unlikely]] if (limit == max_distance) [[unlikely]]
@ -817,13 +851,13 @@ atomic_t<u16>* atomic_wait::root_info::slot_alloc(std::uintptr_t ptr) noexcept
fmt::raw_error("Distance limit (585) exceeded for the atomic wait hashtable."); fmt::raw_error("Distance limit (585) exceeded for the atomic wait hashtable.");
return nullptr; return nullptr;
} }
if (_this == std::end(s_hashtable)) [[unlikely]]
{
_this = s_hashtable;
}
} }
return slot;
}
void atomic_wait::root_info::register_collisions(std::uintptr_t ptr)
{
u32 ptr32 = static_cast<u32>(ptr >> 16); u32 ptr32 = static_cast<u32>(ptr >> 16);
u32 first = first_ptr.load(); u32 first = first_ptr.load();
@ -860,11 +894,9 @@ atomic_t<u16>* atomic_wait::root_info::slot_alloc(std::uintptr_t ptr) noexcept
diff_pop |= 1u << static_cast<u8>((diff >> 16) + diff - 1); diff_pop |= 1u << static_cast<u8>((diff >> 16) + diff - 1);
} }
return slot;
} }
atomic_wait::root_info* atomic_wait::root_info::slot_free(std::uintptr_t, atomic_t<u16>* slot) noexcept void atomic_wait::root_info::slot_free(std::uintptr_t iptr, atomic_t<u16>* slot) noexcept
{ {
const auto begin = reinterpret_cast<std::uintptr_t>(std::begin(s_hashtable)); const auto begin = reinterpret_cast<std::uintptr_t>(std::begin(s_hashtable));
@ -875,7 +907,7 @@ atomic_wait::root_info* atomic_wait::root_info::slot_free(std::uintptr_t, atomic
if (ptr >= sizeof(s_hashtable)) if (ptr >= sizeof(s_hashtable))
{ {
fmt::raw_error("Failed to find slot in hashtable slot deallocation." HERE); fmt::raw_error("Failed to find slot in hashtable slot deallocation." HERE);
return nullptr; return;
} }
root_info* _this = &s_hashtable[ptr / sizeof(root_info)]; root_info* _this = &s_hashtable[ptr / sizeof(root_info)];
@ -883,7 +915,7 @@ atomic_wait::root_info* atomic_wait::root_info::slot_free(std::uintptr_t, atomic
if (!(slot >= _this->slots && slot < std::end(_this->slots))) if (!(slot >= _this->slots && slot < std::end(_this->slots)))
{ {
fmt::raw_error("Failed to find slot in hashtable slot deallocation." HERE); fmt::raw_error("Failed to find slot in hashtable slot deallocation." HERE);
return nullptr; return;
} }
const u32 diff = static_cast<u32>(slot - _this->slots); const u32 diff = static_cast<u32>(slot - _this->slots);
@ -897,14 +929,14 @@ atomic_wait::root_info* atomic_wait::root_info::slot_free(std::uintptr_t, atomic
cond_free(cond_id); cond_free(cond_id);
} }
for (auto entry = this;;) for (hash_engine curr(iptr);; curr++)
{ {
// Reset reference counter and allocation bit in every slot // Reset reference counter and allocation bit in every slot
bits.atomic_op([&](slot_allocator& bits) curr->bits.atomic_op([&](slot_allocator& bits)
{ {
verify(HERE), bits.ref--; verify(HERE), bits.ref--;
if (_this == entry) if (_this == curr.current)
{ {
if (diff < 64) if (diff < 64)
{ {
@ -917,20 +949,11 @@ atomic_wait::root_info* atomic_wait::root_info::slot_free(std::uintptr_t, atomic
} }
}); });
if (_this == entry) if (_this == curr.current)
{ {
break; break;
} }
entry++;
if (entry == std::end(s_hashtable)) [[unlikely]]
{
entry = s_hashtable;
}
} }
return _this;
} }
template <typename F> template <typename F>
@ -939,24 +962,21 @@ FORCE_INLINE auto atomic_wait::root_info::slot_search(std::uintptr_t iptr, u32 s
u32 index = 0; u32 index = 0;
u32 total = 0; u32 total = 0;
for (auto* _this = this;;) for (hash_engine _this(iptr);; _this++)
{ {
const auto bits = _this->bits.load(); const auto bits = _this->bits.load();
if (bits.ref == 0) [[likely]]
{
return;
}
u16 cond_ids[max_threads]; u16 cond_ids[max_threads];
u32 cond_count = 0; u32 cond_count = 0;
u64 high_val = bits.high; u64 high_val = bits.high;
u64 low_val = bits.low; u64 low_val = bits.low;
if (_this == this)
{
if (bits.ref == 0)
{
return;
}
}
for (u64 bits = high_val; bits; bits &= bits - 1) for (u64 bits = high_val; bits; bits &= bits - 1)
{ {
if (u16 cond_id = _this->slots[std::countr_zero(bits)]) if (u16 cond_id = _this->slots[std::countr_zero(bits)])
@ -988,18 +1008,12 @@ FORCE_INLINE auto atomic_wait::root_info::slot_search(std::uintptr_t iptr, u32 s
total += cond_count; total += cond_count;
_this++;
index++; index++;
if (index == max_distance) if (index == max_distance)
{ {
return; return;
} }
if (_this >= std::end(s_hashtable))
{
_this = s_hashtable;
}
} }
} }
@ -1018,14 +1032,10 @@ atomic_wait_engine::wait(const void* data, u32 size, __m128i old_value, u64 time
const std::uintptr_t iptr = reinterpret_cast<std::uintptr_t>(data) & (~s_ref_mask >> 17); const std::uintptr_t iptr = reinterpret_cast<std::uintptr_t>(data) & (~s_ref_mask >> 17);
const auto root = &s_hashtable[iptr % s_hashtable_size];
uint ext_size = 0; uint ext_size = 0;
std::uintptr_t iptr_ext[atomic_wait::max_list - 1]{}; std::uintptr_t iptr_ext[atomic_wait::max_list - 1]{};
atomic_wait::root_info* root_ext[atomic_wait::max_list - 1]{};
if (ext) [[unlikely]] if (ext) [[unlikely]]
{ {
for (auto e = ext; e->data; e++) for (auto e = ext; e->data; e++)
@ -1044,7 +1054,6 @@ atomic_wait_engine::wait(const void* data, u32 size, __m128i old_value, u64 time
} }
iptr_ext[ext_size] = reinterpret_cast<std::uintptr_t>(e->data) & (~s_ref_mask >> 17); iptr_ext[ext_size] = reinterpret_cast<std::uintptr_t>(e->data) & (~s_ref_mask >> 17);
root_ext[ext_size] = &s_hashtable[iptr_ext[ext_size] % s_hashtable_size];
ext_size++; ext_size++;
} }
} }
@ -1058,7 +1067,7 @@ atomic_wait_engine::wait(const void* data, u32 size, __m128i old_value, u64 time
cond_id_ext[i] = cond_alloc(iptr_ext[i], ext[i].mask); cond_id_ext[i] = cond_alloc(iptr_ext[i], ext[i].mask);
} }
const auto slot = root->slot_alloc(iptr); const auto slot = atomic_wait::root_info::slot_alloc(iptr);
std::array<atomic_t<u16>*, atomic_wait::max_list - 1> slot_ext{}; std::array<atomic_t<u16>*, atomic_wait::max_list - 1> slot_ext{};
@ -1067,7 +1076,7 @@ atomic_wait_engine::wait(const void* data, u32 size, __m128i old_value, u64 time
for (u32 i = 0; i < ext_size; i++) for (u32 i = 0; i < ext_size; i++)
{ {
// Allocate slot for cond id location // Allocate slot for cond id location
slot_ext[i] = root_ext[i]->slot_alloc(iptr_ext[i]); slot_ext[i] = atomic_wait::root_info::slot_alloc(iptr_ext[i]);
// Get pointers to the semaphores // Get pointers to the semaphores
cond_ext[i] = s_cond_list + cond_id_ext[i]; cond_ext[i] = s_cond_list + cond_id_ext[i];
@ -1263,10 +1272,10 @@ atomic_wait_engine::wait(const void* data, u32 size, __m128i old_value, u64 time
// Release resources in reverse order // Release resources in reverse order
for (u32 i = ext_size - 1; i != umax; i--) for (u32 i = ext_size - 1; i != umax; i--)
{ {
verify(HERE), root_ext[i] == root_ext[i]->slot_free(iptr_ext[i], slot_ext[i]); atomic_wait::root_info::slot_free(iptr_ext[i], slot_ext[i]);
} }
verify(HERE), root == root->slot_free(iptr, slot); atomic_wait::root_info::slot_free(iptr, slot);
s_tls_wait_cb(data, -1, stamp0); s_tls_wait_cb(data, -1, stamp0);
} }
@ -1460,13 +1469,11 @@ bool atomic_wait_engine::raw_notify(const void* data, u64 thread_id)
const std::uintptr_t iptr = reinterpret_cast<std::uintptr_t>(data) & (~s_ref_mask >> 17); const std::uintptr_t iptr = reinterpret_cast<std::uintptr_t>(data) & (~s_ref_mask >> 17);
auto* const root = &s_hashtable[iptr % s_hashtable_size];
s_tls_notify_cb(data, 0); s_tls_notify_cb(data, 0);
u64 progress = 0; u64 progress = 0;
root->slot_search(iptr, 0, thread_id, _mm_set1_epi64x(-1), [&](u32 cond_id) atomic_wait::root_info::slot_search(iptr, 0, thread_id, _mm_set1_epi64x(-1), [&](u32 cond_id)
{ {
// Forced notification // Forced notification
if (alert_sema(cond_id, data, thread_id, 0, _mm_setzero_si128(), _mm_setzero_si128())) if (alert_sema(cond_id, data, thread_id, 0, _mm_setzero_si128(), _mm_setzero_si128()))
@ -1498,13 +1505,11 @@ atomic_wait_engine::notify_one(const void* data, u32 size, __m128i mask, __m128i
{ {
const std::uintptr_t iptr = reinterpret_cast<std::uintptr_t>(data) & (~s_ref_mask >> 17); const std::uintptr_t iptr = reinterpret_cast<std::uintptr_t>(data) & (~s_ref_mask >> 17);
auto* const root = &s_hashtable[iptr % s_hashtable_size];
s_tls_notify_cb(data, 0); s_tls_notify_cb(data, 0);
u64 progress = 0; u64 progress = 0;
root->slot_search(iptr, size, 0, mask, [&](u32 cond_id) atomic_wait::root_info::slot_search(iptr, size, 0, mask, [&](u32 cond_id)
{ {
if (alert_sema(cond_id, data, -1, size, mask, new_value)) if (alert_sema(cond_id, data, -1, size, mask, new_value))
{ {
@ -1526,8 +1531,6 @@ atomic_wait_engine::notify_all(const void* data, u32 size, __m128i mask, __m128i
{ {
const std::uintptr_t iptr = reinterpret_cast<std::uintptr_t>(data) & (~s_ref_mask >> 17); const std::uintptr_t iptr = reinterpret_cast<std::uintptr_t>(data) & (~s_ref_mask >> 17);
auto* const root = &s_hashtable[iptr % s_hashtable_size];
s_tls_notify_cb(data, 0); s_tls_notify_cb(data, 0);
u64 progress = 0; u64 progress = 0;
@ -1538,7 +1541,7 @@ atomic_wait_engine::notify_all(const void* data, u32 size, __m128i mask, __m128i
// Array itself. // Array itself.
u16 cond_ids[UINT16_MAX + 1]; u16 cond_ids[UINT16_MAX + 1];
root->slot_search(iptr, size, 0, mask, [&](u32 cond_id) atomic_wait::root_info::slot_search(iptr, size, 0, mask, [&](u32 cond_id)
{ {
u32 res = alert_sema<true>(cond_id, data, -1, size, mask, new_value); u32 res = alert_sema<true>(cond_id, data, -1, size, mask, new_value);