diff --git a/rpcs3/util/atomic.hpp b/rpcs3/util/atomic.hpp index d83b1e7763..66d99bdfbe 100644 --- a/rpcs3/util/atomic.hpp +++ b/rpcs3/util/atomic.hpp @@ -1350,465 +1350,3 @@ public: } } }; - -template -class atomic_with_lock_bit -{ - // Simply internal type - using type = std::conditional_t, std::uintptr_t, T>; - - // Used for pointer arithmetics - using ptr_rt = std::conditional_t, ullong, T>; - - static constexpr auto c_lock_bit = BitWidth + 1; - static constexpr auto c_dirty = type{1} << BitWidth; - - // Check space for lock bit - static_assert(BitWidth <= sizeof(T) * 8 - 2, "No space for lock bit"); - static_assert(sizeof(T) <= 8 || (!std::is_pointer_v && !std::is_integral_v), "Not supported"); - static_assert(!std::is_same_v, bool>, "Bool not supported, use integral with size 1."); - static_assert(std::is_pointer_v == (BitWidth == 0), "BitWidth should be 0 for pointers"); - static_assert(!std::is_pointer_v || (alignof(std::remove_pointer_t) >= 4), "Pointer type should have align 4 or more"); - - atomic_t m_data; - -public: - using base_type = T; - - static bool is_locked(type old_val) - { - if constexpr (std::is_signed_v && BitWidth == sizeof(T) * 8 - 2) - { - return old_val < 0; - } - else if constexpr (std::is_pointer_v) - { - return (old_val & 2) != 0; - } - else - { - return (old_val & (type{2} << BitWidth)) != 0; - } - } - - static type clamp_value(type old_val) - { - if constexpr (std::is_pointer_v) - { - return old_val & (~type{0} << 2); - } - else - { - return old_val & ((type{1} << BitWidth) - type{1}); - } - } - - // Define simple type - using simple_type = simple_t; - - atomic_with_lock_bit() noexcept = default; - - atomic_with_lock_bit(const atomic_with_lock_bit&) = delete; - - atomic_with_lock_bit& operator =(const atomic_with_lock_bit&) = delete; - - constexpr atomic_with_lock_bit(T value) noexcept - : m_data(clamp_value(reinterpret_cast(value))) - { - } - - // Unsafe read - type raw_load() const - { - return clamp_value(m_data.load()); - } - - // Unsafe write and unlock - void raw_release(type value) - { - m_data.release(clamp_value(value)); - - // TODO: test dirty bit for notification - if (true) - { - m_data.notify_all(); - } - } - - void lock() - { - while (m_data.bts(c_lock_bit)) [[unlikely]] - { - type old_val = m_data.load(); - - if (is_locked(old_val)) [[likely]] - { - if ((old_val & c_dirty) == 0) - { - // Try to set dirty bit if not set already - if (!m_data.compare_and_swap_test(old_val, old_val | c_dirty)) - { - continue; - } - } - - m_data.wait(old_val | c_dirty); - old_val = m_data.load(); - } - } - } - - bool try_lock() - { - return !m_data.bts(c_lock_bit); - } - - void unlock() - { - type old_val = m_data.load(); - - if constexpr (std::is_pointer_v) - { - m_data.and_fetch(~type{0} << 2); - } - else - { - m_data.and_fetch((type{1} << BitWidth) - type{1}); - } - - // Test dirty bit for notification - if (old_val & c_dirty) - { - m_data.notify_all(); - } - } - - T load() - { - type old_val = m_data.load(); - - while (is_locked(old_val)) [[unlikely]] - { - if ((old_val & c_dirty) == 0) - { - if (!m_data.compare_and_swap_test(old_val, old_val | c_dirty)) - { - old_val = m_data.load(); - continue; - } - } - - m_data.wait(old_val | c_dirty); - old_val = m_data.load(); - } - - return reinterpret_cast(clamp_value(old_val)); - } - - void store(T value) - { - static_cast(exchange(value)); - } - - T exchange(T value) - { - type old_val = m_data.load(); - - while (is_locked(old_val) || !m_data.compare_and_swap_test(old_val, clamp_value(reinterpret_cast(value)))) [[unlikely]] - { - if ((old_val & c_dirty) == 0) - { - if (!m_data.compare_and_swap_test(old_val, old_val | c_dirty)) - { - old_val = m_data.load(); - continue; - } - } - - m_data.wait(old_val); - old_val = m_data.load(); - } - - return reinterpret_cast(clamp_value(old_val)); - } - - T compare_and_swap(T cmp, T exch) - { - static_cast(compare_exchange(cmp, exch)); - return cmp; - } - - bool compare_and_swap_test(T cmp, T exch) - { - return compare_exchange(cmp, exch); - } - - bool compare_exchange(T& cmp_and_old, T exch) - { - type old_val = m_data.load(); - type expected = clamp_value(reinterpret_cast(cmp_and_old)); - type new_val = clamp_value(reinterpret_cast(exch)); - - while (is_locked(old_val) || (old_val == expected && !m_data.compare_and_swap_test(expected, new_val))) [[unlikely]] - { - if (old_val == expected) - { - old_val = m_data.load(); - continue; - } - - if ((old_val & c_dirty) == 0) - { - if (!m_data.compare_and_swap_test(old_val, old_val | c_dirty)) - { - old_val = m_data.load(); - continue; - } - } - - m_data.wait(old_val); - old_val = m_data.load(); - } - - cmp_and_old = reinterpret_cast(clamp_value(old_val)); - - return clamp_value(old_val) == expected; - } - - template > - RT atomic_op(F func) - { - type _new, old; - old = m_data.load(); - - while (true) - { - if (is_locked(old)) [[unlikely]] - { - if ((old & c_dirty) == 0) - { - if (!m_data.compare_and_swap_test(old, old | c_dirty)) - { - old = m_data.load(); - continue; - } - } - - m_data.wait(old); - old = m_data.load(); - continue; - } - - _new = old; - - if constexpr (std::is_void_v) - { - std::invoke(func, reinterpret_cast(_new)); - - if (atomic_storage::compare_exchange(m_data.raw(), old, clamp_value(_new))) [[likely]] - { - return; - } - } - else - { - RT result = std::invoke(func, reinterpret_cast(_new)); - - if (atomic_storage::compare_exchange(m_data.raw(), old, clamp_value(_new))) [[likely]] - { - return result; - } - } - } - } - - auto fetch_add(const ptr_rt& rhs) - { - return atomic_op([&](T& v) - { - return std::exchange(v, (v += rhs)); - }); - } - - auto operator +=(const ptr_rt& rhs) - { - return atomic_op([&](T& v) - { - return v += rhs; - }); - } - - auto fetch_sub(const ptr_rt& rhs) - { - return atomic_op([&](T& v) - { - return std::exchange(v, (v -= rhs)); - }); - } - - auto operator -=(const ptr_rt& rhs) - { - return atomic_op([&](T& v) - { - return v -= rhs; - }); - } - - auto fetch_and(const T& rhs) - { - return atomic_op([&](T& v) - { - return std::exchange(v, (v &= rhs)); - }); - } - - auto operator &=(const T& rhs) - { - return atomic_op([&](T& v) - { - return v &= rhs; - }); - } - - auto fetch_or(const T& rhs) - { - return atomic_op([&](T& v) - { - return std::exchange(v, (v |= rhs)); - }); - } - - auto operator |=(const T& rhs) - { - return atomic_op([&](T& v) - { - return v |= rhs; - }); - } - - auto fetch_xor(const T& rhs) - { - return atomic_op([&](T& v) - { - return std::exchange(v, (v ^= rhs)); - }); - } - - auto operator ^=(const T& rhs) - { - return atomic_op([&](T& v) - { - return v ^= rhs; - }); - } - - auto operator ++() - { - return atomic_op([](T& v) - { - return ++v; - }); - } - - auto operator --() - { - return atomic_op([](T& v) - { - return --v; - }); - } - - auto operator ++(int) - { - return atomic_op([](T& v) - { - return v++; - }); - } - - auto operator --(int) - { - return atomic_op([](T& v) - { - return v--; - }); - } -}; - -using fat_atomic_u1 = atomic_with_lock_bit; -using fat_atomic_u6 = atomic_with_lock_bit; -using fat_atomic_s6 = atomic_with_lock_bit; -using fat_atomic_u8 = atomic_with_lock_bit; -using fat_atomic_s8 = atomic_with_lock_bit; - -using fat_atomic_u14 = atomic_with_lock_bit; -using fat_atomic_s14 = atomic_with_lock_bit; -using fat_atomic_u16 = atomic_with_lock_bit; -using fat_atomic_s16 = atomic_with_lock_bit; - -using fat_atomic_u30 = atomic_with_lock_bit; -using fat_atomic_s30 = atomic_with_lock_bit; -using fat_atomic_u32 = atomic_with_lock_bit; -using fat_atomic_s32 = atomic_with_lock_bit; -using fat_atomic_u62 = atomic_with_lock_bit; -using fat_atomic_s62 = atomic_with_lock_bit; - -template -using fat_atomic_ptr = atomic_with_lock_bit; - -namespace detail -{ - template - struct mao_func_t - { - template - using RT = typename mao_func_t::template RT; - }; - - template - struct mao_func_t - { - template - using RT = std::invoke_result_t&...>; - }; - - template - using mao_result = typename mao_func_t...>::template RT<>; - - template - RT multi_atomic_op(std::index_sequence, Args&&... args) - { - // Tie all arguments (function is the latest) - auto vars = std::tie(args...); - - // Lock all variables - std::lock(std::get(vars)...); - - // Load initial values - auto values = std::make_tuple(std::get(vars).raw_load()...); - - if constexpr (std::is_void_v) - { - std::invoke(std::get<(sizeof...(Args) - 1)>(vars), reinterpret_cast(vars))>::base_type&>(std::get(values))...); - - // Unlock and return - (std::get(vars).raw_release(std::get(values)), ...); - } - else - { - RT result = std::invoke(std::get<(sizeof...(Args) - 1)>(vars), reinterpret_cast(vars))>::base_type&>(std::get(values))...); - - // Unlock and return the result - (std::get(vars).raw_release(std::get(values)), ...); - - return result; - } - } -} - -// Atomic operation; returns function result value, function is the lambda -template > -RT multi_atomic_op(Args&&... args) -{ - return detail::multi_atomic_op(std::make_index_sequence<(sizeof...(Args) - 1)>(), std::forward(args)...); -}