rsx: Avoid calling any blocking callbacks from threads that are not rsx::thread

- Defers on_notity_memory_unmapped to only run from within rsx context
- Avoids passive_lock + writer_lock deadlock

rpcs3/Emu/RSX/GL/GLGSRender.cpp

@@ -1559,7 +1559,7 @@ bool GLGSRender::on_access_violation(u32 address, bool is_writing)
 	return true;
 }
 
-void GLGSRender::on_notify_memory_unmapped(u32 address_base, u32 size)
+void GLGSRender::on_invalidate_memory_range(u32 address_base, u32 size)
 {
 	//Discard all memory in that range without bothering with writeback (Force it for strict?)
 	if (m_gl_texture_cache.invalidate_range(address_base, size, true, true, false).violation_handled)
@@ -1572,7 +1572,7 @@ void GLGSRender::on_notify_memory_unmapped(u32 address_base, u32 size)
 	}
 }
 
-void GLGSRender::do_local_task(bool /*idle*/)
+void GLGSRender::do_local_task(bool idle)
 {
 	m_frame->clear_wm_events();
 
@@ -1610,6 +1610,8 @@ void GLGSRender::do_local_task(bool /*idle*/)
 			flip((s32)current_display_buffer);
 		}
 	}
+
+	rsx::thread::do_local_task(idle);
 }
 
 work_item& GLGSRender::post_flush_request(u32 address, gl::texture_cache::thrashed_set& flush_data)
@@ -1644,6 +1646,11 @@ void GLGSRender::notify_tile_unbound(u32 tile)
 	//u32 addr = rsx::get_address(tiles[tile].offset, tiles[tile].location);
 	//on_notify_memory_unmapped(addr, tiles[tile].size);
 	//m_rtts.invalidate_surface_address(addr, false);
+
+	{
+		std::lock_guard<shared_mutex> lock(m_sampler_mutex);
+		m_samplers_dirty.store(true);
+	}
 }
 
 void GLGSRender::begin_occlusion_query(rsx::reports::occlusion_query_info* query)

rpcs3/Emu/RSX/GL/GLGSRender.h

@@ -384,7 +384,7 @@ protected:
 	void do_local_task(bool idle) override;
 
 	bool on_access_violation(u32 address, bool is_writing) override;
-	void on_notify_memory_unmapped(u32 address_base, u32 size) override;
+	void on_invalidate_memory_range(u32 address_base, u32 size) override;
 	void notify_tile_unbound(u32 tile) override;
 
 	std::array<std::vector<gsl::byte>, 4> copy_render_targets_to_memory() override;

rpcs3/Emu/RSX/RSXThread.cpp

@@ -1276,6 +1276,19 @@ namespace rsx
 		}
 	}
 
+	void thread::do_local_task(bool /*idle*/)
+	{
+		if (!in_begin_end)
+		{
+			for (const auto& range : m_invalidated_memory_ranges)
+			{
+				on_invalidate_memory_range(range.first, range.second);
+			}
+
+			m_invalidated_memory_ranges.clear();
+		}
+	}
+
 	//std::future<void> thread::add_internal_task(std::function<bool()> callback)
 	//{
 	//	std::lock_guard<shared_mutex> lock{ m_mtx_task };
@@ -2312,6 +2325,11 @@ namespace rsx
 		check_zcull_status(false);
 	}
 
+	void thread::on_notify_memory_unmapped(u32 base_address, u32 size)
+	{
+		m_invalidated_memory_ranges.push_back({ base_address, size });
+	}
+
 	//Pause/cont wrappers for FIFO ctrl. Never call this from rsx thread itself!
 	void thread::pause()
 	{

rpcs3/Emu/RSX/RSXThread.h

@@ -84,9 +84,9 @@ namespace rsx
 	enum FIFO_state : u8
 	{
 		running = 0,
-		empty = 1, //PUT == GET
+		empty = 1,    // PUT == GET
-		spinning = 2, //Puller continuously jumps to self addr (synchronization technique)
+		spinning = 2, // Puller continuously jumps to self addr (synchronization technique)
-		nop = 3, //Puller is processing a NOP command
+		nop = 3,      // Puller is processing a NOP command
 	};
 
 	u32 get_vertex_type_size_on_host(vertex_base_type type, u32 size);
@@ -172,9 +172,9 @@ namespace rsx
 
 	struct vertex_input_layout
 	{
-		std::vector<interleaved_range_info> interleaved_blocks;  //Interleaved blocks to be uploaded as-is
+		std::vector<interleaved_range_info> interleaved_blocks;  // Interleaved blocks to be uploaded as-is
-		std::vector<std::pair<u8, u32>> volatile_blocks;  //Volatile data blocks (immediate draw vertex data for example)
+		std::vector<std::pair<u8, u32>> volatile_blocks;  // Volatile data blocks (immediate draw vertex data for example)
-		std::vector<u8> referenced_registers;  //Volatile register data
+		std::vector<u8> referenced_registers;  // Volatile register data
 
 		std::array<attribute_buffer_placement, 16> attribute_placement;
 	};
@@ -206,11 +206,11 @@ namespace rsx
 
 		struct ZCULL_control
 		{
-			//Delay in 'cycles' before a report update operation is forced to retire
+			// Delay in 'cycles' before a report update operation is forced to retire
 			const u32 max_zcull_cycles_delay = 128;
 			const u32 min_zcull_cycles_delay = 16;
 
-			//Number of occlusion query slots available. Real hardware actually has far fewer units before choking
+			// Number of occlusion query slots available. Real hardware actually has far fewer units before choking
 			const u32 occlusion_query_count = 128;
 
 			bool active = false;
@@ -234,31 +234,31 @@ namespace rsx
 
 			void write(vm::addr_t sink, u32 timestamp, u32 type, u32 value);
 
-			//Read current zcull statistics into the address provided
+			// Read current zcull statistics into the address provided
 			void read_report(class ::rsx::thread* ptimer, vm::addr_t sink, u32 type);
 
-			//Sets up a new query slot and sets it to the current task
+			// Sets up a new query slot and sets it to the current task
 			void allocate_new_query(class ::rsx::thread* ptimer);
 
-			//clears current stat block and increments stat_tag_id
+			// Clears current stat block and increments stat_tag_id
 			void clear(class ::rsx::thread* ptimer);
 
-			//forcefully flushes all
+			// Forcefully flushes all
 			void sync(class ::rsx::thread* ptimer);
 
-			//conditionally sync any pending writes if range overlaps
+			// Conditionally sync any pending writes if range overlaps
 			void read_barrier(class ::rsx::thread* ptimer, u32 memory_address, u32 memory_range);
 
-			//call once every 'tick' to update
+			// Call once every 'tick' to update
 			void update(class ::rsx::thread* ptimer);
 
-			//Draw call notification
+			// Draw call notification
 			void on_draw();
 
-			//Check for pending writes
+			// Check for pending writes
 			bool has_pending() const { return (m_pending_writes.size() != 0); }
 
-			//Backend methods (optional, will return everything as always visible by default)
+			// Backend methods (optional, will return everything as always visible by default)
 			virtual void begin_occlusion_query(occlusion_query_info* /*query*/) {}
 			virtual void end_occlusion_query(occlusion_query_info* /*query*/) {}
 			virtual bool check_occlusion_query_status(occlusion_query_info* /*query*/) { return true; }
@@ -285,17 +285,20 @@ namespace rsx
 		bool supports_multidraw = false;
 		bool supports_native_ui = false;
 
-		//occlusion query
+		// Occlusion query
 		bool zcull_surface_active = false;
 		std::unique_ptr<reports::ZCULL_control> zcull_ctrl;
 
-		//framebuffer setup
+		// Framebuffer setup
 		rsx::gcm_framebuffer_info m_surface_info[rsx::limits::color_buffers_count];
 		rsx::gcm_framebuffer_info m_depth_surface_info;
 		bool framebuffer_status_valid = false;
 
+		// Overlays
 		std::shared_ptr<rsx::overlays::display_manager> m_overlay_manager;
-		std::unique_ptr<rsx::overlays::user_interface> m_invalidated_ui;
+
+		// Invalidated memory range
+		std::vector<std::pair<u32, u32>> m_invalidated_memory_ranges;
 
 	public:
 		RsxDmaControl* ctrl = nullptr;
@@ -304,25 +307,25 @@ namespace rsx
 		atomic_t<bool> external_interrupt_lock{ false };
 		atomic_t<bool> external_interrupt_ack{ false };
 
-		//performance approximation counters
+		// Performance approximation counters
 		struct
 		{
-			atomic_t<u64> idle_time{ 0 };  //Time spent idling in microseconds
+			atomic_t<u64> idle_time{ 0 };  // Time spent idling in microseconds
-			u64 last_update_timestamp = 0; //Timestamp of last load update
+			u64 last_update_timestamp = 0; // Timestamp of last load update
-			u64 FIFO_idle_timestamp = 0; //Timestamp of when FIFO queue becomes idle
+			u64 FIFO_idle_timestamp = 0;   // Timestamp of when FIFO queue becomes idle
 			FIFO_state state = FIFO_state::running;
 			u32 approximate_load = 0;
 			u32 sampled_frames = 0;
 		}
 		performance_counters;
 
-		//native UI interrupts
+		// Native UI interrupts
 		atomic_t<bool> native_ui_flip_request{ false };
 
 		GcmTileInfo tiles[limits::tiles_count];
 		GcmZcullInfo zculls[limits::zculls_count];
 
-		//super memory map (mapped block with r/w permissions)
+		// Super memory map (mapped block with r/w permissions)
 		std::pair<u32, std::shared_ptr<u8>> super_memory_map;
 
 		bool capture_current_frame = false;
@@ -416,7 +419,7 @@ namespace rsx
 		 * Execute a backend local task queue
 		 * Idle argument checks that the FIFO queue is in an idle state
 		 */
-		virtual void do_local_task(bool /*idle*/) {}
+		virtual void do_local_task(bool idle);
 
 	public:
 		virtual std::string get_name() const override;
@@ -433,16 +436,16 @@ namespace rsx
 		virtual void flip(int buffer) = 0;
 		virtual u64 timestamp() const;
 		virtual bool on_access_violation(u32 /*address*/, bool /*is_writing*/) { return false; }
-		virtual void on_notify_memory_unmapped(u32 /*address_base*/, u32 /*size*/) {}
+		virtual void on_invalidate_memory_range(u32 /*address*/, u32 /*range*/) {}
 		virtual void notify_tile_unbound(u32 /*tile*/) {}
 
-		//zcull
+		// zcull
 		void notify_zcull_info_changed();
 		void clear_zcull_stats(u32 type);
 		void check_zcull_status(bool framebuffer_swap);
 		void get_zcull_stats(u32 type, vm::addr_t sink);
 
-		//sync
+		// sync
 		void sync();
 		void read_barrier(u32 memory_address, u32 memory_range);
 		
@@ -504,8 +507,6 @@ namespace rsx
 	public:
 		//std::future<void> add_internal_task(std::function<bool()> callback);
 		//void invoke(std::function<bool()> callback);
-		void add_user_interface(std::shared_ptr<rsx::overlays::user_interface> iface);
-		void remove_user_interface();
 
 		/**
 		 * Fill buffer with 4x4 scale offset matrix.
@@ -516,8 +517,7 @@ namespace rsx
 
 		/**
 		 * Fill buffer with user clip information
-		*/
+		 */
-
 		void fill_user_clip_data(void *buffer) const;
 
 		/**
@@ -533,12 +533,18 @@ namespace rsx
 		void fill_fragment_state_buffer(void *buffer, const RSXFragmentProgram &fragment_program);
 
 		/**
-		* Write inlined array data to buffer.
+		 * Write inlined array data to buffer.
-		* The storage of inlined data looks different from memory stored arrays.
+		 * The storage of inlined data looks different from memory stored arrays.
-		* There is no swapping required except for 4 u8 (according to Bleach Soul Resurection)
+		 * There is no swapping required except for 4 u8 (according to Bleach Soul Resurection)
-		*/
+		 */
 		void write_inline_array_to_buffer(void *dst_buffer);
 
+		/**
+		 * Notify that a section of memory has been unmapped
+		 * Any data held in the defined range is discarded
+		 */
+		void on_notify_memory_unmapped(u32 address_base, u32 size);
+
 		/**
 		 * Copy rtt values to buffer.
 		 * TODO: It's more efficient to combine multiple call of this function into one.

rpcs3/Emu/RSX/VK/VKGSRender.cpp

@@ -880,7 +880,7 @@ bool VKGSRender::on_access_violation(u32 address, bool is_writing)
 	return false;
 }
 
-void VKGSRender::on_notify_memory_unmapped(u32 address_base, u32 size)
+void VKGSRender::on_invalidate_memory_range(u32 address_base, u32 size)
 {
 	std::lock_guard<shared_mutex> lock(m_secondary_cb_guard);
 	if (m_texture_cache.invalidate_range(address_base, size, true, true, false,
@@ -2082,7 +2082,7 @@ void VKGSRender::process_swap_request(frame_context_t *ctx, bool free_resources)
 	ctx->swap_command_buffer = nullptr;
 }
 
-void VKGSRender::do_local_task(bool /*idle*/)
+void VKGSRender::do_local_task(bool idle)
 {
 	if (m_flush_requests.pending())
 	{
@@ -2201,6 +2201,8 @@ void VKGSRender::do_local_task(bool /*idle*/)
 			flip((s32)current_display_buffer);
 		}
 	}
+
+	rsx::thread::do_local_task(idle);
 }
 
 bool VKGSRender::do_method(u32 cmd, u32 arg)

rpcs3/Emu/RSX/VK/VKGSRender.h

@@ -425,5 +425,5 @@ protected:
 	void notify_tile_unbound(u32 tile) override;
 
 	bool on_access_violation(u32 address, bool is_writing) override;
-	void on_notify_memory_unmapped(u32 address_base, u32 size) override;
+	void on_invalidate_memory_range(u32 address_base, u32 size) override;
 };

rpcs3/Emu/RSX/rsx_utils.h

@@ -159,7 +159,7 @@ namespace rsx
 					{
 						// Head
 						write_offset = (offset - base_offset);
-						write_length = std::min<u32>(block.second - write_offset, remaining_bytes);
+						write_length = std::min<u32>(block.second - write_offset, (u32)remaining_bytes);
 					}
 					else if (base_offset > offset && block_end <= write_end)
 					{
@@ -171,7 +171,7 @@ namespace rsx
 					{
 						// Tail
 						write_offset = 0;
-						write_length = remaining_bytes;
+						write_length = (u32)remaining_bytes;
 					}
 					else
 					{