vk: Restructure commandbuffer submission into tagged event IDs

- Tagged eventIDs can be used to safely delete resources that are no
longer used
- TODO: Expand gc to collect images as well
- TODO: Fix the texture cache to avoid over-allocating image resources

rpcs3/Emu/RSX/VK/VKGSRender.cpp

@@ -8,6 +8,7 @@
 #include "VKFormats.h"
 #include "VKCommonDecompiler.h"
 #include "VKRenderPass.h"
+#include "VKResourceManager.h"
 
 namespace
 {
@@ -640,7 +641,6 @@ VKGSRender::~VKGSRender()
 	m_rtts.destroy();
 	m_texture_cache.destroy();
 
-	m_resource_manager.destroy();
 	m_stencil_mirror_sampler.reset();
 
 	//Overlay text handler
@@ -1372,7 +1372,7 @@ void VKGSRender::end()
 
 					if (replace)
 					{
-						fs_sampler_handles[i] = m_resource_manager.find_sampler(*m_device, wrap_s, wrap_t, wrap_r, false, lod_bias, af_level, min_lod, max_lod,
+						fs_sampler_handles[i] = vk::get_resource_manager()->find_sampler(*m_device, wrap_s, wrap_t, wrap_r, false, lod_bias, af_level, min_lod, max_lod,
 							min_filter, mag_filter, mip_mode, border_color, compare_enabled, depth_compare_mode);
 					}
 				}
@@ -1422,7 +1422,7 @@ void VKGSRender::end()
 
 					if (replace)
 					{
-						vs_sampler_handles[i] = m_resource_manager.find_sampler(
+						vs_sampler_handles[i] = vk::get_resource_manager()->find_sampler(
 							*m_device,
 							VK_SAMPLER_ADDRESS_MODE_REPEAT, VK_SAMPLER_ADDRESS_MODE_REPEAT, VK_SAMPLER_ADDRESS_MODE_REPEAT,
 							unnormalized_coords,
@@ -2159,6 +2159,11 @@ void VKGSRender::flush_command_queue(bool hard_sync)
 		m_current_cb_index = (m_current_cb_index + 1) % VK_MAX_ASYNC_CB_COUNT;
 		m_current_command_buffer = &m_primary_cb_list[m_current_cb_index];
 
+		if (!m_current_command_buffer->poke())
+		{
+			LOG_ERROR(RSX, "CB chain has run out of free entries!");
+		}
+
 		m_current_command_buffer->reset();
 
 		// Just in case a queued frame holds a ref to this cb, drain the present queue

rpcs3/Emu/RSX/VK/VKGSRender.h

@@ -72,7 +72,7 @@ struct command_buffer_chunk: public vk::command_buffer
 	VkDevice m_device = VK_NULL_HANDLE;
 
 	std::atomic_bool pending = { false };
-	std::atomic<u64> last_sync = { 0 };
+	u64 eid_tag = 0;
 	shared_mutex guard_mutex;
 
 	command_buffer_chunk() = default;
@@ -96,7 +96,7 @@ struct command_buffer_chunk: public vk::command_buffer
 
 	void tag()
 	{
-		last_sync = get_system_time();
+		eid_tag = vk::get_event_id();
 	}
 
 	void reset()
@@ -123,8 +123,11 @@ struct command_buffer_chunk: public vk::command_buffer
 
 			if (pending)
 			{
-				pending = false;
 				vk::reset_fence(&submit_fence);
+				vk::on_event_completed(eid_tag);
+
+				pending = false;
+				eid_tag = 0;
 			}
 		}
 
@@ -145,7 +148,10 @@ struct command_buffer_chunk: public vk::command_buffer
 		if (pending)
 		{
 			vk::reset_fence(&submit_fence);
+			vk::on_event_completed(eid_tag);
+
 			pending = false;
+			eid_tag = 0;
 		}
 
 		return ret;
@@ -288,66 +294,6 @@ struct flush_request_task
 	}
 };
 
-// TODO: This class will be expanded into a global allocator/collector eventually
-class resource_manager
-{
-private:
-	std::unordered_multimap<u64, std::unique_ptr<vk::sampler>> m_sampler_pool;
-
-	bool value_compare(const f32& a, const f32& b)
-	{
-		return fabsf(a - b) < 0.0000001f;
-	}
-
-public:
-
-	resource_manager() = default;
-	~resource_manager() = default;
-
-	void destroy()
-	{
-		m_sampler_pool.clear();
-	}
-
-	vk::sampler* find_sampler(VkDevice dev, VkSamplerAddressMode clamp_u, VkSamplerAddressMode clamp_v, VkSamplerAddressMode clamp_w,
-		VkBool32 unnormalized_coordinates, float mipLodBias, float max_anisotropy, float min_lod, float max_lod,
-		VkFilter min_filter, VkFilter mag_filter, VkSamplerMipmapMode mipmap_mode, VkBorderColor border_color,
-		VkBool32 depth_compare = VK_FALSE, VkCompareOp depth_compare_mode = VK_COMPARE_OP_NEVER)
-	{
-		u64 key = u16(clamp_u) | u64(clamp_v) << 3 | u64(clamp_w) << 6;
-		key |= u64(unnormalized_coordinates) << 9; // 1 bit
-		key |= u64(min_filter) << 10 | u64(mag_filter) << 11; // 1 bit each
-		key |= u64(mipmap_mode) << 12; // 1 bit
-		key |= u64(border_color) << 13; // 3 bits
-		key |= u64(depth_compare) << 16; // 1 bit
-		key |= u64(depth_compare_mode) << 17; // 3 bits
-
-		const auto found = m_sampler_pool.equal_range(key);
-		for (auto It = found.first; It != found.second; ++It)
-		{
-			const auto& info = It->second->info;
-			if (!value_compare(info.mipLodBias, mipLodBias) ||
-				!value_compare(info.maxAnisotropy, max_anisotropy) ||
-				!value_compare(info.minLod, min_lod) ||
-				!value_compare(info.maxLod, max_lod))
-			{
-				continue;
-			}
-
-			return It->second.get();
-		}
-
-		auto result = std::make_unique<vk::sampler>(
-			dev, clamp_u, clamp_v, clamp_w, unnormalized_coordinates,
-			mipLodBias, max_anisotropy, min_lod, max_lod,
-			min_filter, mag_filter, mipmap_mode, border_color,
-			depth_compare, depth_compare_mode);
-
-		auto It = m_sampler_pool.emplace(key, std::move(result));
-		return It->second.get();
-	}
-};
-
 class VKGSRender : public GSRender, public ::rsx::reports::ZCULL_control
 {
 private:
@@ -379,8 +325,6 @@ private:
 	std::unique_ptr<vk::buffer_view> m_volatile_attribute_storage;
 	std::unique_ptr<vk::buffer_view> m_vertex_layout_storage;
 
-	resource_manager m_resource_manager;
-
 public:
 	//vk::fbo draw_fbo;
 	std::unique_ptr<vk::vertex_cache> m_vertex_cache;
@@ -459,8 +403,6 @@ private:
 	shared_mutex m_flush_queue_mutex;
 	flush_request_task m_flush_requests;
 
-	std::atomic<u64> m_last_sync_event = { 0 };
-
 	bool m_render_pass_open = false;
 	u64  m_current_renderpass_key = 0;
 	VkRenderPass m_cached_renderpass = VK_NULL_HANDLE;

rpcs3/Emu/RSX/VK/VKHelpers.cpp

@@ -4,6 +4,7 @@
 #include "VKRenderPass.h"
 #include "VKFramebuffer.h"
 #include "VKResolveHelper.h"
+#include "VKResourceManager.h"
 #include "Utilities/mutex.h"
 
 namespace vk
@@ -247,6 +248,7 @@ namespace vk
 		vk::clear_renderpass_cache(dev);
 		vk::clear_framebuffer_cache();
 		vk::clear_resolve_helpers();
+		vk::get_resource_manager()->destroy();
 
 		g_null_texture.reset();
 		g_null_image_view.reset();

rpcs3/Emu/RSX/VK/VKResourceManager.cpp

@@ -0,0 +1,29 @@
+#include "stdafx.h"
+#include "VKResourceManager.h"
+
+namespace vk
+{
+	resource_manager g_resource_manager;
+	atomic_t<u64> g_event_ctr;
+
+	resource_manager* get_resource_manager()
+	{
+		return &g_resource_manager;
+	}
+
+	u64 get_event_id()
+	{
+		return g_event_ctr++;
+	}
+
+	u64 current_event_id()
+	{
+		return g_event_ctr.load();
+	}
+
+	void on_event_completed(u64 event_id)
+	{
+		// TODO: Offload this to a secondary thread
+		g_resource_manager.eid_completed(event_id);
+	}
+}

rpcs3/Emu/RSX/VK/VKResourceManager.h

@@ -0,0 +1,129 @@
+#pragma once
+#include "VKHelpers.h"
+
+namespace vk
+{
+	u64 get_event_id();
+	u64 current_event_id();
+	void on_event_completed(u64 event_id);
+
+	class resource_manager
+	{
+	private:
+		std::unordered_multimap<u64, std::unique_ptr<vk::sampler>> m_sampler_pool;
+		std::unordered_map<u64, std::vector<std::unique_ptr<vk::buffer>>> m_buffers_pool;
+		std::unordered_map<u64, rsx::simple_array<VkEvent>> m_events_pool;
+
+		bool value_compare(const f32& a, const f32& b)
+		{
+			return fabsf(a - b) < 0.0000001f;
+		}
+
+	public:
+
+		resource_manager() = default;
+		~resource_manager() = default;
+
+		void destroy()
+		{
+			auto& dev = *vk::get_current_renderer();
+
+			for (auto &e : m_events_pool)
+			{
+				for (auto &ev : e.second)
+				{
+					vkDestroyEvent(dev, ev, nullptr);
+				}
+			}
+
+			m_sampler_pool.clear();
+			m_buffers_pool.clear();
+			m_events_pool.clear();
+		}
+
+		vk::sampler* find_sampler(VkDevice dev, VkSamplerAddressMode clamp_u, VkSamplerAddressMode clamp_v, VkSamplerAddressMode clamp_w,
+			VkBool32 unnormalized_coordinates, float mipLodBias, float max_anisotropy, float min_lod, float max_lod,
+			VkFilter min_filter, VkFilter mag_filter, VkSamplerMipmapMode mipmap_mode, VkBorderColor border_color,
+			VkBool32 depth_compare = VK_FALSE, VkCompareOp depth_compare_mode = VK_COMPARE_OP_NEVER)
+		{
+			u64 key = u16(clamp_u) | u64(clamp_v) << 3 | u64(clamp_w) << 6;
+			key |= u64(unnormalized_coordinates) << 9; // 1 bit
+			key |= u64(min_filter) << 10 | u64(mag_filter) << 11; // 1 bit each
+			key |= u64(mipmap_mode) << 12; // 1 bit
+			key |= u64(border_color) << 13; // 3 bits
+			key |= u64(depth_compare) << 16; // 1 bit
+			key |= u64(depth_compare_mode) << 17; // 3 bits
+
+			const auto found = m_sampler_pool.equal_range(key);
+			for (auto It = found.first; It != found.second; ++It)
+			{
+				const auto& info = It->second->info;
+				if (!value_compare(info.mipLodBias, mipLodBias) ||
+					!value_compare(info.maxAnisotropy, max_anisotropy) ||
+					!value_compare(info.minLod, min_lod) ||
+					!value_compare(info.maxLod, max_lod))
+				{
+					continue;
+				}
+
+				return It->second.get();
+			}
+
+			auto result = std::make_unique<vk::sampler>(
+				dev, clamp_u, clamp_v, clamp_w, unnormalized_coordinates,
+				mipLodBias, max_anisotropy, min_lod, max_lod,
+				min_filter, mag_filter, mipmap_mode, border_color,
+				depth_compare, depth_compare_mode);
+
+			auto It = m_sampler_pool.emplace(key, std::move(result));
+			return It->second.get();
+		}
+
+		void dispose(std::unique_ptr<vk::buffer>& buf)
+		{
+			m_buffers_pool[current_event_id()].emplace_back(std::move(buf));
+		}
+
+		void dispose(VkEvent& event)
+		{
+			m_events_pool[current_event_id()].push_back(event);
+			event = VK_NULL_HANDLE;
+		}
+
+		void eid_completed(u64 eid)
+		{
+			auto& dev = *vk::get_current_renderer();
+
+			for (auto It = m_buffers_pool.begin(); It != m_buffers_pool.end();)
+			{
+				if (It->first <= eid)
+				{
+					It = m_buffers_pool.erase(It);
+				}
+				else
+				{
+					++It;
+				}
+			}
+
+			for (auto It = m_events_pool.begin(); It != m_events_pool.end();)
+			{
+				if (It->first <= eid)
+				{
+					for (auto &ev : It->second)
+					{
+						vkDestroyEvent(dev, ev, nullptr);
+					}
+
+					It = m_events_pool.erase(It);
+				}
+				else
+				{
+					++It;
+				}
+			}
+		}
+	};
+
+	resource_manager* get_resource_manager();
+}

rpcs3/Emu/RSX/VK/VKTextureCache.h

@@ -3,6 +3,7 @@
 #include "VKRenderTargets.h"
 #include "VKGSRender.h"
 #include "VKCompute.h"
+#include "VKResourceManager.h"
 #include "Emu/System.h"
 #include "../Common/TextureUtils.h"
 #include "Utilities/mutex.h"
@@ -36,7 +37,6 @@ namespace vk
 
 		//DMA relevant data
 		VkEvent dma_fence = VK_NULL_HANDLE;
-		VkEvent prev_event = VK_NULL_HANDLE;
 		vk::render_device* m_device = nullptr;
 		vk::viewable_image *vram_texture = nullptr;
 		std::unique_ptr<vk::buffer> dma_buffer;
@@ -73,9 +73,8 @@ namespace vk
 				if (!flushed)
 				{
 					// Reset fence
-					verify(HERE), m_device, dma_buffer, dma_fence != VK_NULL_HANDLE;
+					verify(HERE), m_device, dma_buffer, dma_fence;
-					vkDestroyEvent(*m_device, dma_fence, nullptr);
+					vk::get_resource_manager()->dispose(dma_fence);
-					dma_fence = VK_NULL_HANDLE;
 				}
 
 				synchronized = false;
@@ -91,15 +90,9 @@ namespace vk
 		{
 			if (dma_buffer)
 			{
-				dma_buffer.reset();
+				auto gc = vk::get_resource_manager();
-
+				gc->dispose(dma_buffer);
-				if (dma_fence != VK_NULL_HANDLE)
+				gc->dispose(dma_fence);
-				{
-					vkDestroyEvent(*m_device, dma_fence, nullptr);
-					dma_fence = VK_NULL_HANDLE;
-				}
-
-				verify(HERE), prev_event == VK_NULL_HANDLE;
 			}
 		}
 
@@ -288,11 +281,11 @@ namespace vk
 
 			if (UNLIKELY(synchronized))
 			{
-				// Save old event for deletion
+				verify(HERE), miss;
-				verify(HERE), miss, prev_event == VK_NULL_HANDLE;
-				prev_event = dma_fence;
 
 				// Replace the wait event with a new one to avoid premature signaling!
+				vk::get_resource_manager->dispose(dma_fence);
+
 				VkEventCreateInfo createInfo = {};
 				createInfo.sType = VK_STRUCTURE_TYPE_EVENT_CREATE_INFO;
 				vkCreateEvent(*m_device, &createInfo, nullptr, &dma_fence);
@@ -326,13 +319,6 @@ namespace vk
 			vk::wait_for_event(dma_fence, GENERAL_WAIT_TIMEOUT);
 			vkResetEvent(*m_device, dma_fence);
 
-			if (prev_event)
-			{
-				// Remove the stale event if it still exists
-				vkDestroyEvent(*m_device, prev_event, nullptr);
-				prev_event = VK_NULL_HANDLE;
-			}
-
 			return dma_buffer->map(offset, size);
 		}
 

rpcs3/VKGSRender.vcxproj

@@ -35,6 +35,7 @@
     <ClInclude Include="Emu\RSX\VK\VKRenderPass.h" />
     <ClInclude Include="Emu\RSX\VK\VKRenderTargets.h" />
     <ClInclude Include="Emu\RSX\VK\VKResolveHelper.h" />
+    <ClInclude Include="Emu\RSX\VK\VKResourceManager.h" />
     <ClInclude Include="Emu\RSX\VK\VKTextOut.h" />
     <ClInclude Include="Emu\RSX\VK\VKTextureCache.h" />
     <ClInclude Include="Emu\RSX\VK\VKVertexProgram.h" />
@@ -50,6 +51,7 @@
     <ClCompile Include="Emu\RSX\VK\VKProgramPipeline.cpp" />
     <ClCompile Include="Emu\RSX\VK\VKRenderPass.cpp" />
     <ClCompile Include="Emu\RSX\VK\VKResolveHelper.cpp" />
+    <ClCompile Include="Emu\RSX\VK\VKResourceManager.cpp" />
     <ClCompile Include="Emu\RSX\VK\VKTexture.cpp" />
     <ClCompile Include="Emu\RSX\VK\VKVertexBuffers.cpp" />
     <ClCompile Include="Emu\RSX\VK\VKVertexProgram.cpp" />

rpcs3/VKGSRender.vcxproj.filters

@@ -52,6 +52,9 @@
     <ClInclude Include="Emu\RSX\VK\VKResolveHelper.h">
       <Filter>Source Files</Filter>
     </ClInclude>
+    <ClInclude Include="Emu\RSX\VK\VKResourceManager.h">
+      <Filter>Source Files</Filter>
+    </ClInclude>
     <ClInclude Include="Emu\RSX\VK\VKFramebuffer.h">
       <Filter>Source Files</Filter>
     </ClInclude>
@@ -96,6 +99,9 @@
     <ClCompile Include="Emu\RSX\VK\VKResolveHelper.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="Emu\RSX\VK\VKResourceManager.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="Emu\RSX\VK\VKFramebuffer.cpp">
       <Filter>Source Files</Filter>
     </ClCompile>