vk: Enable deferred descriptor updates via descriptor-indexing

rpcs3/Emu/RSX/VK/VKCommandStream.cpp

@@ -1,9 +1,12 @@
 #include "stdafx.h"
 #include "VKCommandStream.h"
+#include "vkutils/descriptors.h"
 #include "vkutils/sync.h"
+
 #include "Emu/IdManager.h"
-#include "Emu/system_config.h"
 #include "Emu/RSX/RSXOffload.h"
+#include "Emu/RSX/RSXThread.h"
+#include "Emu/system_config.h"
 
 namespace vk
 {
@@ -22,6 +25,11 @@ namespace vk
 
 	void queue_submit(VkQueue queue, const VkSubmitInfo* info, fence* pfence, VkBool32 flush)
 	{
+		if (rsx::get_current_renderer()->is_current_thread())
+		{
+			vk::descriptors::flush();
+		}
+
 		if (!flush && g_cfg.video.multithreaded_rsx)
 		{
 			auto packet = new submit_packet(queue, pfence, info);

rpcs3/Emu/RSX/VK/VKCompute.cpp

@@ -39,13 +39,7 @@ namespace vk
 
 		// Reserve descriptor pools
 		m_descriptor_pool.create(*g_render_device, descriptor_pool_sizes.data(), ::size32(descriptor_pool_sizes), VK_MAX_COMPUTE_TASKS, 3);
-
+		m_descriptor_layout = vk::descriptors::create_layout(bindings);
-		VkDescriptorSetLayoutCreateInfo infos = {};
-		infos.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO;
-		infos.pBindings = bindings.data();
-		infos.bindingCount = ::size32(bindings);
-
-		CHECK_RESULT(vkCreateDescriptorSetLayout(*g_render_device, &infos, nullptr, &m_descriptor_layout));
 
 		VkPipelineLayoutCreateInfo layout_info = {};
 		layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO;

rpcs3/Emu/RSX/VK/VKGSRender.cpp

@@ -301,13 +301,7 @@ namespace
 			push_constants[0].size = 20;
 		}
 
-		VkDescriptorSetLayoutCreateInfo infos = {};
+		const auto set_layout = vk::descriptors::create_layout(bindings);
-		infos.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO;
-		infos.pBindings = bindings.data();
-		infos.bindingCount = static_cast<u32>(bindings.size());
-
-		VkDescriptorSetLayout set_layout;
-		CHECK_RESULT(vkCreateDescriptorSetLayout(dev, &infos, nullptr, &set_layout));
 
 		VkPipelineLayoutCreateInfo layout_info = {};
 		layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO;

rpcs3/Emu/RSX/VK/VKHelpers.cpp

@@ -167,6 +167,8 @@ namespace vk
 				vkDestroyBuffer(*g_render_device, tmp, nullptr);
 			}
 		}
+
+		descriptors::init();
 	}
 
 	VkFlags get_heap_compatible_buffer_types()

rpcs3/Emu/RSX/VK/VKOverlays.cpp

@@ -95,13 +95,7 @@ namespace vk
 		}
 
 		ensure(descriptor_index == num_bindings);
-
+		m_descriptor_layout = vk::descriptors::create_layout(bindings);
-		VkDescriptorSetLayoutCreateInfo infos = {};
-		infos.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO;
-		infos.pBindings = bindings.data();
-		infos.bindingCount = ::size32(bindings);
-
-		CHECK_RESULT(vkCreateDescriptorSetLayout(*m_device, &infos, nullptr, &m_descriptor_layout));
 
 		VkPipelineLayoutCreateInfo layout_info = {};
 		layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO;

rpcs3/Emu/RSX/VK/VKShaderInterpreter.cpp

@@ -357,13 +357,7 @@ namespace vk
 			push_constants[0].size = 20;
 		}
 
-		VkDescriptorSetLayoutCreateInfo infos = {};
+		const auto set_layout = vk::descriptors::create_layout(bindings);
-		infos.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO;
-		infos.pBindings = bindings.data();
-		infos.bindingCount = static_cast<u32>(bindings.size());
-
-		VkDescriptorSetLayout set_layout;
-		CHECK_RESULT(vkCreateDescriptorSetLayout(dev, &infos, nullptr, &set_layout));
 
 		VkPipelineLayoutCreateInfo layout_info = {};
 		layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO;

rpcs3/Emu/RSX/VK/VKTextOut.h

@@ -45,23 +45,21 @@ namespace vk
 				{ VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 120 },
 			};
 
-			//Reserve descriptor pools
+			// Reserve descriptor pools
 			m_descriptor_pool.create(dev, descriptor_pools, 1, 120, 2);
 
-			VkDescriptorSetLayoutBinding bindings[1] = {};
+			// Scale and offset data plus output color
-
+			std::vector<VkDescriptorSetLayoutBinding> bindings = 
-			//Scale and offset data plus output color
+			{
-			bindings[0].descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER;
+				{
-			bindings[0].descriptorCount = 1;
+					.binding = 0,
-			bindings[0].stageFlags = VK_SHADER_STAGE_VERTEX_BIT;
+					.descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER,
-			bindings[0].binding = 0;
+					.descriptorCount = 1,
-
+					.stageFlags = VK_SHADER_STAGE_VERTEX_BIT,
-			VkDescriptorSetLayoutCreateInfo infos = {};
+					.pImmutableSamplers = nullptr
-			infos.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO;
+				}
-			infos.pBindings = bindings;
+			};
-			infos.bindingCount = 1;
+			m_descriptor_layout = vk::descriptors::create_layout(bindings);
-
-			CHECK_RESULT(vkCreateDescriptorSetLayout(dev, &infos, nullptr, &m_descriptor_layout));
 
 			VkPipelineLayoutCreateInfo layout_info = {};
 			layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO;

rpcs3/Emu/RSX/VK/vkutils/descriptors.cpp

@@ -1,13 +1,81 @@
+#include "Emu/IdManager.h"
 #include "descriptors.h"
 
 namespace vk
 {
+	namespace descriptors
+	{
+		class dispatch_manager
+		{
+		public:
+			void flush_all()
+			{
+				for (auto& set : m_notification_list)
+				{
+					set->flush();
+				}
+			}
+
+			void notify(descriptor_set* set)
+			{
+				m_notification_list.push_back(set);
+				rsx_log.error("Now monitoring %u descriptor sets", m_notification_list.size());
+			}
+
+			dispatch_manager() = default;
+
+		private:
+			rsx::simple_array<descriptor_set*> m_notification_list;
+
+			dispatch_manager(const dispatch_manager&) = delete;
+			dispatch_manager& operator = (const dispatch_manager&) = delete;
+		};
+
+		void init()
+		{
+			g_fxo->init<dispatch_manager>();
+		}
+
+		void flush()
+		{
+			g_fxo->get<dispatch_manager>().flush_all();
+		}
+
+		VkDescriptorSetLayout create_layout(const std::vector<VkDescriptorSetLayoutBinding>& bindings)
+		{
+			VkDescriptorSetLayoutCreateInfo infos = {};
+			infos.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO;
+			infos.pBindings = bindings.data();
+			infos.bindingCount = ::size32(bindings);
+
+			VkDescriptorSetLayoutBindingFlagsCreateInfo binding_infos = {};
+			std::vector<VkDescriptorBindingFlags> binding_flags;
+
+			if (g_render_device->get_descriptor_indexing_support())
+			{
+				binding_flags.resize(bindings.size(), VK_DESCRIPTOR_BINDING_UPDATE_AFTER_BIND_BIT_EXT);
+
+				binding_infos.sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO_EXT;
+				binding_infos.pNext = nullptr;
+				binding_infos.bindingCount = ::size32(binding_flags);
+				binding_infos.pBindingFlags = binding_flags.data();
+
+				infos.pNext = &binding_infos;
+				infos.flags |= VK_DESCRIPTOR_SET_LAYOUT_CREATE_UPDATE_AFTER_BIND_POOL_BIT_EXT;
+			}
+
+			VkDescriptorSetLayout result;
+			CHECK_RESULT(vkCreateDescriptorSetLayout(*g_render_device, &infos, nullptr, &result));
+			return result;
+		}
+	}
+
 	void descriptor_pool::create(const vk::render_device& dev, VkDescriptorPoolSize* sizes, u32 size_descriptors_count, u32 max_sets, u8 subpool_count)
 	{
 		ensure(subpool_count);
 
 		VkDescriptorPoolCreateInfo infos = {};
-		infos.flags = 0;
+		infos.flags = dev.get_descriptor_indexing_support() ? VK_DESCRIPTOR_POOL_CREATE_UPDATE_AFTER_BIND_BIT : 0;
 		infos.maxSets = max_sets;
 		infos.poolSizeCount = size_descriptors_count;
 		infos.pPoolSizes = sizes;
@@ -57,44 +125,49 @@ namespace vk
 	descriptor_set::descriptor_set(VkDescriptorSet set)
 	{
 		flush();
-		init();
 		m_handle = set;
 	}
 
-	descriptor_set::descriptor_set()
+	void descriptor_set::init(VkDescriptorSet new_set)
 	{
-		init();
+		if (!m_in_use) [[unlikely]]
-	}
-
-	void descriptor_set::init()
-	{
-		if (m_image_info_pool.capacity() == 0)
 		{
 			m_image_info_pool.reserve(max_cache_size + 16);
 			m_buffer_view_pool.reserve(max_cache_size + 16);
 			m_buffer_info_pool.reserve(max_cache_size + 16);
+
+			m_in_use = true;
+			m_update_after_bind = g_render_device->get_descriptor_indexing_support();
+
+			if (m_update_after_bind)
+			{
+				g_fxo->get<descriptors::dispatch_manager>().notify(this);
+			}
 		}
+		else if (!m_update_after_bind)
+		{
+			flush();
+		}
+
+		m_handle = new_set;
 	}
 
 	void descriptor_set::swap(descriptor_set& other)
 	{
+		const auto other_handle = other.m_handle;
 		other.flush();
-		flush();
+		other.m_handle = m_handle;
-
+		init(other_handle);
-		std::swap(m_handle, other.m_handle);
 	}
 
 	descriptor_set& descriptor_set::operator = (VkDescriptorSet set)
 	{
-		flush();
+		init(set);
-		m_handle = set;
 		return *this;
 	}
 
 	VkDescriptorSet* descriptor_set::ptr()
 	{
-		// TODO: You shouldn't need this
-		// ensure(m_handle == VK_NULL_HANDLE);
 		return &m_handle;
 	}
 
@@ -105,6 +178,11 @@ namespace vk
 
 	void descriptor_set::push(const VkBufferView& buffer_view, VkDescriptorType type, u32 binding)
 	{
+		if (m_pending_writes.size() > max_cache_size)
+		{
+			flush();
+		}
+
 		m_buffer_view_pool.push_back(buffer_view);
 		m_pending_writes.push_back(
 		{
@@ -123,6 +201,11 @@ namespace vk
 
 	void descriptor_set::push(const VkDescriptorBufferInfo& buffer_info, VkDescriptorType type, u32 binding)
 	{
+		if (m_pending_writes.size() > max_cache_size)
+		{
+			flush();
+		}
+
 		m_buffer_info_pool.push_back(buffer_info);
 		m_pending_writes.push_back(
 		{
@@ -141,6 +224,11 @@ namespace vk
 
 	void descriptor_set::push(const VkDescriptorImageInfo& image_info, VkDescriptorType type, u32 binding)
 	{
+		if (m_pending_writes.size() >= max_cache_size)
+		{
+			flush();
+		}
+
 		m_image_info_pool.push_back(image_info);
 		m_pending_writes.push_back(
 		{
@@ -192,7 +280,11 @@ namespace vk
 
 	void descriptor_set::bind(VkCommandBuffer cmd, VkPipelineBindPoint bind_point, VkPipelineLayout layout)
 	{
-		flush();
+		if (!m_update_after_bind)
+		{
+			flush();
+		}
+
 		vkCmdBindDescriptorSets(cmd, bind_point, layout, 0, 1, &m_handle, 0, nullptr);
 	}
 

rpcs3/Emu/RSX/VK/vkutils/descriptors.h

@@ -1,6 +1,8 @@
 #pragma once
 
 #include "../VulkanAPI.h"
+#include "Utilities/mutex.h"
+
 #include "commands.h"
 #include "device.h"
 
@@ -32,13 +34,11 @@ namespace vk
 	class descriptor_set
 	{
 		const size_t max_cache_size = 16384;
-
+		void init(VkDescriptorSet new_set);
-		void flush();
-		void init();
 
 	public:
 		descriptor_set(VkDescriptorSet set);
-		descriptor_set();
+		descriptor_set() = default;
 		~descriptor_set() = default;
 
 		descriptor_set(const descriptor_set&) = delete;
@@ -57,8 +57,12 @@ namespace vk
 		void bind(VkCommandBuffer cmd, VkPipelineBindPoint bind_point, VkPipelineLayout layout);
 		void bind(const command_buffer& cmd, VkPipelineBindPoint bind_point, VkPipelineLayout layout);
 
+		void flush();
+
 	private:
 		VkDescriptorSet m_handle = VK_NULL_HANDLE;
+		bool m_update_after_bind = false;
+		bool m_in_use = false;
 
 		rsx::simple_array<VkBufferView> m_buffer_view_pool;
 		rsx::simple_array<VkDescriptorBufferInfo> m_buffer_info_pool;
@@ -68,5 +72,11 @@ namespace vk
 		rsx::simple_array<VkCopyDescriptorSet> m_pending_copies;
 	};
 
-	void flush_descriptor_updates();
+	namespace descriptors
+	{
+		void init();
+		void flush();
+
+		VkDescriptorSetLayout create_layout(const std::vector<VkDescriptorSetLayoutBinding>& bindings);
+	}
 }

rpcs3/Emu/RSX/VK/vkutils/device.cpp

@@ -326,6 +326,7 @@ namespace vk
 
 		if (pgpu->descriptor_indexing_support)
 		{
+			requested_extensions.push_back(VK_KHR_MAINTENANCE3_EXTENSION_NAME);
 			requested_extensions.push_back(VK_EXT_DESCRIPTOR_INDEXING_EXTENSION_NAME);
 		}
 
@@ -460,6 +461,17 @@ namespace vk
 			rsx_log.notice("GPU/driver lacks support for float16 data types. All float16_t arithmetic will be emulated with float32_t.");
 		}
 
+		VkPhysicalDeviceDescriptorIndexingFeatures indexing_features{};
+		if (pgpu->descriptor_indexing_support)
+		{
+			indexing_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_FEATURES_EXT;
+			indexing_features.descriptorBindingUniformTexelBufferUpdateAfterBind = VK_TRUE;
+			indexing_features.descriptorBindingUniformBufferUpdateAfterBind = VK_TRUE;
+			indexing_features.descriptorBindingStorageBufferUpdateAfterBind = VK_TRUE;
+			indexing_features.descriptorBindingSampledImageUpdateAfterBind = VK_TRUE;
+			device.pNext = &indexing_features;
+		}
+
 		CHECK_RESULT_EX(vkCreateDevice(*pgpu, &device, nullptr, &dev), message_on_error);
 
 		// Initialize queues