vk: Implement multithreaded command submission

- A few nagging issues remain, specifically that partial command stream
  largely caused by poor synchronization structures for partial CS flush
  and also the fact that occlusion map entries wait on a command buffer
  and not an EID!

rpcs3/Emu/CMakeLists.txt

@@ -385,6 +385,7 @@ target_sources(rpcs3_emu PRIVATE
 
 if(TARGET 3rdparty_vulkan)
 	target_sources(rpcs3_emu PRIVATE
+		RSX/VK/VKCommandStream.cpp
 		RSX/VK/VKCommonDecompiler.cpp
 		RSX/VK/VKDMA.cpp
 		RSX/VK/VKFormats.cpp

rpcs3/Emu/RSX/RSXOffload.cpp

@@ -57,6 +57,9 @@ namespace rsx
 								static_cast<rsx::primitive_type>(m_current_job->aux_param0),
 								m_current_job->length);
 							break;
+						case callback:
+							rsx::get_current_renderer()->renderctl(m_current_job->aux_param0, m_current_job->src);
+							break;
 						default:
 							ASSUME(0);
 							fmt::throw_exception("Unreachable" HERE);
@@ -119,6 +122,15 @@ namespace rsx
 		}
 	}
 
+	// Backend callback
+	void dma_manager::backend_ctrl(u32 request_code, void* args)
+	{
+		verify(HERE), g_cfg.video.multithreaded_rsx;
+
+		++m_enqueued_count;
+		m_work_queue.push(request_code, args);
+	}
+
 	// Synchronization
 	bool dma_manager::is_current_thread() const
 	{

rpcs3/Emu/RSX/RSXOffload.h

@@ -17,7 +17,8 @@ namespace rsx
 		{
 			raw_copy = 0,
 			vector_copy = 1,
-			index_emulate = 2
+			index_emulate = 2,
+			callback = 3
 		};
 
 		struct transport_packet
@@ -41,6 +42,10 @@ namespace rsx
 			transport_packet(void *_dst, rsx::primitive_type prim, u32 len)
 				: dst(_dst), aux_param0(static_cast<u8>(prim)), length(len), type(op::index_emulate)
 			{}
+
+			transport_packet(u32 command, void* args)
+				: aux_param0(command), src(args), type(op::callback)
+			{}
 		};
 
 		lf_queue<transport_packet> m_work_queue;
@@ -67,6 +72,9 @@ namespace rsx
 		// Vertex utilities
 		void emulate_as_indexed(void *dst, rsx::primitive_type primitive, u32 count);
 
+		// Renderer callback
+		void backend_ctrl(u32 request_code, void* args);
+
 		// Synchronization
 		bool is_current_thread() const;
 		void sync();

rpcs3/Emu/RSX/RSXThread.cpp

@@ -446,10 +446,10 @@ namespace rsx
 
 		rsx::overlays::reset_performance_overlay();
 
+		g_dma_manager.init();
 		on_init_thread();
 
 		method_registers.init();
-		g_dma_manager.init();
 		m_profiler.enabled = !!g_cfg.video.overlay;
 
 		if (!zcull_ctrl)

rpcs3/Emu/RSX/RSXThread.h

@@ -753,6 +753,9 @@ namespace rsx
 		virtual void on_invalidate_memory_range(const address_range & /*range*/, rsx::invalidation_cause) {}
 		virtual void notify_tile_unbound(u32 /*tile*/) {}
 
+		// control
+		virtual void renderctl(u32 request_code, void* args) {}
+
 		// zcull
 		void notify_zcull_info_changed();
 		void clear_zcull_stats(u32 type);

rpcs3/Emu/RSX/VK/VKCommandStream.cpp

@@ -0,0 +1,36 @@
+#include "stdafx.h"
+#include "VKCommandStream.h"
+
+namespace vk
+{
+	// global submit guard to prevent race condition on queue submit
+	shared_mutex g_submit_mutex;
+
+	void acquire_global_submit_lock()
+	{
+		g_submit_mutex.lock();
+	}
+
+	void release_global_submit_lock()
+	{
+		g_submit_mutex.unlock();
+	}
+
+	void queue_submit(VkQueue queue, const VkSubmitInfo* info, fence* pfence, VkBool32 flush)
+	{
+		if (!flush && g_cfg.video.multithreaded_rsx)
+		{
+			auto packet = new submit_packet(queue, pfence, info);
+			rsx::g_dma_manager.backend_ctrl(rctrl_queue_submit, packet);
+		}
+		else
+		{
+			acquire_global_submit_lock();
+			vkQueueSubmit(queue, 1, info, pfence->handle);
+			release_global_submit_lock();
+
+			// Signal fence
+			pfence->flushed = true;
+		}
+	}
+}

rpcs3/Emu/RSX/VK/VKCommandStream.h

@@ -0,0 +1,42 @@
+#pragma once
+
+#include "VKHelpers.h"
+
+namespace vk
+{
+	struct submit_packet
+	{
+		// Core components
+		VkQueue      queue;
+		fence*       pfence;
+		VkSubmitInfo submit_info;
+
+		// Pointer redirection storage
+		VkSemaphore  wait_semaphore;
+		VkSemaphore  signal_semaphore;
+		VkFlags      wait_flags;
+
+		submit_packet(VkQueue _q, fence* _f, const VkSubmitInfo* info) :
+			queue(_q), pfence(_f), submit_info(*info),
+			wait_semaphore(0), signal_semaphore(0), wait_flags(0)
+		{
+			if (info->waitSemaphoreCount)
+			{
+				wait_semaphore = *info->pWaitSemaphores;
+				submit_info.pWaitSemaphores = &wait_semaphore;
+			}
+
+			if (info->signalSemaphoreCount)
+			{
+				signal_semaphore = *info->pSignalSemaphores;
+				submit_info.pSignalSemaphores = &signal_semaphore;
+			}
+
+			if (info->pWaitDstStageMask)
+			{
+				wait_flags = *info->pWaitDstStageMask;
+				submit_info.pWaitDstStageMask = &wait_flags;
+			}
+		}
+	};
+}

rpcs3/Emu/RSX/VK/VKGSRender.cpp

@@ -9,6 +9,7 @@
 #include "VKCommonDecompiler.h"
 #include "VKRenderPass.h"
 #include "VKResourceManager.h"
+#include "VKCommandStream.h"
 
 namespace
 {
@@ -2157,24 +2158,25 @@ void VKGSRender::flush_command_queue(bool hard_sync)
 	}
 	else
 	{
-		// Mark this queue as pending
+		// Mark this queue as pending and proceed
 		m_current_command_buffer->pending = true;
-
-		// Grab next cb in line and make it usable
-		m_current_cb_index = (m_current_cb_index + 1) % VK_MAX_ASYNC_CB_COUNT;
-		m_current_command_buffer = &m_primary_cb_list[m_current_cb_index];
-
-		if (!m_current_command_buffer->poke())
-		{
-			LOG_ERROR(RSX, "CB chain has run out of free entries!");
-		}
-
-		m_current_command_buffer->reset();
-
-		// Just in case a queued frame holds a ref to this cb, drain the present queue
-		check_present_status();
 	}
 
+	// Grab next cb in line and make it usable
+	// NOTE: Even in the case of a hard sync, this is required to free any waiters on the CB (ZCULL)
+	m_current_cb_index = (m_current_cb_index + 1) % VK_MAX_ASYNC_CB_COUNT;
+	m_current_command_buffer = &m_primary_cb_list[m_current_cb_index];
+
+	if (!m_current_command_buffer->poke())
+	{
+		LOG_ERROR(RSX, "CB chain has run out of free entries!");
+	}
+
+	m_current_command_buffer->reset();
+
+	// Just in case a queued frame holds a ref to this cb, drain the present queue
+	check_present_status();
+
 	if (m_occlusion_query_active)
 	{
 		m_current_command_buffer->flags |= vk::command_buffer::cb_load_occluson_task;
@@ -2278,6 +2280,9 @@ void VKGSRender::present(frame_context_t *ctx)
 {
 	verify(HERE), ctx->present_image != UINT32_MAX;
 
+	// Partial CS flush
+	ctx->swap_command_buffer->flush();
+
 	if (!swapchain_unavailable)
 	{
 		switch (VkResult error = m_swapchain->present(ctx->present_wait_semaphore, ctx->present_image))
@@ -2824,11 +2829,9 @@ void VKGSRender::init_buffers(rsx::framebuffer_creation_context context, bool)
 	prepare_rtts(context);
 }
 
-void VKGSRender::close_and_submit_command_buffer(VkFence fence, VkSemaphore wait_semaphore, VkSemaphore signal_semaphore, VkPipelineStageFlags pipeline_stage_flags)
+void VKGSRender::close_and_submit_command_buffer(vk::fence* pFence, VkSemaphore wait_semaphore, VkSemaphore signal_semaphore, VkPipelineStageFlags pipeline_stage_flags)
 {
-	// Wait before sync block below
+	// NOTE: There is no need to wait for dma sync. When MTRSX is enabled, the commands are submitted in order anyway due to CSMT
-	rsx::g_dma_manager.sync();
-
 	if (vk::test_status_interrupt(vk::heap_dirty))
 	{
 		if (m_attrib_ring_info.dirty() ||
@@ -2881,7 +2884,7 @@ void VKGSRender::close_and_submit_command_buffer(VkFence fence, VkSemaphore wait
 	m_current_command_buffer->tag();
 
 	m_current_command_buffer->submit(m_swapchain->get_graphics_queue(),
-		wait_semaphore, signal_semaphore, fence, pipeline_stage_flags);
+		wait_semaphore, signal_semaphore, pFence, pipeline_stage_flags);
 }
 
 void VKGSRender::open_command_buffer()
@@ -3155,16 +3158,11 @@ void VKGSRender::reinitialize_swapchain()
 	}
 
 	//Will have to block until rendering is completed
-	VkFence resize_fence = VK_NULL_HANDLE;
+	vk::fence resize_fence(*m_device);
-	VkFenceCreateInfo infos = {};
-	infos.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
-
-	vkCreateFence((*m_device), &infos, nullptr, &resize_fence);
 
 	//Flush the command buffer
-	close_and_submit_command_buffer(resize_fence);
+	close_and_submit_command_buffer(&resize_fence);
-	vk::wait_for_fence(resize_fence);
+	vk::wait_for_fence(&resize_fence);
-	vkDestroyFence((*m_device), resize_fence, nullptr);
 
 	m_current_command_buffer->reset();
 	open_command_buffer();
@@ -3581,6 +3579,22 @@ void VKGSRender::flip(const rsx::display_flip_info_t& info)
 	rsx::thread::flip(info);
 }
 
+void VKGSRender::renderctl(u32 request_code, void* args)
+{
+	switch (request_code)
+	{
+	case vk::rctrl_queue_submit:
+	{
+		auto packet = reinterpret_cast<vk::submit_packet*>(args);
+		vk::queue_submit(packet->queue, &packet->submit_info, packet->pfence, VK_TRUE);
+		free(packet);
+		break;
+	}
+	default:
+		fmt::throw_exception("Unhandled request code 0x%x" HERE, request_code);
+	}
+}
+
 bool VKGSRender::scaled_image_from_memory(rsx::blit_src_info& src, rsx::blit_dst_info& dst, bool interpolate)
 {
 	if (swapchain_unavailable)
@@ -3675,6 +3689,8 @@ void VKGSRender::get_occlusion_query_result(rsx::reports::occlusion_query_info*
 			busy_wait();
 		}
 
+		data.command_buffer_to_wait->wait();
+
 		// Gather data
 		for (const auto occlusion_id : data.indices)
 		{

rpcs3/Emu/RSX/VK/VKGSRender.h

@@ -67,7 +67,7 @@ enum
 
 struct command_buffer_chunk: public vk::command_buffer
 {
-	VkFence submit_fence = VK_NULL_HANDLE;
+	vk::fence* submit_fence = nullptr;
 	VkDevice m_device = VK_NULL_HANDLE;
 
 	std::atomic_bool pending = { false };
@@ -79,18 +79,13 @@ struct command_buffer_chunk: public vk::command_buffer
 	void init_fence(VkDevice dev)
 	{
 		m_device = dev;
-
+		submit_fence = new vk::fence(dev);
-		VkFenceCreateInfo info = {};
-		info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
-		CHECK_RESULT(vkCreateFence(m_device, &info, nullptr, &submit_fence));
 	}
 
 	void destroy()
 	{
 		vk::command_buffer::destroy();
-
+		delete submit_fence;
-		if (submit_fence != VK_NULL_HANDLE)
-			vkDestroyFence(m_device, submit_fence, nullptr);
 	}
 
 	void tag()
@@ -116,13 +111,16 @@ struct command_buffer_chunk: public vk::command_buffer
 		if (!pending)
 			return true;
 
-		if (vkGetFenceStatus(m_device, submit_fence) == VK_SUCCESS)
+		if (!submit_fence->flushed)
+			return false;
+
+		if (vkGetFenceStatus(m_device, submit_fence->handle) == VK_SUCCESS)
 		{
 			lock.upgrade();
 
 			if (pending)
 			{
-				vk::reset_fence(&submit_fence);
+				vk::reset_fence(submit_fence);
 				vk::on_event_completed(eid_tag);
 
 				pending = false;
@@ -146,7 +144,7 @@ struct command_buffer_chunk: public vk::command_buffer
 
 		if (pending)
 		{
-			vk::reset_fence(&submit_fence);
+			vk::reset_fence(submit_fence);
 			vk::on_event_completed(eid_tag);
 
 			pending = false;
@@ -155,6 +153,11 @@ struct command_buffer_chunk: public vk::command_buffer
 
 		return ret;
 	}
+
+	void flush() const
+	{
+		submit_fence->wait_flush();
+	}
 };
 
 struct occlusion_data
@@ -430,7 +433,7 @@ private:
 
 	void open_command_buffer();
 	void close_and_submit_command_buffer(
-		VkFence fence = VK_NULL_HANDLE,
+		vk::fence* fence = nullptr,
 		VkSemaphore wait_semaphore = VK_NULL_HANDLE,
 		VkSemaphore signal_semaphore = VK_NULL_HANDLE,
 		VkPipelineStageFlags pipeline_stage_flags = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT);
@@ -486,6 +489,8 @@ protected:
 	void on_exit() override;
 	void flip(const rsx::display_flip_info_t& info) override;
 
+	void renderctl(u32 request_code, void* args) override;
+
 	void do_local_task(rsx::FIFO_state state) override;
 	bool scaled_image_from_memory(rsx::blit_src_info& src, rsx::blit_dst_info& dst, bool interpolate) override;
 	void notify_tile_unbound(u32 tile) override;

rpcs3/Emu/RSX/VK/VKHelpers.cpp

@@ -7,7 +7,10 @@
 #include "VKResolveHelper.h"
 #include "VKResourceManager.h"
 #include "VKDMA.h"
+#include "VKCommandStream.h"
+
 #include "Utilities/mutex.h"
+#include "Utilities/lockless.h"
 
 namespace vk
 {
@@ -91,9 +94,6 @@ namespace vk
 	u64 g_num_processed_frames = 0;
 	u64 g_num_total_frames = 0;
 
-	// global submit guard to prevent race condition on queue submit
-	shared_mutex g_submit_mutex;
-
 	VKAPI_ATTR void* VKAPI_CALL mem_realloc(void* pUserData, void* pOriginal, size_t size, size_t alignment, VkSystemAllocationScope allocationScope)
 	{
 #ifdef _MSC_VER
@@ -349,16 +349,6 @@ namespace vk
 		return &g_upload_heap;
 	}
 
-	void acquire_global_submit_lock()
-	{
-		g_submit_mutex.lock();
-	}
-
-	void release_global_submit_lock()
-	{
-		g_submit_mutex.unlock();
-	}
-
 	void reset_compute_tasks()
 	{
 		for (const auto &p : g_compute_tasks)
@@ -836,31 +826,30 @@ namespace vk
 		return (g_num_processed_frames > 0)? g_num_processed_frames - 1: 0;
 	}
 
-	void reset_fence(VkFence *pFence)
+	void reset_fence(fence *pFence)
 	{
 		if (g_drv_disable_fence_reset)
 		{
-			vkDestroyFence(*g_current_renderer, *pFence, nullptr);
+			delete pFence;
-
+			pFence = new fence(*g_current_renderer);
-			VkFenceCreateInfo info = {};
-			info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
-			CHECK_RESULT(vkCreateFence(*g_current_renderer, &info, nullptr, pFence));
 		}
 		else
 		{
-			CHECK_RESULT(vkResetFences(*g_current_renderer, 1, pFence));
+			pFence->reset();
 		}
 	}
 
-	VkResult wait_for_fence(VkFence fence, u64 timeout)
+	VkResult wait_for_fence(fence* pFence, u64 timeout)
 	{
+		pFence->wait_flush();
+
 		if (timeout)
 		{
-			return vkWaitForFences(*g_current_renderer, 1, &fence, VK_FALSE, timeout * 1000ull);
+			return vkWaitForFences(*g_current_renderer, 1, &pFence->handle, VK_FALSE, timeout * 1000ull);
 		}
 		else
 		{
-			while (auto status = vkGetFenceStatus(*g_current_renderer, fence))
+			while (auto status = vkGetFenceStatus(*g_current_renderer, pFence->handle))
 			{
 				switch (status)
 				{

rpcs3/Emu/RSX/VK/VKHelpers.h

@@ -107,6 +107,11 @@ namespace vk
 		VK_REMAP_VIEW_MULTISAMPLED = 0xDEADBEEF     // Special encoding for multisampled images; returns a multisampled image view
 	};
 
+	enum // callback commands
+	{
+		rctrl_queue_submit = 0x80000000
+	};
+
 	class context;
 	class render_device;
 	class swap_chain_image;
@@ -119,6 +124,7 @@ namespace vk
 	class mem_allocator_base;
 	struct memory_type_mapping;
 	struct gpu_formats_support;
+	struct fence;
 
 	const vk::context *get_current_thread_ctx();
 	void set_current_thread_ctx(const vk::context &ctx);
@@ -152,9 +158,10 @@ namespace vk
 	memory_type_mapping get_memory_mapping(const physical_device& dev);
 	gpu_formats_support get_optimal_tiling_supported_formats(const physical_device& dev);
 
-	//Sync helpers around vkQueueSubmit
+	// Sync helpers around vkQueueSubmit
 	void acquire_global_submit_lock();
 	void release_global_submit_lock();
+	void queue_submit(VkQueue queue, const VkSubmitInfo* info, fence* pfence, VkBool32 flush = VK_FALSE);
 
 	template<class T>
 	T* get_compute_task();
@@ -222,8 +229,8 @@ namespace vk
 	const u64 get_last_completed_frame_id();
 
 	// Fence reset with driver workarounds in place
-	void reset_fence(VkFence *pFence);
+	void reset_fence(fence* pFence);
-	VkResult wait_for_fence(VkFence pFence, u64 timeout = 0ull);
+	VkResult wait_for_fence(fence* pFence, u64 timeout = 0ull);
 	VkResult wait_for_event(VkEvent pEvent, u64 timeout = 0ull);
 
 	// Handle unexpected submit with dangling occlusion query
@@ -1022,12 +1029,55 @@ private:
 		}
 	};
 
+	struct fence
+	{
+		volatile bool flushed = false;
+		VkFence handle        = VK_NULL_HANDLE;
+		VkDevice owner        = VK_NULL_HANDLE;
+
+		fence(VkDevice dev)
+		{
+			owner = dev;
+			VkFenceCreateInfo info = {};
+			info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
+			CHECK_RESULT(vkCreateFence(dev, &info, nullptr, &handle));
+		}
+
+		~fence()
+		{
+			if (handle)
+			{
+				vkDestroyFence(owner, handle, nullptr);
+				handle = VK_NULL_HANDLE;
+			}
+		}
+
+		void reset()
+		{
+			vkResetFences(owner, 1, &handle);
+			flushed = false;
+		}
+
+		void wait_flush()
+		{
+			while (!flushed)
+			{
+				_mm_pause();
+			}
+		}
+
+		operator bool() const
+		{
+			return (handle != VK_NULL_HANDLE);
+		}
+	};
+
 	class command_buffer
 	{
 	private:
 		bool is_open = false;
 		bool is_pending = false;
-		VkFence m_submit_fence = VK_NULL_HANDLE;
+		fence* m_submit_fence = nullptr;
 
 	protected:
 		vk::command_pool *pool = nullptr;
@@ -1066,9 +1116,7 @@ private:
 
 			if (auto_reset)
 			{
-				VkFenceCreateInfo info = {};
+				m_submit_fence = new fence(cmd_pool.get_owner());
-				info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
-				CHECK_RESULT(vkCreateFence(cmd_pool.get_owner(), &info, nullptr, &m_submit_fence));
 			}
 
 			pool = &cmd_pool;
@@ -1080,7 +1128,9 @@ private:
 
 			if (m_submit_fence)
 			{
-				vkDestroyFence(pool->get_owner(), m_submit_fence, nullptr);
+				//vkDestroyFence(pool->get_owner(), m_submit_fence, nullptr);
+				delete m_submit_fence;
+				m_submit_fence = nullptr;
 			}
 		}
 
@@ -1116,7 +1166,8 @@ private:
 				wait_for_fence(m_submit_fence);
 				is_pending = false;
 
-				CHECK_RESULT(vkResetFences(pool->get_owner(), 1, &m_submit_fence));
+				//CHECK_RESULT(vkResetFences(pool->get_owner(), 1, &m_submit_fence));
+				reset_fence(m_submit_fence);
 				CHECK_RESULT(vkResetCommandBuffer(commands, 0));
 			}
 
@@ -1146,7 +1197,7 @@ private:
 			is_open = false;
 		}
 
-		void submit(VkQueue queue, VkSemaphore wait_semaphore, VkSemaphore signal_semaphore, VkFence fence, VkPipelineStageFlags pipeline_stage_flags)
+		void submit(VkQueue queue, VkSemaphore wait_semaphore, VkSemaphore signal_semaphore, fence* pfence, VkPipelineStageFlags pipeline_stage_flags)
 		{
 			if (is_open)
 			{
@@ -1157,10 +1208,10 @@ private:
 			// Check for hanging queries to avoid driver hang
 			verify("close and submit of commandbuffer with a hanging query!" HERE), (flags & cb_has_open_query) == 0;
 
-			if (!fence)
+			if (!pfence)
 			{
-				fence = m_submit_fence;
+				pfence = m_submit_fence;
-				is_pending = (fence != VK_NULL_HANDLE);
+				is_pending = bool(pfence);
 			}
 
 			VkSubmitInfo infos = {};
@@ -1181,10 +1232,7 @@ private:
 				infos.pSignalSemaphores = &signal_semaphore;
 			}
 
-			acquire_global_submit_lock();
+			queue_submit(queue, &infos, pfence);
-			CHECK_RESULT(vkQueueSubmit(queue, 1, &infos, fence));
-			release_global_submit_lock();
-
 			clear_flags();
 		}
 	};

rpcs3/Emu/RSX/VK/VKTextureCache.h

@@ -1336,15 +1336,10 @@ namespace vk
 			if (cmd.access_hint != vk::command_buffer::access_type_hint::all)
 			{
 				// Primary access command queue, must restart it after
-				VkFence submit_fence;
+				vk::fence submit_fence(*m_device);
-				VkFenceCreateInfo info{};
+				cmd.submit(m_submit_queue, VK_NULL_HANDLE, VK_NULL_HANDLE, &submit_fence, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT);
-				info.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
-				vkCreateFence(*m_device, &info, nullptr, &submit_fence);
 
-				cmd.submit(m_submit_queue, VK_NULL_HANDLE, VK_NULL_HANDLE, submit_fence, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT);
+				vk::wait_for_fence(&submit_fence, GENERAL_WAIT_TIMEOUT);
-
-				vk::wait_for_fence(submit_fence, GENERAL_WAIT_TIMEOUT);
-				vkDestroyFence(*m_device, submit_fence, nullptr);
 
 				CHECK_RESULT(vkResetCommandBuffer(cmd, 0));
 				cmd.begin();

rpcs3/VKGSRender.vcxproj

@@ -23,6 +23,7 @@
     </ProjectConfiguration>
   </ItemGroup>
   <ItemGroup>
+    <ClInclude Include="Emu\RSX\VK\VKCommandStream.h" />
     <ClInclude Include="Emu\RSX\VK\VKCommonDecompiler.h" />
     <ClInclude Include="Emu\RSX\VK\VKCompute.h" />
     <ClInclude Include="Emu\RSX\VK\VKDMA.h" />
@@ -43,6 +44,7 @@
     <ClInclude Include="Emu\RSX\VK\VulkanAPI.h" />
   </ItemGroup>
   <ItemGroup>
+    <ClCompile Include="Emu\RSX\VK\VKCommandStream.cpp" />
     <ClCompile Include="Emu\RSX\VK\VKCommonDecompiler.cpp" />
     <ClCompile Include="Emu\RSX\VK\VKDMA.cpp" />
     <ClCompile Include="Emu\RSX\VK\VKFormats.cpp" />

rpcs3/VKGSRender.vcxproj.filters

@@ -1,112 +1,42 @@
 <?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <ItemGroup>
-    <Filter Include="Source Files">
+    <ClCompile Include="Emu\RSX\VK\VKCommonDecompiler.cpp" />
-      <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
+    <ClCompile Include="Emu\RSX\VK\VKDMA.cpp" />
-      <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
+    <ClCompile Include="Emu\RSX\VK\VKFormats.cpp" />
-    </Filter>
+    <ClCompile Include="Emu\RSX\VK\VKFragmentProgram.cpp" />
+    <ClCompile Include="Emu\RSX\VK\VKFramebuffer.cpp" />
+    <ClCompile Include="Emu\RSX\VK\VKGSRender.cpp" />
+    <ClCompile Include="Emu\RSX\VK\VKHelpers.cpp" />
+    <ClCompile Include="Emu\RSX\VK\VKProgramPipeline.cpp" />
+    <ClCompile Include="Emu\RSX\VK\VKRenderPass.cpp" />
+    <ClCompile Include="Emu\RSX\VK\VKResolveHelper.cpp" />
+    <ClCompile Include="Emu\RSX\VK\VKResourceManager.cpp" />
+    <ClCompile Include="Emu\RSX\VK\VKTexture.cpp" />
+    <ClCompile Include="Emu\RSX\VK\VKVertexBuffers.cpp" />
+    <ClCompile Include="Emu\RSX\VK\VKVertexProgram.cpp" />
+    <ClCompile Include="Emu\RSX\VK\VKMemAlloc.cpp" />
+    <ClCompile Include="Emu\RSX\VK\VKCommandStream.cpp" />
   </ItemGroup>
-  <ItemGroup>
+  <ItemGroup>
-    <ClInclude Include="Emu\RSX\VK\VKGSRender.h">
+    <ClInclude Include="Emu\RSX\VK\VKCommonDecompiler.h" />
-      <Filter>Source Files</Filter>
+    <ClInclude Include="Emu\RSX\VK\VKCompute.h" />
-    </ClInclude>
+    <ClInclude Include="Emu\RSX\VK\VKDMA.h" />
-    <ClInclude Include="Emu\RSX\VK\VKCommonDecompiler.h">
+    <ClInclude Include="Emu\RSX\VK\VKFormats.h" />
-      <Filter>Source Files</Filter>
+    <ClInclude Include="Emu\RSX\VK\VKFragmentProgram.h" />
-    </ClInclude>
+    <ClInclude Include="Emu\RSX\VK\VKFramebuffer.h" />
-    <ClInclude Include="Emu\RSX\VK\VKFragmentProgram.h">
+    <ClInclude Include="Emu\RSX\VK\VKGSRender.h" />
-      <Filter>Source Files</Filter>
+    <ClInclude Include="Emu\RSX\VK\VKHelpers.h" />
-    </ClInclude>
+    <ClInclude Include="Emu\RSX\VK\VKOverlays.h" />
-    <ClInclude Include="Emu\RSX\VK\VKHelpers.h">
+    <ClInclude Include="Emu\RSX\VK\VKProgramBuffer.h" />
-      <Filter>Source Files</Filter>
+    <ClInclude Include="Emu\RSX\VK\VKRenderPass.h" />
-    </ClInclude>
+    <ClInclude Include="Emu\RSX\VK\VKRenderTargets.h" />
-    <ClInclude Include="Emu\RSX\VK\VKProgramBuffer.h">
+    <ClInclude Include="Emu\RSX\VK\VKResolveHelper.h" />
-      <Filter>Source Files</Filter>
+    <ClInclude Include="Emu\RSX\VK\VKResourceManager.h" />
-    </ClInclude>
+    <ClInclude Include="Emu\RSX\VK\VKTextOut.h" />
-    <ClInclude Include="Emu\RSX\VK\VKRenderTargets.h">
+    <ClInclude Include="Emu\RSX\VK\VKTextureCache.h" />
-      <Filter>Source Files</Filter>
+    <ClInclude Include="Emu\RSX\VK\VKVertexProgram.h" />
-    </ClInclude>
+    <ClInclude Include="Emu\RSX\VK\VulkanAPI.h" />
-    <ClInclude Include="Emu\RSX\VK\VKTextureCache.h">
+    <ClInclude Include="Emu\RSX\VK\VKCommandStream.h" />
-      <Filter>Source Files</Filter>
-    </ClInclude>
-    <ClInclude Include="Emu\RSX\VK\VKVertexProgram.h">
-      <Filter>Source Files</Filter>
-    </ClInclude>
-    <ClInclude Include="Emu\RSX\VK\VulkanAPI.h">
-      <Filter>Source Files</Filter>
-    </ClInclude>
-    <ClInclude Include="Emu\RSX\VK\VKFormats.h">
-      <Filter>Source Files</Filter>
-    </ClInclude>
-    <ClInclude Include="Emu\RSX\VK\VKTextOut.h">
-      <Filter>Source Files</Filter>
-    </ClInclude>
-    <ClInclude Include="Emu\RSX\VK\VKOverlays.h">
-      <Filter>Source Files</Filter>
-    </ClInclude>
-    <ClInclude Include="Emu\RSX\VK\VKCompute.h">
-      <Filter>Source Files</Filter>
-    </ClInclude>
-    <ClInclude Include="Emu\RSX\VK\VKRenderPass.h">
-      <Filter>Source Files</Filter>
-    </ClInclude>
-    <ClInclude Include="Emu\RSX\VK\VKResolveHelper.h">
-      <Filter>Source Files</Filter>
-    </ClInclude>
-    <ClInclude Include="Emu\RSX\VK\VKResourceManager.h">
-      <Filter>Source Files</Filter>
-    </ClInclude>
-    <ClInclude Include="Emu\RSX\VK\VKFramebuffer.h">
-      <Filter>Source Files</Filter>
-    </ClInclude>
-    <ClInclude Include="Emu\RSX\VK\VKDMA.h">
-      <Filter>Source Files</Filter>
-    </ClInclude>
-  </ItemGroup>
-  <ItemGroup>
-    <ClCompile Include="Emu\RSX\VK\VKGSRender.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="Emu\RSX\VK\VKCommonDecompiler.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="Emu\RSX\VK\VKFragmentProgram.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="Emu\RSX\VK\VKHelpers.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="Emu\RSX\VK\VKProgramPipeline.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="Emu\RSX\VK\VKTexture.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="Emu\RSX\VK\VKVertexProgram.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="Emu\RSX\VK\VKVertexBuffers.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="Emu\RSX\VK\VKFormats.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="Emu\RSX\VK\VKMemAlloc.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="Emu\RSX\VK\VKRenderPass.cpp">
-      <Filter>Source Files</Filter>
-    </ClInclude>
-    <ClCompile Include="Emu\RSX\VK\VKResolveHelper.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="Emu\RSX\VK\VKResourceManager.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="Emu\RSX\VK\VKFramebuffer.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
-    <ClCompile Include="Emu\RSX\VK\VKDMA.cpp">
-      <Filter>Source Files</Filter>
-    </ClCompile>
   </ItemGroup>
 </Project>