Add support for Vulkan on Wayland

The variable VK_USE_PLATFORM_WAYLAND_KHR is actually used by the Vulkan
header, so use it here too.

Utilities/variant.hpp

@@ -12,6 +12,7 @@
 #include <utility>
 
 #include "recursive_wrapper.hpp"
+#include "variant_visitor.hpp"
 
 // clang-format off
 // [[deprecated]] is only available in C++14, use this for the time being
@@ -849,6 +850,22 @@ namespace std {
 				return detail::binary_dispatcher<F, V, R, Types...>::apply(v0, v1, std::forward<F>(f));
 			}
 
+			// match
+			// unary
+			template <typename... Fs>
+			auto VARIANT_INLINE match(Fs&&... fs) const
+			-> decltype(variant::visit(*this, ::std::make_visitor(std::forward<Fs>(fs)...)))
+			{
+				return variant::visit(*this, ::std::make_visitor(std::forward<Fs>(fs)...));
+			}
+			// non-const
+			template <typename... Fs>
+			auto VARIANT_INLINE match(Fs&&... fs)
+			-> decltype(variant::visit(*this, ::std::make_visitor(std::forward<Fs>(fs)...)))
+			{
+				return variant::visit(*this, ::std::make_visitor(std::forward<Fs>(fs)...));
+			}
+
 			~variant() noexcept // no-throw destructor
 			{
 				helper_type::destroy(type_index, &data);

Utilities/variant_visitor.hpp

@@ -0,0 +1,45 @@
+#ifndef MAPBOX_UTIL_VARIANT_VISITOR_HPP
+#define MAPBOX_UTIL_VARIANT_VISITOR_HPP
+
+#include <utility>
+
+//namespace mapbox {
+//namespace util {
+namespace std {
+
+template <typename... Fns>
+struct visitor;
+
+template <typename Fn>
+struct visitor<Fn> : Fn
+{
+    using Fn::operator();
+
+    template<typename T>
+    visitor(T&& fn) : Fn(std::forward<T>(fn)) {}
+};
+
+template <typename Fn, typename... Fns>
+struct visitor<Fn, Fns...> : Fn, visitor<Fns...>
+{
+    using Fn::operator();
+    using visitor<Fns...>::operator();
+
+    template<typename T, typename... Ts>
+    visitor(T&& fn, Ts&&... fns)
+        : Fn(std::forward<T>(fn))
+        , visitor<Fns...>(std::forward<Ts>(fns)...) {}
+};
+
+template <typename... Fns>
+visitor<typename std::decay<Fns>::type...> make_visitor(Fns&&... fns)
+{
+    return visitor<typename std::decay<Fns>::type...>
+        (std::forward<Fns>(fns)...);
+}
+
+}
+//} // namespace util
+//} // namespace mapbox
+
+#endif // MAPBOX_UTIL_VARIANT_VISITOR_HPP

rpcs3/CMakeLists.txt

@@ -11,13 +11,14 @@ if(WIN32)
 	find_package(Qt5 5.7 COMPONENTS WinExtras REQUIRED)
 	set(RPCS3_QT_LIBS Qt5::Widgets Qt5::WinExtras Qt5::Network)
 else()
-	find_package(Qt5 5.7 COMPONENTS DBus)
+	find_package(Qt5 5.7 COMPONENTS DBus Gui)
 	if(Qt5DBus_FOUND)
 		set(RPCS3_QT_LIBS Qt5::Widgets Qt5::DBus Qt5::Network)
 		add_definitions(-DHAVE_QTDBUS)
 	else()
 		set(RPCS3_QT_LIBS Qt5::Widgets Qt5::Network)
 	endif()
+	include_directories(${Qt5Gui_PRIVATE_INCLUDE_DIRS})
 endif()
 
 # Let's make sure we have Qt before we continue
@@ -153,6 +154,11 @@ elseif(NOT MSVC AND NOT CMAKE_CXX_FLAGS MATCHES "LIBICONV_PLUG")
 endif()
 if(UNIX AND NOT APPLE)
 	set(ADDITIONAL_LIBS ${ADDITIONAL_LIBS} "X11")
+	find_package(Wayland)
+	if (WAYLAND_FOUND)
+		include_directories(${WAYLAND_INCLUDE_DIR})
+		add_definitions(-DVK_USE_PLATFORM_WAYLAND_KHR)
+	endif()
 endif()
 
 if(NOT RPCS3_SRC_DIR)

rpcs3/Emu/RSX/GSRender.h

@@ -3,6 +3,19 @@
 #include "Emu/RSX/RSXThread.h"
 #include <memory>
 
+#ifdef _WIN32
+#include <windows.h>
+#else
+// Cannot include Xlib.h before Qt5
+// and we don't need all of Xlib anyway
+typedef struct _XDisplay Display;
+typedef unsigned long Window;
+#endif
+
+#ifdef VK_USE_PLATFORM_WAYLAND_KHR
+#include <wayland-client.h>
+#endif
+
 struct RSXDebuggerProgram
 {
 	u32 id;
@@ -34,6 +47,17 @@ using RSXDebuggerPrograms = std::vector<RSXDebuggerProgram>;
 
 using draw_context_t = void*;
 
+#ifdef _WIN32
+	using display_handle_t = HWND;
+#else
+	using display_handle_t = std::variant<
+		std::pair<Display*, Window>
+#ifdef VK_USE_PLATFORM_WAYLAND_KHR
+		, std::pair<wl_display*, wl_surface*>
+#endif
+	>;
+#endif
+
 class GSFrameBase
 {
 public:
@@ -52,7 +76,7 @@ public:
 	virtual int client_width() = 0;
 	virtual int client_height() = 0;
 
-	virtual void* handle() const = 0;
+	virtual display_handle_t handle() const = 0;
 
 protected:
 

rpcs3/Emu/RSX/VK/VKGSRender.cpp

@@ -494,12 +494,8 @@ VKGSRender::VKGSRender() : GSRender()
 		return;
 	}
 
-#ifdef _WIN32
+	std::vector<vk::physical_device>& gpus = m_thread_context.enumerateDevices();
-	HINSTANCE hInstance = NULL;
-	HWND hWnd = (HWND)m_frame->handle();
 
-	std::vector<vk::physical_device>& gpus = m_thread_context.enumerateDevices();	
-	
 	//Actually confirm  that the loader found at least one compatible device
 	//This should not happen unless something is wrong with the driver setup on the target system
 	if (gpus.size() == 0)
@@ -512,11 +508,18 @@ VKGSRender::VKGSRender() : GSRender()
 
 	bool gpu_found = false;
 	std::string adapter_name = g_cfg.video.vk.adapter;
+
+	display_handle_t display = m_frame->handle();
+
+#ifdef _WIN32
+
+	HINSTANCE hInstance = NULL;
+
 	for (auto &gpu : gpus)
 	{
 		if (gpu.name() == adapter_name)
 		{
-			m_swap_chain = m_thread_context.createSwapChain(hInstance, hWnd, gpu);
+			m_swap_chain = m_thread_context.createSwapChain(hInstance, display, gpu);
 			gpu_found = true;
 			break;
 		}
@@ -524,35 +527,18 @@ VKGSRender::VKGSRender() : GSRender()
 
 	if (!gpu_found || adapter_name.empty())
 	{
-		m_swap_chain = m_thread_context.createSwapChain(hInstance, hWnd, gpus[0]);
+		m_swap_chain = m_thread_context.createSwapChain(hInstance, display, gpus[0]);
 	}
-	
+
 #elif HAVE_VULKAN
 
-	Window window = (Window)m_frame->handle();
+	display.match([](std::pair<Display*, Window> p) { XFlush(p.first); }, [](auto _) {});
-	Display *display = XOpenDisplay(0);
 
-	std::vector<vk::physical_device>& gpus = m_thread_context.enumerateDevices();	
-	
-	//Actually confirm  that the loader found at least one compatible device
-	//This should not happen unless something is wrong with the driver setup on the target system
-	if (gpus.size() == 0)
-	{
-		//We can't throw in Emulator::Load, so we show error and return
-		LOG_FATAL(RSX, "No compatible GPU devices found");
-		m_device = VK_NULL_HANDLE;
-		return;
-	}
-
-	XFlush(display);
-
-	bool gpu_found = false;
-	std::string adapter_name = g_cfg.video.vk.adapter;
 	for (auto &gpu : gpus)
 	{
 		if (gpu.name() == adapter_name)
 		{
-			m_swap_chain = m_thread_context.createSwapChain(display, window, gpu);
+			m_swap_chain = m_thread_context.createSwapChain(display, gpu);
 			gpu_found = true;
 			break;
 		}
@@ -560,10 +546,10 @@ VKGSRender::VKGSRender() : GSRender()
 
 	if (!gpu_found || adapter_name.empty())
 	{
-		m_swap_chain = m_thread_context.createSwapChain(display, window, gpus[0]);
+		m_swap_chain = m_thread_context.createSwapChain(display, gpus[0]);
 	}
-	
+
-	m_display_handle = display;
+	display.match([&](std::pair<Display*, Window> p) { m_display_handle = p.first; }, [](auto _) {});
 
 #endif
 

rpcs3/Emu/RSX/VK/VKHelpers.h

@@ -8,10 +8,8 @@
 #include <memory>
 #include <unordered_map>
 
-#ifdef __linux__
+#include "Utilities/variant.hpp"
-#include <X11/Xlib.h>
+#include "Emu/RSX/GSRender.h"
-#endif
-
 #include "Emu/System.h"
 #include "VulkanAPI.h"
 #include "../GCM.h"
@@ -199,7 +197,7 @@ namespace vk
 			//Set up instance information
 			const char *requested_extensions[] =
 			{
-				"VK_KHR_swapchain"
+				VK_KHR_SWAPCHAIN_EXTENSION_NAME
 			};
 
 			std::vector<const char *> layers;
@@ -324,7 +322,7 @@ namespace vk
 			VkDevice dev = (VkDevice)(*owner);
 
 			u32 access_mask = 0;
-			
+
 			if (host_visible)
 				access_mask |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
 
@@ -412,7 +410,7 @@ namespace vk
 
 			VkMemoryRequirements memory_req;
 			vkGetImageMemoryRequirements(m_device, value, &memory_req);
-			
+
 			if (!(memory_req.memoryTypeBits & (1 << memory_type_index)))
 			{
 				//Suggested memory type is incompatible with this memory type.
@@ -953,7 +951,7 @@ namespace vk
 
 			nb_swap_images = 0;
 			getSwapchainImagesKHR(dev, m_vk_swapchain, &nb_swap_images, nullptr);
-			
+
 			if (!nb_swap_images) fmt::throw_exception("Driver returned 0 images for swapchain" HERE);
 
 			std::vector<VkImage> swap_images;
@@ -1185,11 +1183,11 @@ namespace vk
 			m_instance = nullptr;
 			m_vk_instances.resize(0);
 		}
-		
+
 		void enable_debugging()
 		{
 			if (!g_cfg.video.debug_output) return;
-			 
+
 			PFN_vkDebugReportCallbackEXT callback = vk::dbgFunc;
 
 			createDebugReportCallback = (PFN_vkCreateDebugReportCallbackEXT)vkGetInstanceProcAddr(m_instance, "vkCreateDebugReportCallbackEXT");
@@ -1220,13 +1218,16 @@ namespace vk
 			//Set up instance information
 			const char *requested_extensions[] =
 			{
-				"VK_KHR_surface",
+				VK_KHR_SURFACE_EXTENSION_NAME,
 #ifdef _WIN32
-				"VK_KHR_win32_surface",
+				VK_KHR_WIN32_SURFACE_EXTENSION_NAME,
 #else
-				"VK_KHR_xlib_surface",
+				VK_KHR_XLIB_SURFACE_EXTENSION_NAME,
 #endif
-				"VK_EXT_debug_report",
+#ifdef VK_USE_PLATFORM_WAYLAND_KHR
+				VK_KHR_WAYLAND_SURFACE_EXTENSION_NAME,
+#endif
+				VK_EXT_DEBUG_REPORT_EXTENSION_NAME,
 			};
 
 			std::vector<const char *> layers;
@@ -1239,7 +1240,7 @@ namespace vk
 			instance_info.pApplicationInfo = &app;
 			instance_info.enabledLayerCount = static_cast<uint32_t>(layers.size());
 			instance_info.ppEnabledLayerNames = layers.data();
-			instance_info.enabledExtensionCount = fast? 0: 3;
+			instance_info.enabledExtensionCount = fast? 0: sizeof(requested_extensions)/sizeof(char*);
 			instance_info.ppEnabledExtensionNames = fast? nullptr: requested_extensions;
 
 			VkInstance instance;
@@ -1304,8 +1305,8 @@ namespace vk
 		}
 
 #ifdef _WIN32
-		
+
-		vk::swap_chain* createSwapChain(HINSTANCE hInstance, HWND hWnd, vk::physical_device &dev)
+		vk::swap_chain* createSwapChain(HINSTANCE hInstance, display_handle_t hWnd, vk::physical_device &dev)
 		{
 			VkWin32SurfaceCreateInfoKHR createInfo = {};
 			createInfo.sType = VK_STRUCTURE_TYPE_WIN32_SURFACE_CREATE_INFO_KHR;
@@ -1315,16 +1316,31 @@ namespace vk
 			VkSurfaceKHR surface;
 			CHECK_RESULT(vkCreateWin32SurfaceKHR(m_instance, &createInfo, NULL, &surface));
 #elif HAVE_VULKAN
-		
+
-		vk::swap_chain* createSwapChain(Display *display, Window window, vk::physical_device &dev)
+		vk::swap_chain* createSwapChain(display_handle_t ctx, vk::physical_device &dev)
 		{
-			VkXlibSurfaceCreateInfoKHR createInfo = {};
-			createInfo.sType = VK_STRUCTURE_TYPE_XLIB_SURFACE_CREATE_INFO_KHR;
-			createInfo.dpy = display;
-			createInfo.window = window;
-			
 			VkSurfaceKHR surface;
-			CHECK_RESULT(vkCreateXlibSurfaceKHR(m_instance, &createInfo, nullptr, &surface));
+
+			ctx.match(
+				[&](std::pair<Display*, Window> p)
+				{
+					VkXlibSurfaceCreateInfoKHR createInfo = {};
+					createInfo.sType = VK_STRUCTURE_TYPE_XLIB_SURFACE_CREATE_INFO_KHR;
+					createInfo.dpy = p.first;
+					createInfo.window = p.second;
+					CHECK_RESULT(vkCreateXlibSurfaceKHR(this->m_instance, &createInfo, nullptr, &surface));
+				}
+#ifdef VK_USE_PLATFORM_WAYLAND_KHR
+				, [&](std::pair<wl_display*, wl_surface*> p)
+				{
+					VkWaylandSurfaceCreateInfoKHR createInfo = {};
+					createInfo.sType = VK_STRUCTURE_TYPE_WAYLAND_SURFACE_CREATE_INFO_KHR;
+					createInfo.display = p.first;
+					createInfo.surface = p.second;
+					CHECK_RESULT(vkCreateWaylandSurfaceKHR(this->m_instance, &createInfo, nullptr, &surface));
+				}
+#endif
+			);
 #endif
 
 			uint32_t device_queues = dev.get_queue_count();
@@ -1439,7 +1455,7 @@ namespace vk
 		void destroy()
 		{
 			if (!pool) return;
-			
+
 			vkDestroyDescriptorPool((*owner), pool, nullptr);
 			owner = nullptr;
 			pool = nullptr;
@@ -1591,7 +1607,7 @@ namespace vk
 		{
 			::glsl::program_domain domain;
 			program_input_type type;
-			
+
 			bound_buffer as_buffer;
 			bound_sampler as_sampler;
 

rpcs3/cmake_modules/FindWayland.cmake

@@ -0,0 +1,66 @@
+# Try to find Wayland on a Unix system
+#
+# This will define:
+#
+#   WAYLAND_FOUND       - True if Wayland is found
+#   WAYLAND_LIBRARIES   - Link these to use Wayland
+#   WAYLAND_INCLUDE_DIR - Include directory for Wayland
+#   WAYLAND_DEFINITIONS - Compiler flags for using Wayland
+#
+# In addition the following more fine grained variables will be defined:
+#
+#   WAYLAND_CLIENT_FOUND  WAYLAND_CLIENT_INCLUDE_DIR  WAYLAND_CLIENT_LIBRARIES
+#   WAYLAND_SERVER_FOUND  WAYLAND_SERVER_INCLUDE_DIR  WAYLAND_SERVER_LIBRARIES
+#   WAYLAND_EGL_FOUND     WAYLAND_EGL_INCLUDE_DIR     WAYLAND_EGL_LIBRARIES
+#
+# Copyright (c) 2013 Martin Gräßlin <mgraesslin@kde.org>
+#
+# Redistribution and use is allowed according to the terms of the BSD license.
+# For details see the accompanying COPYING-CMAKE-SCRIPTS file.
+
+IF (NOT WIN32)
+  IF (WAYLAND_INCLUDE_DIR AND WAYLAND_LIBRARIES)
+    # In the cache already
+    SET(WAYLAND_FIND_QUIETLY TRUE)
+  ENDIF ()
+
+  # Use pkg-config to get the directories and then use these values
+  # in the FIND_PATH() and FIND_LIBRARY() calls
+  FIND_PACKAGE(PkgConfig)
+  PKG_CHECK_MODULES(PKG_WAYLAND QUIET wayland-client wayland-server wayland-egl wayland-cursor)
+
+  SET(WAYLAND_DEFINITIONS ${PKG_WAYLAND_CFLAGS})
+
+  FIND_PATH(WAYLAND_CLIENT_INCLUDE_DIR  NAMES wayland-client.h HINTS ${PKG_WAYLAND_INCLUDE_DIRS})
+  FIND_PATH(WAYLAND_SERVER_INCLUDE_DIR  NAMES wayland-server.h HINTS ${PKG_WAYLAND_INCLUDE_DIRS})
+  FIND_PATH(WAYLAND_EGL_INCLUDE_DIR     NAMES wayland-egl.h    HINTS ${PKG_WAYLAND_INCLUDE_DIRS})
+  FIND_PATH(WAYLAND_CURSOR_INCLUDE_DIR  NAMES wayland-cursor.h HINTS ${PKG_WAYLAND_INCLUDE_DIRS})
+
+  FIND_LIBRARY(WAYLAND_CLIENT_LIBRARIES NAMES wayland-client   HINTS ${PKG_WAYLAND_LIBRARY_DIRS})
+  FIND_LIBRARY(WAYLAND_SERVER_LIBRARIES NAMES wayland-server   HINTS ${PKG_WAYLAND_LIBRARY_DIRS})
+  FIND_LIBRARY(WAYLAND_EGL_LIBRARIES    NAMES wayland-egl      HINTS ${PKG_WAYLAND_LIBRARY_DIRS})
+  FIND_LIBRARY(WAYLAND_CURSOR_LIBRARIES NAMES wayland-cursor   HINTS ${PKG_WAYLAND_LIBRARY_DIRS})
+
+  set(WAYLAND_INCLUDE_DIR ${WAYLAND_CLIENT_INCLUDE_DIR} ${WAYLAND_SERVER_INCLUDE_DIR} ${WAYLAND_EGL_INCLUDE_DIR} ${WAYLAND_CURSOR_INCLUDE_DIR})
+
+  set(WAYLAND_LIBRARIES ${WAYLAND_CLIENT_LIBRARIES} ${WAYLAND_SERVER_LIBRARIES} ${WAYLAND_EGL_LIBRARIES} ${WAYLAND_CURSOR_LIBRARIES})
+
+  list(REMOVE_DUPLICATES WAYLAND_INCLUDE_DIR)
+
+  include(FindPackageHandleStandardArgs)
+
+  FIND_PACKAGE_HANDLE_STANDARD_ARGS(WAYLAND_CLIENT  DEFAULT_MSG  WAYLAND_CLIENT_LIBRARIES  WAYLAND_CLIENT_INCLUDE_DIR)
+  FIND_PACKAGE_HANDLE_STANDARD_ARGS(WAYLAND_SERVER  DEFAULT_MSG  WAYLAND_SERVER_LIBRARIES  WAYLAND_SERVER_INCLUDE_DIR)
+  FIND_PACKAGE_HANDLE_STANDARD_ARGS(WAYLAND_EGL     DEFAULT_MSG  WAYLAND_EGL_LIBRARIES     WAYLAND_EGL_INCLUDE_DIR)
+  FIND_PACKAGE_HANDLE_STANDARD_ARGS(WAYLAND_CURSOR  DEFAULT_MSG  WAYLAND_CURSOR_LIBRARIES  WAYLAND_CURSOR_INCLUDE_DIR)
+  FIND_PACKAGE_HANDLE_STANDARD_ARGS(WAYLAND         DEFAULT_MSG  WAYLAND_LIBRARIES         WAYLAND_INCLUDE_DIR)
+
+  MARK_AS_ADVANCED(
+        WAYLAND_INCLUDE_DIR         WAYLAND_LIBRARIES
+        WAYLAND_CLIENT_INCLUDE_DIR  WAYLAND_CLIENT_LIBRARIES
+        WAYLAND_SERVER_INCLUDE_DIR  WAYLAND_SERVER_LIBRARIES
+        WAYLAND_EGL_INCLUDE_DIR     WAYLAND_EGL_LIBRARIES
+        WAYLAND_CURSOR_INCLUDE_DIR  WAYLAND_CURSOR_LIBRARIES
+  )
+
+ENDIF ()

rpcs3/rpcs3qt/gs_frame.cpp

@@ -8,6 +8,13 @@
 #include <QTimer>
 #include <QThread>
 
+#ifdef VK_USE_PLATFORM_WAYLAND_KHR
+#include <QGuiApplication>
+#include <qpa/qplatformnativeinterface.h>
+#endif
+
+#include <X11/Xlib.h>
+
 #include <string>
 
 #include "rpcs3_version.h"
@@ -145,12 +152,28 @@ void gs_frame::show()
 	});
 }
 
-void* gs_frame::handle() const
+display_handle_t gs_frame::handle() const
 {
 #ifdef _WIN32
 	return (HWND) this->winId();
 #else
-	return (void *)this->winId();
+#ifdef VK_USE_PLATFORM_WAYLAND_KHR
+	QPlatformNativeInterface *native = QGuiApplication::platformNativeInterface();
+	struct wl_display *wl_dpy = static_cast<struct wl_display *>(
+		native->nativeResourceForWindow("display", NULL));
+	struct wl_surface *wl_surf = static_cast<struct wl_surface *>(
+		native->nativeResourceForWindow("surface", (QWindow *)this));
+	if (wl_dpy != nullptr && wl_surf != nullptr)
+	{
+		return std::make_pair(wl_dpy, wl_surf);
+	}
+	else
+	{
+#endif
+		return std::make_pair(XOpenDisplay(0), (unsigned long)(this->winId()));
+#ifdef VK_USE_PLATFORM_WAYLAND_KHR
+	}
+#endif
 #endif
 }
 

rpcs3/rpcs3qt/gs_frame.h

@@ -40,7 +40,7 @@ protected:
 	void show() override;
 	void mouseDoubleClickEvent(QMouseEvent* ev) override;
 
-	void* handle() const override;
+	display_handle_t handle() const override;
 
 	void flip(draw_context_t context, bool skip_frame=false) override;
 	int client_width() override;