add wayland vulkan support

CMakeLists.txt

@@ -94,8 +94,14 @@ if (NOT TARGET glslang::SPIRV AND TARGET SPIRV)
 	add_library(glslang::SPIRV ALIAS SPIRV)
 endif()
 
+if (NOT TARGET glm::glm AND TARGET glm)
+        add_library(glm::glm ALIAS glm)
+endif()
+
 if (UNIX AND NOT APPLE)
 	find_package(X11 REQUIRED)
+	find_package(Wayland REQUIRED)
+	find_package(GTK3 REQUIRED)
 endif()
 
 if (ENABLE_VULKAN)

cmake/FindGTK3.cmake

@@ -0,0 +1,21 @@
+# SPDX-FileCopyrightText: 2022 Andrea Pappacoda <andrea@pappacoda.it>
+# SPDX-License-Identifier: ISC
+
+include(FindPackageHandleStandardArgs)
+
+find_package(PkgConfig REQUIRED)
+if (PKG_CONFIG_FOUND)
+    pkg_search_module(GTK3 REQUIRED gtk+-3.0)
+    if (GTK3_FOUND)
+        add_library(GTK3::gtk IMPORTED INTERFACE)
+        target_link_libraries(GTK3::gtk INTERFACE ${GTK3_LIBRARIES})
+        target_link_directories(GTK3::gtk INTERFACE ${GTK3_LIBRARY_DIRS})
+        target_include_directories(GTK3::gtk INTERFACE ${GTK3_INCLUDE_DIRS})
+    endif()
+    find_package_handle_standard_args(GTK3
+        REQUIRED_VARS
+            GTK3_LINK_LIBRARIES
+            GTK3_FOUND
+        VERSION_VAR GTK3_VERSION
+    )
+endif()

cmake/FindWayland.cmake

@@ -0,0 +1,45 @@
+IF (NOT WIN32)
+  IF (WAYLAND_INCLUDE_DIR AND WAYLAND_LIBRARIES)
+    # In the cache already
+    SET(WAYLAND_FIND_QUIETLY TRUE)
+  ENDIF ()
+
+  # Use pkg-config to get the directories and then use these values
+  # in the FIND_PATH() and FIND_LIBRARY() calls
+  FIND_PACKAGE(PkgConfig)
+  PKG_CHECK_MODULES(PKG_WAYLAND QUIET wayland-client wayland-server wayland-egl wayland-cursor)
+
+  SET(WAYLAND_DEFINITIONS ${PKG_WAYLAND_CFLAGS})
+
+  FIND_PATH(WAYLAND_CLIENT_INCLUDE_DIR  NAMES wayland-client.h HINTS ${PKG_WAYLAND_INCLUDE_DIRS})
+  FIND_PATH(WAYLAND_SERVER_INCLUDE_DIR  NAMES wayland-server.h HINTS ${PKG_WAYLAND_INCLUDE_DIRS})
+  FIND_PATH(WAYLAND_EGL_INCLUDE_DIR     NAMES wayland-egl.h    HINTS ${PKG_WAYLAND_INCLUDE_DIRS})
+  FIND_PATH(WAYLAND_CURSOR_INCLUDE_DIR  NAMES wayland-cursor.h HINTS ${PKG_WAYLAND_INCLUDE_DIRS})
+
+  FIND_LIBRARY(WAYLAND_CLIENT_LIBRARIES NAMES wayland-client   HINTS ${PKG_WAYLAND_LIBRARY_DIRS})
+  FIND_LIBRARY(WAYLAND_SERVER_LIBRARIES NAMES wayland-server   HINTS ${PKG_WAYLAND_LIBRARY_DIRS})
+  FIND_LIBRARY(WAYLAND_EGL_LIBRARIES    NAMES wayland-egl      HINTS ${PKG_WAYLAND_LIBRARY_DIRS})
+  FIND_LIBRARY(WAYLAND_CURSOR_LIBRARIES NAMES wayland-cursor   HINTS ${PKG_WAYLAND_LIBRARY_DIRS})
+
+  set(WAYLAND_INCLUDE_DIR ${WAYLAND_CLIENT_INCLUDE_DIR} ${WAYLAND_SERVER_INCLUDE_DIR} ${WAYLAND_EGL_INCLUDE_DIR} ${WAYLAND_CURSOR_INCLUDE_DIR})
+
+  set(WAYLAND_LIBRARIES ${WAYLAND_CLIENT_LIBRARIES} ${WAYLAND_SERVER_LIBRARIES} ${WAYLAND_EGL_LIBRARIES} ${WAYLAND_CURSOR_LIBRARIES})
+
+  list(REMOVE_DUPLICATES WAYLAND_INCLUDE_DIR)
+
+  include(FindPackageHandleStandardArgs)
+
+  FIND_PACKAGE_HANDLE_STANDARD_ARGS(WAYLAND_CLIENT  DEFAULT_MSG  WAYLAND_CLIENT_LIBRARIES  WAYLAND_CLIENT_INCLUDE_DIR)
+  FIND_PACKAGE_HANDLE_STANDARD_ARGS(WAYLAND_SERVER  DEFAULT_MSG  WAYLAND_SERVER_LIBRARIES  WAYLAND_SERVER_INCLUDE_DIR)
+  FIND_PACKAGE_HANDLE_STANDARD_ARGS(WAYLAND_EGL     DEFAULT_MSG  WAYLAND_EGL_LIBRARIES     WAYLAND_EGL_INCLUDE_DIR)
+  FIND_PACKAGE_HANDLE_STANDARD_ARGS(WAYLAND_CURSOR  DEFAULT_MSG  WAYLAND_CURSOR_LIBRARIES  WAYLAND_CURSOR_INCLUDE_DIR)
+  FIND_PACKAGE_HANDLE_STANDARD_ARGS(WAYLAND         DEFAULT_MSG  WAYLAND_LIBRARIES         WAYLAND_INCLUDE_DIR)
+
+  MARK_AS_ADVANCED(
+        WAYLAND_INCLUDE_DIR         WAYLAND_LIBRARIES
+        WAYLAND_CLIENT_INCLUDE_DIR  WAYLAND_CLIENT_LIBRARIES
+        WAYLAND_SERVER_INCLUDE_DIR  WAYLAND_SERVER_LIBRARIES
+        WAYLAND_EGL_INCLUDE_DIR     WAYLAND_EGL_LIBRARIES
+        WAYLAND_CURSOR_INCLUDE_DIR  WAYLAND_CURSOR_LIBRARIES
+  )
+ENDIF ()

src/CMakeLists.txt

@@ -27,6 +27,7 @@ elseif(UNIX)
     else()
         add_definitions(-DVK_USE_PLATFORM_XLIB_KHR) # legacy. Do we need to support XLIB surfaces?
         add_definitions(-DVK_USE_PLATFORM_XCB_KHR)
+        add_definitions(-DVK_USE_PLATFORM_WAYLAND_KHR)
     endif()
     add_definitions(-maes)
     # warnings

src/Cafe/CMakeLists.txt

@@ -47,6 +47,8 @@ target_link_libraries(CemuCafe PRIVATE
 	zstd::zstd
 )
 
+target_include_directories(CemuCafe PRIVATE ${WAYLAND_CLIENT_INCLUDE_DIR})
+
 if (ENABLE_WXWIDGETS)
 	target_link_libraries(CemuCafe PRIVATE wx::base wx::core)
 endif()

src/Cafe/HW/Latte/Renderer/Vulkan/VulkanAPI.h

@@ -129,6 +129,7 @@ VKFUNC_DEVICE(vkCmdBindPipeline);
 #if BOOST_OS_LINUX
 VKFUNC_INSTANCE(vkCreateXlibSurfaceKHR);
 VKFUNC_INSTANCE(vkCreateXcbSurfaceKHR);
+VKFUNC_INSTANCE(vkCreateWaylandSurfaceKHR);
 #endif
 
 #if BOOST_OS_WINDOWS

src/Cafe/HW/Latte/Renderer/Vulkan/VulkanRenderer.cpp

@@ -30,6 +30,9 @@
 
 #include <wx/msgdlg.h>
 
+#include <dlfcn.h>
+#include <variant>
+
 #define VK_API_VERSION_MAJOR(version) (((uint32_t)(version) >> 22) & 0x7FU)
 #define VK_API_VERSION_MINOR(version) (((uint32_t)(version) >> 12) & 0x3FFU)
 
@@ -107,6 +110,7 @@ std::vector<VulkanRenderer::DeviceInfo> VulkanRenderer::GetDevices()
 	#if BOOST_OS_WINDOWS
 	requiredExtensions.emplace_back(VK_KHR_WIN32_SURFACE_EXTENSION_NAME);
 	#elif BOOST_OS_LINUX
+	requiredExtensions.emplace_back(VK_KHR_WAYLAND_SURFACE_EXTENSION_NAME);
 	requiredExtensions.emplace_back(VK_KHR_XLIB_SURFACE_EXTENSION_NAME);
 	#elif BOOST_OS_MACOS
 	requiredExtensions.emplace_back(VK_EXT_METAL_SURFACE_EXTENSION_NAME);
@@ -1172,6 +1176,7 @@ std::vector<const char*> VulkanRenderer::CheckInstanceExtensionSupport(FeatureCo
 	#if BOOST_OS_WINDOWS
 	requiredInstanceExtensions.emplace_back(VK_KHR_WIN32_SURFACE_EXTENSION_NAME);
 	#elif BOOST_OS_LINUX
+	requiredInstanceExtensions.emplace_back(VK_KHR_WAYLAND_SURFACE_EXTENSION_NAME);
 	requiredInstanceExtensions.emplace_back(VK_KHR_XLIB_SURFACE_EXTENSION_NAME);
 	#elif BOOST_OS_MACOS
 	requiredInstanceExtensions.emplace_back(VK_EXT_METAL_SURFACE_EXTENSION_NAME);
@@ -1325,6 +1330,27 @@ VkSurfaceKHR VulkanRenderer::CreateXlibSurface(VkInstance instance, Display* dpy
     return result;
 }
 
+VkSurfaceKHR VulkanRenderer::CreateWaylandSurface(VkInstance instance, wl_display* display, wl_surface* window)
+{
+	VkWaylandSurfaceCreateInfoKHR sci{
+		.sType = VK_STRUCTURE_TYPE_WAYLAND_SURFACE_CREATE_INFO_KHR,
+		.flags = 0,
+		.display = display,
+		.surface = window,
+	};
+
+	VkSurfaceKHR result;
+    VkResult err;
+
+    if ((err = vkCreateWaylandSurfaceKHR(instance, &sci, nullptr, &result)) != VK_SUCCESS)
+    {
+		forceLog_printf("Cannot create a wayland Vulkan surface: %d", (sint32)err);
+        throw std::runtime_error(fmt::format("Cannot create a wayland Vulkan surface: {}", err));
+    }
+
+	return result;
+}
+
 VkSurfaceKHR VulkanRenderer::CreateXcbSurface(VkInstance instance, xcb_connection_t* connection, xcb_window_t window)
 {
     VkXcbSurfaceCreateInfoKHR sci{};
@@ -1350,7 +1376,21 @@ VkSurfaceKHR VulkanRenderer::CreateFramebufferSurface(VkInstance instance, struc
 #if BOOST_OS_WINDOWS
 	return CreateWinSurface(instance, windowInfo.hwnd);
 #elif BOOST_OS_LINUX
-	return CreateXlibSurface(instance, windowInfo.xlib_display, windowInfo.xlib_window);
+	switch (windowInfo.info.index()) {
+		case 1:
+		{
+			auto& info = std::get<X11WindowHandleInfo>(windowInfo.info);
+			return CreateXlibSurface(instance, info.display, info.window);
+		}
+		case 2:
+		{
+			auto& info = std::get<WaylandWindowHandleInfo>(windowInfo.info);
+			return CreateWaylandSurface(instance, info.display, info.window);
+		}
+
+		default:
+			return nullptr;
+	}
 #elif BOOST_OS_MACOS
 	return CreateCocoaSurface(instance, windowInfo.handle);
 #endif

src/Cafe/HW/Latte/Renderer/Vulkan/VulkanRenderer.h

@@ -200,6 +200,7 @@ public:
 #if BOOST_OS_LINUX
 	static VkSurfaceKHR CreateXlibSurface(VkInstance instance, Display* dpy, Window window);
     static VkSurfaceKHR CreateXcbSurface(VkInstance instance, xcb_connection_t* connection, xcb_window_t window);
+	static VkSurfaceKHR CreateWaylandSurface(VkInstance instance, wl_display* display, wl_surface* window);
 #endif
 
 	static VkSurfaceKHR CreateFramebufferSurface(VkInstance instance, struct WindowHandleInfo& windowInfo);

src/Common/platform.h

@@ -10,6 +10,7 @@
 #include <X11/Xlib.h>
 #include <X11/extensions/Xrender.h>
 #include <X11/Xutil.h>
+#include <wayland-client-protocol.h>
 #include "Common/unix/platform.h"
 #elif BOOST_OS_MACOS
 #include <libkern/OSByteOrder.h>

src/gui/guiWrapper.cpp

@@ -1,6 +1,13 @@
+#if BOOST_OS_LINUX
+#include <gtk/gtk.h>
+#include <gdk/gdk.h>
+#include <gdk/gdkwindow.h>
+#include <gdk/gdkwayland.h>
+#include <gdk/gdkx.h>
+#endif
+
 #include "gui/wxgui.h"
 #include "gui/guiWrapper.h"
-#include "gui/CemuApp.h"
 #include "gui/MainWindow.h"
 #include "gui/debugger/DebuggerWindow2.h"
 #include "Cafe/HW/Latte/Core/Latte.h"
@@ -9,8 +16,6 @@
 #include "Cafe/HW/Latte/Renderer/Renderer.h"
 #include "Cafe/CafeSystem.h"
 
-#include "wxHelper.h"
-
 WindowInfo g_window_info {};
 
 std::shared_mutex g_mutex;
@@ -152,7 +157,6 @@ bool gui_isPadWindowOpen()
 #include <wx/nativewin.h>
 #include <dlfcn.h>
 
-typedef void GdkDisplay;
 #endif
 
 void gui_initHandleContextFromWxWidgetsWindow(WindowHandleInfo& handleInfoOut, class wxWindow* wxw)
@@ -160,44 +164,33 @@ void gui_initHandleContextFromWxWidgetsWindow(WindowHandleInfo& handleInfoOut, c
 #if BOOST_OS_WINDOWS
 	handleInfoOut.hwnd = wxw->GetHWND();
 #elif BOOST_OS_LINUX
-    /* dynamically retrieve GTK imports so we dont have to include and link the whole lib */
-    void (*dyn_gtk_widget_realize)(GtkWidget *widget);
-    dyn_gtk_widget_realize = (void(*)(GtkWidget* widget))dlsym(RTLD_NEXT, "gtk_widget_realize");
-
-    GdkWindow* (*dyn_gtk_widget_get_window)(GtkWidget *widget);
-    dyn_gtk_widget_get_window = (GdkWindow*(*)(GtkWidget* widget))dlsym(RTLD_NEXT, "gtk_widget_get_window");
-
-    GdkDisplay* (*dyn_gdk_window_get_display)(GdkWindow *widget);
-    dyn_gdk_window_get_display = (GdkDisplay*(*)(GdkWindow* window))dlsym(RTLD_NEXT, "gdk_window_get_display");
-
-    Display* (*dyn_gdk_x11_display_get_xdisplay)(GdkDisplay *display);
-    dyn_gdk_x11_display_get_xdisplay = (Display*(*)(GdkDisplay* display))dlsym(RTLD_NEXT, "gdk_x11_display_get_xdisplay");
-
-    Window (*dyn_gdk_x11_window_get_xid)(GdkWindow *window);
-    dyn_gdk_x11_window_get_xid = (Window(*)(GdkWindow *window))dlsym(RTLD_NEXT, "gdk_x11_window_get_xid");
-
-    if(!dyn_gtk_widget_realize || !dyn_gtk_widget_get_window ||
-    !dyn_gdk_window_get_display || !dyn_gdk_x11_display_get_xdisplay ||
-    !dyn_gdk_x11_window_get_xid)
-    {
-        cemuLog_log(LogType::Force, "Unable to load GDK symbols");
-        return;
-    }
-
     /* end of imports */
 
     // get window
     GtkWidget* gtkWidget = (GtkWidget*)wxw->GetHandle(); // returns GtkWidget
-    dyn_gtk_widget_realize(gtkWidget);
+    gtk_widget_realize(gtkWidget);
-    GdkWindow* gdkWindow = dyn_gtk_widget_get_window(gtkWidget);
-    handleInfoOut.xlib_window = dyn_gdk_x11_window_get_xid(gdkWindow);
 
-    // get display
+    GdkWindow* gdkWindow = gtk_widget_get_window(gtkWidget);
-    GdkDisplay* gdkDisplay = dyn_gdk_window_get_display(gdkWindow);
+	GdkDisplay* gdkDisplay = gdk_window_get_display(gdkWindow);
-    handleInfoOut.xlib_display = dyn_gdk_x11_display_get_xdisplay(gdkDisplay);
+
-    if(!handleInfoOut.xlib_display)
+	if(GDK_IS_WAYLAND_WINDOW(gdkWindow))
+	{
+		handleInfoOut.info = WaylandWindowHandleInfo{
+			.window = gdk_wayland_window_get_wl_surface(gdkWindow),
+			.display = gdk_wayland_display_get_wl_display(GDK_WAYLAND_DISPLAY(gdk_window_get_display(gdkWindow)))
+		};
+	}
+	else if(GDK_IS_X11_WINDOW(gdkWindow))
+	{
+		handleInfoOut.info = X11WindowHandleInfo{
+			.window = gdk_x11_window_get_xid(gdkWindow),
+			.display = gdk_x11_display_get_xdisplay(GDK_WAYLAND_DISPLAY(gdk_window_get_display(gdkWindow)))
+		};
+	}
+
+    if(handleInfoOut.info.index() == 0)
     {
-        cemuLog_log(LogType::Force, "Unable to get xlib display");
+        cemuLog_log(LogType::Force, "Unable to get display");
     }
 #else
 	handleInfoOut.handle = wxw->GetHandle();

src/gui/guiWrapper.h

@@ -1,19 +1,31 @@
 #pragma once
 
 #include <atomic>
+#include <variant>
+
+#include <wayland-client-protocol.h>
 
 #if BOOST_OS_LINUX
-#include "xcb/xproto.h"
+
 #endif
 
+struct X11WindowHandleInfo {
+	Window window{};
+	Display* display{};
+};
+struct WaylandWindowHandleInfo {
+	struct wl_surface* window{};
+	struct wl_display* display{};
+};
+
 struct WindowHandleInfo
 {
 #if BOOST_OS_WINDOWS
 	std::atomic<HWND> hwnd;
 #elif BOOST_OS_LINUX
+
     // XLIB
-    Display* xlib_display{};
+	std::variant<std::monostate, X11WindowHandleInfo, WaylandWindowHandleInfo> info;
-    Window xlib_window{};
 
 	// XCB (not used by GTK so we cant retrieve these without making our own window)
 	//xcb_connection_t* xcb_con{};