Add GDB stub for debugging (#657)

* Implement GDB stub debugger

Can be enabled by using the "--enable-gdbstub" option (and the debugger GUI, although that's untested) which'll pause any game you launch at start-up. Will start at port 1337 although it'll eventually be user-editable. The code is a bit weirdly sorted and also just needs a general cleanup, so expect that eventually too. And uses egyptian braces but formatting was easier to do at the end, so that's also something to do.

It has been tested to work with IDA Pro, Clion and the standalone interface for now, but I plan on writing some instructions in the PR to follow for people who want to use this. Memory breakpoints aren't possible yet, only execution breakpoints.

This code was aimed to be decoupled from the existing debugger to be able to be ported to the Wii U for an equal debugging experience. That's also why it uses the Cafe OS's thread sleep and resuming functions whenever possible instead of using recompiler/interpreter controls.

* Add memory writing and floating point registers support

* Reformat code a bit

* Format code to adhere to Cemu's coding style

* Rework GDB Stub settings in GUI

* Small styling fixes

* Rework execution breakpoints

Should work better in some edge cases now. But this should also allow for adding access breakpoints since it's now more separated.

* Implement access breakpoints

* Fix some issues with breakpoints

* Fix includes for Linux

* Fix unnecessary include

* Tweaks for Linux compatibility

* Use std::thread instead of std::jthread to fix MacOS support

* Enable GDB read/write breakpoints on x86 only

* Fix compilation for GCC compilers at least

The thread type varies on some platforms, so supporting this is hell... but let's get it to compile on MacOS first.

* Disable them for MacOS due to lack of ptrace

---------

Co-authored-by: Exzap <13877693+Exzap@users.noreply.github.com>

src/Cafe/OS/libs/coreinit/coreinit_Thread.cpp

@@ -671,6 +671,11 @@ namespace coreinit
 		__OSUnlockScheduler();
 	}
 
+    void __OSSuspendThreadNolock(OSThread_t* thread)
+    {
+        __OSSuspendThreadInternal(thread);
+    }
+
 	void OSSleepThread(OSThreadQueue* threadQueue)
 	{
 		__OSLockScheduler();
@@ -798,7 +803,18 @@ namespace coreinit
 		return suspendCounter > 0;
 	}
 
-	void OSCancelThread(OSThread_t* thread)
+    bool OSIsThreadRunning(OSThread_t* thread)
+    {
+        bool isRunning = false;
+        __OSLockScheduler();
+        if (thread->state == OSThread_t::THREAD_STATE::STATE_RUNNING)
+            isRunning = true;
+        __OSUnlockScheduler();
+        return isRunning;
+    }
+
+
+    void OSCancelThread(OSThread_t* thread)
 	{
 		__OSLockScheduler();
 		cemu_assert_debug(thread->requestFlags == 0 || thread->requestFlags == OSThread_t::REQUEST_FLAG_CANCEL); // todo - how to handle cases where other flags are already set?
@@ -1315,6 +1331,7 @@ namespace coreinit
 		cafeExportRegister("coreinit", OSResumeThread, LogType::CoreinitThread);
 		cafeExportRegister("coreinit", OSContinueThread, LogType::CoreinitThread);
 		cafeExportRegister("coreinit", OSSuspendThread, LogType::CoreinitThread);
+		cafeExportRegister("coreinit", __OSSuspendThreadNolock, LogType::CoreinitThread);
 		cafeExportRegister("coreinit", OSSleepThread, LogType::CoreinitThread);
 		cafeExportRegister("coreinit", OSWakeupThread, LogType::CoreinitThread);
 

src/Cafe/OS/libs/coreinit/coreinit_Thread.h

@@ -461,10 +461,10 @@ struct OSThread_t
 	/* +0x628 */ uint64								wakeTimeRelatedUkn2;
 
 	// set via OSSetExceptionCallback
-	/* +0x630 */ MPTR								ukn630Callback[Espresso::CORE_COUNT];
-	/* +0x63C */ MPTR								ukn63CCallback[Espresso::CORE_COUNT];
-	/* +0x648 */ MPTR								ukn648Callback[Espresso::CORE_COUNT];
-	/* +0x654 */ MPTR								ukn654Callback[Espresso::CORE_COUNT];
+	/* +0x630 */ MPTR								dsiCallback[Espresso::CORE_COUNT];
+	/* +0x63C */ MPTR								isiCallback[Espresso::CORE_COUNT];
+	/* +0x648 */ MPTR								programCallback[Espresso::CORE_COUNT];
+	/* +0x654 */ MPTR								perfMonCallback[Espresso::CORE_COUNT];
 
 	/* +0x660 */ uint32								ukn660;
 
@@ -514,6 +514,7 @@ namespace coreinit
 	sint32 OSResumeThread(OSThread_t* thread);
 	void OSContinueThread(OSThread_t* thread);
 	void __OSSuspendThreadInternal(OSThread_t* thread);
+	void __OSSuspendThreadNolock(OSThread_t* thread);
 	void OSSuspendThread(OSThread_t* thread);
 	void OSSleepThread(OSThreadQueue* threadQueue);
 	void OSWakeupThread(OSThreadQueue* threadQueue);
@@ -525,6 +526,7 @@ namespace coreinit
 
 	bool OSIsThreadTerminated(OSThread_t* thread);
 	bool OSIsThreadSuspended(OSThread_t* thread);
+	bool OSIsThreadRunning(OSThread_t* thread);
 
 	// OSThreadQueue
 	void OSInitThreadQueue(OSThreadQueue* threadQueue);

src/Cafe/OS/libs/coreinit/coreinit_ThreadQueue.cpp

@@ -135,7 +135,7 @@ namespace coreinit
 		while (OSThread_t* thread = takeFirstFromQueue(offsetof(OSThread_t, waitQueueLink)))
 		{
 			cemu_assert_debug(thread->state == OSThread_t::THREAD_STATE::STATE_WAITING);
-			cemu_assert_debug(thread->suspendCounter == 0);
+			//cemu_assert_debug(thread->suspendCounter == 0);
 			thread->state = OSThread_t::THREAD_STATE::STATE_READY;
 			thread->currentWaitQueue = nullptr;
 			coreinit::__OSAddReadyThreadToRunQueue(thread);

src/Cafe/OS/libs/nsysnet/nsysnet.cpp

@@ -73,6 +73,15 @@ void nsysnetExport_socket_lib_init(PPCInterpreter_t* hCPU)
 	osLib_returnFromFunction(hCPU, 0); // 0 -> Success
 }
 
+void nsysnetExport_socket_lib_finish(PPCInterpreter_t* hCPU)
+{
+	sockLibReady = false;
+#if BOOST_OS_WINDOWS
+	WSACleanup();
+#endif // BOOST_OS_WINDOWS
+	osLib_returnFromFunction(hCPU, 0); // 0 -> Success
+}
+
 uint32* __gh_errno_ptr()
 {
 	OSThread_t* osThread = coreinitThread_getCurrentThreadDepr(PPCInterpreter_getCurrentInstance());
@@ -2120,6 +2129,7 @@ void nsysnet_load()
 {
 	
 	osLib_addFunction("nsysnet", "socket_lib_init", nsysnetExport_socket_lib_init);
+	osLib_addFunction("nsysnet", "socket_lib_finish", nsysnetExport_socket_lib_finish);
 	
 	// socket API
 	osLib_addFunction("nsysnet", "socket", nsysnetExport_socket);