Improve GDB debug server (#4027)

* Made GDB debugger working with IDA

* Added async interrupts support

* Report proper thread after pausing

* Support attaching debugger before running app

Utilities/GDBDebugServer.cpp

@@ -166,9 +166,10 @@ char GDBDebugServer::read_char()
 
 u8 GDBDebugServer::read_hexbyte()
 {
-	char buf[2];
+	std::string s = "";
-	read(buf, 2);
+	s += read_char();
-	return static_cast<u8>(strtol(buf, nullptr, 16));
+	s += read_char();
+	return hex_to_u8(s);
 }
 
 void GDBDebugServer::try_read_cmd(gdb_cmd & out_cmd)
@@ -176,7 +177,7 @@ void GDBDebugServer::try_read_cmd(gdb_cmd & out_cmd)
 	char c = read_char();
 	//interrupt
 	if (UNLIKELY(c == 0x03)) {
-		out_cmd.cmd = "\0x03";
+		out_cmd.cmd = '\x03';
 		out_cmd.data = "";
 		out_cmd.checksum = 0;
 		return;
@@ -223,7 +224,7 @@ void GDBDebugServer::try_read_cmd(gdb_cmd & out_cmd)
 	}
 	out_cmd.checksum = read_hexbyte();
 	if (out_cmd.checksum != checksum) {
-		throw new wrong_checksum_exception("Wrong checksum for packet" HERE);
+		throw wrong_checksum_exception("Wrong checksum for packet" HERE);
 	}
 }
 
@@ -427,6 +428,25 @@ bool GDBDebugServer::send_reason()
 	return send_cmd_ack("S05");
 }
 
+void GDBDebugServer::wait_with_interrupts() {
+	char c;
+	while (!paused) {
+		int result = recv(client_socket, &c, 1, 0);
+
+		if (result == SOCKET_ERROR) {
+			if (check_errno_again()) {
+				thread_ctrl::wait_for(50);
+				continue;
+			}
+
+			gdbDebugServer.error("Error during socket read");
+			fmt::throw_exception("Error during socket read" HERE);
+		} else if (c == 0x03) {
+			paused = true;
+		}
+	}
+}
+
 bool GDBDebugServer::cmd_extended_mode(gdb_cmd & cmd)
 {
 	return send_cmd_ack("OK");
@@ -453,8 +473,8 @@ bool GDBDebugServer::cmd_thread_info(gdb_cmd & cmd)
 		result += u64_to_padded_hex(static_cast<u64>(cpu.id));
 	};
 	idm::select<ppu_thread>(on_select);
-	idm::select<RawSPUThread>(on_select);
+	//idm::select<RawSPUThread>(on_select);
-	idm::select<SPUThread>(on_select);
+	//idm::select<SPUThread>(on_select);
 
 	//todo: this may exceed max command length
 	result = "m" + result + "l";
@@ -464,12 +484,14 @@ bool GDBDebugServer::cmd_thread_info(gdb_cmd & cmd)
 
 bool GDBDebugServer::cmd_current_thread(gdb_cmd & cmd)
 {
-	return send_cmd_ack(selected_thread.expired() ? "" : u64_to_padded_hex(selected_thread.lock()->id));
+	return send_cmd_ack(selected_thread.expired() ? "" : ("QC" + u64_to_padded_hex(selected_thread.lock()->id)));
 }
 
 bool GDBDebugServer::cmd_read_register(gdb_cmd & cmd)
 {
-	select_thread(general_ops_thread_id);
+	if (!select_thread(general_ops_thread_id)) {
+		return send_cmd_ack("E02");
+	}
 	auto th = selected_thread.lock();
 	if (th->id_type() == 1) {
 		auto ppu = std::static_pointer_cast<ppu_thread>(th);
@@ -487,7 +509,9 @@ bool GDBDebugServer::cmd_read_register(gdb_cmd & cmd)
 
 bool GDBDebugServer::cmd_write_register(gdb_cmd & cmd)
 {
-	select_thread(general_ops_thread_id);
+	if (!select_thread(general_ops_thread_id)) {
+		return send_cmd_ack("E02");
+	}
 	auto th = selected_thread.lock();
 	if (th->id_type() == 1) {
 		auto ppu = std::static_pointer_cast<ppu_thread>(th);
@@ -516,7 +540,7 @@ bool GDBDebugServer::cmd_read_memory(gdb_cmd & cmd)
 	std::string result;
 	result.reserve(len * 2);
 	for (u32 i = 0; i < len; ++i) {
-		if (vm::check_addr(addr + i)) {
+		if (vm::check_addr(addr, 1, vm::page_info_t::page_readable)) {
 			result += to_hexbyte(vm::read8(addr + i));
 		} else {
 			break;
@@ -542,7 +566,7 @@ bool GDBDebugServer::cmd_write_memory(gdb_cmd & cmd)
 	u32 len = hex_to_u32(cmd.data.substr(s + 1, s2 - s - 1));
 	const char* data_ptr = (cmd.data.c_str()) + s2 + 1;
 	for (u32 i = 0; i < len; ++i) {
-		if (vm::check_addr(addr + i)) {
+		if (vm::check_addr(addr + i, 1, vm::page_info_t::page_writable)) {
 			u8 val;
 			int res = sscanf_s(data_ptr, "%02hhX", &val);
 			if (!res) {
@@ -633,30 +657,31 @@ bool GDBDebugServer::cmd_vcont(gdb_cmd & cmd)
 {
 	//todo: handle multiple actions and thread ids
 	this->from_breakpoint = false;
-	if (cmd.data[1] == 'c') {
+	if (cmd.data[1] == 'c' || cmd.data[1] == 's') {
-		select_thread(continue_ops_thread_id);
-		auto ppu = std::static_pointer_cast<ppu_thread>(selected_thread.lock());
-		ppu->state -= cpu_flag::dbg_pause;
-		if (Emu.IsPaused()) {
-			Emu.Resume();
-		}
-		thread_ctrl::wait();
-		//we are in all-stop mode
-		Emu.Pause();
-		return send_reason();
-	} else if (cmd.data[1] == 's') {
 		select_thread(continue_ops_thread_id);
 		auto ppu = std::static_pointer_cast<ppu_thread>(selected_thread.lock());
+		paused = false;
+		if (cmd.data[1] == 's') {
 			ppu->state += cpu_flag::dbg_step;
+		}
 		ppu->state -= cpu_flag::dbg_pause;
+		//special case if app didn't start yet (only loaded)
+		if (!Emu.IsPaused() && !Emu.IsRunning()) {
+			Emu.Run();
+		}
 		if (Emu.IsPaused()) {
 			Emu.Resume();
 		} else {
 			ppu->notify();
 		}
-		thread_ctrl::wait();
+		wait_with_interrupts();
 		//we are in all-stop mode
 		Emu.Pause();
+		select_thread(pausedBy);
+		// we have to remove dbg_pause from thread that paused execution, otherwise
+		// it will be paused forever (Emu.Resume only removes dbg_global_pause)
+		ppu = std::static_pointer_cast<ppu_thread>(selected_thread.lock());
+		ppu->state -= cpu_flag::dbg_pause;
 		return send_reason();
 	}
 	return send_cmd_ack("");
@@ -728,7 +753,9 @@ void GDBDebugServer::on_task()
 			return;
 		}
 		//stop immediately
+		if (Emu.IsRunning()) {
 			Emu.Pause();
+		}
 
 		try {
 			char hostbuf[32];
@@ -804,6 +831,15 @@ void GDBDebugServer::on_stop()
 	named_thread::on_stop();
 }
 
+void GDBDebugServer::pause_from(cpu_thread* t) {
+	if (paused) {
+		return;
+	}
+	paused = true;
+	pausedBy = t->id;
+	notify();
+}
+
 u32 g_gdb_debugger_id = 0;
 
 #ifndef _WIN32

Utilities/GDBDebugServer.h

@@ -50,7 +50,7 @@ class GDBDebugServer : public named_thread {
 
 	//initialize server socket and start listening
 	void start_server();
-	//read at most cnt bytes to buf, returns nubmer of bytes actually read
+	//read at most cnt bytes to buf, returns number of bytes actually read
 	int read(void* buf, int cnt);
 	//reads one character
 	char read_char();
@@ -90,6 +90,8 @@ class GDBDebugServer : public named_thread {
 	//send reason of stop, returns false if sending response failed
 	bool send_reason();
 
+	void wait_with_interrupts();
+
 	//commands
 	bool cmd_extended_mode(gdb_cmd& cmd);
 	bool cmd_reason(gdb_cmd& cmd);
@@ -115,17 +117,24 @@ protected:
 	void on_exit() override final;
 
 public:
-	static const u32 id_base = 1;
-	static const u32 id_step = 1;
-	static const u32 id_count = 0x100000;
-
 	bool from_breakpoint = true;
 	bool stop = false;
+	bool paused = false;
+	u64 pausedBy;
 
 	virtual std::string get_name() const;
 	virtual void on_stop() override final;
+	void pause_from(cpu_thread* t);
 };
 
 extern u32 g_gdb_debugger_id;
 
+template <>
+struct id_manager::on_stop<GDBDebugServer> {
+	static inline void func(GDBDebugServer* ptr)
+	{
+		if (ptr) ptr->on_stop();
+	}
+};
+
 #endif

rpcs3/Emu/CPU/CPUThread.cpp

@@ -101,7 +101,7 @@ bool cpu_thread::check_state()
 {
 #ifdef WITH_GDB_DEBUGGER
 	if (test(state, cpu_flag::dbg_pause)) {
-		fxm::get<GDBDebugServer>()->notify();
+		fxm::get<GDBDebugServer>()->pause_from(this);
 	}
 #endif
 

rpcs3/Emu/Cell/PPUThread.cpp

@@ -296,7 +296,7 @@ static bool ppu_break(ppu_thread& ppu, ppu_opcode_t op)
 	// Pause and wait if necessary
 	bool status = ppu.state.test_and_set(cpu_flag::dbg_pause);
 #ifdef WITH_GDB_DEBUGGER
-	fxm::get<GDBDebugServer>()->notify();
+	fxm::get<GDBDebugServer>()->pause_from(&ppu);
 #endif
 	if (!status && ppu.check_state())
 	{