Make stopping emulation not pause or crash UI

* Make the UI and main thread available when stopping emulation.
* Make BlockingCallFromMainThread always execute, preventing bugs when it unexpectedly did not.
* Add error code for when starting emulation when Emu.Kill() is in progress.

rpcs3/Emu/Cell/lv2/sys_process.cpp

@@ -425,7 +425,6 @@ void lv2_exitspawn(ppu_thread& ppu, std::vector<std::string>& argv, std::vector<
 
 		std::string path = vfs::get(argv[0]);
 		std::string hdd1 = vfs::get("/dev_hdd1/");
-		std::string old_config = Emu.GetUsedConfig();
 
 		const u128 klic = g_fxo->get<loaded_npdrm_keys>().last_key();
 
@@ -457,7 +456,9 @@ void lv2_exitspawn(ppu_thread& ppu, std::vector<std::string>& argv, std::vector<
 			g_fxo->init<lv2_memory_container>(std::min(old_size - total_size, sdk_suggested_mem) + total_size);
 		};
 
-		Emu.Kill(false);
+		Emu.after_kill_callback = [func = std::move(func), argv = std::move(argv), envp = std::move(envp), data = std::move(data),
+			disc = std::move(disc), path = std::move(path), hdd1 = std::move(hdd1), old_config = Emu.GetUsedConfig(), klic]() mutable
+		{
 			Emu.argv = std::move(argv);
 			Emu.envp = std::move(envp);
 			Emu.data = std::move(data);
@@ -478,6 +479,9 @@ void lv2_exitspawn(ppu_thread& ppu, std::vector<std::string>& argv, std::vector<
 			{
 				sys_process.fatal("Failed to boot from exitspawn! (path=\"%s\", error=%s)", path, res);
 			}
+		};
+
+		Emu.Kill(false);
 	});
 
 	// Wait for GUI thread

rpcs3/Emu/RSX/Overlays/HomeMenu/overlay_home_menu_main_menu.cpp

@@ -52,12 +52,15 @@ namespace rsx
 
 				Emu.CallFromMainThread([suspend_mode]()
 				{
-					Emu.Kill(false, true);
-
 					if (!suspend_mode)
+					{
+						Emu.after_kill_callback = []()
 						{
 							Emu.Restart();
+						};
 					}
+
+					Emu.Kill(false, true);
 				});
 
 				return page_navigation::exit;

rpcs3/Emu/System.cpp

@@ -177,10 +177,10 @@ void Emulator::BlockingCallFromMainThread(std::function<void()>&& func) const
 
 	CallFromMainThread(std::move(func), &wake_up);
 
-	while (!wake_up && !IsStopped())
+	while (!wake_up)
 	{
 		ensure(thread_ctrl::get_current());
-		thread_ctrl::wait_on(wake_up, false);
+		wake_up.wait(false);
 	}
 }
 
@@ -635,6 +635,11 @@ std::string Emulator::GetBackgroundPicturePath() const
 
 bool Emulator::BootRsxCapture(const std::string& path)
 {
+	if (m_state != system_state::stopped)
+	{
+		return false;
+	}
+
 	fs::file in_file(path);
 
 	if (!in_file)
@@ -781,6 +786,11 @@ void Emulator::SetForceBoot(bool force_boot)
 
 game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch, usz recursion_count)
 {
+	if (m_state != system_state::stopped)
+	{
+		return game_boot_result::still_running;
+	}
+
 	m_ar.reset();
 
 	{
@@ -817,11 +827,6 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch,
 		}
 	}
 
-	if (!IsStopped())
-	{
-		Kill();
-	}
-
 	if (!title_id.empty())
 	{
 		m_title_id = title_id;
@@ -2422,8 +2427,14 @@ void Emulator::GracefulShutdown(bool allow_autoexit, bool async_op, bool savesta
 {
 	const auto old_state = m_state.load();
 
-	if (old_state == system_state::stopped)
+	if (old_state == system_state::stopped || old_state == system_state::stopping)
 	{
+		while (!async_op && m_state != system_state::stopped)
+		{
+			process_qt_events();
+			std::this_thread::sleep_for(16ms);
+		}
+
 		return;
 	}
 
@@ -2438,6 +2449,13 @@ void Emulator::GracefulShutdown(bool allow_autoexit, bool async_op, bool savesta
 	{
 		// The callback has been rudely ignored, we have no other option but to force termination
 		Kill(allow_autoexit && !savestate, savestate);
+
+		while (!async_op && m_state != system_state::stopped)
+		{
+			process_qt_events();
+			std::this_thread::sleep_for(16ms);
+		}
+
 		return;
 	}
 
@@ -2483,22 +2501,26 @@ void Emulator::GracefulShutdown(bool allow_autoexit, bool async_op, bool savesta
 	else
 	{
 		perform_kill();
+
+		while (m_state != system_state::stopped)
+		{
+			process_qt_events();
+			std::this_thread::sleep_for(16ms);
+		}
 	}
 }
 
 extern bool try_lock_vdec_context_creation();
 extern bool try_lock_spu_threads_in_a_state_compatible_with_savestates(bool revert_lock = false);
 
-std::shared_ptr<utils::serial> Emulator::Kill(bool allow_autoexit, bool savestate)
+void Emulator::Kill(bool allow_autoexit, bool savestate)
 {
-	std::shared_ptr<utils::serial> to_ar;
+	if (!IsStopped() && savestate && !try_lock_spu_threads_in_a_state_compatible_with_savestates())
-
-	if (savestate && !try_lock_spu_threads_in_a_state_compatible_with_savestates())
 	{
 		sys_log.error("Failed to savestate: failed to lock SPU threads execution.");
 	}
 
-	if (savestate && !try_lock_vdec_context_creation())
+	if (!IsStopped() && savestate && !try_lock_vdec_context_creation())
 	{
 		try_lock_spu_threads_in_a_state_compatible_with_savestates(true);
 
@@ -2507,7 +2529,7 @@ std::shared_ptr<utils::serial> Emulator::Kill(bool allow_autoexit, bool savestat
 			"\nYou need to close the game for to take effect."
 			"\nIf you cannot close the game due to losing important progress your best chance is to skip the current cutscenes if any are played and retry.");
 
-		return to_ar;
+		return;
 	}
 
 	g_tls_log_prefix = []()
@@ -2515,8 +2537,23 @@ std::shared_ptr<utils::serial> Emulator::Kill(bool allow_autoexit, bool savestat
 		return std::string();
 	};
 
-	if (m_state.exchange(system_state::stopped) == system_state::stopped)
+	if (system_state old_state = m_state.fetch_op([](system_state& state)
 	{
+		if (state == system_state::stopping || state == system_state::stopped)
+		{
+			return false;
+		}
+
+		state = system_state::stopping;
+		return true;
+	}).first; old_state <= system_state::stopping)
+	{
+		if (old_state == system_state::stopping)
+		{
+			// Termination is in progress
+			return;
+		}
+
 		// Ensure clean state
 		m_ar.reset();
 		argv.clear();
@@ -2526,11 +2563,12 @@ std::shared_ptr<utils::serial> Emulator::Kill(bool allow_autoexit, bool savestat
 		klic.clear();
 		hdd1.clear();
 		init_mem_containers = nullptr;
+		after_kill_callback = nullptr;
 		m_config_path.clear();
 		m_config_mode = cfg_mode::custom;
 		read_used_savestate_versions();
 		m_savestate_extension_flags1 = {};
-		return to_ar;
+		return;
 	}
 
 	sys_log.notice("Stopping emulator...");
@@ -2546,28 +2584,6 @@ std::shared_ptr<utils::serial> Emulator::Kill(bool allow_autoexit, bool savestat
 		}
 	}
 
-	named_thread stop_watchdog("Stop Watchdog", [&]()
-	{
-		for (int i = 0; thread_ctrl::state() != thread_state::aborting;)
-		{
-			// We don't need accurate timekeeping, using clocks may interfere with debugging
-			if (i >= (savestate ? 2000 : 1000))
-			{
-				// Total amount of waiting: about 5s
-				report_fatal_error("Stopping emulator took too long."
-					"\nSome thread has probably deadlocked. Aborting.");
-			}
-
-			thread_ctrl::wait_for(5'000);
-
-			if (!g_watchdog_hold_ctr)
-			{
-				// Don't count if there are still uninterruptable threads like PPU LLVM workers
-				i++;
-			}
-		}
-	});
-
 	// Signal threads
 
 	if (auto rsx = g_fxo->try_get<rsx::thread>())
@@ -2588,7 +2604,48 @@ std::shared_ptr<utils::serial> Emulator::Kill(bool allow_autoexit, bool savestat
 	// Wait fot newly created cpu_thread to see that emulation has been stopped
 	id_manager::g_mutex.lock_unlock();
 
-	GetCallbacks().on_stop();
+	// Type-less smart pointer container for thread (cannot know its type with this approach)
+	// There is no race condition because it is only accessed by the same thread
+	std::shared_ptr<std::shared_ptr<void>> join_thread = std::make_shared<std::shared_ptr<void>>();
+
+	auto make_ptr = [](auto ptr)
+	{
+		return std::shared_ptr<std::remove_pointer_t<decltype(ptr)>>(ptr);
+	};
+
+	*join_thread = make_ptr(new named_thread("Emulation Join Thread"sv, [join_thread, savestate, allow_autoexit, this]() mutable
+	{
+		named_thread stop_watchdog("Stop Watchdog"sv, [this]()
+		{
+			const auto closed_sucessfully = std::make_shared<atomic_t<bool>>(false);
+
+			for (int i = 0; thread_ctrl::state() != thread_state::aborting;)
+			{
+				// We don't need accurate timekeeping, using clocks may interfere with debugging
+				if (i >= 2000)
+				{
+					// Total amount of waiting: about 10s
+					GetCallbacks().on_emulation_stop_no_response(closed_sucessfully, 10);
+
+					while (thread_ctrl::state() != thread_state::aborting)
+					{
+						thread_ctrl::wait_for(5'000);
+					}
+
+					break;
+				}
+
+				thread_ctrl::wait_for(5'000);
+
+				if (!g_watchdog_hold_ctr)
+				{
+					// Don't count if there are still uninterruptable threads like PPU LLVM workers
+					i++;
+				}
+			}
+
+			*closed_sucessfully = true;
+		});
 
 		// Join threads
 		for (const auto& [type, data] : *g_fxo)
@@ -2602,10 +2659,10 @@ std::shared_ptr<utils::serial> Emulator::Kill(bool allow_autoexit, bool savestat
 		// Save it first for maximum timing accuracy
 		const u64 timestamp = get_timebased_time();
 
-	stop_watchdog = thread_state::aborting;
-
 		sys_log.notice("All threads have been stopped.");
 
+		std::unique_ptr<utils::serial> to_ar;
+
 		if (savestate)
 		{
 			to_ar = std::make_unique<utils::serial>();
@@ -2724,47 +2781,7 @@ std::shared_ptr<utils::serial> Emulator::Kill(bool allow_autoexit, bool savestat
 			}
 		}
 
-	cpu_thread::cleanup();
+		stop_watchdog = thread_state::aborting;
-
-	initialize_timebased_time(0, true);
-
-	lv2_obj::cleanup();
-
-	g_fxo->reset();
-
-	sys_log.notice("Objects cleared...");
-
-	vm::close();
-
-	jit_runtime::finalize();
-
-	perf_stat_base::report();
-
-	static u64 aw_refs = 0;
-	static u64 aw_colm = 0;
-	static u64 aw_colc = 0;
-	static u64 aw_used = 0;
-
-	aw_refs = 0;
-	aw_colm = 0;
-	aw_colc = 0;
-	aw_used = 0;
-
-	atomic_wait::parse_hashtable([](u64 /*id*/, u32 refs, u64 ptr, u32 maxc) -> bool
-	{
-		aw_refs += refs != 0;
-		aw_used += ptr != 0;
-
-		aw_colm = std::max<u64>(aw_colm, maxc);
-		aw_colc += maxc != 0;
-
-		return false;
-	});
-
-	sys_log.notice("Atomic wait hashtable stats: [in_use=%u, used=%u, max_collision_weight=%u, total_collisions=%u]", aw_refs, aw_used, aw_colm, aw_colc);
-
-	m_stop_ctr++;
-	m_stop_ctr.notify_all();
 
 		if (savestate)
 		{
@@ -2816,6 +2833,51 @@ std::shared_ptr<utils::serial> Emulator::Kill(bool allow_autoexit, bool savestat
 			ar.set_reading_state();
 		}
 
+		// Final termination from main thread (move the last ownership of join thread in order to destroy it)
+		CallFromMainThread([join_thread = std::move(join_thread), allow_autoexit, this]() mutable
+		{
+			cpu_thread::cleanup();
+
+			initialize_timebased_time(0, true);
+
+			lv2_obj::cleanup();
+
+			g_fxo->reset();
+
+			sys_log.notice("Objects cleared...");
+
+			vm::close();
+
+			jit_runtime::finalize();
+
+			perf_stat_base::report();
+
+			static u64 aw_refs = 0;
+			static u64 aw_colm = 0;
+			static u64 aw_colc = 0;
+			static u64 aw_used = 0;
+
+			aw_refs = 0;
+			aw_colm = 0;
+			aw_colc = 0;
+			aw_used = 0;
+
+			atomic_wait::parse_hashtable([](u64 /*id*/, u32 refs, u64 ptr, u32 maxc) -> bool
+			{
+				aw_refs += refs != 0;
+				aw_used += ptr != 0;
+
+				aw_colm = std::max<u64>(aw_colm, maxc);
+				aw_colc += maxc != 0;
+
+				return false;
+			});
+
+			sys_log.notice("Atomic wait hashtable stats: [in_use=%u, used=%u, max_collision_weight=%u, total_collisions=%u]", aw_refs, aw_used, aw_colm, aw_colc);
+
+			m_stop_ctr++;
+			m_stop_ctr.notify_all();
+
 			// Boot arg cleanup (preserved in the case restarting)
 			argv.clear();
 			envp.clear();
@@ -2830,6 +2892,10 @@ std::shared_ptr<utils::serial> Emulator::Kill(bool allow_autoexit, bool savestat
 			read_used_savestate_versions();
 			m_savestate_extension_flags1 = {};
 
+			// Complete the operation
+			m_state = system_state::stopped;
+			GetCallbacks().on_stop();
+
 			// Always Enable display sleep, not only if it was prevented.
 			enable_display_sleep();
 
@@ -2838,7 +2904,13 @@ std::shared_ptr<utils::serial> Emulator::Kill(bool allow_autoexit, bool savestat
 				Quit(g_cfg.misc.autoexit.get());
 			}
 
-	return to_ar;
+			if (after_kill_callback)
+			{
+				after_kill_callback();
+				after_kill_callback = nullptr;
+			}
+		});
+	}));
 }
 
 game_boot_result Emulator::Restart(bool graceful)

rpcs3/Emu/System.h

@@ -21,6 +21,7 @@ enum class video_renderer;
 enum class system_state : u32
 {
 	stopped,
+	stopping,
 	running,
 	paused,
 	frozen, // paused but cannot resume
@@ -43,6 +44,7 @@ enum class game_boot_result : u32
 	unsupported_disc_type,
 	savestate_corrupted,
 	savestate_version_unsupported,
+	still_running,
 };
 
 constexpr bool is_error(game_boot_result res)
@@ -59,6 +61,7 @@ struct EmuCallbacks
 	std::function<void()> on_stop;
 	std::function<void()> on_ready;
 	std::function<bool()> on_missing_fw;
+	std::function<void(std::shared_ptr<atomic_t<bool>>, int)> on_emulation_stop_no_response;
 	std::function<void(bool enabled)> enable_disc_eject;
 	std::function<void(bool enabled)> enable_disc_insert;
 	std::function<bool(bool, std::function<void()>)> try_to_quit; // (force_quit, on_exit) Try to close RPCS3
@@ -215,6 +218,7 @@ public:
 	std::string disc;
 	std::string hdd1;
 	std::function<void(u32)> init_mem_containers;
+	std::function<void()> after_kill_callback;
 
 	u32 m_boot_source_type = 0; // CELL_GAME_GAMETYPE_SYS
 
@@ -322,17 +326,17 @@ public:
 	bool Pause(bool freeze_emulation = false, bool show_resume_message = true);
 	void Resume();
 	void GracefulShutdown(bool allow_autoexit = true, bool async_op = false, bool savestate = false);
-	std::shared_ptr<utils::serial> Kill(bool allow_autoexit = true, bool savestate = false);
+	void Kill(bool allow_autoexit = true, bool savestate = false);
 	game_boot_result Restart(bool graceful = true);
 	bool Quit(bool force_quit);
 	static void CleanUp();
 
 	bool IsRunning() const { return m_state == system_state::running; }
 	bool IsPaused()  const { return m_state >= system_state::paused; } // ready/starting are also considered paused by this function
-	bool IsStopped() const { return m_state == system_state::stopped; }
+	bool IsStopped() const { return m_state <= system_state::stopping; }
 	bool IsReady()   const { return m_state == system_state::ready; }
 	bool IsStarting() const { return m_state == system_state::starting; }
-	auto GetStatus(bool fixup = true) const { system_state state = m_state; return fixup && state == system_state::frozen ? system_state::paused : state; }
+	auto GetStatus(bool fixup = true) const { system_state state = m_state; return fixup && state == system_state::frozen ? system_state::paused : fixup && state == system_state::stopping ? system_state::stopped : state; }
 
 	bool HasGui() const { return m_has_gui; }
 	void SetHasGui(bool has_gui) { m_has_gui = has_gui; }

rpcs3/headless_application.cpp

@@ -10,6 +10,8 @@
 
 #include <clocale>
 
+[[noreturn]] void report_fatal_error(std::string_view text, bool is_html = false, bool include_help_text = true);
+
 // For now, a trivial constructor/destructor. May add command line usage later.
 headless_application::headless_application(int& argc, char** argv) : QCoreApplication(argc, argv)
 {
@@ -137,6 +139,14 @@ void headless_application::InitializeCallbacks()
 	callbacks.on_resume = []() {};
 	callbacks.on_stop   = []() {};
 	callbacks.on_ready  = []() {};
+	callbacks.on_emulation_stop_no_response = [](std::shared_ptr<atomic_t<bool>> closed_successfully, int /*seconds_waiting_already*/)
+	{
+		if (!closed_successfully || !*closed_successfully)
+		{
+			report_fatal_error(tr("Stopping emulator took too long."
+						"\nSome thread has probably deadlocked. Aborting.").toStdString());
+		}
+	};
 
 	callbacks.enable_disc_eject  = [](bool) {};
 	callbacks.enable_disc_insert = [](bool) {};

rpcs3/rpcs3qt/gs_frame.cpp

@@ -605,10 +605,13 @@ void gs_frame::close()
 		if (!Emu.IsStopped())
 		{
 			// Blocking shutdown request. Obsolete, but I'm keeping it here as last resort.
-			Emu.GracefulShutdown(true, false);
+			Emu.after_kill_callback = [this](){ deleteLater(); };
+			Emu.GracefulShutdown(true);
 		}
-
+		else
+		{
 			deleteLater();
+		}
 	});
 }
 

rpcs3/rpcs3qt/gui_application.cpp

@@ -49,6 +49,8 @@
 
 LOG_CHANNEL(gui_log, "GUI");
 
+[[noreturn]] void report_fatal_error(std::string_view text, bool is_html = false, bool include_help_text = true);
+
 gui_application::gui_application(int& argc, char** argv) : QApplication(argc, argv)
 {
 }
@@ -569,6 +571,61 @@ void gui_application::InitializeCallbacks()
 		};
 	}
 
+	callbacks.on_emulation_stop_no_response = [this](std::shared_ptr<atomic_t<bool>> closed_successfully, int seconds_waiting_already)
+	{
+		const std::string terminate_message = tr("Stopping emulator took too long."
+			"\nSome thread has probably deadlocked. Aborting.").toStdString();
+
+		if (!closed_successfully)
+		{
+			report_fatal_error(terminate_message);
+		}
+
+		Emu.CallFromMainThread([this, closed_successfully, seconds_waiting_already, terminate_message]
+		{
+			const auto seconds = std::make_shared<int>(seconds_waiting_already);
+
+			QMessageBox* mb = new QMessageBox();
+			mb->setWindowTitle(tr("PS3 Game/Application Is Unresponsive"));
+			mb->setIcon(QMessageBox::Critical);
+			mb->setStandardButtons(QMessageBox::Yes | QMessageBox::No);
+			mb->setDefaultButton(QMessageBox::No);
+			mb->button(QMessageBox::Yes)->setText(tr("Terminate RPCS3"));
+			mb->button(QMessageBox::No)->setText(tr("Keep Waiting"));
+
+			QString text_base = tr("Waiting for %0 second(s) already to stop emulation without success."
+			                       "\nKeep waiting or terminate RPCS3 unsafely at your own risk?");
+
+			mb->setText(text_base.arg(10));
+			mb->layout()->setSizeConstraint(QLayout::SetFixedSize);
+			mb->setAttribute(Qt::WA_DeleteOnClose);
+
+			QTimer* update_timer = new QTimer(mb);
+
+			connect(update_timer, &QTimer::timeout, [mb, seconds, text_base, closed_successfully]()
+			{
+				*seconds += 1;
+				mb->setText(text_base.arg(*seconds));
+
+				if (*closed_successfully)
+				{
+					mb->reject();
+				}
+			});
+
+			connect(mb, &QDialog::accepted, mb, [closed_successfully, terminate_message]
+			{
+				if (!*closed_successfully)
+				{
+					report_fatal_error(terminate_message);
+				}
+			});
+
+			mb->open();
+			update_timer->start(1000);
+		});
+	};
+
 	Emu.SetCallbacks(std::move(callbacks));
 }
 

rpcs3/rpcs3qt/main_window.cpp

@@ -439,6 +439,9 @@ void main_window::show_boot_error(game_boot_result status)
 	case game_boot_result::savestate_version_unsupported:
 		message = tr("Savestate versioning data differes from your RPCS3 build.");
 		break;
+	case game_boot_result::still_running:
+		message = tr("A game or PS3 application is still running or has yet to be fully stopped.");
+		break;
 	case game_boot_result::firmware_missing: // Handled elsewhere
 	case game_boot_result::no_errors:
 		return;
@@ -1854,8 +1857,6 @@ void main_window::OnEmuStop()
 	const QString title = GetCurrentTitle();
 	const QString play_tooltip = tr("Play %0").arg(title);
 
-	m_debugger_frame->UpdateUI();
-
 	ui->sysPauseAct->setText(tr("&Play"));
 	ui->sysPauseAct->setIcon(m_icon_play);
 #ifdef _WIN32