Savestates: Compressed state files

Utilities/File.cpp

@@ -2277,6 +2277,12 @@ bool fs::pending_file::open(std::string_view path)
 
 		if (file.open(m_path, fs::create + fs::write + fs::read + fs::excl))
 		{
+#ifdef _WIN32
+			// Auto-delete pending log file
+			FILE_DISPOSITION_INFO disp;
+			disp.DeleteFileW = true;
+			SetFileInformationByHandle(file.get_handle(), FileDispositionInfo, &disp, sizeof(disp));
+#endif
 			m_dest = path;
 			break;
 		}
@@ -2314,6 +2320,14 @@ bool fs::pending_file::commit(bool overwrite)
 	}
 
 #endif
+
+#ifdef _WIN32
+	// Disable auto-delete
+	FILE_DISPOSITION_INFO disp;
+	disp.DeleteFileW = false;
+	SetFileInformationByHandle(file.get_handle(), FileDispositionInfo, &disp, sizeof(disp));
+#endif
+
 	file.close();
 
 #ifdef _WIN32

rpcs3/Crypto/unzip.h

@@ -2,32 +2,32 @@
 
 std::vector<u8> unzip(const void* src, usz size);
 
-template<typename T>
+template <typename T>
-std::vector<u8> unzip(const T& src)
+inline std::vector<u8> unzip(const T& src)
 {
 	return unzip(src.data(), src.size());
 }
 
 bool unzip(const void* src, usz size, fs::file& out);
 
-template<typename T>
+template <typename T>
-bool unzip(const std::vector<u8>& src, fs::file& out)
+inline bool unzip(const std::vector<u8>& src, fs::file& out)
 {
 	return unzip(src.data(), src.size(), out);
 }
 
 std::vector<u8> zip(const void* src, usz size);
 
-template<typename T>
+template <typename T>
-std::vector<u8> zip(const T& src)
+inline std::vector<u8> zip(const T& src)
 {
 	return zip(src.data(), src.size());
 }
 
 bool zip(const void* src, usz size, fs::file& out);
 
-template<typename T>
+template <typename T>
-bool zip(const T& src, fs::file& out)
+inline bool zip(const T& src, fs::file& out)
 {
 	return zip(src.data(), src.size(), out);
 }

rpcs3/Emu/Cell/Modules/sceNpTrophy.cpp

@@ -43,7 +43,7 @@ struct trophy_context_t
 	trophy_context_t() = default;
 
 	trophy_context_t(utils::serial& ar)
-		: trp_name(ar.operator std::string())
+		: trp_name(ar.pop<std::string>())
 	{
 		std::string trophy_path = vfs::get(Emu.GetDir() + "TROPDIR/" + trp_name + "/TROPHY.TRP");
 		fs::file trp_stream(trophy_path);
@@ -55,7 +55,7 @@ struct trophy_context_t
 			trp_stream.open(trophy_path);
 		}
 
-		if (!ar.operator bool())
+		if (!ar.pop<bool>())
 		{
 			ar(read_only);
 			return;

rpcs3/Emu/Cell/Modules/sys_io_.cpp

@@ -52,6 +52,7 @@ void config_event_entry(ppu_thread& ppu)
 	{
 		if (ppu.is_stopped())
 		{
+			ppu.state += cpu_flag::again;
 			return;
 		}
 
@@ -73,6 +74,8 @@ void config_event_entry(ppu_thread& ppu)
 		}
 	}
 
+	sys_io.notice("config_event_entry(): Exited with the following error code: %s", CellError{static_cast<u32>(ppu.gpr[3])});
+
 	ppu_execute<&sys_ppu_thread_exit>(ppu, 0);
 }
 

rpcs3/Emu/Cell/PPUModule.cpp

@@ -358,7 +358,7 @@ static void ppu_initialize_modules(ppu_linkage_info* link, utils::serial* ar = n
 
 			while (true)
 			{
-				const std::string name = ar.operator std::string();
+				const std::string name = ar.pop<std::string>();
 
 				if (name.empty())
 				{
@@ -370,10 +370,10 @@ static void ppu_initialize_modules(ppu_linkage_info* link, utils::serial* ar = n
 
 				auto& variable = _module->variables;
 
-				for (u32 i = 0, end = ar.operator usz(); i < end; i++)
+				for (u32 i = 0, end = ar.pop<usz>(); i < end; i++)
 				{
-					auto* ptr = &::at32(variable, ar.operator u32());
+					auto* ptr = &::at32(variable, ar.pop<u32>());
-					ptr->addr = ar.operator u32();
+					ptr->addr = ar.pop<u32>();
 					ensure(!!ptr->var);
 				}
 			}
@@ -1052,7 +1052,12 @@ void init_ppu_functions(utils::serial* ar, bool full = false)
 
 	if (ar)
 	{
-		ensure(vm::check_addr(g_fxo->init<ppu_function_manager>(*ar)->addr));
+		const u32 addr = g_fxo->init<ppu_function_manager>(*ar)->addr;
+
+		if (addr % 0x1000 || !vm::check_addr(addr))
+		{
+			fmt::throw_exception("init_ppu_functions(): Failure to initialize function manager. (addr=0x%x, %s)", addr, *ar);
+		}
 	}
 	else
 		g_fxo->init<ppu_function_manager>();

rpcs3/Emu/Cell/PPUThread.cpp

@@ -2352,6 +2352,16 @@ void ppu_thread::serialize_common(utils::serial& ar)
 
 	ar(gpr, fpr, cr, fpscr.bits, lr, ctr, vrsave, cia, xer, sat, nj, prio.raw().all);
 
+	if (cia % 4 || !vm::check_addr(cia))
+	{
+		fmt::throw_exception("Failed to serialize PPU thread ID=0x%x (cia=0x%x, ar=%s)", this->id, cia, ar);
+	}
+
+	if (ar.is_writing())
+	{
+		ppu_log.notice("Saving PPU Thread [0x%x: %s]: cia=0x%x, state=%s", id, *ppu_tname.load(), cia, +state);
+	}
+
 	ar(optional_savestate_state, vr);
 
 	if (optional_savestate_state->data.empty())
@@ -2364,7 +2374,7 @@ ppu_thread::ppu_thread(utils::serial& ar)
 	: cpu_thread(idm::last_id()) // last_id() is showed to constructor on serialization
 	, stack_size(ar)
 	, stack_addr(ar)
-	, joiner(ar.operator ppu_join_status())
+	, joiner(ar.pop<ppu_join_status>())
 	, entry_func(std::bit_cast<ppu_func_opd_t, u64>(ar))
 	, is_interrupt_thread(ar)
 {
@@ -2397,7 +2407,7 @@ ppu_thread::ppu_thread(utils::serial& ar)
 		}
 	};
 
-	switch (const u32 status = ar.operator u32())
+	switch (const u32 status = ar.pop<u32>())
 	{
 	case PPU_THREAD_STATUS_IDLE:
 	{
@@ -2490,7 +2500,9 @@ ppu_thread::ppu_thread(utils::serial& ar)
 		state += cpu_flag::memory;
 	}
 
-	ppu_tname = make_single<std::string>(ar.operator std::string());
+	ppu_tname = make_single<std::string>(ar.pop<std::string>());
+
+	ppu_log.notice("Loading PPU Thread [0x%x: %s]: cia=0x%x, state=%s", id, *ppu_tname.load(), cia, +state);
 }
 
 void ppu_thread::save(utils::serial& ar)
@@ -2506,12 +2518,6 @@ void ppu_thread::save(utils::serial& ar)
 		_joiner = ppu_join_status::joinable;
 	}
 
-	if (state & cpu_flag::again)
-	{
-		std::memcpy(&gpr[3], syscall_args, sizeof(syscall_args));
-		cia -= 4;
-	}
-
 	ar(stack_size, stack_addr, _joiner, entry, is_interrupt_thread);
 	serialize_common(ar);
 
@@ -2685,6 +2691,13 @@ void ppu_thread::fast_call(u32 addr, u64 rtoc, bool is_thread_entry)
 			std::memcpy(syscall_args, &gpr[3], sizeof(syscall_args));
 		}
 
+		if (!old_cia && state & cpu_flag::again)
+		{
+			// Fixup argument registers and CIA for reloading
+			std::memcpy(&gpr[3], syscall_args, sizeof(syscall_args));
+			cia -= 4;
+		}
+
 		current_function = old_func;
 		g_tls_log_prefix = old_fmt;
 		state -= cpu_flag::ret;

rpcs3/Emu/Cell/SPUThread.cpp

@@ -1981,7 +1981,7 @@ spu_thread::spu_thread(utils::serial& ar, lv2_spu_group* group)
 	: cpu_thread(idm::last_id())
 	, group(group)
 	, index(ar)
-	, thread_type(group ? spu_type::threaded : ar.operator u8() ? spu_type::isolated : spu_type::raw)
+	, thread_type(group ? spu_type::threaded : ar.pop<u8>() ? spu_type::isolated : spu_type::raw)
 	, shm(ensure(vm::get(vm::spu)->peek(vm_offset()).second))
 	, ls(map_ls(*this->shm))
 	, option(ar)
@@ -2029,12 +2029,12 @@ spu_thread::spu_thread(utils::serial& ar, lv2_spu_group* group)
 		for (auto& pair : spuq)
 		{
 			ar(pair.first);
-			pair.second = idm::get_unlocked<lv2_obj, lv2_event_queue>(ar.operator u32());
+			pair.second = idm::get_unlocked<lv2_obj, lv2_event_queue>(ar.pop<u32>());
 		}
 
 		for (auto& q : spup)
 		{
-			q = idm::get_unlocked<lv2_obj, lv2_event_queue>(ar.operator u32());
+			q = idm::get_unlocked<lv2_obj, lv2_event_queue>(ar.pop<u32>());
 		}
 	}
 	else
@@ -2042,7 +2042,7 @@ spu_thread::spu_thread(utils::serial& ar, lv2_spu_group* group)
 		for (spu_int_ctrl_t& ctrl : int_ctrl)
 		{
 			ar(ctrl.mask, ctrl.stat);
-			ctrl.tag = idm::get_unlocked<lv2_obj, lv2_int_tag>(ar.operator u32());
+			ctrl.tag = idm::get_unlocked<lv2_obj, lv2_int_tag>(ar.pop<u32>());
 		}
 
 		g_raw_spu_ctr++;

rpcs3/Emu/Cell/lv2/sys_event.cpp

@@ -59,7 +59,7 @@ void lv2_event_queue::save_ptr(utils::serial& ar, lv2_event_queue* q)
 
 std::shared_ptr<lv2_event_queue> lv2_event_queue::load_ptr(utils::serial& ar, std::shared_ptr<lv2_event_queue>& queue, std::string_view msg)
 {
-	const u32 id = ar.operator u32();
+	const u32 id = ar.pop<u32>();
 
 	if (!id)
 	{

rpcs3/Emu/Cell/lv2/sys_fs.cpp

@@ -445,7 +445,7 @@ lv2_file::lv2_file(utils::serial& ar)
 
 	g_fxo->get<loaded_npdrm_keys>().npdrm_fds.raw() += type != lv2_file_type::regular;
 
-	if (ar.operator bool()) // see lv2_file::save in_mem
+	if (ar.pop<bool>()) // see lv2_file::save in_mem
 	{
 		const fs::stat_t stat = ar;
 

rpcs3/Emu/Cell/lv2/sys_lwcond.cpp

@@ -12,10 +12,10 @@
 LOG_CHANNEL(sys_lwcond);
 
 lv2_lwcond::lv2_lwcond(utils::serial& ar)
-	: name(ar.operator be_t<u64>())
+	: name(ar.pop<be_t<u64>>())
 	, lwid(ar)
 	, protocol(ar)
-	, control(ar.operator decltype(control)())
+	, control(ar.pop<decltype(control)>())
 {
 }
 

rpcs3/Emu/Cell/lv2/sys_lwmutex.cpp

@@ -12,8 +12,8 @@ LOG_CHANNEL(sys_lwmutex);
 
 lv2_lwmutex::lv2_lwmutex(utils::serial& ar)
 	: protocol(ar)
-	, control(ar.operator decltype(control)())
+	, control(ar.pop<decltype(control)>())
-	, name(ar.operator be_t<u64>())
+	, name(ar.pop<be_t<u64>>())
 {
 	ar(lv2_control.raw().signaled);
 }

rpcs3/Emu/Cell/lv2/sys_mmapper.cpp

@@ -50,7 +50,7 @@ lv2_memory::lv2_memory(utils::serial& ar)
 	, flags(ar)
 	, key(ar)
 	, pshared(ar)
-	, ct(lv2_memory_container::search(ar.operator u32()))
+	, ct(lv2_memory_container::search(ar.pop<u32>()))
 	, shm([&](u32 addr)
 	{
 		if (addr)
@@ -61,7 +61,7 @@ lv2_memory::lv2_memory(utils::serial& ar)
 		const auto _shm = std::make_shared<utils::shm>(size, 1);
 		ar(std::span(_shm->map_self(), size));
 		return _shm;
-	}(ar.operator u32()))
+	}(ar.pop<u32>()))
 	, counter(ar)
 {
 #ifndef _WIN32

rpcs3/Emu/Cell/lv2/sys_overlay.cpp

@@ -72,7 +72,7 @@ fs::file make_file_view(fs::file&& file, u64 offset, u64 size);
 
 std::shared_ptr<void> lv2_overlay::load(utils::serial& ar)
 {
-	const std::string path = vfs::get(ar.operator std::string());
+	const std::string path = vfs::get(ar.pop<std::string>());
 	const s64 offset = ar;
 
 	std::shared_ptr<lv2_overlay> ovlm;

rpcs3/Emu/Cell/lv2/sys_prx.cpp

@@ -292,7 +292,7 @@ std::shared_ptr<void> lv2_prx::load(utils::serial& ar)
 {
 	[[maybe_unused]] const s32 version = GET_SERIALIZATION_VERSION(lv2_prx_overlay);
 
-	const std::string path = vfs::get(ar.operator std::string());
+	const std::string path = vfs::get(ar.pop<std::string>());
 	const s64 offset = ar;
 	const u32 state = ar;
 

rpcs3/Emu/Cell/lv2/sys_spu.cpp

@@ -202,13 +202,13 @@ void sys_spu_image::deploy(u8* loc, std::span<const sys_spu_segment> segs, bool
 }
 
 lv2_spu_group::lv2_spu_group(utils::serial& ar) noexcept
-	: name(ar.operator std::string())
+	: name(ar.pop<std::string>())
 	, id(idm::last_id())
 	, max_num(ar)
 	, mem_size(ar)
 	, type(ar) // SPU Thread Group Type
 	, ct(lv2_memory_container::search(ar))
-	, has_scheduler_context(ar.operator u8())
+	, has_scheduler_context(ar.pop<u8>())
 	, max_run(ar)
 	, init(ar)
 	, prio([&ar]()
@@ -219,12 +219,12 @@ lv2_spu_group::lv2_spu_group(utils::serial& ar) noexcept
 
 		return prio;
 	}())
-	, run_state(ar.operator spu_group_status())
+	, run_state(ar.pop<spu_group_status>())
 	, exit_status(ar)
 {
 	for (auto& thread : threads)
 	{
-		if (ar.operator u8())
+		if (ar.pop<bool>())
 		{
 			ar(id_manager::g_id);
 			thread = std::make_shared<named_thread<spu_thread>>(ar, this);
@@ -239,7 +239,7 @@ lv2_spu_group::lv2_spu_group(utils::serial& ar) noexcept
 
 	for (auto ep : {&ep_run, &ep_exception, &ep_sysmodule})
 	{
-		*ep = idm::get_unlocked<lv2_obj, lv2_event_queue>(ar.operator u32());
+		*ep = idm::get_unlocked<lv2_obj, lv2_event_queue>(ar.pop<u32>());
 	}
 
 	waiter_spu_index = -1;
@@ -328,7 +328,7 @@ void lv2_spu_group::save(utils::serial& ar)
 
 lv2_spu_image::lv2_spu_image(utils::serial& ar)
 	: e_entry(ar)
-	, segs(ar.operator decltype(segs)())
+	, segs(ar.pop<decltype(segs)>())
 	, nsegs(ar)
 {
 }

rpcs3/Emu/Cell/lv2/sys_usbd.cpp

@@ -70,7 +70,7 @@ public:
 
 	usb_handler_thread(utils::serial& ar) : usb_handler_thread()
 	{
-		is_init = !!ar.operator u8();
+		is_init = !!ar.pop<u8>();
 	}
 
 	void save(utils::serial& ar)

rpcs3/Emu/IdManager.h

@@ -270,7 +270,7 @@ namespace id_manager
 		{
 			vec.resize(T::id_count);
 
-			u32 i = ar.operator u32();
+			u32 i = ar.pop<u32>();
 
 			ensure(i <= T::id_count);
 
@@ -309,7 +309,7 @@ namespace id_manager
 		void save(utils::serial& ar) requires IdmSavable<T>
 		{
 			u32 obj_count = 0;
-			usz obj_count_offs = ar.data.size();
+			usz obj_count_offs = ar.pos;
 
 			// To be patched at the end of the function
 			ar(obj_count);
@@ -340,7 +340,7 @@ namespace id_manager
 			}
 
 			// Patch object count
-			std::memcpy(ar.data.data() + obj_count_offs, &obj_count, sizeof(obj_count));
+			ar.patch_raw_data(obj_count_offs, &obj_count, sizeof(obj_count));
 		}
 
 		id_map& operator=(thread_state state) noexcept requires (std::is_assignable_v<T&, thread_state>)

rpcs3/Emu/Memory/vm.cpp

@@ -1620,45 +1620,52 @@ namespace vm
 		return _7 == v128{};
 	}
 
-	static void save_memory_bytes(utils::serial& ar, const u8* ptr, usz size)
+	static void serialize_memory_bytes(utils::serial& ar, u8* ptr, usz size)
 	{
-		AUDIT(ar.is_writing() && !(size % 1024));
+		ensure((size % 4096) == 0);
 
-		for (; size; ptr += 128 * 8, size -= 128 * 8)
+		for (; size; ptr += 128 * 8)
 		{
-			ar(u8{}); // bitmap of 1024 bytes (bit is 128-byte)
+			const usz process_size = std::min<usz>(size, 128 * 8);
-			u8 bitmap = 0, count = 0;
+			size -= process_size;
 
-			for (usz i = 0, end = std::min<usz>(size, 128 * 8); i < end; i += 128)
+			u8 bitmap = 0;
+
+			if (ar.is_writing())
+			{
+				for (usz i = 0; i < process_size; i += 128)
 				{
 					if (!check_cache_line_zero(ptr + i))
 					{
 						bitmap |= 1u << (i / 128);
-					count++;
+					}
-					ar(std::span(ptr + i, 128));
 				}
 			}
 
-			// Patch bitmap with correct value
+			// bitmap of 1024 bytes (bit is 128-byte)
-			*std::prev(&ar.data.back(), count * 128) = bitmap;
+			ar(bitmap);
-		}
+
+			for (usz i = 0; i < process_size;)
+			{
+				usz block_count = 0;
+
+				for (usz bit = i / 128; bit < sizeof(bitmap) * 8 && (bitmap & (1u << bit)) != 0;)
+				{
+					bit++;
+					block_count++;
 				}
 
-	static void load_memory_bytes(utils::serial& ar, u8* ptr, usz size)
+				if (!block_count)
 				{
-		AUDIT(!ar.is_writing() && !(size % 128));
+					i += 128;
-
+					continue;
-		for (; size; ptr += 128 * 8, size -= 128 * 8)
-		{
-			const u8 bitmap{ar};
-
-			for (usz i = 0, end = std::min<usz>(size, 128 * 8); i < end; i += 128)
-			{
-				if (bitmap & (1u << (i / 128)))
-				{
-					ar(std::span(ptr + i, 128));
 				}
+
+				ar(std::span<u8>(ptr + i, block_count * 128));
+				i += block_count * 128;
 			}
+
+			ar.breathe();
 		}
 	}
 
@@ -1682,14 +1689,15 @@ namespace vm
 				if (is_memory_compatible_for_copy_from_executable_optimization(addr, shm.first))
 				{
 					// Revert changes
-					ar.data.resize(ar.seek_end(sizeof(u32) * 2 + sizeof(memory_page)));
+					ar.data.resize(ar.data.size() - (sizeof(u32) * 2 + sizeof(memory_page)));
+					ar.seek_end();
 					vm_log.success("Removed memory block matching the memory of the executable from savestate. (addr=0x%x, size=0x%x)", addr, shm.first);
 					continue;
 				}
 
 				// Save raw binary image
 				const u32 guard_size = flags & stack_guarded ? 0x1000 : 0;
-				save_memory_bytes(ar, vm::get_super_ptr<const u8>(addr + guard_size), shm.first - guard_size * 2);
+				serialize_memory_bytes(ar, vm::get_super_ptr<u8>(addr + guard_size), shm.first - guard_size * 2);
 			}
 			else
 			{
@@ -1758,13 +1766,13 @@ namespace vm
 
 			// Map the memory through the same method as alloc() and falloc()
 			// Copy the shared handle unconditionally
-			ensure(try_alloc(addr0, pflags, size0, ::as_rvalue(flags & preallocated ? null_shm : shared[ar.operator usz()])));
+			ensure(try_alloc(addr0, pflags, size0, ::as_rvalue(flags & preallocated ? null_shm : shared[ar.pop<usz>()])));
 
 			if (flags & preallocated)
 			{
 				// Load binary image
 				const u32 guard_size = flags & stack_guarded ? 0x1000 : 0;
-				load_memory_bytes(ar, vm::get_super_ptr<u8>(addr0 + guard_size), size0 - guard_size * 2);
+				serialize_memory_bytes(ar, vm::get_super_ptr<u8>(addr0 + guard_size), size0 - guard_size * 2);
 			}
 		}
 	}
@@ -2224,9 +2232,8 @@ namespace vm
 			//  Save shared memory
 			ar(shm->flags());
 
-			// TODO: string_view serialization (even with load function, so the loaded address points to a position of the stream's buffer)
 			ar(shm->size());
-			save_memory_bytes(ar, vm::get_super_ptr<u8>(addr), shm->size());
+			serialize_memory_bytes(ar, vm::get_super_ptr<u8>(addr), shm->size());
 		}
 
 		// TODO: Serialize std::vector direcly
@@ -2249,19 +2256,19 @@ namespace vm
 	void load(utils::serial& ar)
 	{
 		std::vector<std::shared_ptr<utils::shm>> shared;
-		shared.resize(ar.operator usz());
+		shared.resize(ar.pop<usz>());
 
 		for (auto& shm : shared)
 		{
 			// Load shared memory
 
-			const u32 flags = ar;
+			const u32 flags = ar.pop<u32>();
-			const u64 size = ar;
+			const u64 size = ar.pop<u64>();
 			shm = std::make_shared<utils::shm>(size, flags);
 
 			// Load binary image
 			// elad335: I'm not proud about it as well.. (ideal situation is to not call map_self())
-			load_memory_bytes(ar, shm->map_self(), shm->size());
+			serialize_memory_bytes(ar, shm->map_self(), shm->size());
 		}
 
 		for (auto& block : g_locations)
@@ -2270,11 +2277,11 @@ namespace vm
 		}
 
 		g_locations.clear();
-		g_locations.resize(ar.operator usz());
+		g_locations.resize(ar.pop<usz>());
 
 		for (auto& loc : g_locations)
 		{
-			const u8 has = ar;
+			const u8 has = ar.pop<u8>();
 
 			if (has)
 			{

rpcs3/Emu/RSX/RSXThread.cpp

@@ -4,6 +4,7 @@
 #include "Emu/Cell/PPUCallback.h"
 #include "Emu/Cell/SPUThread.h"
 #include "Emu/Cell/timers.hpp"
+#include "Emu/savestate_utils.hpp"
 
 #include "Capture/rsx_capture.h"
 #include "Common/BufferUtils.h"
@@ -19,6 +20,7 @@
 #include "Emu/Cell/lv2/sys_event.h"
 #include "Emu/Cell/lv2/sys_time.h"
 #include "Emu/Cell/Modules/cellGcmSys.h"
+#include "Emu/savestate_utils.hpp"
 #include "Overlays/overlay_perf_metrics.h"
 #include "Overlays/overlay_message.h"
 #include "Program/GLSLCommon.h"
@@ -26,7 +28,6 @@
 #include "Utilities/StrUtil.h"
 #include "Crypto/unzip.h"
 
-#include "util/serialization.hpp"
 #include "util/asm.hpp"
 
 #include <span>
@@ -3574,6 +3575,8 @@ namespace rsx
 
 	void thread::on_frame_end(u32 buffer, bool forced)
 	{
+		bool pause_emulator = false;
+
 		// Marks the end of a frame scope GPU-side
 		if (g_user_asked_for_frame_capture.exchange(false) && !capture_current_frame)
 		{
@@ -3594,36 +3597,34 @@ namespace rsx
 		{
 			capture_current_frame = false;
 
-			std::string file_path = fs::get_config_dir() + "captures/" + Emu.GetTitleID() + "_" + date_time::current_time_narrow() + "_capture.rrc";
+			std::string file_path = fs::get_config_dir() + "captures/" + Emu.GetTitleID() + "_" + date_time::current_time_narrow() + "_capture.rrc.gz";
-
-			utils::serial save_manager;
-			save_manager.reserve(0x800'0000); // 128MB
-
-			save_manager(frame_capture);
-
-			if (std::vector<u8> zipped = zip(save_manager.data); !zipped.empty())
-			{
-				file_path += ".gz";
-				save_manager.data = std::move(zipped);
-			}
-			else
-			{
-				rsx_log.error("Failed to compress capture");
-			}
 
 			fs::pending_file temp(file_path);
 
-			if (temp.file && (temp.file.write(save_manager.data), temp.commit(false)))
+			utils::serial save_manager;
+
+			if (temp.file)
+			{
+				save_manager.m_file_handler = make_compressed_serialization_file_handler(temp.file);
+				save_manager(frame_capture);
+
+				save_manager.m_file_handler->finalize(save_manager);
+
+				if (temp.commit(false))
 				{
 					rsx_log.success("Capture successful: %s", file_path);
+					frame_capture.reset();
+					pause_emulator = true;
+				}
+				else
+				{
+					rsx_log.error("Capture failed: %s (%s)", file_path, fs::g_tls_error);
+				}
 			}
 			else
 			{
 				rsx_log.fatal("Capture failed: %s (%s)", file_path, fs::g_tls_error);
 			}
-
-			frame_capture.reset();
-			Emu.Pause();
 		}
 
 		if (zcull_ctrl->has_pending())
@@ -3673,6 +3674,12 @@ namespace rsx
 			}
 		}
 
+		if (pause_emulator)
+		{
+			Emu.Pause();
+			thread_ctrl::wait_for(30'000);
+		}
+
 		// Reset current stats
 		m_frame_stats = {};
 		m_profiler.enabled = !!g_cfg.video.overlay;

rpcs3/Emu/System.cpp

@@ -83,9 +83,6 @@ extern void ppu_unload_prx(const lv2_prx&);
 extern std::shared_ptr<lv2_prx> ppu_load_prx(const ppu_prx_object&, bool virtual_load, const std::string&, s64 = 0, utils::serial* = nullptr);
 extern std::pair<std::shared_ptr<lv2_overlay>, CellError> ppu_load_overlay(const ppu_exec_object&, bool virtual_load, const std::string& path, s64 = 0, utils::serial* = nullptr);
 extern bool ppu_load_rel_exec(const ppu_rel_object&);
-extern bool is_savestate_version_compatible(const std::vector<std::pair<u16, u16>>& data, bool is_boot_check);
-extern std::vector<std::pair<u16, u16>> read_used_savestate_versions();
-std::string get_savestate_file(std::string_view title_id, std::string_view boot_path, s64 abs_id, s64 rel_id);
 
 extern void send_close_home_menu_cmds();
 
@@ -219,7 +216,7 @@ void init_fxo_for_exec(utils::serial* ar, bool full = false)
 
 		const usz advance = (Emu.m_savestate_extension_flags1 & Emulator::SaveStateExtentionFlags1::SupportsMenuOpenResume ? 32 : 31);
 
-		ar->pos += advance; // Reserved area
+		load_and_check_reserved(*ar, advance); // Reserved area
 	}
 }
 
@@ -719,8 +716,11 @@ bool Emulator::BootRsxCapture(const std::string& path)
 
 	if (fmt::to_lower(path).ends_with(".gz"))
 	{
-		load.data = unzip(in_file.to_vector<u8>());
+		load.m_file_handler = make_compressed_serialization_file_handler(std::move(in_file));
-		in_file.close();
+
+		// Forcefully read some data to check validity
+		load.pop<uchar>();
+		load.pos -= sizeof(uchar);
 
 		if (load.data.empty())
 		{
@@ -896,13 +896,31 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch,
 		}
 		else
 		{
-			if (fs::file save{m_path, fs::isfile + fs::read}; save && save.size() >= 8 && save.read<u64>() == "RPCS3SAV"_u64)
+			fs::file save{m_path, fs::isfile + fs::read};
+
+			if (!m_path.ends_with(".gz") && save && save.size() >= 8 && save.read<u64>() == "RPCS3SAV"_u64)
 			{
 				m_ar = std::make_shared<utils::serial>();
 				m_ar->set_reading_state();
 
 				m_ar->m_file_handler = make_uncompressed_serialization_file_handler(std::move(save));
 			}
+			else if (save && m_path.ends_with(".gz"))
+			{
+				m_ar = std::make_shared<utils::serial>();
+				m_ar->set_reading_state();
+
+				m_ar->m_file_handler = make_compressed_serialization_file_handler(std::move(save));
+
+				if (m_ar->try_read<u64>().second != "RPCS3SAV"_u64)
+				{
+					m_ar.reset();
+				}
+				else
+				{
+					m_ar->pos = 0;
+				}
+			}
 
 			m_boot_source_type = CELL_GAME_GAMETYPE_SYS;
 		}
@@ -952,7 +970,7 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch,
 				bool LE_format;
 				bool state_inspection_support;
 				nse_t<u64, 1> offset;
-				std::array<u8, 32> reserved;
+				b8 flag_versions_is_following_data;
 			};
 
 			const auto header = m_ar->try_read<file_header>().second;
@@ -969,15 +987,24 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch,
 
 			g_cfg.savestate.state_inspection_mode.set(header.state_inspection_support);
 
+			if (header.flag_versions_is_following_data)
+			{
+				ensure(header.offset == m_ar->pos);
+
+				if (!is_savestate_version_compatible(m_ar->pop<std::vector<version_entry>>(), true))
+				{
+					return game_boot_result::savestate_version_unsupported;
+				}
+			}
+			else
 			{
 				// Read data on another container to keep the existing data
 				utils::serial ar_temp;
-				ar_temp.set_reading_state();
+				ar_temp.set_reading_state({}, true);
 				ar_temp.swap_handler(*m_ar);
 				ar_temp.seek_pos(header.offset);
-				ar_temp.m_avoid_large_prefetch = true;
 
-				if (!is_savestate_version_compatible(ar_temp.operator std::vector<std::pair<u16, u16>>(), true))
+				if (!is_savestate_version_compatible(ar_temp.pop<std::vector<version_entry>>(), true))
 				{
 					return game_boot_result::savestate_version_unsupported;
 				}
@@ -986,11 +1013,16 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch,
 				ar_temp.swap_handler(*m_ar);
 			}
 
+			if (!load_and_check_reserved(*m_ar, header.flag_versions_is_following_data ? 32 : 31))
+			{
+				return game_boot_result::savestate_version_unsupported;
+			}
+
 			argv.clear();
 			klic.clear();
 
 			std::string disc_info;
-			(*m_ar)(argv.emplace_back(), disc_info, klic.emplace_back(), m_game_dir, hdd1);
+			m_ar->serialize(argv.emplace_back(), disc_info, klic.emplace_back(), m_game_dir, hdd1);
 
 			if (!klic[0])
 			{
@@ -1026,12 +1058,16 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch,
 			{
 				const usz size = *m_ar;
 
+				fs::remove_all(path, size == 0);
+
 				if (size)
 				{
-					fs::remove_all(path, false);
 					m_ar->breathe(true);
-					ensure(tar_object(make_file_view(*static_cast<uncompressed_serialization_file_handler*>(m_ar->m_file_handler.get())->m_file, m_ar->data_offset, size)).extract(path));
+					m_ar->m_max_data = m_ar->pos + size;
-					m_ar->seek_pos(m_ar->pos + size, size >= 4096);
+					ensure(tar_object(*m_ar).extract(path));
+					m_ar->seek_pos(m_ar->m_max_data, size >= 4096);
+					m_ar->m_max_data = umax;
+					m_ar->breathe();
 				}
 			};
 
@@ -1053,7 +1089,11 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch,
 				load_tar(hdd0_game + game_data);
 			}
 
-			m_ar->pos += 32; // Reserved area
+			// Reserved area
+			if (!load_and_check_reserved(*m_ar, 32))
+			{
+				return game_boot_result::savestate_version_unsupported;
+			}
 
 			if (disc_info.starts_with("/"sv))
 			{
@@ -2261,7 +2301,7 @@ void Emulator::FixGuestTime()
 {
 	if (m_ar)
 	{
-		initialize_timebased_time(m_ar->operator u64());
+		initialize_timebased_time(m_ar->pop<u64>());
 
 		g_cfg.savestate.state_inspection_mode.set(m_state_inspection_savestate);
 
@@ -2862,6 +2902,13 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s
 		{
 			path = get_savestate_file(m_title_id, m_path, 0, 0);
 
+			// The function is meant for reading files, so if there is no GZ file it would not return compressed file path
+			// So this is the only place where the result is edited if need to be
+			if (!path.ends_with(".gz"))
+			{
+				path += ".gz";
+			}
+
 			if (!fs::create_path(fs::get_parent_dir(path)))
 			{
 				sys_log.error("Failed to create savestate directory! (path='%s', %s)", fs::get_parent_dir(path), fs::g_tls_error);
@@ -2877,7 +2924,7 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s
 			}
 
 			to_ar = std::make_unique<utils::serial>();
-			to_ar->m_file_handler = make_uncompressed_serialization_file_handler(std::move(file.file));
+			to_ar->m_file_handler = make_compressed_serialization_file_handler(file.file);
 			break;
 		}
 
@@ -2899,31 +2946,31 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s
 				// Avoid duplicating TAR object memory because it can be very large
 				auto save_tar = [&](const std::string& path)
 				{
-					const usz old_data_start = ar.data_offset;
+					if (!fs::is_dir(path))
-					const usz old_pos = ar.seek_end();
+					{
+						ar(usz{});
+						return;
+					}
 
+					// Cached file list from the first call
+					std::vector<fs::dir_entry> dir_entries;
+
+					// Calculate memory requirements
+					utils::serial ar_null;
+					ar_null.m_file_handler = make_null_serialization_file_handler();
+					tar_object::save_directory(path, ar_null, {}, std::move(dir_entries), false);
+					ar(ar_null.pos);
 					ar.breathe();
 
-					ar(usz{}); // Reserve memory to be patched later with correct size
+					const usz old_pos = ar.seek_end();
-					tar_object::save_directory(path, ar);
+					tar_object::save_directory(path, ar, {}, std::move(dir_entries), true);
-
 					const usz new_pos = ar.seek_end();
-					const usz tar_size = new_pos - old_pos - sizeof(usz);
 
-					// Check if breathe() actually did something, in this case memory needs to be discarded
+					const usz tar_size = new_pos - old_pos;
-					const bool was_emptied = old_data_start != ar.data_offset;
 
-					if (was_emptied)
+					if (tar_size != ar_null.pos)
 					{
-						ensure(ar.data_offset > old_data_start);
+						fmt::throw_exception("Unexpected TAR entry size (size=0x%x, expected=0x%x, entries=0x%x)", tar_size, ar_null.pos, dir_entries.size());
-
-						// Write to file directly (slower)
-						ar.m_file_handler->handle_file_op(ar, old_pos, sizeof(tar_size), &tar_size);
-					}
-					else
-					{
-						// If noty written to file, simply write to memory
-						std::memcpy(ar.data.data() + old_pos - old_data_start, &tar_size, sizeof(usz));
 					}
 
 					sys_log.success("Saved the contents of directory '%s' (size=0x%x)", path, tar_size);
@@ -2969,7 +3016,18 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s
 				ar("RPCS3SAV"_u64);
 				ar(std::endian::native == std::endian::little);
 				ar(g_cfg.savestate.state_inspection_mode.get());
-				ar(usz{0}); // Offset of versioning data, to be overwritten at the end of saving
+
+				ar(usz{10 + sizeof(usz) + sizeof(u8)}); // Offset of versioning data (fixed to the following data)
+
+				{
+					// Gather versions because with compressed format going back and patching offset is not efficient
+					utils::serial ar_temp;
+					ar_temp.m_file_handler = make_null_serialization_file_handler();
+					g_fxo->save(ar_temp);
+					ar(u8{1});
+					ar(read_used_savestate_versions());
+				}
+
 				ar(std::array<u8, 32>{}); // Reserved for future use
 
 				if (auto dir = vfs::get("/dev_bdvd/PS3_GAME"); fs::is_dir(dir) && !fs::is_file(fs::get_parent_dir(dir) + "/PS3_DISC.SFB"))
@@ -3022,27 +3080,11 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s
 
 		if (savestate)
 		{
-			// Identifer -> version
-			std::vector<std::pair<u16, u16>> used_serial = read_used_savestate_versions();
-
 			auto& ar = *to_ar;
 
-			const usz pos = ar.seek_end();
-
-			// Patch offset with a direct write
-			ar.m_file_handler->handle_file_op(ar, 10, sizeof(usz), &pos);
-
-			// Write the version data at the end
-			ar(used_serial);
-
 			// Final file write, the file is ready to be committed
-			ar.breathe(true);
+			ar.seek_end();
-
+			ar.m_file_handler->finalize(ar);
-#ifndef _WIN32
-			// The temporary file's contents must be on disk before rename
-			// Flush to file
-			ar.m_file_handler->handle_file_op(ar, umax, umax, nullptr);
-#endif
 
 			if (!file.commit())
 			{

rpcs3/Emu/savestate_utils.cpp

@@ -2,6 +2,8 @@
 #include "util/types.hpp"
 #include "util/logs.hpp"
 #include "util/asm.hpp"
+#include "util/v128.hpp"
+#include "util/simd.hpp"
 #include "Utilities/File.h"
 #include "Utilities/StrFmt.h"
 #include "system_config.h"
@@ -10,6 +12,9 @@
 #include "System.h"
 
 #include <set>
+#include <span>
+
+#include <zlib.h>
 
 LOG_CHANNEL(sys_log, "SYS");
 
@@ -18,14 +23,14 @@ void fmt_class_string<utils::serial>::format(std::string& out, u64 arg)
 {
 	const utils::serial& ar = get_object(arg);
 
-	fmt::append(out, "{ %s, 0x%x/0%x, memory=0x%x }", ar.is_writing() ? "writing" : "reading", ar.pos, ar.data_offset + ar.data.size(), ar.data.size());
+	fmt::append(out, "{ %s, 0x%x/0x%x, memory=0x%x }", ar.is_writing() ? "writing" : "reading", ar.pos, ar.data_offset + ar.data.size(), ar.data.size());
 }
 
 struct serial_ver_t
 {
 	bool used = false;
-	s32 current_version = 0;
+	u16 current_version = 0;
-	std::set<s32> compatible_versions;
+	std::set<u16> compatible_versions;
 };
 
 static std::array<serial_ver_t, 26> s_serial_versions;
@@ -89,7 +94,7 @@ SERIALIZATION_VER(sys_io, 23,                                   1)
 SERIALIZATION_VER(LLE, 24,                                      1)
 SERIALIZATION_VER(HLE, 25,                                      1)
 
-std::vector<std::pair<u16, u16>> get_savestate_versioning_data(fs::file&& file)
+std::vector<version_entry> get_savestate_versioning_data(fs::file&& file, std::string_view filepath)
 {
 	if (!file)
 	{
@@ -98,31 +103,36 @@ std::vector<std::pair<u16, u16>> get_savestate_versioning_data(fs::file&& file)
 
 	file.seek(0);
 
-	if (u64 r = 0; !file.read(r) || r != "RPCS3SAV"_u64)
+	utils::serial ar;
+	ar.set_reading_state();
+
+	ar.m_file_handler = filepath.ends_with(".gz") ? static_cast<std::unique_ptr<utils::serialization_file_handler>>(make_compressed_serialization_file_handler(std::move(file)))
+		: make_uncompressed_serialization_file_handler(std::move(file));
+
+	if (u64 r = 0; ar.try_read(r) != 0 || r != "RPCS3SAV"_u64)
 	{
 		return {};
 	}
 
-	file.seek(10);
+	ar.pos = 10;
 
-	u64 offs = 0;
+	u64 offs = ar.try_read<u64>().second;
-	file.read(offs);
 
-	const usz fsize = file.size();
+	const usz fsize = ar.get_size(offs);
 
 	if (!offs || fsize <= offs)
 	{
 		return {};
 	}
 
-	utils::serial ar;
-	ar.set_reading_state();
-	ar.m_file_handler = make_uncompressed_serialization_file_handler(std::move(file));
 	ar.seek_pos(offs);
-	return ar;
+	ar.breathe(true);
+
+	std::vector<version_entry> ver_data = ar.pop<std::vector<version_entry>>();
+	return std::move(ver_data);
 }
 
-bool is_savestate_version_compatible(const std::vector<std::pair<u16, u16>>& data, bool is_boot_check)
+bool is_savestate_version_compatible(const std::vector<version_entry>& data, bool is_boot_check)
 {
 	if (data.empty())
 	{
@@ -178,24 +188,31 @@ std::string get_savestate_file(std::string_view title_id, std::string_view boot_
 	// While not needing to keep a 59 chars long suffix at all times for this purpose
 	const char prefix = ::at32("0123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"sv, save_id.size());
 
-	return fs::get_cache_dir() + "/savestates/" + title + "/" + title + '_' + prefix + '_' + save_id + ".SAVESTAT";
+	std::string path = fs::get_cache_dir() + "/savestates/" + title + "/" + title + '_' + prefix + '_' + save_id + ".SAVESTAT";
+
+	if (std::string path_compressed = path + ".gz"; fs::is_file(path_compressed))
+	{
+		return std::move(path_compressed);
+	}
+
+	return std::move(path);
 }
 
-bool is_savestate_compatible(fs::file&& file)
+bool is_savestate_compatible(fs::file&& file, std::string_view filepath)
 {
-	return is_savestate_version_compatible(get_savestate_versioning_data(std::move(file)), false);
+	return is_savestate_version_compatible(get_savestate_versioning_data(std::move(file), filepath), false);
 }
 
-std::vector<std::pair<u16, u16>> read_used_savestate_versions()
+std::vector<version_entry> read_used_savestate_versions()
 {
-	std::vector<std::pair<u16, u16>> used_serial;
+	std::vector<version_entry> used_serial;
 	used_serial.reserve(s_serial_versions.size());
 
 	for (serial_ver_t& ver : s_serial_versions)
 	{
 		if (std::exchange(ver.used, false))
 		{
-			used_serial.emplace_back(&ver - s_serial_versions.data(), *ver.compatible_versions.rbegin());
+			used_serial.push_back(version_entry{static_cast<u16>(&ver - s_serial_versions.data()), *ver.compatible_versions.rbegin()});
 		}
 
 		ver.current_version = 0;
@@ -208,8 +225,6 @@ bool boot_last_savestate(bool testing)
 {
 	if (!g_cfg.savestate.suspend_emu && !Emu.GetTitleID().empty() && (Emu.IsRunning() || Emu.GetStatus() == system_state::paused))
 	{
-		extern bool is_savestate_compatible(fs::file&& file);
-
 		const std::string save_dir = fs::get_cache_dir() + "/savestates/";
 
 		std::string savestate_path;
@@ -225,7 +240,12 @@ bool boot_last_savestate(bool testing)
 			// Find the latest savestate file compatible with the game (TODO: Check app version and anything more)
 			if (entry.name.find(Emu.GetTitleID()) != umax && mtime <= entry.mtime)
 			{
-				if (std::string path = save_dir + entry.name; is_savestate_compatible(fs::file(path)))
+				if (std::string path = save_dir + entry.name + ".gz"; is_savestate_compatible(fs::file(path), path))
+				{
+					savestate_path = std::move(path);
+					mtime = entry.mtime;
+				}
+				else if (std::string path = save_dir + entry.name; is_savestate_compatible(fs::file(path), path))
 				{
 					savestate_path = std::move(path);
 					mtime = entry.mtime;
@@ -282,9 +302,9 @@ bool uncompressed_serialization_file_handler::handle_file_op(utils::serial& ar,
 			m_file->sync();
 		}
 
-		ar.data_offset += ar.data.size();
-		ar.data.clear();
 		ar.seek_end();
+		ar.data_offset = ar.pos;
+		ar.data.clear();
 		return true;
 	}
 
@@ -320,7 +340,7 @@ bool uncompressed_serialization_file_handler::handle_file_op(utils::serial& ar,
 		}
 
 		// Discard all loaded data
-		ar.data_offset += ar.data.size();
+		ar.data_offset = ar.pos;
 		ar.data.clear();
 
 		if (ar.data.capacity() >= 0x200'0000)
@@ -332,7 +352,7 @@ bool uncompressed_serialization_file_handler::handle_file_op(utils::serial& ar,
 		return true;
 	}
 
-	if (~size < pos)
+	if (~pos < size - 1)
 	{
 		// Overflow
 		return false;
@@ -340,11 +360,11 @@ bool uncompressed_serialization_file_handler::handle_file_op(utils::serial& ar,
 
 	if (ar.data.empty() && pos != ar.pos)
 	{
-		// Relocate instead oof over-fetch
+		// Relocate instead of over-fetch
 		ar.seek_pos(pos);
 	}
 
-	const usz read_pre_buffer = utils::sub_saturate<usz>(ar.data_offset, pos);
+	const usz read_pre_buffer = ar.data.empty() ? 0 : utils::sub_saturate<usz>(ar.data_offset, pos);
 
 	if (read_pre_buffer)
 	{
@@ -358,7 +378,10 @@ bool uncompressed_serialization_file_handler::handle_file_op(utils::serial& ar,
 		ar.data_offset -= read_pre_buffer;
 	}
 
-	const usz read_past_buffer = utils::sub_saturate<usz>(pos + size, ar.data_offset + ar.data.size());
+	// Adjustment to prevent overflow
+	const usz subtrahend = ar.data.empty() ? 0 : 1;
+	const usz read_past_buffer = utils::sub_saturate<usz>(pos + (size - subtrahend), ar.data_offset + (ar.data.size() - subtrahend));
+	const usz read_limit = utils::sub_saturate<usz>(ar.m_max_data, ar.data_offset); 
 
 	if (read_past_buffer)
 	{
@@ -368,14 +391,13 @@ bool uncompressed_serialization_file_handler::handle_file_op(utils::serial& ar,
 		const usz old_size = ar.data.size();
 
 		// Try to prefetch data by reading more than requested
-		ar.data.resize(std::max<usz>({ ar.data.capacity(), ar.data.size() + read_past_buffer * 3 / 2, ar.m_avoid_large_prefetch ? usz{4096} : usz{0x10'0000} }));
+		ar.data.resize(std::min<usz>(read_limit, std::max<usz>({ ar.data.capacity(), ar.data.size() + read_past_buffer * 3 / 2, ar.m_avoid_large_prefetch ? usz{4096} : usz{0x10'0000} })));
 		ar.data.resize(m_file->read_at(old_size + ar.data_offset, data ? const_cast<void*>(data) : ar.data.data() + old_size, ar.data.size() - old_size) + old_size);
 	}
 
 	return true;
 }
 
-
 usz uncompressed_serialization_file_handler::get_size(const utils::serial& ar, usz recommended) const
 {
 	if (ar.is_writing())
@@ -394,6 +416,416 @@ usz uncompressed_serialization_file_handler::get_size(const utils::serial& ar, u
 	return std::max<usz>(m_file->size(), memory_available);
 }
 
+void uncompressed_serialization_file_handler::finalize(utils::serial& ar)
+{
+	ar.seek_end();
+	handle_file_op(ar, 0, umax, nullptr);
+	ar.data = {}; // Deallocate and clear
+}
+
+void compressed_serialization_file_handler::initialize(utils::serial& ar)
+{
+	if (!m_stream.has_value())
+	{
+		m_stream.emplace<z_stream>();
+	}
+
+	z_stream& m_zs = std::any_cast<z_stream&>(m_stream);
+
+	if (ar.is_writing())
+	{
+		if (m_write_inited)
+		{
+			return;
+		}
+
+#ifndef _MSC_VER
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wold-style-cast"
+#endif
+		if (m_read_inited)
+		{
+			finalize(ar);
+		}
+
+		m_zs = {};
+		ensure(deflateInit2(&m_zs, 9, Z_DEFLATED, 16 + 15, 9, Z_DEFAULT_STRATEGY) == Z_OK);
+#ifndef _MSC_VER
+#pragma GCC diagnostic pop
+#endif
+		m_write_inited = true;
+	}
+	else
+	{
+		if (m_read_inited)
+		{
+			return;
+		}
+
+		if (m_write_inited)
+		{
+			finalize(ar);
+		}
+
+		m_zs.avail_in = 0;
+		m_zs.avail_out = 0;
+		m_zs.next_in = nullptr;
+		m_zs.next_out = nullptr;
+	#ifndef _MSC_VER
+	#pragma GCC diagnostic push
+	#pragma GCC diagnostic ignored "-Wold-style-cast"
+	#endif
+		ensure(inflateInit2(&m_zs, 16 + 15) == Z_OK);
+		m_read_inited = true;
+	}
+}
+
+bool compressed_serialization_file_handler::handle_file_op(utils::serial& ar, usz pos, usz size, const void* data)
+{
+	if (ar.is_writing())
+	{
+		initialize(ar);
+
+		z_stream& m_zs = std::any_cast<z_stream&>(m_stream);
+
+		if (data)
+		{
+			ensure(false);
+		}
+
+		// Writing not at the end is forbidden
+		ensure(ar.pos == ar.data_offset + ar.data.size());
+
+		m_zs.avail_in = static_cast<uInt>(ar.data.size());
+		m_zs.next_in  = ar.data.data();
+
+		do
+		{
+			m_stream_data.resize(std::max<usz>(m_stream_data.size(), ::compressBound(m_zs.avail_in)));
+			m_zs.avail_out = static_cast<uInt>(m_stream_data.size());
+			m_zs.next_out  = m_stream_data.data();
+
+			if (deflate(&m_zs, Z_NO_FLUSH) == Z_STREAM_ERROR || m_file->write(m_stream_data.data(), m_stream_data.size() - m_zs.avail_out) != m_stream_data.size() - m_zs.avail_out)
+			{
+				deflateEnd(&m_zs);
+				//m_file->close();
+				break;
+			}
+		}
+		while (m_zs.avail_out == 0);
+
+		ar.seek_end();
+		ar.data_offset = ar.pos;
+		ar.data.clear();
+
+		if (pos == umax && size == umax && *m_file)
+		{
+			// Request to flush the file to disk
+			m_file->sync();
+		}
+
+		return true;
+	}
+
+	initialize(ar);
+
+	if (!size)
+	{
+		return true;
+	}
+
+	if (pos == 0 && size == umax)
+	{
+		// Discard loaded data until pos if profitable
+		const usz limit = ar.data_offset + ar.data.size();
+
+		if (ar.pos > ar.data_offset && ar.pos < limit)
+		{
+			const usz may_discard_bytes = ar.pos - ar.data_offset;
+			const usz moved_byte_count_on_discard = limit - ar.pos;
+
+			// Cheeck profitability (check recycled memory and std::memmove costs)
+			if (may_discard_bytes >= 0x50'0000 || (may_discard_bytes >= 0x20'0000 && moved_byte_count_on_discard / may_discard_bytes < 3))
+			{
+				ar.data_offset += may_discard_bytes;
+				ar.data.erase(ar.data.begin(), ar.data.begin() + may_discard_bytes);
+
+				if (ar.data.capacity() >= 0x200'0000)
+				{
+					// Discard memory
+					ar.data.shrink_to_fit();
+				}
+			}
+
+			return true;
+		}
+
+		// Discard all loaded data
+		ar.data_offset += ar.data.size();
+		ensure(ar.pos >= ar.data_offset);
+		ar.data.clear();
+
+		if (ar.data.capacity() >= 0x200'0000)
+		{
+			// Discard memory
+			ar.data.shrink_to_fit();
+		}
+
+		return true;
+	}
+
+	if (~pos < size - 1)
+	{
+		// Overflow
+		return false;
+	}
+
+	// TODO: Investigate if this optimization is worth an implementation for compressed stream
+	// if (ar.data.empty() && pos != ar.pos)
+	// {
+	// 	// Relocate instead of over-fetch
+	// 	ar.seek_pos(pos);
+	// }
+
+	const usz read_pre_buffer = utils::sub_saturate<usz>(ar.data_offset, pos);
+
+	if (read_pre_buffer)
+	{
+		// Not allowed with compressed data for now
+		// Unless someone implements mechanism for it
+		ensure(false);
+	}
+
+	// Adjustment to prevent overflow
+	const usz subtrahend = ar.data.empty() ? 0 : 1;
+	const usz read_past_buffer = utils::sub_saturate<usz>(pos + (size - subtrahend), ar.data_offset + (ar.data.size() - subtrahend));
+	const usz read_limit = utils::sub_saturate<usz>(ar.m_max_data, ar.data_offset); 
+
+	if (read_past_buffer)
+	{
+		// Read proceeding data
+		// More lightweight operation, this is the common operation
+		// Allowed to fail, if memory is truly needed an assert would take place later
+		const usz old_size = ar.data.size();
+
+		// Try to prefetch data by reading more than requested
+		ar.data.resize(std::min<usz>(read_limit, std::max<usz>({ ar.data.capacity(), ar.data.size() + read_past_buffer * 3 / 2, ar.m_avoid_large_prefetch ? usz{4096} : usz{0x10'0000} })));
+		ar.data.resize(this->read_at(ar, old_size + ar.data_offset, data ? const_cast<void*>(data) : ar.data.data() + old_size, ar.data.size() - old_size) + old_size);
+	}
+
+	return true;
+}
+
+usz compressed_serialization_file_handler::read_at(utils::serial& ar, usz read_pos, void* data, usz size)
+{
+	ensure(read_pos == ar.data.size() + ar.data_offset - size);
+
+	if (!size)
+	{
+		return 0;
+	}
+
+	initialize(ar);
+
+	z_stream& m_zs = std::any_cast<z_stream&>(m_stream);
+
+	const usz total_to_read = size;
+	usz read_size = 0;
+	u8* out_data = static_cast<u8*>(data);
+
+	auto adjust_for_uint = [](usz size)
+	{
+		return static_cast<uInt>(std::min<usz>(uInt{umax}, size));
+	};
+
+	for (; read_size < total_to_read;)
+	{
+		// Drain extracted memory stash (also before first file read)
+		out_data = static_cast<u8*>(data) + read_size;
+		m_zs.avail_in = adjust_for_uint(m_stream_data.size() - m_stream_data_index);
+		m_zs.next_in = reinterpret_cast<const u8*>(m_stream_data.data() + m_stream_data_index);
+		m_zs.next_out = out_data;
+		m_zs.avail_out = adjust_for_uint(size - read_size);
+
+		while (read_size < total_to_read && m_zs.avail_in)
+		{
+			const int res = inflate(&m_zs, Z_BLOCK);
+
+			bool need_more_file_memory = false;
+
+			switch (res)
+			{
+			case Z_OK:
+			case Z_STREAM_END:
+				break;
+			case Z_BUF_ERROR:
+			{
+				if (m_zs.avail_in)
+				{
+					need_more_file_memory = true;
+					break;
+				}
+
+				[[fallthrough]];
+			}
+			default:
+				inflateEnd(&m_zs);
+				m_read_inited = false;
+				return read_size;
+			}
+
+			read_size = m_zs.next_out - static_cast<u8*>(data);
+			m_stream_data_index = m_zs.avail_in ? m_zs.next_in - m_stream_data.data() : m_stream_data.size();
+
+			// Adjust again in case the values simply did not fit into uInt
+			m_zs.avail_out = adjust_for_uint(utils::sub_saturate<usz>(total_to_read, read_size));
+			m_zs.avail_in = adjust_for_uint(m_stream_data.size() - m_stream_data_index);
+
+			if (need_more_file_memory)
+			{
+				break;
+			}
+		}
+
+		if (read_size >= total_to_read)
+		{
+			break;
+		}
+
+		const usz add_size = ar.m_avoid_large_prefetch ? 0x1'0000 : 0x10'0000;
+		const usz old_file_buf_size = m_stream_data.size();
+
+		m_stream_data.resize(old_file_buf_size + add_size);
+		m_stream_data.resize(old_file_buf_size + m_file->read_at(m_file_read_index, m_stream_data.data() + old_file_buf_size, add_size));
+
+		if (m_stream_data.size() == old_file_buf_size)
+		{
+			// EOF
+			break;
+		}
+
+		m_file_read_index += m_stream_data.size() - old_file_buf_size;
+	}
+
+	if (m_stream_data.size() - m_stream_data_index <= m_stream_data_index / 5)
+	{
+		// Shrink to required memory size
+		m_stream_data.erase(m_stream_data.begin(), m_stream_data.begin() + m_stream_data_index);
+
+		if (m_stream_data.capacity() >= 0x200'0000)
+		{
+			// Discard memory
+			m_stream_data.shrink_to_fit();
+		}
+
+		m_stream_data_index = 0;
+	}
+
+	return read_size;
+}
+
+void compressed_serialization_file_handler::skip_until(utils::serial& ar)
+{
+	ensure(!ar.is_writing() && ar.pos >= ar.data_offset);
+
+	if (ar.pos > ar.data_offset)
+	{
+		handle_file_op(ar, ar.data_offset, ar.pos - ar.data_offset, nullptr);
+	}
+}
+
+void compressed_serialization_file_handler::finalize(utils::serial& ar)
+{
+	handle_file_op(ar, 0, umax, nullptr);
+
+	if (!m_stream.has_value())
+	{
+		return;
+	}
+
+	z_stream& m_zs = std::any_cast<z_stream&>(m_stream);
+
+	if (m_read_inited)
+	{
+		m_read_inited = false;
+		ensure(inflateEnd(&m_zs) == Z_OK);
+		return;
+	}
+
+	m_write_inited = false;
+
+	m_zs.avail_in = 0;
+	m_zs.next_in  = nullptr;
+
+	m_stream_data.resize(m_zs.avail_out);
+
+	do
+	{
+		m_zs.avail_out = static_cast<uInt>(m_stream_data.size());
+		m_zs.next_out  = m_stream_data.data();
+
+		if (deflate(&m_zs, Z_FINISH) == Z_STREAM_ERROR)
+		{
+			break;
+		}
+
+		m_file->write(m_stream_data.data(), m_stream_data.size() - m_zs.avail_out);
+	}
+	while (m_zs.avail_out == 0);
+
+	m_stream_data = {};
+	ensure(deflateEnd(&m_zs) == Z_OK);
+	ar.data = {}; // Deallocate and clear
+}
+
+usz compressed_serialization_file_handler::get_size(const utils::serial& ar, usz recommended) const
+{
+	if (ar.is_writing())
+	{
+		return m_file->size();
+	}
+
+	const usz memory_available = ar.data_offset + ar.data.size();
+
+	if (memory_available >= recommended)
+	{
+		// Avoid calling size() if possible
+		return memory_available;
+	}
+
+	return std::max<usz>(utils::mul_saturate<usz>(m_file->size(), 6), memory_available);
+}
+
+bool null_serialization_file_handler::handle_file_op(utils::serial&, usz, usz, const void*)
+{
+	return true;
+}
+
+void null_serialization_file_handler::finalize(utils::serial&)
+{
+}
+
+bool load_and_check_reserved(utils::serial& ar, usz size)
+{
+	u8 bytes[4096];
+	std::memset(&bytes[size & (0 - sizeof(v128))], 0, sizeof(v128));
+	ensure(size <= std::size(bytes));
+
+	const usz old_pos = ar.pos;
+	ar(std::span<u8>(bytes, size));
+
+	// Check if all are 0
+	for (usz i = 0; i < size; i += sizeof(v128))
+	{
+		if (v128::loadu(&bytes[i]) != v128{})
+		{
+			return false;
+		}
+	}
+
+	return old_pos + size == ar.pos;
+}
+
 namespace stx
 {
 	extern void serial_breathe(utils::serial& ar)

rpcs3/Emu/savestate_utils.hpp

@@ -1,10 +1,22 @@
+#pragma once
+
 #include "util/serialization.hpp"
 
+#include <any>
+
 namespace fs
 {
 	class file;
 }
 
+struct version_entry
+{
+	u16 type;
+	u16 version;
+
+	ENABLE_BITWISE_SERIALIZATION;
+};
+
 // Uncompressed file serialization handler
 struct uncompressed_serialization_file_handler : utils::serialization_file_handler
 {
@@ -25,21 +37,102 @@ struct uncompressed_serialization_file_handler : utils::serialization_file_handl
 	{
 	}
 
+	uncompressed_serialization_file_handler(const uncompressed_serialization_file_handler&) = delete;
+
 	// Handle file read and write requests
 	bool handle_file_op(utils::serial& ar, usz pos, usz size, const void* data) override;
 
 	// Get available memory or file size
 	// Preferably memory size if is already greater/equal to recommended to avoid additional file ops
 	usz get_size(const utils::serial& ar, usz recommended) const override;
+
+	void finalize(utils::serial& ar) override;
 };
 
-inline std::unique_ptr<uncompressed_serialization_file_handler> make_uncompressed_serialization_file_handler(fs::file&& file)
+template <typename File> requires (std::is_same_v<std::remove_cvref_t<File>, fs::file>)
+inline std::unique_ptr<uncompressed_serialization_file_handler> make_uncompressed_serialization_file_handler(File&& file)
 {
-	return std::make_unique<uncompressed_serialization_file_handler>(std::move(file));
+	ensure(file);
+	return std::make_unique<uncompressed_serialization_file_handler>(std::forward<File>(file));
 }
 
-inline std::unique_ptr<uncompressed_serialization_file_handler> make_uncompressed_serialization_file_handler(const fs::file& file)
+// Compressed file serialization handler
+struct compressed_serialization_file_handler : utils::serialization_file_handler
 {
-	return std::make_unique<uncompressed_serialization_file_handler>(file);
+	const std::unique_ptr<fs::file> m_file_storage;
+	const std::add_pointer_t<const fs::file> m_file;
+	std::vector<u8> m_stream_data;
+	usz m_stream_data_index = 0;
+	usz m_file_read_index = 0;
+	bool m_write_inited = false;
+	bool m_read_inited = false;
+	std::any m_stream;
+
+	explicit compressed_serialization_file_handler(fs::file&& file) noexcept
+		: utils::serialization_file_handler()
+		, m_file_storage(std::make_unique<fs::file>(std::move(file)))
+		, m_file(m_file_storage.get())
+	{
+	}
+
+	explicit compressed_serialization_file_handler(const fs::file& file) noexcept
+		: utils::serialization_file_handler()
+		, m_file_storage(nullptr)
+		, m_file(std::addressof(file))
+	{
+	}
+
+	compressed_serialization_file_handler(const compressed_serialization_file_handler&) = delete;
+
+	// Handle file read and write requests
+	bool handle_file_op(utils::serial& ar, usz pos, usz size, const void* data) override;
+
+	// Get available memory or file size
+	// Preferably memory size if is already greater/equal to recommended to avoid additional file ops
+	usz get_size(const utils::serial& ar, usz recommended) const override;
+	void skip_until(utils::serial& ar) override;
+
+	void finalize(utils::serial& ar) override;
+
+private:
+	usz read_at(utils::serial& ar, usz read_pos, void* data, usz size);
+	void initialize(utils::serial& ar);
+};
+
+template <typename File> requires (std::is_same_v<std::remove_cvref_t<File>, fs::file>)
+inline std::unique_ptr<compressed_serialization_file_handler> make_compressed_serialization_file_handler(File&& file)
+{
+	ensure(file);
+	return std::make_unique<compressed_serialization_file_handler>(std::forward<File>(file));
 }
 
+// Null file serialization handler
+struct null_serialization_file_handler : utils::serialization_file_handler
+{
+	explicit null_serialization_file_handler() noexcept
+	{
+	}
+
+	// Handle file read and write requests
+	bool handle_file_op(utils::serial& ar, usz pos, usz size, const void* data) override;
+
+	void finalize(utils::serial& ar) override;
+
+	bool is_null() const override
+	{
+		return true;
+	}
+};
+
+inline std::unique_ptr<null_serialization_file_handler> make_null_serialization_file_handler()
+{
+	return std::make_unique<null_serialization_file_handler>();
+}
+
+bool load_and_check_reserved(utils::serial& ar, usz size);
+bool is_savestate_version_compatible(const std::vector<version_entry>& data, bool is_boot_check);
+std::vector<version_entry> get_savestate_versioning_data(fs::file&& file, std::string_view filepath);
+bool is_savestate_compatible(fs::file&& file, std::string_view filepath);
+std::vector<version_entry> read_used_savestate_versions();
+std::string get_savestate_file(std::string_view title_id, std::string_view boot_path, s64 abs_id, s64 rel_id);
+

rpcs3/Loader/TAR.cpp

@@ -8,14 +8,30 @@
 #include "TAR.h"
 
 #include "util/asm.hpp"
-#include "util/serialization.hpp"
+
+#include "Emu/savestate_utils.hpp"
 
 #include <charconv>
+#include <span>
 
 LOG_CHANNEL(tar_log, "TAR");
 
+fs::file make_file_view(const fs::file& file, u64 offset, u64 size);
+
+// File constructor
 tar_object::tar_object(const fs::file& file)
-	: m_file(file)
+	: m_file(std::addressof(file))
+	, m_ar(nullptr)
+	, m_ar_tar_start(umax)
+{
+	ensure(*m_file);
+}
+
+// Stream (pipe-like) constructor
+tar_object::tar_object(utils::serial& ar)
+	: m_file(nullptr)
+	, m_ar(std::addressof(ar))
+	, m_ar_tar_start(ar.pos)
 {
 }
 
@@ -23,12 +39,14 @@ TARHeader tar_object::read_header(u64 offset) const
 {
 	TARHeader header{};
 
-	if (m_file.seek(offset) != offset)
+	if (m_ar)
 	{
+		ensure(m_ar->pos == m_ar_tar_start + offset);
+		m_ar->serialize(header);
 		return header;
 	}
 
-	if (!m_file.read(header))
+	if (m_file->read_at(offset, &header, sizeof(header)) != sizeof(header))
 	{
 		std::memset(&header, 0, sizeof(header));
 	}
@@ -38,13 +56,13 @@ TARHeader tar_object::read_header(u64 offset) const
 
 u64 octal_text_to_u64(std::string_view sv)
 {
-	u64 i = -1;
+	u64 i = umax;
 	const auto ptr = std::from_chars(sv.data(), sv.data() + sv.size(), i, 8).ptr;
 
 	// Range must be terminated with either NUL or space
 	if (ptr == sv.data() + sv.size() || (*ptr && *ptr != ' '))
 	{
-		i = -1;
+		i = umax;
 	}
 
 	return i;
@@ -53,36 +71,32 @@ u64 octal_text_to_u64(std::string_view sv)
 std::vector<std::string> tar_object::get_filenames()
 {
 	std::vector<std::string> vec;
+
 	get_file("");
+
 	for (auto it = m_map.cbegin(); it != m_map.cend(); ++it)
 	{
 		vec.push_back(it->first);
 	}
+
 	return vec;
 }
 
-fs::file tar_object::get_file(const std::string& path)
+std::unique_ptr<utils::serial> tar_object::get_file(const std::string& path, std::string* new_file_path)
 {
-	if (!m_file) return fs::file();
+	std::unique_ptr<utils::serial> m_out;
 
-	if (auto it = m_map.find(path); it != m_map.end())
+	auto emplace_single_entry = [&](usz& offset, const usz max_size) -> std::pair<usz, std::string>
 	{
-		u64 size = 0;
+		if (offset >= max_size)
-		std::memcpy(&size, it->second.second.size, sizeof(size));
+		{
-		std::vector<u8> buf(size);
+			return {};
-		m_file.seek(it->second.first);
-		m_file.read(buf, size);
-		return fs::make_stream(std::move(buf));
 		}
-	else //continue scanning from last file entered
-	{
-		const u64 max_size = m_file.size();
 
-		while (largest_offset < max_size)
+		TARHeader header = read_header(offset);
-		{
+		offset += 512;
-			TARHeader header = read_header(largest_offset);
 
-			u64 size = -1;
+		u64 size = umax;
 
 		std::string filename;
 
@@ -93,7 +107,7 @@ fs::file tar_object::get_file(const std::string& path)
 			size = octal_text_to_u64(size_sv);
 
 			// Check for overflows and if surpasses file size
-				if ((header.name[0] || header.prefix[0]) && size + 512 > size && max_size >= size + 512 && max_size - size - 512 >= largest_offset)
+			if ((header.name[0] || header.prefix[0]) && ~size >= 512 && max_size >= size && max_size - size >= offset)
 			{
 				// Cache size in native u64 format
 				static_assert(sizeof(size) < sizeof(header.size));
@@ -109,12 +123,17 @@ fs::file tar_object::get_file(const std::string& path)
 				filename += name;
 
 				// Save header and offset
-					m_map.insert_or_assign(filename, std::make_pair(largest_offset + 512, header));
+				m_map.insert_or_assign(filename, std::make_pair(offset, header));
+
+				if (new_file_path)
+				{
+					*new_file_path = filename;
+				}
+
+				return { size, std::move(filename) };
 			}
 			else
 			{
-					// Invalid
-					size = -1;
 				tar_log.error("tar_object::get_file() failed to convert header.size=%s, filesize=0x%x", size_sv, max_size);
 			}
 		}
@@ -123,44 +142,95 @@ fs::file tar_object::get_file(const std::string& path)
 			tar_log.trace("tar_object::get_file() failed to parse header: offset=0x%x, filesize=0x%x", largest_offset, max_size);
 		}
 
+		return { size, {} };
+	};
+
+	if (auto it = m_map.find(path); it != m_map.end())
+	{
+		u64 size = 0;
+		std::memcpy(&size, it->second.second.size, sizeof(size));
+
+		if (m_file)
+		{
+			m_out = std::make_unique<utils::serial>();
+			m_out->set_reading_state();
+			m_out->m_file_handler = make_uncompressed_serialization_file_handler(make_file_view(*m_file, it->second.first, size));
+		}
+		else
+		{
+			m_out = std::make_unique<utils::serial>();
+			*m_out = std::move(*m_ar);
+			m_out->m_max_data = m_ar_tar_start + it->second.first + size;
+		}
+
+		return m_out;
+	}
+	else if (m_ar && path.empty())
+	{
+		const u64 size = emplace_single_entry(largest_offset, m_ar->get_size(umax) - m_ar_tar_start).first;
+
+		// Advance offset to next block
+		largest_offset += utils::align(size, 512);
+	}
+	// Continue scanning from last file entered
+	else if (m_file)
+	{
+		const u64 max_size = m_file->size();
+
+		while (largest_offset < max_size)
+		{
+			const auto [size, filename] = emplace_single_entry(largest_offset, max_size);
+
 			if (size == umax)
 			{
-				largest_offset += 512;
 				continue;
 			}
 
 			// Advance offset to next block
-			largest_offset += utils::align(size, 512) + 512;
+			largest_offset += utils::align(size, 512);
 
 			if (!path.empty() && path == filename)
 			{
-				// Path is equal, read file and advance offset to start of next block
+				// Path is equal, return handle to the file data
-				std::vector<u8> buf(size);
+				return get_file(path);
-
-				if (m_file.read(buf, size))
-				{
-					return fs::make_stream(std::move(buf));
 			}
-
-				tar_log.error("tar_object::get_file() failed to read file entry %s (size=0x%x)", filename, size);
-				largest_offset -= utils::align(size, 512);
 		}
 	}
 
-		return fs::file();
+	return m_out;
-	}
 }
 
 bool tar_object::extract(std::string prefix_path, bool is_vfs)
 {
-	if (!m_file) return false;
+	std::vector<u8> filedata_buffer(0x80'0000);
+	std::span<u8> filedata_span{filedata_buffer.data(), filedata_buffer.size()};
 
-	get_file(""); // Make sure we have scanned all files
+	auto iter = m_map.begin();
 
-	for (auto& iter : m_map)
+	auto get_next = [&](bool is_first)
 	{
-		const TARHeader& header = iter.second.second;
+		if (m_ar)
-		const std::string& name = iter.first;
+		{
+			ensure(!is_first || m_map.empty()); // Must be empty on first call
+			std::string name_iter;
+			get_file("", &name_iter); // Get next entry
+			return m_map.find(name_iter);
+		}
+		else if (is_first)
+		{
+			get_file(""); // Scan entries
+			return m_map.begin();
+		}
+		else
+		{
+			return std::next(iter);
+		}
+	};
+
+	for (iter = get_next(true); iter != m_map.end(); iter = get_next(false))
+	{
+		const TARHeader& header = iter->second.second;
+		const std::string& name = iter->first;
 
 		std::string result = name;
 
@@ -210,15 +280,53 @@ bool tar_object::extract(std::string prefix_path, bool is_vfs)
 				return false;
 			}
 
-			auto data = get_file(name).release();
+			// For restoring m_ar->m_max_data
+			usz restore_limit = umax;
+
+			if (!m_file)
+			{
+				// Restore m_ar (remove limit)
+				restore_limit = m_ar->m_max_data;
+			}
+
+			std::unique_ptr<utils::serial> file_data = get_file(name);
 
 			fs::file file(result, fs::rewrite);
 
-			if (file)
+			if (file && file_data)
 			{
-				file.write(static_cast<fs::container_stream<std::vector<u8>>*>(data.get())->obj);
+				while (true)
+				{
+					const usz unread_size = file_data->try_read(filedata_span);
+
+					if (unread_size == 0)
+					{
+						file.write(filedata_span.data(), filedata_span.size());
+						continue;
+					}
+
+					// Tail data
+
+					if (usz read_size = filedata_span.size() - unread_size)
+					{
+						ensure(file_data->try_read(filedata_span.first(read_size)) == 0);
+						file.write(filedata_span.data(), read_size);
+					}
+
+					break;
+				}
+
 				file.close();
 
+				file_data->seek_pos(m_ar_tar_start + largest_offset, true);
+
+				if (!m_file)
+				{
+					// Restore m_ar
+					*m_ar = std::move(*file_data);
+					m_ar->m_max_data = restore_limit;
+				}
+
 				if (mtime != umax && !fs::utime(result, atime, mtime))
 				{
 					tar_log.error("TAR Loader: fs::utime failed on %s (%s)", result, fs::g_tls_error);
@@ -229,6 +337,13 @@ bool tar_object::extract(std::string prefix_path, bool is_vfs)
 				break;
 			}
 
+			if (!m_file)
+			{
+				// Restore m_ar
+				*m_ar = std::move(*file_data);
+				m_ar->m_max_data = restore_limit;
+			}
+
 			const auto old_error = fs::g_tls_error;
 			tar_log.error("TAR Loader: failed to write file %s (%s) (fs::exists=%s)", name, old_error, fs::exists(result));
 			return false;
@@ -259,32 +374,14 @@ bool tar_object::extract(std::string prefix_path, bool is_vfs)
 	return true;
 }
 
-void tar_object::save_directory(const std::string& src_dir, utils::serial& ar, const process_func& func, std::string full_path)
+void tar_object::save_directory(const std::string& target_path, utils::serial& ar, const process_func& func, std::vector<fs::dir_entry>&& entries, bool has_evaluated_results, usz src_dir_pos)
 {
-	const std::string& target_path = full_path.empty() ? src_dir : full_path;
+	const bool is_null = ar.m_file_handler && ar.m_file_handler->is_null();
+	const bool reuse_entries = !is_null || has_evaluated_results;
 
-	fs::stat_t stat{};
+	if (reuse_entries)
-	if (!fs::get_stat(target_path, stat))
 	{
-		return;
+		ensure(!entries.empty());
-	}
-
-	if (stat.is_directory)
-	{
-		bool has_items = false;
-
-		for (auto& entry : fs::dir(target_path))
-		{
-			if (entry.name.find_first_not_of('.') == umax) continue;
-
-			save_directory(src_dir, ar, func, target_path + '/' + entry.name);
-			has_items = true;
-		}
-
-		if (has_items)
-		{
-			return;
-		}
 	}
 
 	auto write_octal = [](char* ptr, u64 i)
@@ -303,39 +400,106 @@ void tar_object::save_directory(const std::string& src_dir, utils::serial& ar, c
 		}
 	};
 
-	std::string saved_path{target_path.data() + src_dir.size(), target_path.size() - src_dir.size()};
+	auto save_file = [&](const fs::stat_t& file_stat, const std::string& file_name)
-
-	const u64 old_size = ar.data.size();
-	ar.data.resize(old_size + sizeof(TARHeader));
-
-	if (!stat.is_directory)
 	{
-		fs::file fd(target_path);
+		if (!file_stat.size)
+		{
+			return;
+		}
 
-		const u64 old_size2 = ar.data.size();
+		if (is_null && !func)
+		{
+			ar.pos += utils::align(file_stat.size, 512);
+			return;
+		}
+
+		if (fs::file fd{file_name})
+		{
+			const u64 old_pos = ar.pos;
+			const usz old_size = ar.data.size();
 
 			if (func)
 			{
+				std::string saved_path{&::at32(file_name, src_dir_pos), file_name.size() - src_dir_pos};
+
 				// Use custom function for file saving if provided
 				// Allows for example to compress PNG files as JPEG in the TAR itself
 				if (!func(fd, saved_path, ar))
 				{
 					// Revert (this entry should not be included if func returns false)
+
+					if (is_null)
+					{
+						ar.pos = old_pos;
+						return;
+					}
+
 					ar.data.resize(old_size);
+					ar.seek_end();
+					return;
+				}
+
+				if (is_null)
+				{
+					// Align
+					ar.pos += utils::align(ar.pos - old_pos, 512);
 					return;
 				}
 			}
 			else
 			{
-			ar.data.resize(ar.data.size() + stat.size);
+				constexpr usz transfer_block_size = 0x100'0000;
-			ensure(fd.read(ar.data.data() + old_size2, stat.size) == stat.size);
+
+				for (usz read_index = 0; read_index < file_stat.size; read_index += transfer_block_size)
+				{
+					const usz read_size = std::min<usz>(transfer_block_size, file_stat.size - read_index);
+
+					// Read file data
+					ar.data.resize(ar.data.size() + read_size);
+					ensure(fd.read_at(read_index, ar.data.data() + old_size, read_size) == read_size);
+
+					// Set position to the end of data, so breathe() would work correctly
+					ar.seek_end();
+
+					// Allow flushing to file if needed
+					ar.breathe();
+				}
 			}
 
 			// Align
-		ar.data.resize(old_size2 + utils::align(ar.data.size() - old_size2, 512));
+			const usz diff = ar.pos - old_pos;
+			ar.data.resize(ar.data.size() + utils::align(diff, 512) - diff);
+			ar.seek_end();
 
 			fd.close();
-		fs::utime(target_path, stat.atime, stat.mtime);
+			ensure(fs::utime(file_name, file_stat.atime, file_stat.mtime));
+		}
+		else
+		{
+			ensure(false);
+		}
+	};
+
+	auto save_header = [&](const fs::stat_t& stat, const std::string& name)
+	{
+		static_assert(sizeof(TARHeader) == 512);
+
+		std::string_view saved_path{name.size() == src_dir_pos ? name.c_str() : &::at32(name, src_dir_pos), name.size() - src_dir_pos};
+
+		if (is_null)
+		{
+			ar.pos += sizeof(TARHeader);
+			return;
+		}
+
+		if (usz pos = saved_path.find_first_not_of(fs::delim); pos != umax)
+		{
+			saved_path = saved_path.substr(pos, saved_path.size());
+		}
+		else
+		{
+			// Target the destination directory, I do not know if this is compliant with TAR format
+			saved_path = "/"sv;
 		}
 
 		TARHeader header{};
@@ -353,10 +517,107 @@ void tar_object::save_directory(const std::string& src_dir, utils::serial& ar, c
 		write_octal(header.padding, stat.atime);
 		header.filetype = stat.is_directory ? '5' : '0';
 
-	std::memcpy(ar.data.data() + old_size, &header, sizeof(header));
+		ar(header);
+		ar.breathe();
+	};
+
+	fs::stat_t stat{};
+
+	if (src_dir_pos == umax)
+	{
+		// First call, get source directory string size so it can be cut from entry paths
+		src_dir_pos = target_path.size();
+	}
+
+	if (has_evaluated_results)
+	{
+		// Save from cached data by previous call
+		for (auto&& entry : entries)
+		{
+			ensure(entry.name.starts_with(target_path));
+			save_header(entry, entry.name);
+
+			if (!entry.is_directory)
+			{
+				save_file(entry, entry.name);
+			}
+		}
+	}
+	else
+	{
+		if (entries.empty())
+		{
+			if (!fs::get_stat(target_path, stat))
+			{
+				return;
+			}
+
+			save_header(stat, target_path);
+
+			// Optimization: avoid saving to list if this is not an evaluation call
+			if (is_null)
+			{
+				static_cast<fs::stat_t&>(entries.emplace_back()) = stat;
+				entries.back().name = target_path;
+			}
+		}
+		else
+		{
+			stat = entries.back();
+			save_header(stat, entries.back().name);
+		}
+
+		if (stat.is_directory)
+		{
+			bool exists = false;
+
+			for (auto&& entry : fs::dir(target_path))
+			{
+				exists = true;
+
+				if (entry.name.find_first_not_of('.') == umax)
+				{
+					continue;
+				}
+
+				entry.name = target_path.ends_with('/') ? target_path + entry.name : target_path + '/' + entry.name;
+
+				if (!entry.is_directory)
+				{
+					save_header(entry, entry.name);
+					save_file(entry, entry.name);
 
 					// TAR is an old format which does not depend on previous data so memory ventilation is trivial here
 					ar.breathe();
+
+					entries.emplace_back(std::move(entry));
+				}
+				else
+				{
+					if (!is_null)
+					{
+						// Optimization: avoid saving to list if this is not an evaluation call
+						entries.clear();
+					}
+
+					entries.emplace_back(std::move(entry));
+					save_directory(::as_rvalue(entries.back().name), ar, func, std::move(entries), false, src_dir_pos);
+				}
+			}
+
+			ensure(exists);
+		}
+		else
+		{
+			fs::dir_entry entry{};
+			entry.name = target_path;
+			static_cast<fs::stat_t&>(entry) = stat;
+
+			save_file(entry, entry.name);
+		}
+
+		ar.breathe();
+	}
 }
 
 bool extract_tar(const std::string& file_path, const std::string& dir_path, fs::file file)

rpcs3/Loader/TAR.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "util/types.hpp"
+
 #include <map>
 
 struct TARHeader
@@ -15,11 +17,14 @@ struct TARHeader
 	char dontcare2[82];
 	char prefix[155];
 	char padding[12]; // atime for RPCS3
+
+	ENABLE_BITWISE_SERIALIZATION;
 };
 
 namespace fs
 {
 	class file;
+	struct dir_entry;
 }
 
 namespace utils
@@ -29,7 +34,9 @@ namespace utils
 
 class tar_object
 {
-	const fs::file& m_file;
+	const fs::file* m_file;
+	utils::serial* m_ar;
+	const usz m_ar_tar_start;
 
 	usz largest_offset = 0; // We store the largest offset so we can continue to scan from there.
 	std::map<std::string, std::pair<u64, TARHeader>> m_map{}; // Maps path to offset of file data and its header
@@ -38,10 +45,11 @@ class tar_object
 
 public:
 	tar_object(const fs::file& file);
+	tar_object(utils::serial& ar);
 
 	std::vector<std::string> get_filenames();
 
-	fs::file get_file(const std::string& path);
+	std::unique_ptr<utils::serial> get_file(const std::string& path, std::string* new_file_path = nullptr);
 
 	using process_func = std::function<bool(const fs::file&, std::string&, utils::serial&)>;
 
@@ -49,7 +57,7 @@ public:
 	// Allow to optionally specify explicit mount point (which may be directory meant for extraction)
 	bool extract(std::string prefix_path = {}, bool is_vfs = false);
 
-	static void save_directory(const std::string& src_dir, utils::serial& ar, const process_func& func = {}, std::string append_path = {});
+	static void save_directory(const std::string& src_dir, utils::serial& ar, const process_func& func = {}, std::vector<fs::dir_entry>&& = std::vector<fs::dir_entry>{}, bool has_evaluated_results = false, usz src_dir_pos = umax);
 };
 
 bool extract_tar(const std::string& file_path, const std::string& dir_path, fs::file file = {});

rpcs3/rpcs3qt/game_list_frame.cpp

@@ -1078,9 +1078,9 @@ void game_list_frame::ShowContextMenu(const QPoint &pos)
 		});
 	}
 
-	extern bool is_savestate_compatible(fs::file&& file);
+	extern bool is_savestate_compatible(fs::file&& file, std::string_view filepath);
 
-	if (const std::string sstate = get_savestate_file(current_game.serial, current_game.path, 0, 0); is_savestate_compatible(fs::file(sstate)))
+	if (const std::string sstate = get_savestate_file(current_game.serial, current_game.path, 0, 0); is_savestate_compatible(fs::file(sstate), sstate))
 	{
 		QAction* boot_state = menu.addAction(is_current_running_game
 			? tr("&Reboot with savestate")

rpcs3/rpcs3qt/main_window.cpp

@@ -56,6 +56,7 @@
 #include "Emu/vfs_config.h"
 #include "Emu/System.h"
 #include "Emu/system_utils.hpp"
+#include "Emu/savestate_utils.hpp"
 
 #include "Crypto/unpkg.h"
 #include "Crypto/unself.h"
@@ -589,7 +590,7 @@ void main_window::BootSavestate()
 	}
 
 	const QString file_path = QFileDialog::getOpenFileName(this, tr("Select Savestate To Boot"), qstr(fs::get_cache_dir() + "/savestates/"), tr(
-		"Savestate files (*.SAVESTAT);;"
+		"Savestate files (*.SAVESTAT *.SAVESTAT.gz);;"
 		"All files (*.*)"),
 		Q_NULLPTR, QFileDialog::DontResolveSymlinks);
 
@@ -1529,7 +1530,18 @@ void main_window::HandlePupInstallation(const QString& file_path, const QString&
 		{
 			for (const auto& update_filename : update_filenames)
 			{
-				fs::file update_file = update_files.get_file(update_filename);
+				auto update_file_stream = update_files.get_file(update_filename);
+
+				if (update_file_stream->m_file_handler)
+				{
+					// Forcefully read all the data
+					update_file_stream->pop<char>();
+					update_file_stream->pos = umax;
+					update_file_stream->pos /= 2; // Avoid internal overflows
+					update_file_stream->m_file_handler->handle_file_op(*update_file_stream, update_file_stream->pos, 1, nullptr);
+				}
+
+				fs::file update_file = fs::make_stream(std::move(update_file_stream->data));
 
 				SCEDecrypter self_dec(update_file);
 				self_dec.LoadHeaders();
@@ -3558,7 +3570,7 @@ main_window::drop_type main_window::IsValidFile(const QMimeData& md, QStringList
 				type = drop_type::drop_rrc;
 			}
 			// The emulator allows to execute ANY filetype, just not from drag-and-drop because it is confusing to users
-			else if (suffix_lo == "savestat" || suffix_lo == "sprx" || suffix_lo == "self" || suffix_lo == "bin" || suffix_lo == "prx" || suffix_lo == "elf" || suffix_lo == "o")
+			else if (path.toLower().endsWith(".savestat.gz") || suffix_lo == "savestat" || suffix_lo == "sprx" || suffix_lo == "self" || suffix_lo == "bin" || suffix_lo == "prx" || suffix_lo == "elf" || suffix_lo == "o")
 			{
 				type = drop_type::drop_game;
 			}

rpcs3/util/serialization.hpp

@@ -39,8 +39,31 @@ namespace utils
 		serialization_file_handler() = default;
 		virtual ~serialization_file_handler() = default;
 
+		// Handle read/write operations
 		virtual bool handle_file_op(serial& ar, usz pos, usz size, const void* data = nullptr) = 0;
-		virtual usz get_size(const utils::serial& ar, usz recommended = umax) const = 0;
+
+		// Obtain data size (targets to be only higher than 'recommended' and thus may not be accurate)
+		virtual usz get_size(const utils::serial& /*ar*/, usz /*recommended*/) const
+		{
+			return 0;
+		}
+
+		// Skip reading some (compressed) data
+		virtual void skip_until(utils::serial& /*ar*/)
+		{
+		}
+
+		// Detect empty stream (TODO: Clean this, instead perhaps use a magic static representing empty stream)
+		virtual bool is_null() const
+		{
+			return false;
+		}
+
+		virtual void finalize_block(utils::serial& /*ar*/)
+		{
+		}
+
+		virtual void finalize(utils::serial&) = 0;
 	};
 
 	struct serial
@@ -48,12 +71,16 @@ namespace utils
 		std::vector<u8> data;
 		usz data_offset = 0;
 		usz pos = 0;
+		usz m_max_data = umax;
 		bool m_is_writing = true;
 		bool m_avoid_large_prefetch = false;
 		std::unique_ptr<serialization_file_handler> m_file_handler;
 
 		serial() noexcept = default;
 		serial(const serial&) = delete;
+		serial& operator=(const serial&) = delete;
+		explicit serial(serial&&) noexcept = default;
+		serial& operator=(serial&&) noexcept = default;
 		~serial() noexcept = default;
 
 		// Checks if this instance is currently used for serialization
@@ -80,27 +107,34 @@ namespace utils
 				return true;
 			}
 
+			if (m_file_handler && m_file_handler->is_null())
+			{
+				// Instead of doing nothing at all, increase pos so it would be possible to estimate memory requirements
+				pos += size;
+				return true;
+			}
+
 			// Overflow check
-			ensure(~pos >= size);
+			ensure(~pos >= size - 1);
 
 			if (is_writing())
 			{
 				ensure(pos >= data_offset);
 				const auto ptr = reinterpret_cast<const u8*>(memory_provider());
-				data.insert(data.begin() + pos - data_offset, ptr, ptr + size);
+				data.insert(data.begin() + (pos - data_offset), ptr, ptr + size);
 				pos += size;
 				return true;
 			}
 
-			if (data.empty() || pos < data_offset || pos + size > data.size() + data_offset)
+			if (data.empty() || pos < data_offset || pos + (size - 1) > (data.size() - 1) + data_offset)
 			{
 				// Load from file
 				ensure(m_file_handler);
 				ensure(m_file_handler->handle_file_op(*this, pos, size, nullptr));
-				ensure(!data.empty() && pos >= data_offset && pos + size <= data.size() + data_offset);
+				ensure(!data.empty() && pos >= data_offset && pos + (size - 1) <= (data.size() - 1) + data_offset);
 			}
 
-			std::memcpy(const_cast<void*>(static_cast<const void*>(memory_provider())), data.data() + pos - data_offset, size);
+			std::memcpy(const_cast<void*>(static_cast<const void*>(memory_provider())), data.data() + (pos - data_offset), size);
 			pos += size;
 			return true;
 		}
@@ -169,7 +203,8 @@ namespace utils
 		}
 
 		// std::vector, std::basic_string
-		template <typename T> requires FastRandomAccess<T> && ListAlike<T>
+		// Discourage using std::pair/tuple with vectors because it eliminates the possibility of bitwise optimization
+		template <typename T> requires FastRandomAccess<T> && ListAlike<T> && (!TupleAlike<typename T::value_type>)
 		bool serialize(T& obj)
 		{
 			if (is_writing())
@@ -194,6 +229,13 @@ namespace utils
 				return true;
 			}
 
+			obj.clear();
+
+			if (m_file_handler && m_file_handler->is_null())
+			{
+				return true;
+			}
+
 			usz size = 0;
 			if (!deserialize_vle(size))
 			{
@@ -211,7 +253,6 @@ namespace utils
 			else
 			{
 				// TODO: Postpone resizing to after file bounds checks
-				obj.clear();
 				obj.resize(size);
 
 				for (auto&& value : obj)
@@ -270,6 +311,11 @@ namespace utils
 
 			obj.clear();
 
+			if (m_file_handler && m_file_handler->is_null())
+			{
+				return true;
+			}
+
 			usz size = 0;
 			if (!deserialize_vle(size))
 			{
@@ -326,16 +372,23 @@ namespace utils
 		}
 
 		// Wrapper for serialize(T&), allows to pass multiple objects at once
-		template <typename... Args>
+		template <typename... Args> requires (sizeof...(Args) != 0)
-		bool operator()(Args&&... args)
+		bool operator()(Args&&... args) noexcept
 		{
 			return ((AUDIT(!std::is_const_v<std::remove_reference_t<Args>> || is_writing())
 				, serialize(const_cast<std::remove_cvref_t<Args>&>(static_cast<const Args&>(args)))), ...);
 		}
 
+		// Code style utility, for when utils::serial is a pointer for example
+		template <typename... Args> requires (sizeof...(Args) > 1 || !(std::is_convertible_v<Args&&, Args&> && ...))
+		bool serialize(Args&&... args)
+		{
+			return this->operator()(std::forward<Args>(args)...);
+		}
+
 		// Convert serialization manager to deserializion manager
 		// If no arg is provided reuse saved buffer
-		void set_reading_state(std::vector<u8>&& _data = std::vector<u8>{})
+		void set_reading_state(std::vector<u8>&& _data = std::vector<u8>{}, bool avoid_large_prefetch = false)
 		{
 			if (!_data.empty())
 			{
@@ -343,7 +396,7 @@ namespace utils
 			}
 
 			m_is_writing = false;
-			m_avoid_large_prefetch = false;
+			m_avoid_large_prefetch = avoid_large_prefetch;
 			pos = 0;
 			data_offset = 0;
 		}
@@ -365,15 +418,19 @@ namespace utils
 			return pos;
 		}
 
-		usz seek_pos(usz val, bool empty_data = false)
+		usz seek_pos(usz val, bool cleanup = false)
 		{
 			const usz old_pos = std::exchange(pos, val);
 
-			if (empty_data || data.empty())
+			if (cleanup || data.empty())
 			{
 				// Relocate future data
-				data.clear();
+				if (m_file_handler)
-				data_offset = pos;
+				{
+					m_file_handler->skip_until(*this);
+				}
+
+				breathe();
 			}
 
 			return old_pos;
@@ -403,7 +460,7 @@ namespace utils
 
 		template <typename T> requires (std::is_copy_constructible_v<std::remove_const_t<T>>) && (std::is_constructible_v<std::remove_const_t<T>> || Bitcopy<std::remove_const_t<T>> ||
 			std::is_constructible_v<std::remove_const_t<T>, stx::exact_t<serial&>> || TupleAlike<std::remove_const_t<T>>)
-		operator T()
+		operator T() noexcept
 		{
 			AUDIT(!is_writing());
 
@@ -433,6 +490,13 @@ namespace utils
 			}
 		}
 
+		// Code style utility wrapper for operator T()
+		template <typename T>
+		T pop()
+		{
+			return this->operator T();
+		}
+
 		void swap_handler(serial& ar)
 		{
 			std::swap(ar.m_file_handler, this->m_file_handler);
@@ -440,7 +504,40 @@ namespace utils
 
 		usz get_size(usz recommended = umax) const
 		{
-			return m_file_handler ? m_file_handler->get_size(*this, recommended) : data_offset + data.size();
+			recommended = std::min<usz>(recommended, m_max_data);
+			return std::min<usz>(m_max_data, m_file_handler ? m_file_handler->get_size(*this, recommended) : data_offset + data.size());
+		}
+
+		template <typename T> requires (Bitcopy<T>)
+		usz predict_object_size(const T&)
+		{
+			return sizeof(T);
+		}
+
+		template <typename T> requires FastRandomAccess<T> && (!ListAlike<T>) && (!Bitcopy<T>)
+		usz predict_object_size(const T& obj)
+		{
+			return std::size(obj) * sizeof(obj[0]);
+		}
+
+		template <typename T> requires (std::is_copy_constructible_v<std::remove_reference_t<T>> && std::is_constructible_v<std::remove_reference_t<T>>)
+		usz try_read(T&& obj)
+		{
+			if (is_writing())
+			{
+				return 0;
+			}
+
+			const usz end_pos = pos + predict_object_size(std::forward<T>(obj));
+			const usz size = get_size(end_pos);
+
+			if (size >= end_pos)
+			{
+				serialize(std::forward<T>(obj));
+				return 0;
+			}
+
+			return end_pos - size;
 		}
 
 		template <typename T> requires (std::is_copy_constructible_v<T> && std::is_constructible_v<T> && Bitcopy<T>)
@@ -457,14 +554,27 @@ namespace utils
 
 			if (size >= end_pos)
 			{
-				u8 buf[sizeof(type)]{};
+				return {true, this->operator T()};
-				ensure(raw_serialize(buf, sizeof(buf)));
-				return {true, std::bit_cast<type>(buf)};
 			}
 
 			return {};
 		}
 
+		void patch_raw_data(usz pos, const void* data, usz size)
+		{
+			if (m_file_handler && m_file_handler->is_null())
+			{
+				return;
+			}
+
+			if (!size)
+			{
+				return;
+			}
+
+			std::memcpy(&::at32(this->data, pos - data_offset + size - 1) - (size - 1), data, size);
+		}
+
 		// Returns true if valid, can be invalidated by setting pos to umax
 		// Used when an invalid state is encountered somewhere in a place we can't check success code such as constructor)
 		bool is_valid() const