Crypto/PKG installer: Fix potential RAM shortage when extracing EDAT files

rpcs3/Crypto/decrypt_binaries.cpp

@@ -139,7 +139,16 @@ usz decrypt_binaries_t::decrypt(std::string klic_input)
 
 				if (fs::file new_file{new_path, fs::rewrite})
 				{
-					new_file.write(elf_file.to_string());
+					// 16MB buffer
+					std::vector<u8> buffer(std::min<usz>(elf_file.size(), 1u << 24));
+
+					elf_file.seek(0);
+
+					while (usz read_size = elf_file.read(buffer.data(), buffer.size()))
+					{
+						new_file.write(buffer.data(), read_size);
+					}
+
 					dec_log.success("Decrypted %s -> %s", old_path, new_path);
 					std::cout << "Decrypted " << old_path << " -> " << new_path << std::endl; // For CLI
 					m_index++;

rpcs3/Crypto/unedat.cpp

@@ -610,21 +610,25 @@ bool validate_dev_klic(const u8* klicensee, NPD_HEADER *npd)
 	return cmac_hash_compare(reinterpret_cast<uchar*>(&key), 0x10, dev, 0x60, npd->dev_hash, 0x10);
 }
 
-bool validate_npd_hashes(const char* file_name, const u8* klicensee, NPD_HEADER *npd, EDAT_HEADER* edat, bool verbose)
+bool validate_npd_hashes(std::string_view file_name, const u8* klicensee, NPD_HEADER* npd, EDAT_HEADER* edat, bool verbose)
 {
-	if (!file_name)
-	{
-		fmt::throw_exception("Empty filename");
-	}
-
 	// Ignore header validation in DEBUG data.
 	if (edat->flags & EDAT_DEBUG_DATA_FLAG)
 	{
 		return true;
 	}
 
-	const usz file_name_length = std::strlen(file_name);
+	if (!validate_dev_klic(klicensee, npd))
-	const usz buf_len = 0x30 + file_name_length;
+	{
+		return false;
+	}
+
+	if (file_name.empty())
+	{
+		return true;
+	}
+
+	const usz buf_len = 0x30 + file_name.size();
 
 	std::unique_ptr<u8[]> buf(new u8[buf_len]);
 	std::unique_ptr<u8[]> buf_lower(new u8[buf_len]);
@@ -632,12 +636,12 @@ bool validate_npd_hashes(const char* file_name, const u8* klicensee, NPD_HEADER
 
 	// Build the title buffer (content_id + file_name).
 	std::memcpy(buf.get(), npd->content_id, 0x30);
-	std::memcpy(buf.get() + 0x30, file_name, file_name_length);
+	std::memcpy(buf.get() + 0x30, file_name.data(), file_name.size());
 
 	std::memcpy(buf_lower.get(), buf.get(), buf_len);
 	std::memcpy(buf_upper.get(), buf.get(), buf_len);
 
-	for (usz i = std::basic_string_view<u8>(buf.get() + 0x30, file_name_length).find_last_of('.'); i < buf_len; i++)
+	for (usz i = std::basic_string_view<u8>(buf.get() + 0x30, file_name.size()).find_last_of('.'); i < buf_len; i++)
 	{
 		const u8 c = static_cast<u8>(buf[i]);
 		buf_upper[i] = std::toupper(c);
@@ -659,17 +663,17 @@ bool validate_npd_hashes(const char* file_name, const u8* klicensee, NPD_HEADER
 			edat_log.warning("NPD title hash is invalid!");
 	}
 
-	const bool dev_hash_result = validate_dev_klic(klicensee, npd);
+	return title_hash_result;
-
-	return title_hash_result && dev_hash_result;
 }
 
 void read_npd_edat_header(const fs::file* input, NPD_HEADER& NPD, EDAT_HEADER& EDAT)
 {
-	char npd_header[0x80];
+	char npd_header[0x80]{};
-	char edat_header[0x10];
+	char edat_header[0x10]{};
-	input->read(npd_header, sizeof(npd_header));
+
-	input->read(edat_header, sizeof(edat_header));
+	usz pos = 0;
+	pos = input->read_at(pos, npd_header, sizeof(npd_header));
+	input->read_at(pos, edat_header, sizeof(edat_header));
 
 	std::memcpy(&NPD.magic, npd_header, 4);
 	NPD.version = read_from_ptr<be_t<s32>>(npd_header, 4);
@@ -739,7 +743,7 @@ bool extract_all_data(const fs::file* input, const fs::file* output, const char*
 		}
 
 		// Perform header validation (EDAT only).
-		char real_file_name[CRYPTO_MAX_PATH];
+		char real_file_name[CRYPTO_MAX_PATH]{};
 		extract_file_name(input_file_name, real_file_name);
 		if (!validate_npd_hashes(real_file_name, devklic, &NPD, &EDAT, verbose))
 		{
@@ -839,7 +843,7 @@ bool VerifyEDATHeaderWithKLicense(const fs::file& input, const std::string& inpu
 	}
 
 	// Perform header validation (EDAT only).
-	char real_file_name[CRYPTO_MAX_PATH];
+	char real_file_name[CRYPTO_MAX_PATH]{};
 	extract_file_name(input_file_name.c_str(), real_file_name);
 	if (!validate_npd_hashes(real_file_name, custom_klic, &NPD, &EDAT, false))
 	{
@@ -915,20 +919,21 @@ fs::file DecryptEDAT(const fs::file& input, const std::string& input_file_name,
 	}
 
 	// Delete the bad output file if any errors arise.
-	fs::file output = fs::make_stream<std::vector<u8>>();
+	auto data = std::make_unique<EDATADecrypter>(input, devklic, input_file_name, false);
-	if (extract_all_data(&input, &output, input_file_name.c_str(), reinterpret_cast<uchar*>(&devklic), verbose))
+
+	if (!data->ReadHeader())
 	{
-		output.release();
 		return fs::file{};
 	}
 
-	output.seek(0);
+	fs::file output;
+	output.reset(std::move(data));
+
 	return output;
 }
 
 bool EDATADecrypter::ReadHeader()
 {
-	edata_file.seek(0);
 	// Read in the NPD and EDAT/SDAT headers.
 	read_npd_edat_header(&edata_file, npdHeader, edatHeader);
 
@@ -945,14 +950,45 @@ bool EDATADecrypter::ReadHeader()
 	}
 	else
 	{
-		// verify key
+		// extract key from RIF
-		if (!validate_dev_klic(reinterpret_cast<uchar*>(&dec_key), &npdHeader))
+		char real_file_name[CRYPTO_MAX_PATH]{};
+		extract_file_name(m_file_name.c_str(), real_file_name);
+
+		if (!validate_npd_hashes(real_file_name, reinterpret_cast<const u8*>(&dec_key), &npdHeader, &edatHeader, false))
 		{
-			edat_log.error("Failed validating klic");
+			edat_log.error("NPD hash validation failed!");
-			return false;
+			return true;
 		}
 
-		// Use provided dec_key
+		// Select EDAT key.
+		if (m_is_key_final)
+		{
+			// Already provided
+		}
+		// Type 3: Use supplied dec_key.
+		else if ((npdHeader.license & 0x3) == 0x3)
+		{
+			//
+		}
+		// Type 2: Use key from RAP file (RIF key). (also used for type 1 at the moment)
+		else 
+		{
+			const std::string rap_path = rpcs3::utils::get_rap_file_path(npdHeader.content_id);
+
+			if (fs::file rap{rap_path}; rap && rap.size() >= sizeof(dec_key))
+			{
+				dec_key = GetEdatRifKeyFromRapFile(rap);
+			}
+
+			// Make sure we don't have an empty RIF key.
+			if (!dec_key)
+			{
+				edat_log.error("A valid RAP file is needed for this EDAT file! (license=%d)", npdHeader.license);
+				return true;
+			}
+
+			edat_log.trace("RIFKEY: %s", std::bit_cast<be_t<u128>>(dec_key));
+		}
 	}
 
 	edata_file.seek(0);

rpcs3/Crypto/unedat.h

@@ -71,7 +71,10 @@ u128 GetEdatRifKeyFromRapFile(const fs::file& rap_file);
 struct EDATADecrypter final : fs::file_base
 {
 	// file stream
-	fs::file edata_file;
+	fs::file m_edata_file;
+	const fs::file& edata_file;
+	std::string m_file_name;
+	bool m_is_key_final = true;
 	u64 file_size{0};
 	u32 total_blocks{0};
 	u64 pos{0};
@@ -82,8 +85,20 @@ struct EDATADecrypter final : fs::file_base
 	u128 dec_key{};
 
 public:
-	EDATADecrypter(fs::file&& input, u128 dec_key = {})
+	EDATADecrypter(fs::file&& input, u128 dec_key = {}, std::string file_name = {}, bool is_key_final = true) noexcept
-		: edata_file(std::move(input))
+		: m_edata_file(std::move(input))
+		, edata_file(m_edata_file)
+		, m_file_name(std::move(file_name))
+		, m_is_key_final(is_key_final)
+		, dec_key(dec_key)
+	{
+	}
+
+	EDATADecrypter(const fs::file& input, u128 dec_key = {}, std::string file_name = {}, bool is_key_final = true) noexcept
+		: m_edata_file(fs::file{})
+		, edata_file(input)
+		, m_file_name(std::move(file_name))
+		, m_is_key_final(is_key_final)
 		, dec_key(dec_key)
 	{
 	}

rpcs3/Crypto/unpkg.cpp

@@ -866,6 +866,8 @@ fs::file DecryptEDAT(const fs::file& input, const std::string& input_file_name,
 
 void package_reader::extract_worker(thread_key thread_data_key)
 {
+	std::vector<u8> read_cache;
+
 	while (m_num_failures == 0 && !m_aborted)
 	{
 		// Make sure m_entry_indexer does not exceed m_install_entries
@@ -941,11 +943,18 @@ void package_reader::extract_worker(thread_key thread_data_key)
 				pkg_log.warning("NPDRM EDAT!");
 			}
 
-			if (fs::file out = is_buffered ? fs::make_stream<std::vector<u8>>() : fs::file{ path, did_overwrite ? fs::rewrite : fs::write_new })
+			if (fs::file out{ path, did_overwrite ? fs::rewrite : fs::write_new })
 			{
 				bool extract_success = true;
-				for (u64 pos = 0; pos < entry.file_size; pos += BUF_SIZE)
+
+				auto read_op = [&](usz pos, usz size) -> std::span<const char>
 				{
+					if (pos >= entry.file_size)
+					{
+						return {};
+					}
+
+					// Because that is the length of the buffer at the moment
 					const u64 block_size = std::min<u64>(BUF_SIZE, entry.file_size - pos);
 
 					const std::span<const char> data_span = decrypt(entry.file_offset + pos, block_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data(), thread_data_key);
@@ -954,29 +963,196 @@ void package_reader::extract_worker(thread_key thread_data_key)
 					{
 						extract_success = false;
 						pkg_log.error("Failed to extract file %s (data_span.size=%d, block_size=%d)", path, data_span.size(), block_size);
-						break;
 					}
 
-					if (out.write(data_span.data(), block_size) != block_size)
+					return data_span;
+				};
+
+				struct pkg_file_reader : fs::file_base
+				{
+					const std::function<u64(u64, void*, u64)> m_read_func;
+					const install_entry& m_entry;
+					usz m_pos;
+
+					explicit pkg_file_reader(std::function<u64(u64, void* buffer, u64)> read_func, const install_entry& entry) noexcept
+						: m_read_func(std::move(read_func))
+						, m_entry(entry)
+						, m_pos(0)
 					{
-						extract_success = false;
-						pkg_log.error("Failed to write file %s (error=%s)", path, fs::g_tls_error);
-						break;
 					}
 
-					m_written_bytes += block_size;
+					fs::stat_t get_stat() override
-				}
+					{
+						fs::stat_t stat{};
+						stat.size = m_entry.file_size;
+						return stat;
+					}
+
+					bool trunc(u64) override
+					{
+						return false;
+					}
+
+					u64 read(void* buffer, u64 size) override
+					{
+						const u64 result = pkg_file_reader::read_at(m_pos, buffer, size);
+						m_pos += result;
+						return result;
+					}
+
+					u64 read_at(u64 offset, void* buffer, u64 size) override
+					{
+						return m_read_func(offset, buffer, size);
+					}
+
+					u64 write(const void*, u64) override
+					{
+						return 0;
+					}
+
+					u64 seek(s64 offset, fs::seek_mode whence) override
+					{
+						const s64 new_pos =
+							whence == fs::seek_set ? offset :
+							whence == fs::seek_cur ? offset + m_pos :
+							whence == fs::seek_end ? offset + size() : -1;
+
+						if (new_pos < 0)
+						{
+							fs::g_tls_error = fs::error::inval;
+							return -1;
+						}
+
+						m_pos = new_pos;
+						return m_pos;
+					}
+
+					u64 size() override
+					{
+						return m_entry.file_size;
+					}
+
+					fs::file_id get_id() override
+					{
+						fs::file_id id{};
+
+						id.type.insert(0, "pkg_file_reader: "sv);
+						return id;
+					}
+				};
+
+				read_cache.clear();
+
+				auto reader = std::make_unique<pkg_file_reader>([&, cache_off = u64{umax}](usz pos, void* ptr, usz size) mutable -> u64
+				{
+					if (pos >= entry.file_size || !size)
+					{
+						return 0;
+					}
+
+					size = std::min<u64>(entry.file_size - pos, size);
+
+					u64 size_cache_end = 0;
+					u64 read_size = 0;
+
+					// Check if exists in cache
+					if (!read_cache.empty() && cache_off <= pos && pos < cache_off + read_cache.size())
+					{
+						read_size = std::min<u64>(pos + size, cache_off + read_cache.size()) - pos;
+
+						std::memcpy(ptr, read_cache.data() + (pos - cache_off), read_size);
+						pos += read_size;
+					}
+					else if (!read_cache.empty() && cache_off < pos + size && cache_off + read_cache.size() >= pos + size)
+					{
+						size_cache_end = size - (std::max<u64>(cache_off, pos) - pos);
+
+						std::memcpy(static_cast<u8*>(ptr) + (cache_off - pos), read_cache.data(), size_cache_end);
+						size -= size_cache_end;
+					}
+
+					if (pos >= entry.file_size || !size)
+					{
+						return read_size + size_cache_end;
+					}
+
+					// Try to cache for later
+					if (size <= BUF_SIZE && !size_cache_end && !read_size)
+					{
+						const u64 block_size = std::min<u64>({BUF_SIZE, std::max<u64>(size * 5 / 3, 65536), entry.file_size - pos});
+
+						read_cache.resize(block_size);
+						cache_off = pos;
+
+						const std::span<const char> data_span = decrypt(entry.file_offset + pos, block_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data(), thread_data_key);
+
+						if (data_span.empty())
+						{
+							cache_off = umax;
+							read_cache.clear();
+							return 0;
+						}
+
+						read_cache.resize(data_span.size());
+						std::memcpy(read_cache.data(), data_span.data(), data_span.size());
+
+						size = std::min<usz>(data_span.size(), size);
+						std::memcpy(ptr, data_span.data(), size);
+						return size;
+					}
+
+					while (read_size < size)
+					{
+						const u64 block_size = std::min<u64>(BUF_SIZE, size - read_size);
+
+						const std::span<const char> data_span = decrypt(entry.file_offset + pos, block_size, is_psp ? PKG_AES_KEY2 : m_dec_key.data(), thread_data_key);
+
+						if (data_span.empty())
+						{
+							break;
+						}
+
+						std::memcpy(static_cast<u8*>(ptr) + read_size, data_span.data(), data_span.size());
+
+						read_size += data_span.size();
+						pos += data_span.size();
+					}
+
+					return read_size + size_cache_end;
+				}, entry);
+
+				fs::file in_data;
+				in_data.reset(std::move(reader));
+
+				fs::file final_data;
 
 				if (is_buffered)
 				{
-					out = DecryptEDAT(out, name, 1, reinterpret_cast<u8*>(&m_header.klicensee), true);
+					final_data = DecryptEDAT(in_data, name, 1, reinterpret_cast<u8*>(&m_header.klicensee), true);
-					if (!out || !fs::write_file(path, fs::rewrite, static_cast<fs::container_stream<std::vector<u8>>*>(out.release().get())->obj))
-					{
-						m_num_failures++;
-						pkg_log.error("Failed to create file %s (error=%s)", path, fs::g_tls_error);
-						break;
-					}
 				}
+				else
+				{
+					final_data = std::move(in_data);
+				}
+
+				if (!final_data)
+				{
+					m_num_failures++;
+					pkg_log.error("Failed to decrypt EDAT file %s (error=%s)", path, fs::g_tls_error);
+					break;
+				}
+
+				// 16MB buffer
+				std::vector<u8> buffer(std::min<usz>(entry.file_size, 1u << 24));
+
+				while (usz read_size = final_data.read(buffer.data(), buffer.size()))
+				{
+					out.write(buffer.data(), read_size);
+					m_written_bytes += read_size;
+				}
+
+				final_data.close();
+				out.close();
 
 				if (extract_success)
 				{

rpcs3/Emu/Cell/lv2/sys_fs.cpp

@@ -915,7 +915,7 @@ lv2_file::open_raw_result_t lv2_file::open_raw(const std::string& local_path, s3
 					if (!edata_file->ReadHeader())
 					{
 						// Prepare file for the next iteration
-						file = std::move(edata_file->edata_file);
+						file = std::move(edata_file->m_edata_file);
 						continue;
 					}