Add support for WUHB file format (#1190)

src/Cafe/Filesystem/WUHB/RomFSStructs.h

@@ -0,0 +1,40 @@
+#pragma once
+
+struct romfs_header_t
+{
+	uint32 header_magic;
+	uint32be header_size;
+	uint64be dir_hash_table_ofs;
+	uint64be dir_hash_table_size;
+	uint64be dir_table_ofs;
+	uint64be dir_table_size;
+	uint64be file_hash_table_ofs;
+	uint64be file_hash_table_size;
+	uint64be file_table_ofs;
+	uint64be file_table_size;
+	uint64be file_partition_ofs;
+};
+
+struct romfs_direntry_t
+{
+	uint32be parent;
+	uint32be listNext;	   // offset to next directory entry in linked list of parent directory (aka "sibling")
+	uint32be dirListHead;  // offset to first entry in linked list of directory entries (aka "child")
+	uint32be fileListHead; // offset to first entry in linked list of file entries (aka "file")
+	uint32be hash;
+	uint32be name_size;
+	std::string name;
+};
+
+struct romfs_fentry_t
+{
+	uint32be parent;
+	uint32be listNext; // offset to next file entry in linked list of parent directory (aka "sibling")
+	uint64be offset;
+	uint64be size;
+	uint32be hash;
+	uint32be name_size;
+	std::string name;
+};
+
+#define ROMFS_ENTRY_EMPTY 0xFFFFFFFF

src/Cafe/Filesystem/WUHB/WUHBReader.cpp

@@ -0,0 +1,224 @@
+#include "WUHBReader.h"
+WUHBReader* WUHBReader::FromPath(const fs::path& path)
+{
+	FileStream* fileIn{FileStream::openFile2(path)};
+	if (!fileIn)
+		return nullptr;
+
+	WUHBReader* ret = new WUHBReader(fileIn);
+	if (!ret->CheckMagicValue())
+	{
+		delete ret;
+		return nullptr;
+	}
+
+	if (!ret->ReadHeader())
+	{
+		delete ret;
+		return nullptr;
+	}
+
+	return ret;
+}
+
+static const romfs_direntry_t fallbackDirEntry{
+	.parent = ROMFS_ENTRY_EMPTY,
+	.listNext = ROMFS_ENTRY_EMPTY,
+	.dirListHead = ROMFS_ENTRY_EMPTY,
+	.fileListHead = ROMFS_ENTRY_EMPTY,
+	.hash = ROMFS_ENTRY_EMPTY,
+	.name_size = 0,
+	.name = ""
+};
+static const romfs_fentry_t fallbackFileEntry{
+	.parent = ROMFS_ENTRY_EMPTY,
+	.listNext = ROMFS_ENTRY_EMPTY,
+	.offset = 0,
+	.size = 0,
+	.hash = ROMFS_ENTRY_EMPTY,
+	.name_size = 0,
+	.name = ""
+};
+template<bool File>
+const WUHBReader::EntryType<File>& WUHBReader::GetFallback()
+{
+	if constexpr (File)
+		return fallbackFileEntry;
+	else
+		return fallbackDirEntry;
+}
+
+template<bool File>
+WUHBReader::EntryType<File> WUHBReader::GetEntry(uint32 offset) const
+{
+	auto fallback = GetFallback<File>();
+	if(offset == ROMFS_ENTRY_EMPTY)
+		return fallback;
+
+	const char* typeName = File ? "fentry" : "direntry";
+	EntryType<File> ret;
+	if (offset >= (File ? m_header.file_table_size : m_header.dir_table_size))
+	{
+		cemuLog_log(LogType::Force, "WUHB {} offset exceeds table size declared in header", typeName);
+		return fallback;
+	}
+
+	// read the entry
+	m_fileIn->SetPosition((File ? m_header.file_table_ofs : m_header.dir_table_ofs) + offset);
+	auto read = m_fileIn->readData(&ret, offsetof(EntryType<File>, name));
+	if (read != offsetof(EntryType<File>, name))
+	{
+		cemuLog_log(LogType::Force, "failed to read WUHB {} at offset: {}", typeName, offset);
+		return fallback;
+	}
+
+	// read the name
+	ret.name.resize(ret.name_size);
+	read = m_fileIn->readData(ret.name.data(), ret.name_size);
+	if (read != ret.name_size)
+	{
+		cemuLog_log(LogType::Force, "failed to read WUHB {} name", typeName);
+		return fallback;
+	}
+
+	return ret;
+}
+
+romfs_direntry_t WUHBReader::GetDirEntry(uint32 offset) const
+{
+	return GetEntry<false>(offset);
+}
+romfs_fentry_t WUHBReader::GetFileEntry(uint32 offset) const
+{
+	return GetEntry<true>(offset);
+}
+
+uint64 WUHBReader::GetFileSize(uint32 entryOffset) const
+{
+	return GetFileEntry(entryOffset).size;
+}
+
+uint64 WUHBReader::ReadFromFile(uint32 entryOffset, uint64 fileOffset, uint64 length, void* buffer) const
+{
+	const auto fileEntry = GetFileEntry(entryOffset);
+	if (fileOffset >= fileEntry.size)
+		return 0;
+	const uint64 readAmount = std::min(length, fileEntry.size - fileOffset);
+	const uint64 wuhbOffset = m_header.file_partition_ofs + fileEntry.offset + fileOffset;
+	m_fileIn->SetPosition(wuhbOffset);
+	return m_fileIn->readData(buffer, readAmount);
+}
+
+uint32 WUHBReader::GetHashTableEntryOffset(uint32 hash, bool isFile) const
+{
+	const uint64 hash_table_size = (isFile ? m_header.file_hash_table_size : m_header.dir_hash_table_size);
+	const uint64 hash_table_ofs = (isFile ? m_header.file_hash_table_ofs : m_header.dir_hash_table_ofs);
+
+	const uint64 hash_table_entry_count = hash_table_size / sizeof(uint32);
+	const uint64 hash_table_entry_offset = hash_table_ofs + (hash % hash_table_entry_count) * sizeof(uint32);
+
+	m_fileIn->SetPosition(hash_table_entry_offset);
+	uint32 tableOffset;
+	if (!m_fileIn->readU32(tableOffset))
+	{
+		cemuLog_log(LogType::Force, "failed to read WUHB hash table entry at file offset: {}", hash_table_entry_offset);
+		return ROMFS_ENTRY_EMPTY;
+	}
+
+	return uint32be::from_bevalue(tableOffset);
+}
+
+template<bool T>
+bool WUHBReader::SearchHashList(uint32& entryOffset, const fs::path& targetName) const
+{
+	for (;;)
+	{
+		if (entryOffset == ROMFS_ENTRY_EMPTY)
+			return false;
+		auto entry = GetEntry<T>(entryOffset);
+
+		if (entry.name == targetName)
+			return true;
+		entryOffset = entry.hash;
+	}
+	return false;
+}
+
+uint32 WUHBReader::Lookup(const std::filesystem::path& path, bool isFile) const
+{
+	uint32 currentEntryOffset = 0;
+	auto look = [&](const fs::path& part, bool lookInFileHT) {
+		const auto partString = part.string();
+		currentEntryOffset = GetHashTableEntryOffset(CalcPathHash(currentEntryOffset, partString.c_str(), 0, partString.size()), lookInFileHT);
+		if (lookInFileHT)
+			return SearchHashList<true>(currentEntryOffset, part);
+		else
+			return SearchHashList<false>(currentEntryOffset, part);
+	};
+	// look for the root entry
+	if (!look("", false))
+		return ROMFS_ENTRY_EMPTY;
+
+	auto it = path.begin();
+	while (it != path.end())
+	{
+		fs::path part = *it;
+		++it;
+		// no need to recurse after trailing forward slash (e.g. directory/)
+		if (part.empty() && !isFile)
+			break;
+		// skip leading forward slash
+		if (part == "/")
+			continue;
+
+		// if the lookup target is a file and this is the last iteration, look in the file hash table instead.
+		if (!look(part, it == path.end() && isFile))
+			return ROMFS_ENTRY_EMPTY;
+	}
+	return currentEntryOffset;
+}
+bool WUHBReader::CheckMagicValue() const
+{
+	uint8 magic[4];
+	m_fileIn->SetPosition(0);
+	int read = m_fileIn->readData(magic, 4);
+	if (read != 4)
+	{
+		cemuLog_log(LogType::Force, "Failed to read WUHB magic numbers");
+		return false;
+	}
+	static_assert(sizeof(magic) == s_headerMagicValue.size());
+	return std::memcmp(&magic, s_headerMagicValue.data(), sizeof(magic)) == 0;
+}
+bool WUHBReader::ReadHeader()
+{
+	m_fileIn->SetPosition(0);
+	auto read = m_fileIn->readData(&m_header, sizeof(m_header));
+	auto readSuccess = read == sizeof(m_header);
+	if (!readSuccess)
+		cemuLog_log(LogType::Force, "Failed to read WUHB header");
+	return readSuccess;
+}
+unsigned char WUHBReader::NormalizeChar(unsigned char c)
+{
+	if (c >= 'a' && c <= 'z')
+	{
+		return c + 'A' - 'a';
+	}
+	else
+	{
+		return c;
+	}
+}
+uint32 WUHBReader::CalcPathHash(uint32 parent, const char* path, uint32 start, size_t path_len)
+{
+	cemu_assert(path != nullptr || path_len == 0);
+	uint32 hash = parent ^ 123456789;
+	for (uint32 i = 0; i < path_len; i++)
+	{
+		hash = (hash >> 5) | (hash << 27);
+		hash ^= NormalizeChar(path[start + i]);
+	}
+
+	return hash;
+}

src/Cafe/Filesystem/WUHB/WUHBReader.h

@@ -0,0 +1,45 @@
+#pragma once
+#include <Common/FileStream.h>
+#include "RomFSStructs.h"
+class WUHBReader
+{
+  public:
+	static WUHBReader* FromPath(const fs::path& path);
+
+	romfs_direntry_t GetDirEntry(uint32 offset) const;
+	romfs_fentry_t GetFileEntry(uint32 offset) const;
+
+	uint64 GetFileSize(uint32 entryOffset) const;
+
+	uint64 ReadFromFile(uint32 entryOffset, uint64 fileOffset, uint64 length, void* buffer) const;
+
+	uint32 Lookup(const std::filesystem::path& path, bool isFile) const;
+
+  private:
+	WUHBReader(FileStream* file)
+		: m_fileIn(file)
+	{
+		cemu_assert_debug(file != nullptr);
+	};
+	WUHBReader() = delete;
+
+	romfs_header_t m_header;
+	std::unique_ptr<FileStream> m_fileIn;
+	constexpr static std::string_view s_headerMagicValue = "WUHB";
+	bool ReadHeader();
+	bool CheckMagicValue() const;
+
+	static inline unsigned char NormalizeChar(unsigned char c);
+	static uint32 CalcPathHash(uint32 parent, const char* path, uint32 start, size_t path_len);
+
+	template<bool File>
+	using EntryType = std::conditional_t<File, romfs_fentry_t, romfs_direntry_t>;
+	template<bool File>
+	static const EntryType<File>& GetFallback();
+	template<bool File>
+	EntryType<File> GetEntry(uint32 offset) const;
+
+	template<bool T>
+	bool SearchHashList(uint32& entryOffset, const fs::path& targetName) const;
+	uint32 GetHashTableEntryOffset(uint32 hash, bool isFile) const;
+};

src/Cafe/Filesystem/fsc.h

@@ -204,6 +204,9 @@ bool FSCDeviceWUD_Mount(std::string_view mountPath, std::string_view destination
 // wua device
 bool FSCDeviceWUA_Mount(std::string_view mountPath, std::string_view destinationBaseDir, class ZArchiveReader* archive, sint32 priority);
 
+// wuhb device
+bool FSCDeviceWUHB_Mount(std::string_view mountPath, std::string_view destinationBaseDir, class WUHBReader* wuhbReader, sint32 priority);
+
 // hostFS device
 bool FSCDeviceHostFS_Mount(std::string_view mountPath, std::string_view hostTargetPath, sint32 priority);
 

src/Cafe/Filesystem/fscDeviceWuhb.cpp

@@ -0,0 +1,151 @@
+#include "Filesystem/WUHB/WUHBReader.h"
+#include "Cafe/Filesystem/fsc.h"
+#include "Cafe/Filesystem/FST/FST.h"
+
+class FSCDeviceWuhbFileCtx : public FSCVirtualFile
+{
+  public:
+	FSCDeviceWuhbFileCtx(WUHBReader* reader, uint32 entryOffset, uint32 fscType)
+		: m_wuhbReader(reader), m_entryOffset(entryOffset), m_fscType(fscType)
+	{
+		cemu_assert(entryOffset != ROMFS_ENTRY_EMPTY);
+		if (fscType == FSC_TYPE_DIRECTORY)
+		{
+			romfs_direntry_t entry = reader->GetDirEntry(entryOffset);
+			m_dirIterOffset = entry.dirListHead;
+			m_fileIterOffset = entry.fileListHead;
+		}
+	}
+	sint32 fscGetType() override
+	{
+		return m_fscType;
+	}
+	uint64 fscQueryValueU64(uint32 id) override
+	{
+		if (m_fscType == FSC_TYPE_FILE)
+		{
+			if (id == FSC_QUERY_SIZE)
+				return m_wuhbReader->GetFileSize(m_entryOffset);
+			else if (id == FSC_QUERY_WRITEABLE)
+				return 0; // WUHB images are read-only
+			else
+				cemu_assert_error();
+		}
+		else
+		{
+			cemu_assert_unimplemented();
+		}
+		return 0;
+	}
+	uint32 fscWriteData(void* buffer, uint32 size) override
+	{
+		cemu_assert_error();
+		return 0;
+	}
+	uint32 fscReadData(void* buffer, uint32 size) override
+	{
+		if (m_fscType != FSC_TYPE_FILE)
+			return 0;
+		auto read = m_wuhbReader->ReadFromFile(m_entryOffset, m_seek, size, buffer);
+		m_seek += read;
+		return read;
+	}
+	void fscSetSeek(uint64 seek) override
+	{
+		m_seek = seek;
+	}
+	uint64 fscGetSeek() override
+	{
+		if (m_fscType != FSC_TYPE_FILE)
+			return 0;
+		return m_seek;
+	}
+	void fscSetFileLength(uint64 endOffset) override
+	{
+		cemu_assert_error();
+	}
+	bool fscDirNext(FSCDirEntry* dirEntry) override
+	{
+		if (m_dirIterOffset != ROMFS_ENTRY_EMPTY)
+		{
+			romfs_direntry_t entry = m_wuhbReader->GetDirEntry(m_dirIterOffset);
+			m_dirIterOffset = entry.listNext;
+			if(entry.name_size > 0)
+			{
+				dirEntry->isDirectory = true;
+				dirEntry->isFile = false;
+				dirEntry->fileSize = 0;
+				std::strncpy(dirEntry->path, entry.name.c_str(), FSC_MAX_DIR_NAME_LENGTH);
+				return true;
+			}
+		}
+		if (m_fileIterOffset != ROMFS_ENTRY_EMPTY)
+		{
+			romfs_fentry_t entry = m_wuhbReader->GetFileEntry(m_fileIterOffset);
+			m_fileIterOffset = entry.listNext;
+			if(entry.name_size > 0)
+			{
+				dirEntry->isDirectory = false;
+				dirEntry->isFile = true;
+				dirEntry->fileSize = entry.size;
+				std::strncpy(dirEntry->path, entry.name.c_str(), FSC_MAX_DIR_NAME_LENGTH);
+				return true;
+			}
+		}
+
+		return false;
+	}
+
+  private:
+	WUHBReader* m_wuhbReader{};
+	uint32 m_fscType;
+	uint32 m_entryOffset = ROMFS_ENTRY_EMPTY;
+	uint32 m_dirIterOffset = ROMFS_ENTRY_EMPTY;
+	uint32 m_fileIterOffset = ROMFS_ENTRY_EMPTY;
+	uint64 m_seek = 0;
+};
+
+class fscDeviceWUHB : public fscDeviceC
+{
+	FSCVirtualFile* fscDeviceOpenByPath(std::string_view path, FSC_ACCESS_FLAG accessFlags, void* ctx, sint32* fscStatus) override
+	{
+		WUHBReader* reader = (WUHBReader*)ctx;
+		cemu_assert_debug(!HAS_FLAG(accessFlags, FSC_ACCESS_FLAG::WRITE_PERMISSION)); // writing to WUHB is not supported
+
+		bool isFile;
+		uint32 table_offset = ROMFS_ENTRY_EMPTY;
+
+		if (table_offset == ROMFS_ENTRY_EMPTY && HAS_FLAG(accessFlags, FSC_ACCESS_FLAG::OPEN_DIR))
+		{
+			table_offset = reader->Lookup(path, false);
+			isFile = false;
+		}
+		if (table_offset == ROMFS_ENTRY_EMPTY && HAS_FLAG(accessFlags, FSC_ACCESS_FLAG::OPEN_FILE))
+		{
+			table_offset = reader->Lookup(path, true);
+			isFile = true;
+		}
+
+		if (table_offset == ROMFS_ENTRY_EMPTY)
+		{
+			*fscStatus = FSC_STATUS_FILE_NOT_FOUND;
+			return nullptr;
+		}
+
+		*fscStatus = FSC_STATUS_OK;
+		return new FSCDeviceWuhbFileCtx(reader, table_offset, isFile ? FSC_TYPE_FILE : FSC_TYPE_DIRECTORY);
+	}
+
+	// singleton
+  public:
+	static fscDeviceWUHB& instance()
+	{
+		static fscDeviceWUHB _instance;
+		return _instance;
+	}
+};
+
+bool FSCDeviceWUHB_Mount(std::string_view mountPath, std::string_view destinationBaseDir, WUHBReader* wuhbReader, sint32 priority)
+{
+	return fsc_mount(mountPath, destinationBaseDir, &fscDeviceWUHB::instance(), wuhbReader, priority) == FSC_STATUS_OK;
+}