The rest

Utilities/Atomic.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "types.h"
+#include "Platform.h"
 
 // Helper class, provides access to compiler-specific atomic intrinsics
 template<typename T, std::size_t Size>
@@ -744,9 +745,9 @@ public:
 
 		while (true)
 		{
-			func(_new = old, args...);
+			func((_new = old), args...);
 
-			if (atomic_storage<type>::compare_exchange(m_data, old, _new)) return old;
+			if (LIKELY(atomic_storage<type>::compare_exchange(m_data, old, _new))) return old;
 		}
 	}
 
@@ -765,9 +766,9 @@ public:
 
 		while (true)
 		{
-			func(_new = old, args...);
+			func((_new = old), args...);
 
-			if (atomic_storage<type>::compare_exchange(m_data, old, _new)) return _new;
+			if (LIKELY(atomic_storage<type>::compare_exchange(m_data, old, _new))) return _new;
 		}
 	}
 
@@ -786,9 +787,9 @@ public:
 
 		while (true)
 		{
-			RT&& result = func(_new = old, args...);
+			RT&& result = func((_new = old), args...);
 
-			if (atomic_storage<type>::compare_exchange(m_data, old, _new)) return std::move(result);
+			if (LIKELY(atomic_storage<type>::compare_exchange(m_data, old, _new))) return std::move(result);
 		}
 	}
 
@@ -800,9 +801,9 @@ public:
 
 		while (true)
 		{
-			func(_new = old, args...);
+			func((_new = old), args...);
 
-			if (atomic_storage<type>::compare_exchange(m_data, old, _new)) return;
+			if (LIKELY(atomic_storage<type>::compare_exchange(m_data, old, _new))) return;
 		}
 	}
 

Utilities/AutoPause.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <unordered_set>
+
 // Regarded as a Debugger Enchantment
 namespace debug
 {

Utilities/Config.cpp

@@ -5,7 +5,7 @@
 
 namespace cfg
 {
-	_log::channel cfg("CFG", _log::level::notice);
+	logs::channel cfg("CFG", logs::level::notice);
 
 	entry_base::entry_base(type _type)
 		: m_type(_type)

Utilities/Config.h

@@ -179,7 +179,7 @@ namespace cfg
 			for (const auto& v : init)
 			{
 				// Ensure elements are unique
-				ASSERT(map.emplace(v.first, v.second).second);
+				VERIFY(map.emplace(v.first, v.second).second);
 			}
 
 			return map;
@@ -529,4 +529,4 @@ namespace cfg
 }
 
 // Registered log channel
-#define LOG_CHANNEL(name) _log::channel name(#name, _log::level::notice); namespace _log { cfg::enum_entry<_log::level, true> name(cfg::root.log, #name, ::name.enabled); }
+#define LOG_CHANNEL(name) extern logs::channel name; namespace logs { static cfg::enum_entry<logs::level, true> name(cfg::root.log, #name, ::name.enabled); }

Utilities/File.cpp

@@ -69,6 +69,19 @@ static time_t to_time(const FILETIME& ft)
 	return to_time(v);
 }
 
+static fs::error to_error(DWORD e)
+{
+	switch (e)
+	{
+	case ERROR_FILE_NOT_FOUND: return fs::error::noent;
+	case ERROR_PATH_NOT_FOUND: return fs::error::noent;
+	case ERROR_ALREADY_EXISTS: return fs::error::exist;
+	case ERROR_FILE_EXISTS: return fs::error::exist;
+	case ERROR_NEGATIVE_SEEK: return fs::error::inval;
+	default: throw fmt::exception("Unknown Win32 error: %u.", e);
+	}
+}
+
 #else
 
 #include <sys/mman.h>
@@ -87,11 +100,22 @@ static time_t to_time(const FILETIME& ft)
 #include <sys/sendfile.h>
 #endif
 
+static fs::error to_error(int e)
+{
+	switch (e)
+	{
+	case ENOENT: return fs::error::noent;
+	case EEXIST: return fs::error::exist;
+	case EINVAL: return fs::error::inval;
+	default: throw fmt::exception("Unknown system error: %d.", e);
+	}
+}
+
 #endif
 
 namespace fs
 {
-	thread_local uint error = 0;
+	thread_local error g_tls_error = error::ok;
 
 	class device_manager final
 	{
@@ -146,7 +170,7 @@ std::shared_ptr<fs::device_base> fs::get_virtual_device(const std::string& path)
 
 std::shared_ptr<fs::device_base> fs::set_virtual_device(const std::string& name, const std::shared_ptr<device_base>& device)
 {
-	Expects(name.size() > 2 && name[0] == '/' && name[1] == '/' && name.find('/', 2) == -1);
+	EXPECTS(name.size() > 2 && name[0] == '/' && name[1] == '/' && name.find('/', 2) == -1);
 
 	return get_device_manager().set_device(name, device);
 }
@@ -178,26 +202,26 @@ std::string fs::get_parent_dir(const std::string& path)
 static const auto test_get_parent_dir = []() -> bool
 {
 	// Success:
-	ASSERT(fs::get_parent_dir("/x/y///") == "/x");
-	ASSERT(fs::get_parent_dir("/x/y/") == "/x");
-	ASSERT(fs::get_parent_dir("/x/y") == "/x");
-	ASSERT(fs::get_parent_dir("x:/y") == "x:");
-	ASSERT(fs::get_parent_dir("//x/y") == "//x");
+	VERIFY(fs::get_parent_dir("/x/y///") == "/x");
+	VERIFY(fs::get_parent_dir("/x/y/") == "/x");
+	VERIFY(fs::get_parent_dir("/x/y") == "/x");
+	VERIFY(fs::get_parent_dir("x:/y") == "x:");
+	VERIFY(fs::get_parent_dir("//x/y") == "//x");
 
 	// Failure:
-	ASSERT(fs::get_parent_dir("").empty());
-	ASSERT(fs::get_parent_dir("x/").empty());
-	ASSERT(fs::get_parent_dir("x").empty());
-	ASSERT(fs::get_parent_dir("x///").empty());
-	ASSERT(fs::get_parent_dir("/x/").empty());
-	ASSERT(fs::get_parent_dir("/x").empty());
-	ASSERT(fs::get_parent_dir("/").empty());
-	ASSERT(fs::get_parent_dir("//").empty());
-	ASSERT(fs::get_parent_dir("//x").empty());
-	ASSERT(fs::get_parent_dir("//x/").empty());
-	ASSERT(fs::get_parent_dir("///").empty());
-	ASSERT(fs::get_parent_dir("///x").empty());
-	ASSERT(fs::get_parent_dir("///x/").empty());
+	VERIFY(fs::get_parent_dir("").empty());
+	VERIFY(fs::get_parent_dir("x/").empty());
+	VERIFY(fs::get_parent_dir("x").empty());
+	VERIFY(fs::get_parent_dir("x///").empty());
+	VERIFY(fs::get_parent_dir("/x/").empty());
+	VERIFY(fs::get_parent_dir("/x").empty());
+	VERIFY(fs::get_parent_dir("/").empty());
+	VERIFY(fs::get_parent_dir("//").empty());
+	VERIFY(fs::get_parent_dir("//x").empty());
+	VERIFY(fs::get_parent_dir("//x/").empty());
+	VERIFY(fs::get_parent_dir("///").empty());
+	VERIFY(fs::get_parent_dir("///x").empty());
+	VERIFY(fs::get_parent_dir("///x/").empty());
 
 	return false;
 }();
@@ -213,14 +237,7 @@ bool fs::stat(const std::string& path, stat_t& info)
 	WIN32_FILE_ATTRIBUTE_DATA attrs;
 	if (!GetFileAttributesExW(to_wchar(path).get(), GetFileExInfoStandard, &attrs))
 	{
-		// TODO: convert Win32 error code to errno
-		switch (DWORD error = GetLastError())
-		{
-		case ERROR_FILE_NOT_FOUND: fs::error = ENOENT; break;
-		case ERROR_PATH_NOT_FOUND: fs::error = ENOENT; break;
-		default: throw fmt::exception("Unknown Win32 error: %u (%s)." HERE, error, path);
-		}
-
+		g_tls_error = to_error(GetLastError());
 		return false;
 	}
 
@@ -234,7 +251,7 @@ bool fs::stat(const std::string& path, stat_t& info)
 	struct ::stat file_info;
 	if (::stat(path.c_str(), &file_info) != 0)
 	{
-		fs::error = errno;
+		g_tls_error = to_error(errno);
 		return false;
 	}
 
@@ -260,14 +277,7 @@ bool fs::exists(const std::string& path)
 #ifdef _WIN32
 	if (GetFileAttributesW(to_wchar(path).get()) == INVALID_FILE_ATTRIBUTES)
 	{
-		// TODO: convert Win32 error code to errno
-		switch (DWORD error = GetLastError())
-		{
-		case ERROR_FILE_NOT_FOUND: fs::error = ENOENT; break;
-		case ERROR_PATH_NOT_FOUND: fs::error = ENOENT; break;
-		default: throw fmt::exception("Unknown Win32 error: %u (%s)." HERE, error, path);
-		}
-
+		g_tls_error = to_error(GetLastError());
 		return false;
 	}
 
@@ -276,7 +286,7 @@ bool fs::exists(const std::string& path)
 	struct ::stat file_info;
 	if (::stat(path.c_str(), &file_info) != 0)
 	{
-		fs::error = errno;
+		g_tls_error = to_error(errno);
 		return false;
 	}
 
@@ -296,7 +306,7 @@ bool fs::is_file(const std::string& path)
 
 		if (info.is_directory)
 		{
-			fs::error = EEXIST;
+			g_tls_error = error::exist;
 			return false;
 		}
 
@@ -307,21 +317,14 @@ bool fs::is_file(const std::string& path)
 	const DWORD attrs = GetFileAttributesW(to_wchar(path).get());
 	if (attrs == INVALID_FILE_ATTRIBUTES)
 	{
-		// TODO: convert Win32 error code to errno
-		switch (DWORD error = GetLastError())
-		{
-		case ERROR_FILE_NOT_FOUND: fs::error = ENOENT; break;
-		case ERROR_PATH_NOT_FOUND: fs::error = ENOENT; break;
-		default: throw fmt::exception("Unknown Win32 error: %u (%s)." HERE, error, path);
-		}
-
+		g_tls_error = to_error(GetLastError());
 		return false;
 	}
 #else
 	struct ::stat file_info;
 	if (::stat(path.c_str(), &file_info) != 0)
 	{
-		fs::error = errno;
+		g_tls_error = to_error(errno);
 		return false;
 	}
 #endif
@@ -333,7 +336,7 @@ bool fs::is_file(const std::string& path)
 	if (S_ISDIR(file_info.st_mode))
 #endif
 	{
-		fs::error = EEXIST;
+		g_tls_error = error::exist;
 		return false;
 	}
 
@@ -352,7 +355,7 @@ bool fs::is_dir(const std::string& path)
 
 		if (info.is_directory == false)
 		{
-			fs::error = EEXIST;
+			g_tls_error = error::exist;
 			return false;
 		}
 
@@ -363,21 +366,14 @@ bool fs::is_dir(const std::string& path)
 	const DWORD attrs = GetFileAttributesW(to_wchar(path).get());
 	if (attrs == INVALID_FILE_ATTRIBUTES)
 	{
-		// TODO: convert Win32 error code to errno
-		switch (DWORD error = GetLastError())
-		{
-		case ERROR_FILE_NOT_FOUND: fs::error = ENOENT; break;
-		case ERROR_PATH_NOT_FOUND: fs::error = ENOENT; break;
-		default: throw fmt::exception("Unknown Win32 error: %u (%s)." HERE, error, path);
-		}
-
+		g_tls_error = to_error(GetLastError());
 		return false;
 	}
 #else
 	struct ::stat file_info;
 	if (::stat(path.c_str(), &file_info) != 0)
 	{
-		fs::error = errno;
+		g_tls_error = to_error(errno);
 		return false;
 	}
 #endif
@@ -388,7 +384,7 @@ bool fs::is_dir(const std::string& path)
 	if (!S_ISDIR(file_info.st_mode))
 #endif
 	{
-		fs::error = EEXIST;
+		g_tls_error = error::exist;
 		return false;
 	}
 
@@ -405,14 +401,7 @@ bool fs::create_dir(const std::string& path)
 #ifdef _WIN32
 	if (!CreateDirectoryW(to_wchar(path).get(), NULL))
 	{
-		// TODO: convert Win32 error code to errno
-		switch (DWORD error = GetLastError())
-		{
-		case ERROR_ALREADY_EXISTS: fs::error = EEXIST; break;
-		case ERROR_PATH_NOT_FOUND: fs::error = ENOENT; break;
-		default: throw fmt::exception("Unknown Win32 error: %u (%s)." HERE, error, path);
-		}
-
+		g_tls_error = to_error(GetLastError());
 		return false;
 	}
 
@@ -420,7 +409,7 @@ bool fs::create_dir(const std::string& path)
 #else
 	if (::mkdir(path.c_str(), S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH) != 0)
 	{
-		fs::error = errno;
+		g_tls_error = to_error(errno);
 		return false;
 	}
 
@@ -450,13 +439,7 @@ bool fs::remove_dir(const std::string& path)
 #ifdef _WIN32
 	if (!RemoveDirectoryW(to_wchar(path).get()))
 	{
-		// TODO: convert Win32 error code to errno
-		switch (DWORD error = GetLastError())
-		{
-		case ERROR_PATH_NOT_FOUND: fs::error = ENOENT; break;
-		default: throw fmt::exception("Unknown Win32 error: %u (%s)." HERE, error, path);
-		}
-
+		g_tls_error = to_error(GetLastError());
 		return false;
 	}
 
@@ -464,7 +447,7 @@ bool fs::remove_dir(const std::string& path)
 #else
 	if (::rmdir(path.c_str()) != 0)
 	{
-		fs::error = errno;
+		g_tls_error = to_error(errno);
 		return false;
 	}
 
@@ -489,13 +472,7 @@ bool fs::rename(const std::string& from, const std::string& to)
 #ifdef _WIN32
 	if (!MoveFileW(to_wchar(from).get(), to_wchar(to).get()))
 	{
-		// TODO: convert Win32 error code to errno
-		switch (DWORD error = GetLastError())
-		{
-		case ERROR_PATH_NOT_FOUND: fs::error = ENOENT; break;
-		default: throw fmt::exception("Unknown Win32 error: %u.\nFrom: %s\nTo: %s" HERE, error, from, to);
-		}
-
+		g_tls_error = to_error(GetLastError());
 		return false;
 	}
 
@@ -503,7 +480,7 @@ bool fs::rename(const std::string& from, const std::string& to)
 #else
 	if (::rename(from.c_str(), to.c_str()) != 0)
 	{
-		fs::error = errno;
+		g_tls_error = to_error(errno);
 		return false;
 	}
 
@@ -523,13 +500,7 @@ bool fs::copy_file(const std::string& from, const std::string& to, bool overwrit
 #ifdef _WIN32
 	if (!CopyFileW(to_wchar(from).get(), to_wchar(to).get(), !overwrite))
 	{
-		// TODO: convert Win32 error code to errno
-		switch (DWORD error = GetLastError())
-		{
-		case ERROR_PATH_NOT_FOUND: fs::error = ENOENT; break;
-		default: throw fmt::exception("Unknown Win32 error: %u.\nFrom: %s\nTo: %s" HERE, error, from, to);
-		}
-
+		g_tls_error = to_error(GetLastError());
 		return false;
 	}
 
@@ -540,7 +511,7 @@ bool fs::copy_file(const std::string& from, const std::string& to, bool overwrit
 	const int input = ::open(from.c_str(), O_RDONLY);
 	if (input == -1)
 	{
-		fs::error = errno;
+		g_tls_error = to_error(errno);
 		return false;
 	}
 
@@ -550,7 +521,7 @@ bool fs::copy_file(const std::string& from, const std::string& to, bool overwrit
 		const int err = errno;
 
 		::close(input);
-		fs::error = err;
+		g_tls_error = to_error(err);
 		return false;
 	}
 
@@ -569,7 +540,7 @@ bool fs::copy_file(const std::string& from, const std::string& to, bool overwrit
 
 		::close(input);
 		::close(output);
-		fs::error = err;
+		g_tls_error = to_error(err);
 		return false;
 	}
 
@@ -589,14 +560,7 @@ bool fs::remove_file(const std::string& path)
 #ifdef _WIN32
 	if (!DeleteFileW(to_wchar(path).get()))
 	{
-		// TODO: convert Win32 error code to errno
-		switch (DWORD error = GetLastError())
-		{
-		case ERROR_FILE_NOT_FOUND: fs::error = ENOENT; break;
-		case ERROR_PATH_NOT_FOUND: fs::error = ENOENT; break;
-		default: throw fmt::exception("Unknown Win32 error: %u (%s)." HERE, error, path);
-		}
-
+		g_tls_error = to_error(GetLastError());
 		return false;
 	}
 
@@ -604,7 +568,7 @@ bool fs::remove_file(const std::string& path)
 #else
 	if (::unlink(path.c_str()) != 0)
 	{
-		fs::error = errno;
+		g_tls_error = to_error(errno);
 		return false;
 	}
 
@@ -624,14 +588,7 @@ bool fs::truncate_file(const std::string& path, u64 length)
 	const auto handle = CreateFileW(to_wchar(path).get(), GENERIC_WRITE, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
 	if (handle == INVALID_HANDLE_VALUE)
 	{
-		// TODO: convert Win32 error code to errno
-		switch (DWORD error = GetLastError())
-		{
-		case ERROR_FILE_NOT_FOUND: fs::error = ENOENT; break;
-		case ERROR_PATH_NOT_FOUND: fs::error = ENOENT; break;
-		default: throw fmt::exception("Unknown Win32 error: %u (%s)." HERE, error, path);
-		}
-
+		g_tls_error = to_error(GetLastError());
 		return false;
 	}
 
@@ -641,13 +598,7 @@ bool fs::truncate_file(const std::string& path, u64 length)
 	// Seek and truncate
 	if (!SetFilePointerEx(handle, distance, NULL, FILE_BEGIN) || !SetEndOfFile(handle))
 	{
-		// TODO: convert Win32 error code to errno
-		switch (DWORD error = GetLastError())
-		{
-		case ERROR_NEGATIVE_SEEK: fs::error = EINVAL; break;
-		default: throw fmt::exception("Unknown Win32 error: %u (length=0x%llx)." HERE, error, length);
-		}
-
+		g_tls_error = to_error(GetLastError());
 		CloseHandle(handle);
 		return false;
 	}
@@ -657,7 +608,7 @@ bool fs::truncate_file(const std::string& path, u64 length)
 #else
 	if (::truncate(path.c_str(), length) != 0)
 	{
-		fs::error = errno;
+		g_tls_error = to_error(errno);
 		return false;
 	}
 
@@ -672,7 +623,7 @@ void fs::file::xnull() const
 
 void fs::file::xfail() const
 {
-	throw fmt::exception("Unexpected fs::file error %u", fs::error);
+	throw fmt::exception("Unexpected fs::error %u", g_tls_error);
 }
 
 bool fs::file::open(const std::string& path, bitset_t<open_mode> mode)
@@ -704,7 +655,7 @@ bool fs::file::open(const std::string& path, bitset_t<open_mode> mode)
 	{
 		if (mode & fs::excl)
 		{
-			fs::error = EINVAL;
+			g_tls_error = error::inval;
 			return false;
 		}
 
@@ -715,15 +666,7 @@ bool fs::file::open(const std::string& path, bitset_t<open_mode> mode)
 
 	if (handle == INVALID_HANDLE_VALUE)
 	{
-		// TODO: convert Win32 error code to errno
-		switch (DWORD error = GetLastError())
-		{
-		case ERROR_FILE_NOT_FOUND: fs::error = ENOENT; break;
-		case ERROR_PATH_NOT_FOUND: fs::error = ENOENT; break;
-		case ERROR_FILE_EXISTS:    fs::error = EEXIST; break;
-		default: throw fmt::exception("Unknown Win32 error: %u (%s)." HERE, error, path);
-		}
-
+		g_tls_error = to_error(GetLastError());
 		return false;
 	}
 
@@ -747,12 +690,7 @@ bool fs::file::open(const std::string& path, bitset_t<open_mode> mode)
 			FILE_BASIC_INFO basic_info;
 			if (!GetFileInformationByHandleEx(m_handle, FileBasicInfo, &basic_info, sizeof(FILE_BASIC_INFO)))
 			{
-				// TODO: convert Win32 error code to errno
-				switch (DWORD error = GetLastError())
-				{
-				case 0:
-				default: throw fmt::exception("Win32 error: %u." HERE, error);
-				}
+				throw fmt::exception("Win32 error: %u." HERE, GetLastError());
 			}
 
 			stat_t info;
@@ -773,47 +711,22 @@ bool fs::file::open(const std::string& path, bitset_t<open_mode> mode)
 			pos.QuadPart = 0;
 			if (!SetFilePointerEx(m_handle, pos, &old, FILE_CURRENT)) // get old position
 			{
-				// TODO: convert Win32 error code to errno
-				switch (DWORD error = GetLastError())
-				{
-				case 0:
-				default: throw fmt::exception("Unknown Win32 error: %u." HERE, error);
-				}
-
+				g_tls_error = to_error(GetLastError());
 				return false;
 			}
 
 			pos.QuadPart = length;
 			if (!SetFilePointerEx(m_handle, pos, NULL, FILE_BEGIN)) // set new position
 			{
-				// TODO: convert Win32 error code to errno
-				switch (DWORD error = GetLastError())
-				{
-				case ERROR_NEGATIVE_SEEK: fs::error = EINVAL; break;
-				default: throw fmt::exception("Unknown Win32 error: %u." HERE, error);
-				}
-
+				g_tls_error = to_error(GetLastError());
 				return false;
 			}
 
 			const BOOL result = SetEndOfFile(m_handle); // change file size
 
-			if (!result)
+			if (!result || !SetFilePointerEx(m_handle, old, NULL, FILE_BEGIN)) // restore position
 			{
-				// TODO: convert Win32 error code to errno
-				switch (DWORD error = GetLastError())
-				{
-				case 0:
-				default: throw fmt::exception("Unknown Win32 error: %u." HERE, error);
-				}
-			}
-
-			if (!SetFilePointerEx(m_handle, old, NULL, FILE_BEGIN) && result) // restore position
-			{
-				if (DWORD error = GetLastError())
-				{
-					throw fmt::exception("Win32 error: %u." HERE, error);
-				}
+				throw fmt::exception("Win32 error: %u." HERE, GetLastError());
 			}
 
 			return result != FALSE;
@@ -823,16 +736,12 @@ bool fs::file::open(const std::string& path, bitset_t<open_mode> mode)
 		{
 			// TODO (call ReadFile multiple times if count is too big)
 			const int size = ::narrow<int>(count, "Too big count" HERE);
-			Expects(size >= 0);
+			EXPECTS(size >= 0);
 
 			DWORD nread;
 			if (!ReadFile(m_handle, buffer, size, &nread, NULL))
 			{
-				switch (DWORD error = GetLastError())
-				{
-				case 0:
-				default: throw fmt::exception("Win32 error: %u." HERE, error);
-				}
+				throw fmt::exception("Win32 error: %u." HERE, GetLastError());
 			}
 
 			return nread;
@@ -842,16 +751,12 @@ bool fs::file::open(const std::string& path, bitset_t<open_mode> mode)
 		{
 			// TODO (call WriteFile multiple times if count is too big)
 			const int size = ::narrow<int>(count, "Too big count" HERE);
-			Expects(size >= 0);
+			EXPECTS(size >= 0);
 
 			DWORD nwritten;
 			if (!WriteFile(m_handle, buffer, size, &nwritten, NULL))
 			{
-				switch (DWORD error = GetLastError())
-				{
-				case 0:
-				default: throw fmt::exception("Win32 error: %u." HERE, error);
-				}
+				throw fmt::exception("Win32 error: %u." HERE, GetLastError());
 			}
 
 			return nwritten;
@@ -870,11 +775,7 @@ bool fs::file::open(const std::string& path, bitset_t<open_mode> mode)
 
 			if (!SetFilePointerEx(m_handle, pos, &pos, mode))
 			{
-				switch (DWORD error = GetLastError())
-				{
-				case 0:
-				default: throw fmt::exception("Win32 error: %u." HERE, error);
-				}
+				throw fmt::exception("Win32 error: %u." HERE, GetLastError());
 			}
 
 			return pos.QuadPart;
@@ -885,11 +786,7 @@ bool fs::file::open(const std::string& path, bitset_t<open_mode> mode)
 			LARGE_INTEGER size;
 			if (!GetFileSizeEx(m_handle, &size))
 			{
-				switch (DWORD error = GetLastError())
-				{
-				case 0:
-				default: throw fmt::exception("Win32 error: %u." HERE, error);
-				}
+				throw fmt::exception("Win32 error: %u." HERE, GetLastError());
 			}
 
 			return size.QuadPart;
@@ -913,8 +810,7 @@ bool fs::file::open(const std::string& path, bitset_t<open_mode> mode)
 
 	if (fd == -1)
 	{
-		// TODO: errno
-		fs::error = errno;
+		g_tls_error = to_error(errno);
 		return false;
 	}
 
@@ -938,11 +834,7 @@ bool fs::file::open(const std::string& path, bitset_t<open_mode> mode)
 			struct ::stat file_info;
 			if (::fstat(m_fd, &file_info) != 0)
 			{
-				switch (int error = errno)
-				{
-				case 0:
-				default: throw fmt::exception("Unknown error: %d." HERE, error);
-				}
+				throw fmt::exception("System error: %d." HERE, errno);
 			}
 
 			stat_t info;
@@ -960,12 +852,7 @@ bool fs::file::open(const std::string& path, bitset_t<open_mode> mode)
 		{
 			if (::ftruncate(m_fd, length) != 0)
 			{
-				switch (int error = errno)
-				{
-				case 0:
-				default: throw fmt::exception("Unknown error: %d." HERE, error);
-				}
-
+				g_tls_error = to_error(errno);
 				return false;
 			}
 
@@ -977,11 +864,7 @@ bool fs::file::open(const std::string& path, bitset_t<open_mode> mode)
 			const auto result = ::read(m_fd, buffer, count);
 			if (result == -1)
 			{
-				switch (int error = errno)
-				{
-				case 0:
-				default: throw fmt::exception("Unknown error: %d." HERE, error);
-				}
+				throw fmt::exception("System error: %d." HERE, errno);
 			}
 
 			return result;
@@ -992,11 +875,7 @@ bool fs::file::open(const std::string& path, bitset_t<open_mode> mode)
 			const auto result = ::write(m_fd, buffer, count);
 			if (result == -1)
 			{
-				switch (int error = errno)
-				{
-				case 0:
-				default: throw fmt::exception("Unknown error: %d." HERE, error);
-				}
+				throw fmt::exception("System error: %d." HERE, errno);
 			}
 
 			return result;
@@ -1013,11 +892,7 @@ bool fs::file::open(const std::string& path, bitset_t<open_mode> mode)
 			const auto result = ::lseek(m_fd, offset, mode);
 			if (result == -1)
 			{
-				switch (int error = errno)
-				{
-				case 0:
-				default: throw fmt::exception("Unknown error: %d." HERE, error);
-				}
+				throw fmt::exception("System error: %d." HERE, errno);
 			}
 
 			return result;
@@ -1028,11 +903,7 @@ bool fs::file::open(const std::string& path, bitset_t<open_mode> mode)
 			struct ::stat file_info;
 			if (::fstat(m_fd, &file_info) != 0)
 			{
-				switch (int error = errno)
-				{
-				case 0:
-				default: throw fmt::exception("Unknown error: %d." HERE, error);
-				}
+				throw fmt::exception("System error: %d." HERE, errno);
 			}
 
 			return file_info.st_size;
@@ -1049,7 +920,7 @@ fs::file::file(const void* ptr, std::size_t size)
 {
 	class memory_stream : public file_base
 	{
-		u64 m_pos{}; // TODO: read/seek could modify m_pos atomically
+		u64 m_pos{};
 
 		const char* const m_ptr;
 		const u64 m_size;
@@ -1063,26 +934,26 @@ fs::file::file(const void* ptr, std::size_t size)
 
 		fs::stat_t stat() override
 		{
-			throw std::logic_error("memory_stream doesn't support stat()");
+			throw std::logic_error("Not supported" HERE);
 		}
 
 		bool trunc(u64 length) override
 		{
-			throw std::logic_error("memory_stream doesn't support trunc()");
+			throw std::logic_error("Not allowed" HERE);
 		}
 
 		u64 read(void* buffer, u64 count) override
 		{
 			const u64 start = m_pos;
 			const u64 end = seek(count, fs::seek_cur);
-			const u64 read_size = end >= start ? end - start : throw std::logic_error("memory_stream::read(): overflow");
+			const u64 read_size = end >= start ? end - start : throw std::logic_error("Stream overflow" HERE);
 			std::memcpy(buffer, m_ptr + start, read_size);
 			return read_size;
 		}
 
 		u64 write(const void* buffer, u64 count) override
 		{
-			throw std::logic_error("memory_stream is not writable");
+			throw std::logic_error("Not allowed" HERE);
 		}
 
 		u64 seek(s64 offset, fs::seek_mode whence) override
@@ -1091,7 +962,7 @@ fs::file::file(const void* ptr, std::size_t size)
 				whence == fs::seek_set ? m_pos = std::min<u64>(offset, m_size) :
 				whence == fs::seek_cur ? m_pos = std::min<u64>(offset + m_pos, m_size) :
 				whence == fs::seek_end ? m_pos = std::min<u64>(offset + m_size, m_size) :
-				throw std::logic_error("memory_stream::seek(): invalid whence");
+				throw fmt::exception("Invalid whence (0x%x)" HERE, whence);
 		}
 
 		u64 size() override
@@ -1127,14 +998,7 @@ bool fs::dir::open(const std::string& path)
 
 	if (handle == INVALID_HANDLE_VALUE)
 	{
-		// TODO: convert Win32 error code to errno
-		switch (DWORD error = GetLastError())
-		{
-		case ERROR_FILE_NOT_FOUND: fs::error = ENOENT; break;
-		case ERROR_PATH_NOT_FOUND: fs::error = ENOENT; break;
-		default: throw fmt::exception("Unknown Win32 error: %u." HERE, error);
-		}
-
+		g_tls_error = to_error(GetLastError());
 		return false;
 	}
 
@@ -1179,11 +1043,12 @@ bool fs::dir::open(const std::string& path)
 				WIN32_FIND_DATAW found;
 				if (!FindNextFileW(m_handle, &found))
 				{
-					switch (DWORD error = GetLastError())
+					if (ERROR_NO_MORE_FILES == GetLastError())
 					{
-					case ERROR_NO_MORE_FILES: return false;
-					default: throw fmt::exception("Unknown Win32 error: %u." HERE, error);
+						return false;
 					}
+
+					throw fmt::exception("Win32 error: %u." HERE, GetLastError());
 				}
 
 				add_entry(found);
@@ -1205,8 +1070,7 @@ bool fs::dir::open(const std::string& path)
 
 	if (!ptr)
 	{
-		// TODO: errno
-		fs::error = errno;
+		g_tls_error = to_error(errno);
 		return false;
 	}
 
@@ -1236,11 +1100,7 @@ bool fs::dir::open(const std::string& path)
 			struct ::stat file_info;
 			if (::fstatat(::dirfd(m_dd), found->d_name, &file_info, 0) != 0)
 			{
-				switch (int error = errno)
-				{
-				case 0:
-				default: throw fmt::exception("Unknown error: %d." HERE, error);
-				}
+				throw fmt::exception("System error: %d." HERE, errno);
 			}
 
 			info.name = found->d_name;

Utilities/File.h

@@ -10,9 +10,6 @@
 
 namespace fs
 {
-	// Error code returned
-	extern thread_local uint error;
-
 	// File open mode flags
 	enum struct open_mode : u32
 	{
@@ -265,6 +262,13 @@ namespace fs
 			return read(&str[0], str.size()) == str.size();
 		}
 
+		// Read std::string
+		bool read(std::string& str, std::size_t size) const
+		{
+			str.resize(size);
+			return read(&str[0], size) == size;
+		}
+
 		// Read POD, sizeof(T) is used
 		template<typename T>
 		std::enable_if_t<std::is_pod<T>::value && !std::is_pointer<T>::value, bool> read(T& data) const
@@ -279,6 +283,14 @@ namespace fs
 			return read(vec.data(), sizeof(T) * vec.size()) == sizeof(T) * vec.size();
 		}
 
+		// Read POD std::vector
+		template<typename T>
+		std::enable_if_t<std::is_pod<T>::value && !std::is_pointer<T>::value, bool> read(std::vector<T>& vec, std::size_t size) const
+		{
+			vec.resize(size);
+			return read(vec.data(), sizeof(T) * size) == sizeof(T) * size;
+		}
+
 		// Read POD (experimental)
 		template<typename T>
 		std::enable_if_t<std::is_pod<T>::value && !std::is_pointer<T>::value, T> read() const
@@ -432,4 +444,16 @@ namespace fs
 
 	// Get size of all files recursively
 	u64 get_dir_size(const std::string& path);
+
+	enum class error : uint
+	{
+		ok = 0,
+
+		inval,
+		noent,
+		exist,
+	};
+
+	// Error code returned
+	extern thread_local error g_tls_error;
 }

Utilities/GSL.h

@@ -3,12 +3,8 @@
 #define GSL_THROW_ON_CONTRACT_VIOLATION
 
 #pragma push_macro("new")
-#pragma push_macro("Expects")
-#pragma push_macro("Ensures")
 #undef new
+#include <gsl.h>
+#pragma pop_macro("new")
 #undef Expects
 #undef Ensures
-#include <gsl.h>
-#pragma pop_macro("Ensures")
-#pragma pop_macro("Expects")
-#pragma pop_macro("new")

Utilities/Log.cpp

@@ -8,7 +8,11 @@
 // Thread-specific log prefix provider
 thread_local std::string(*g_tls_log_prefix)() = nullptr;
 
-namespace _log
+#ifndef _MSC_VER
+constexpr DECLARE(bijective<logs::level, const char*>::map);
+#endif
+
+namespace logs
 {
 	struct listener
 	{
@@ -65,7 +69,7 @@ namespace _log
 	channel ARMv7("ARMv7");
 }
 
-void _log::channel::broadcast(const _log::channel& ch, _log::level sev, const char* fmt...)
+void logs::channel::broadcast(const logs::channel& ch, logs::level sev, const char* fmt...)
 {
 	va_list args;
 	va_start(args, fmt);
@@ -75,13 +79,13 @@ void _log::channel::broadcast(const _log::channel& ch, _log::level sev, const ch
 
 [[noreturn]] extern void catch_all_exceptions();
 
-_log::file_writer::file_writer(const std::string& name)
+logs::file_writer::file_writer(const std::string& name)
 {
 	try
 	{
 		if (!m_file.open(fs::get_config_dir() + name, fs::rewrite + fs::append))
 		{
-			throw fmt::exception("Can't create log file %s (error %d)", name, fs::error);
+			throw fmt::exception("Can't create log file %s (error %d)", name, fs::g_tls_error);
 		}
 	}
 	catch (...)
@@ -90,17 +94,17 @@ _log::file_writer::file_writer(const std::string& name)
 	}
 }
 
-void _log::file_writer::log(const std::string& text)
+void logs::file_writer::log(const std::string& text)
 {
 	m_file.write(text);
 }
 
-std::size_t _log::file_writer::size() const
+std::size_t logs::file_writer::size() const
 {
 	return m_file.pos();
 }
 
-void _log::file_listener::log(const _log::channel& ch, _log::level sev, const std::string& text)
+void logs::file_listener::log(const logs::channel& ch, logs::level sev, const std::string& text)
 {
 	std::string msg; msg.reserve(text.size() + 200);
 

Utilities/Log.h

@@ -3,7 +3,7 @@
 #include "types.h"
 #include "Atomic.h"
 
-namespace _log
+namespace logs
 {
 	enum class level : uint
 	{
@@ -79,27 +79,27 @@ namespace _log
 }
 
 template<>
-struct bijective<_log::level, const char*>
+struct bijective<logs::level, const char*>
 {
-	static constexpr bijective_pair<_log::level, const char*> map[]
+	static constexpr bijective_pair<logs::level, const char*> map[]
 	{
-		{ _log::level::always, "Nothing" },
-		{ _log::level::fatal, "Fatal" },
-		{ _log::level::error, "Error" },
-		{ _log::level::todo, "TODO" },
-		{ _log::level::success, "Success" },
-		{ _log::level::warning, "Warning" },
-		{ _log::level::notice, "Notice" },
-		{ _log::level::trace, "Trace" },
+		{ logs::level::always, "Nothing" },
+		{ logs::level::fatal, "Fatal" },
+		{ logs::level::error, "Error" },
+		{ logs::level::todo, "TODO" },
+		{ logs::level::success, "Success" },
+		{ logs::level::warning, "Warning" },
+		{ logs::level::notice, "Notice" },
+		{ logs::level::trace, "Trace" },
 	};
 };
 
 // Legacy:
 
-#define LOG_SUCCESS(ch, fmt, ...) _log::ch.success(fmt, ##__VA_ARGS__)
-#define LOG_NOTICE(ch, fmt, ...)  _log::ch.notice (fmt, ##__VA_ARGS__)
-#define LOG_WARNING(ch, fmt, ...) _log::ch.warning(fmt, ##__VA_ARGS__)
-#define LOG_ERROR(ch, fmt, ...)   _log::ch.error  (fmt, ##__VA_ARGS__)
-#define LOG_TODO(ch, fmt, ...)    _log::ch.todo   (fmt, ##__VA_ARGS__)
-#define LOG_TRACE(ch, fmt, ...)   _log::ch.trace  (fmt, ##__VA_ARGS__)
-#define LOG_FATAL(ch, fmt, ...)   _log::ch.fatal  (fmt, ##__VA_ARGS__)
+#define LOG_SUCCESS(ch, fmt, ...) logs::ch.success(fmt, ##__VA_ARGS__)
+#define LOG_NOTICE(ch, fmt, ...)  logs::ch.notice (fmt, ##__VA_ARGS__)
+#define LOG_WARNING(ch, fmt, ...) logs::ch.warning(fmt, ##__VA_ARGS__)
+#define LOG_ERROR(ch, fmt, ...)   logs::ch.error  (fmt, ##__VA_ARGS__)
+#define LOG_TODO(ch, fmt, ...)    logs::ch.todo   (fmt, ##__VA_ARGS__)
+#define LOG_TRACE(ch, fmt, ...)   logs::ch.trace  (fmt, ##__VA_ARGS__)
+#define LOG_FATAL(ch, fmt, ...)   logs::ch.fatal  (fmt, ##__VA_ARGS__)

Utilities/Macro.h

@@ -41,6 +41,7 @@ constexpr std::uint32_t size32(const T(&)[Size])
 #define CHECK_ALIGN(type, align) static_assert(alignof(type) == align, "Invalid " #type " type alignment")
 #define CHECK_MAX_SIZE(type, size) static_assert(sizeof(type) <= size, #type " type size is too big")
 #define CHECK_SIZE_ALIGN(type, size, align) CHECK_SIZE(type, size); CHECK_ALIGN(type, align)
+#define CHECK_STORAGE(type, storage) static_assert(sizeof(type) <= sizeof(storage) && alignof(type) <= alignof(decltype(storage)), #type " is too small")
 
 // Return 32 bit sizeof() to avoid widening/narrowing conversions with size_t
 #define SIZE_32(type) static_cast<std::uint32_t>(sizeof(type))
@@ -60,20 +61,22 @@ constexpr std::uint32_t size32(const T(&)[Size])
 #define STRINGIZE_DETAIL(x) #x
 #define STRINGIZE(x) STRINGIZE_DETAIL(x)
 
-// Macro set, allows to hide "return" in simple lambda expressions.
+// Macro set, wraps an expression into lambda
 #define WRAP_EXPR(expr, ...) [&](__VA_ARGS__) { return expr; }
 #define COPY_EXPR(expr, ...) [=](__VA_ARGS__) { return expr; }
 #define PURE_EXPR(expr, ...) [] (__VA_ARGS__) { return expr; }
 
+#define return_ return
+
 #define HERE "\n(in file " __FILE__ ":" STRINGIZE(__LINE__) ")"
 
 // Ensure that the expression is evaluated to true. Always evaluated and allowed to have side effects (unlike assert() macro).
-#define ASSERT(expr) if (!(expr)) throw std::runtime_error("Assertion failed: " #expr HERE)
+#define VERIFY(expr) do { if (!(expr)) throw std::runtime_error("Verification failed: " #expr HERE); } while (0)
 
-// Expects() and Ensures() are intended to check function arguments and results.
-// Expressions are not guaranteed to evaluate. Redefinition with ASSERT macro for better unification.
-#define Expects ASSERT
-#define Ensures ASSERT
+// EXPECTS() and ENSURES() are intended to check function arguments and results.
+// Expressions are not guaranteed to evaluate.
+#define EXPECTS(expr) do { if (!(expr)) throw std::runtime_error("Precondition failed: " #expr HERE); } while (0)
+#define ENSURES(expr) do { if (!(expr)) throw std::runtime_error("Postcondition failed: " #expr HERE); } while (0)
 
 #define DECLARE(...) decltype(__VA_ARGS__) __VA_ARGS__
 

Utilities/Semaphore.h

@@ -35,7 +35,7 @@ public:
 
 	bool try_wait()
 	{
-		return LIKELY(m_value.compare_and_swap_test(1, 0));
+		return m_value.compare_and_swap_test(1, 0);
 	}
 
 	void post()

Utilities/SharedMutex.cpp

@@ -140,6 +140,34 @@ void shared_mutex::unlock_notify()
 	}
 }
 
+void shared_mutex::lock_upgrade_hard()
+{
+	unlock_shared();
+	lock();
+}
+
+void shared_mutex::lock_degrade_hard()
+{
+	initialize_once();
+
+	std::unique_lock<std::mutex> lock(m_data->mutex);
+
+	m_ctrl -= SM_WRITER_LOCK - 1;
+
+	if (m_data->rq_size)
+	{
+		// Notify all readers
+		lock.unlock();
+		m_data->rcv.notify_all();
+	}
+	else if (m_data->wq_size)
+	{
+		// Notify next exclusive owner
+		lock.unlock();
+		m_data->wcv.notify_one();
+	}
+}
+
 void shared_mutex::initialize_once()
 {
 	if (UNLIKELY(!m_data))

Utilities/SharedMutex.h

@@ -32,6 +32,9 @@ class shared_mutex final
 	void lock_hard();
 	void unlock_notify();
 
+	void lock_upgrade_hard();
+	void lock_degrade_hard();
+
 public:
 	constexpr shared_mutex() = default;
 
@@ -42,15 +45,9 @@ public:
 
 	bool try_lock_shared()
 	{
-		auto ctrl = m_ctrl.load();
+		const u32 ctrl = m_ctrl.load();
 
-		if (UNLIKELY(ctrl >= SM_READER_MAX))
-		{
-			ctrl = 0;
-		}
-
-		// Weak attempt
-		return LIKELY(m_ctrl.compare_and_swap_test(ctrl, ctrl + 1));
+		return ctrl < SM_READER_MAX && m_ctrl.compare_and_swap_test(ctrl, ctrl + 1);
 	}
 
 	void lock_shared()
@@ -72,12 +69,12 @@ public:
 
 	bool try_lock()
 	{
-		return LIKELY(m_ctrl.compare_and_swap_test(0, SM_WRITER_LOCK));
+		return !m_ctrl && m_ctrl.compare_and_swap_test(0, SM_WRITER_LOCK);
 	}
 
 	void lock()
 	{
-		if (UNLIKELY(!try_lock()))
+		if (UNLIKELY(!m_ctrl.compare_and_swap_test(0, SM_WRITER_LOCK)))
 		{
 			lock_hard();
 		}
@@ -85,11 +82,39 @@ public:
 
 	void unlock()
 	{
-		if (UNLIKELY(m_ctrl.fetch_sub(SM_WRITER_LOCK) != SM_WRITER_LOCK))
+		m_ctrl &= ~SM_WRITER_LOCK;
+
+		if (UNLIKELY(m_ctrl))
 		{
 			unlock_notify();
 		}
 	}
+
+	bool try_lock_upgrade()
+	{
+		return m_ctrl == 1 && m_ctrl.compare_and_swap_test(1, SM_WRITER_LOCK);
+	}
+
+	bool try_lock_degrade()
+	{
+		return m_ctrl == SM_WRITER_LOCK && m_ctrl.compare_and_swap_test(SM_WRITER_LOCK, 1);
+	}
+
+	void lock_upgrade()
+	{
+		if (UNLIKELY(!m_ctrl.compare_and_swap_test(1, SM_WRITER_LOCK)))
+		{
+			lock_upgrade_hard();
+		}
+	}
+
+	void lock_degrade()
+	{
+		if (UNLIKELY(!m_ctrl.compare_and_swap_test(SM_WRITER_LOCK, 1)))
+		{
+			lock_degrade_hard();
+		}
+	}
 };
 
 //! Simplified shared (reader) lock implementation.
@@ -133,3 +158,23 @@ public:
 		m_mutex.unlock();
 	}
 };
+
+// Exclusive (writer) lock in the scope of shared (reader) lock.
+class upgraded_lock final
+{
+	shared_mutex& m_mutex;
+
+public:
+	upgraded_lock(const writer_lock&) = delete;
+
+	upgraded_lock(shared_mutex& mutex)
+		: m_mutex(mutex)
+	{
+		m_mutex.lock_upgrade();
+	}
+
+	~upgraded_lock()
+	{
+		m_mutex.lock_degrade();
+	}
+};

Utilities/SleepQueue.h

@@ -59,17 +59,27 @@ public:
 	// Remove thread from the sleep queue
 	void leave()
 	{
-		auto it = std::find(m_queue.begin(), m_queue.end(), &m_thread);
-		
-		if (it != m_queue.end())
+		for (auto it = m_queue.begin(), end = m_queue.end(); it != end; it++)
 		{
-			m_queue.erase(it);
+			if (*it == &m_thread)
+			{
+				m_queue.erase(it);
+				return;
+			}
 		}
 	}
 
 	// Check whether the thread exists in the sleep queue
 	explicit operator bool() const
 	{
-		return std::find(m_queue.begin(), m_queue.end(), &m_thread) != m_queue.end();
+		for (auto it = m_queue.begin(), end = m_queue.end(); it != end; it++)
+		{
+			if (*it == &m_thread)
+			{
+				return true;
+			}
+		}
+
+		return false;
 	}
 };

Utilities/StrFmt.cpp

@@ -5,6 +5,7 @@
 #include <cassert>
 #include <array>
 #include <memory>
+#include <algorithm>
 
 std::string v128::to_hex() const
 {

Utilities/StrFmt.h

@@ -1,8 +1,8 @@
 #pragma once
 
 #include <cstdarg>
-#include <string>
 #include <exception>
+#include <string>
 
 #include "Platform.h"
 #include "types.h"
@@ -14,7 +14,7 @@ namespace fmt
 
 	// Formatting function
 	template<typename... Args>
-	inline std::string format(const char* fmt, const Args&... args) noexcept
+	inline std::string format(const char* fmt, const Args&... args)
 	{
 		return unsafe_format(fmt, ::unveil<Args>::get(args)...);
 	}
@@ -34,7 +34,6 @@ namespace fmt
 	class exception : public exception_base
 	{
 	public:
-		// Formatting constructor
 		template<typename... Args>
 		exception(const char* fmt, const Args&... args)
 			: exception_base(fmt, ::unveil<Args>::get(args)...)

Utilities/Thread.cpp

@@ -1263,9 +1263,11 @@ const bool s_self_test = []() -> bool
 	return true;
 }();
 
+#include <thread>
 #include <mutex>
 #include <condition_variable>
 #include <exception>
+#include <chrono>
 
 thread_local DECLARE(thread_ctrl::g_tls_this_thread) = nullptr;
 
@@ -1278,17 +1280,20 @@ struct thread_ctrl::internal
 	task_stack atexit;
 
 	std::exception_ptr exception; // Caught exception
+
+	std::chrono::high_resolution_clock::time_point time_limit;
 };
 
-// Temporarily until better interface is implemented
-extern std::condition_variable& get_current_thread_cv()
-{
-	return thread_ctrl::get_current()->get_data()->cond;
-}
+thread_local thread_ctrl::internal* g_tls_internal = nullptr;
 
 extern std::mutex& get_current_thread_mutex()
 {
-	return thread_ctrl::get_current()->get_data()->mutex;
+	return g_tls_internal->mutex;
+}
+
+extern std::condition_variable& get_current_thread_cv()
+{
+	return g_tls_internal->cond;
 }
 
 // TODO
@@ -1296,10 +1301,64 @@ extern atomic_t<u32> g_thread_count(0);
 
 extern thread_local std::string(*g_tls_log_prefix)();
 
+void thread_ctrl::start(const std::shared_ptr<thread_ctrl>& ctrl, task_stack task)
+{
+	reinterpret_cast<std::thread&>(ctrl->m_thread) = std::thread([ctrl, task = std::move(task)]
+	{
+		try
+		{
+			ctrl->initialize();
+			task.exec();
+		}
+		catch (...)
+		{
+			ctrl->initialize_once();
+			ctrl->m_data->exception = std::current_exception();
+		}
+
+		ctrl->finalize();
+	});
+}
+
+void thread_ctrl::wait_start(u64 timeout)
+{
+	initialize_once();
+
+	m_data->time_limit = std::chrono::high_resolution_clock::now() + std::chrono::microseconds(timeout);
+}
+
+bool thread_ctrl::wait_wait(u64 timeout)
+{
+	initialize_once();
+
+	std::unique_lock<std::mutex> lock(m_data->mutex, std::adopt_lock);
+
+	if (timeout && m_data->cond.wait_until(lock, m_data->time_limit) == std::cv_status::timeout)
+	{
+		lock.release();
+		return false;
+	}
+
+	m_data->cond.wait(lock);
+	lock.release();
+	return true;
+}
+
+void thread_ctrl::test()
+{
+	if (m_data && m_data->exception)
+	{
+		std::rethrow_exception(m_data->exception);
+	}
+}
+
 void thread_ctrl::initialize()
 {
+	initialize_once(); // TODO (temporarily)
+
 	// Initialize TLS variable
 	g_tls_this_thread = this;
+	g_tls_internal = this->m_data;
 
 	g_tls_log_prefix = []
 	{
@@ -1339,12 +1398,6 @@ void thread_ctrl::initialize()
 #endif
 }
 
-void thread_ctrl::set_exception() noexcept
-{
-	initialize_once();
-	m_data->exception = std::current_exception();
-}
-
 void thread_ctrl::finalize() noexcept
 {
 	// TODO
@@ -1355,30 +1408,43 @@ void thread_ctrl::finalize() noexcept
 
 	--g_thread_count;
 
-#ifdef _MSC_VER
+#ifdef _WIN32
 	ULONG64 time;
-	QueryThreadCycleTime(m_thread.native_handle(), &time);
+	QueryThreadCycleTime(GetCurrentThread(), &time);
 	LOG_NOTICE(GENERAL, "Thread time: %f Gc", time / 1000000000.);
 #endif
 }
 
-task_stack& thread_ctrl::get_atexit() const
+void thread_ctrl::push_atexit(task_stack task)
 {
 	initialize_once();
-	return m_data->atexit;
+	m_data->atexit.push(std::move(task));
+}
+
+thread_ctrl::thread_ctrl(std::string&& name)
+	: m_name(std::move(name))
+{
+	CHECK_STORAGE(std::thread, m_thread);
+
+#pragma push_macro("new")
+#undef new
+	new (&m_thread) std::thread;
+#pragma pop_macro("new")
 }
 
 thread_ctrl::~thread_ctrl()
 {
-	if (m_thread.joinable())
+	if (reinterpret_cast<std::thread&>(m_thread).joinable())
 	{
-		m_thread.detach();
+		reinterpret_cast<std::thread&>(m_thread).detach();
 	}
 
 	delete m_data;
+
+	reinterpret_cast<std::thread&>(m_thread).~thread();
 }
 
-void thread_ctrl::initialize_once() const
+void thread_ctrl::initialize_once()
 {
 	if (UNLIKELY(!m_data))
 	{
@@ -1393,33 +1459,37 @@ void thread_ctrl::initialize_once() const
 
 void thread_ctrl::join()
 {
-	if (LIKELY(m_thread.joinable()))
+	// Increase contention counter
+	const u32 _j = m_joining++;
+
+	if (LIKELY(_j >= 0x80000000))
 	{
-		// Increase contention counter
-		if (UNLIKELY(m_joining++))
+		// Already joined (signal condition)
+		m_joining = 0x80000000;
+	}
+	else if (LIKELY(_j == 0))
+	{
+		// Winner joins the thread
+		reinterpret_cast<std::thread&>(m_thread).join();
+
+		// Notify others if necessary
+		if (UNLIKELY(m_joining.exchange(0x80000000) != 1))
 		{
-			// Hard way
 			initialize_once();
 
-			std::unique_lock<std::mutex> lock(m_data->mutex);
-			m_data->join.wait(lock, WRAP_EXPR(!m_thread.joinable()));
+			// Serialize for reliable notification
+			m_data->mutex.lock();
+			m_data->mutex.unlock();
+			m_data->join.notify_all();
 		}
-		else
-		{
-			// Winner joins the thread
-			m_thread.join();
+	}
+	else
+	{
+		// Hard way
+		initialize_once();
 
-			// Notify others if necessary
-			if (UNLIKELY(m_joining > 1))
-			{
-				initialize_once();
-
-				// Serialize for reliable notification
-				m_data->mutex.lock();
-				m_data->mutex.unlock();
-				m_data->join.notify_all();
-			}
-		}
+		std::unique_lock<std::mutex> lock(m_data->mutex);
+		m_data->join.wait(lock, WRAP_EXPR(m_joining >= 0x80000000));
 	}
 
 	if (UNLIKELY(m_data && m_data->exception))
@@ -1428,7 +1498,18 @@ void thread_ctrl::join()
 	}
 }
 
-void thread_ctrl::lock_notify() const
+void thread_ctrl::lock()
+{
+	initialize_once();
+	m_data->mutex.lock();
+}
+
+void thread_ctrl::unlock()
+{
+	m_data->mutex.unlock();
+}
+
+void thread_ctrl::lock_notify()
 {
 	if (UNLIKELY(g_tls_this_thread == this))
 	{
@@ -1443,16 +1524,19 @@ void thread_ctrl::lock_notify() const
 	m_data->cond.notify_one();
 }
 
-void thread_ctrl::notify() const
+void thread_ctrl::notify()
 {
-	initialize_once();
 	m_data->cond.notify_one();
 }
 
-thread_ctrl::internal* thread_ctrl::get_data() const
+void thread_ctrl::set_exception(std::exception_ptr e)
 {
-	initialize_once();
-	return m_data;
+	m_data->exception = e;
+}
+
+void thread_ctrl::sleep(u64 useconds)
+{
+	std::this_thread::sleep_for(std::chrono::microseconds(useconds));
 }
 
 
@@ -1462,7 +1546,6 @@ named_thread::named_thread()
 
 named_thread::~named_thread()
 {
-	LOG_TRACE(GENERAL, "%s", __func__);
 }
 
 std::string named_thread::get_name() const

Utilities/Thread.h

@@ -1,8 +1,8 @@
 #pragma once
 
+#include <exception>
 #include <string>
 #include <memory>
-#include <thread>
 
 #include "Platform.h"
 #include "Atomic.h"
@@ -28,38 +28,41 @@ class task_stack
 		}
 	};
 
+	template<typename F>
+	struct task_type : task_base
+	{
+		std::remove_reference_t<F> func;
+
+		task_type(F&& func)
+			: func(std::forward<F>(func))
+		{
+		}
+
+		void exec() override
+		{
+			func();
+			task_base::exec();
+		}
+	};
+
 	std::unique_ptr<task_base> m_stack;
 
 public:
+	task_stack() = default;
+
 	template<typename F>
-	void push(F&& func)
+	task_stack(F&& func)
+		: m_stack(new task_type<F>(std::forward<F>(func)))
 	{
-		struct task_t : task_base
-		{
-			std::remove_reference_t<F> func;
+	}
 
-			task_t(F&& func)
-				: func(std::forward<F>(func))
-			{
-			}
-
-			void exec() override
-			{
-				func();
-				task_base::exec();
-			}
-		};
-
-		auto _top = new task_t(std::forward<F>(func));
+	void push(task_stack stack)
+	{
+		auto _top = stack.m_stack.release();
 		auto _next = m_stack.release();
 		m_stack.reset(_top);
-#ifndef _MSC_VER
+		while (UNLIKELY(_top->next)) _top = _top->next.get();
 		_top->next.reset(_next);
-#else
-		auto& next = _top->next;
-		next.release();
-		next.reset(_next);
-#endif
 	}
 
 	void reset()
@@ -79,42 +82,48 @@ public:
 // Thread control class
 class thread_ctrl final
 {
+public: // TODO
 	struct internal;
 
+private:
 	static thread_local thread_ctrl* g_tls_this_thread;
 
-	// Thread handle
-	std::thread m_thread;
+	// Thread handle storage
+	std::aligned_storage_t<16> m_thread;
 
 	// Thread join contention counter
-	atomic_t<uint> m_joining{};
+	atomic_t<u32> m_joining{};
+
+	// Thread internals
+	atomic_t<internal*> m_data{};
 
 	// Fixed name
 	std::string m_name;
 
-	// Thread internals
-	mutable atomic_t<internal*> m_data{};
+	// Start thread
+	static void start(const std::shared_ptr<thread_ctrl>&, task_stack);
 
 	// Called at the thread start
 	void initialize();
 
-	// Set std::current_exception
-	void set_exception() noexcept;
-
 	// Called at the thread end
 	void finalize() noexcept;
 
 	// Get atexit function
-	task_stack& get_atexit() const;
+	void push_atexit(task_stack);
+
+	// Start waiting
+	void wait_start(u64 timeout);
+
+	// Proceed waiting
+	bool wait_wait(u64 timeout);
+
+	// Check exception
+	void test();
 
 public:
-	template<typename N>
-	thread_ctrl(N&& name)
-		: m_name(std::forward<N>(name))
-	{
-	}
+	thread_ctrl(std::string&& name);
 
-	// Disable copy/move constructors and operators
 	thread_ctrl(const thread_ctrl&) = delete;
 
 	~thread_ctrl();
@@ -126,54 +135,139 @@ public:
 	}
 
 	// Initialize internal data
-	void initialize_once() const;
+	void initialize_once();
 
 	// Get thread result (may throw, simultaneous joining allowed)
 	void join();
 
+	// Lock thread mutex
+	void lock();
+
+	// Lock conditionally (double-checked)
+	template<typename F>
+	bool lock_if(F&& pred)
+	{
+		if (pred())
+		{
+			lock();
+
+			try
+			{
+				if (LIKELY(pred()))
+				{
+					return true;
+				}
+				else
+				{
+					unlock();
+					return false;
+				}
+			}
+			catch (...)
+			{
+				unlock();
+				throw;
+			}
+		}
+		else
+		{
+			return false;
+		}
+	}
+
+	// Unlock thread mutex (internal data must be initialized)
+	void unlock();
+
 	// Lock, unlock, notify the thread (required if the condition changed locklessly)
-	void lock_notify() const;
+	void lock_notify();
 
-	// Notify the thread, beware the condition change
-	void notify() const;
+	// Notify the thread (internal data must be initialized)
+	void notify();
 
-	//
-	internal* get_data() const;
+	// Set exception (internal data must be initialized, thread mutex must be locked)
+	void set_exception(std::exception_ptr);
+
+	// Current thread sleeps for specified amount of microseconds.
+	// Wrapper for std::this_thread::sleep, doesn't require valid thread_ctrl.
+	[[deprecated]] static void sleep(u64 useconds);
+
+	// Wait until pred(). Abortable, may throw. Thread must be locked.
+	// Timeout in microseconds (zero means infinite).
+	template<typename F>
+	static inline auto wait(u64 useconds, F&& pred)
+	{
+		g_tls_this_thread->wait_start(useconds);
+
+		while (true)
+		{
+			g_tls_this_thread->test();
+
+			if (auto&& result = pred())
+			{
+				return result;
+			}
+			else if (!g_tls_this_thread->wait_wait(useconds) && useconds)
+			{
+				return result;
+			}
+		}
+	}
+
+	// Wait until pred(). Abortable, may throw. Thread must be locked.
+	template<typename F>
+	static inline auto wait(F&& pred)
+	{
+		while (true)
+		{
+			g_tls_this_thread->test();
+
+			if (auto&& result = pred())
+			{
+				return result;
+			}
+
+			g_tls_this_thread->wait_wait(0);
+		}
+	}
+
+	// Wait once. Thread must be locked.
+	static inline void wait()
+	{
+		g_tls_this_thread->test();
+		g_tls_this_thread->wait_wait(0);
+		g_tls_this_thread->test();
+	}
+
+	// Wait unconditionally until aborted. Thread must be locked.
+	[[noreturn]] static inline void eternalize()
+	{
+		while (true)
+		{
+			g_tls_this_thread->test();
+			g_tls_this_thread->wait_wait(0);
+		}
+	}
 
 	// Get current thread (may be nullptr)
-	static const thread_ctrl* get_current()
+	static thread_ctrl* get_current()
 	{
 		return g_tls_this_thread;
 	}
 
 	// Register function at thread exit (for the current thread)
 	template<typename F>
-	static inline void at_exit(F&& func)
+	static inline void atexit(F&& func)
 	{
-		return g_tls_this_thread->get_atexit().push(std::forward<F>(func));
+		return g_tls_this_thread->push_atexit(std::forward<F>(func));
 	}
 
 	// Named thread factory
-	template<typename N, typename F, typename... Args>
-	static inline std::shared_ptr<thread_ctrl> spawn(N&& name, F&& func, Args&&... args)
+	template<typename N, typename F>
+	static inline std::shared_ptr<thread_ctrl> spawn(N&& name, F&& func)
 	{
 		auto ctrl = std::make_shared<thread_ctrl>(std::forward<N>(name));
 
-		ctrl->m_thread = std::thread([ctrl, task = std::forward<F>(func)](Args&&... args)
-		{
-			try
-			{
-				ctrl->initialize();
-				task(std::forward<Args>(args)...);
-			}
-			catch (...)
-			{
-				ctrl->set_exception();
-			}
-
-			ctrl->finalize();
-
-		}, std::forward<Args>(args)...);
+		thread_ctrl::start(ctrl, std::forward<F>(func));
 
 		return ctrl;
 	}
@@ -218,22 +312,43 @@ public:
 		m_thread->join();
 	}
 
-	// Get thread_ctrl
-	const thread_ctrl* operator->() const
+	// Access thread_ctrl
+	thread_ctrl* operator->() const
 	{
 		return m_thread.get();
 	}
+};
 
-	// Lock mutex, notify condition variable
-	void lock_notify() const
+// Simple thread mutex locker
+class thread_lock final
+{
+	thread_ctrl* m_thread;
+
+public:
+	thread_lock(const thread_lock&) = delete;
+
+	// Lock specified thread
+	thread_lock(thread_ctrl* thread)
+		: m_thread(thread)
 	{
-		m_thread->lock_notify();
+		m_thread->lock();
 	}
 
-	// Notify condition variable
-	void notify() const
+	// Lock specified named_thread
+	thread_lock(named_thread& thread)
+		: thread_lock(thread.operator->())
 	{
-		m_thread->notify();
+	}
+
+	// Lock current thread
+	thread_lock()
+		: thread_lock(thread_ctrl::get_current())
+	{
+	}
+
+	~thread_lock()
+	{
+		m_thread->unlock();
 	}
 };
 

Utilities/VirtualMemory.cpp

@@ -17,10 +17,10 @@ namespace memory_helper
 	{
 #ifdef _WIN32
 		void* ret = VirtualAlloc(NULL, size, MEM_RESERVE, PAGE_NOACCESS);
-		Ensures(ret != NULL);
+		ENSURES(ret != NULL);
 #else
 		void* ret = mmap(nullptr, size, PROT_NONE, MAP_ANON | MAP_PRIVATE, -1, 0);
-		Ensures(ret != 0);
+		ENSURES(ret != 0);
 #endif
 		return ret;
 	}
@@ -28,18 +28,18 @@ namespace memory_helper
 	void commit_page_memory(void* pointer, size_t page_size)
 	{
 #ifdef _WIN32
-		ASSERT(VirtualAlloc((u8*)pointer, page_size, MEM_COMMIT, PAGE_READWRITE) != NULL);
+		VERIFY(VirtualAlloc((u8*)pointer, page_size, MEM_COMMIT, PAGE_READWRITE) != NULL);
 #else
-		ASSERT(mprotect((u8*)pointer, page_size, PROT_READ | PROT_WRITE) != -1);
+		VERIFY(mprotect((u8*)pointer, page_size, PROT_READ | PROT_WRITE) != -1);
 #endif
 	}
 
 	void free_reserved_memory(void* pointer, size_t size)
 	{
 #ifdef _WIN32
-		ASSERT(VirtualFree(pointer, 0, MEM_RELEASE) != 0);
+		VERIFY(VirtualFree(pointer, 0, MEM_RELEASE) != 0);
 #else
-		ASSERT(munmap(pointer, size) == 0);
+		VERIFY(munmap(pointer, size) == 0);
 #endif
 	}
 }

Utilities/lockless.h

@@ -0,0 +1,105 @@
+#pragma once
+
+#include "types.h"
+#include "Atomic.h"
+#include "Platform.h"
+
+//! Simple sizeless array base for concurrent access. Cannot shrink, only growths automatically.
+//! There is no way to know the current size. The smaller index is, the faster it's accessed.
+//! 
+//! T is the type of elements. Currently, default constructor of T shall be constexpr.
+//! N is initial element count, available without any memory allocation and only stored contiguously.
+template<typename T, std::size_t N>
+class lf_array
+{
+	// Data (default-initialized)
+	T m_data[N]{};
+
+	// Next array block
+	atomic_t<lf_array*> m_next{};
+
+public:
+	constexpr lf_array() = default;
+
+	~lf_array()
+	{
+		for (auto ptr = m_next.raw(); UNLIKELY(ptr);)
+		{
+			delete std::exchange(ptr, std::exchange(ptr->m_next.raw(), nullptr));
+		}
+	}
+
+	T& operator [](std::size_t index)
+	{
+		if (LIKELY(index < N))
+		{
+			return m_data[index];
+		}
+		else if (UNLIKELY(!m_next))
+		{
+			// Create new array block. It's not a full-fledged once-synchronization, unlikely needed.
+			for (auto _new = new lf_array, ptr = this; UNLIKELY(ptr);)
+			{
+				// Install the pointer. If failed, go deeper.
+				ptr = ptr->m_next.compare_and_swap(nullptr, _new);
+			}
+		}
+
+		// Access recursively
+		return (*m_next)[index - N];
+	}
+};
+
+//! Simple lock-free FIFO queue base. Based on lf_array<T, N> itself. Currently uses 32-bit counters.
+//! There is no "push_end" or "pop_begin" provided, the queue element must signal its state on its own.
+template<typename T, std::size_t N>
+class lf_fifo : public lf_array<T, N>
+{
+	struct alignas(8) ctrl_t
+	{
+		u32 push;
+		u32 pop;
+	};
+
+	atomic_t<ctrl_t> m_ctrl{};
+
+public:
+	constexpr lf_fifo() = default;
+
+	// Get current "push" position
+	u32 size()
+	{
+		return reinterpret_cast<atomic_t<u32>&>(m_ctrl).load(); // Hack
+	}
+	
+	// Acquire the place for one or more elements.
+	u32 push_begin(u32 count = 1)
+	{
+		return reinterpret_cast<atomic_t<u32>&>(m_ctrl).fetch_add(count); // Hack
+	}
+
+	// Get current "pop" position
+	u32 peek()
+	{
+		return m_ctrl.load().pop;
+	}
+
+	// Acknowledge processed element, return number of the next one.
+	// Perform clear if possible, zero is returned in this case.
+	u32 pop_end(u32 count = 1)
+	{
+		return m_ctrl.atomic_op([&](ctrl_t& ctrl)
+		{
+			ctrl.pop += count;
+
+			if (ctrl.pop == ctrl.push)
+			{
+				// Clean if possible
+				ctrl.push = 0;
+				ctrl.pop = 0;
+			}
+
+			return ctrl.pop;
+		});
+	}
+};

Utilities/sync.h

@@ -0,0 +1,192 @@
+#pragma once
+
+/* For internal use. Don't include. */
+
+#include "types.h"
+#include "Macro.h"
+#include "Atomic.h"
+
+#ifdef _WIN32
+
+#include <Windows.h>
+
+#define DYNAMIC_IMPORT(handle, name) do { name = reinterpret_cast<decltype(name)>(GetProcAddress(handle, #name)); } while (0)
+
+static NTSTATUS(*NtSetTimerResolution)(ULONG DesiredResolution, BOOLEAN SetResolution, PULONG CurrentResolution);
+static NTSTATUS(*NtWaitForKeyedEvent)(HANDLE Handle, PVOID Key, BOOLEAN Alertable, PLARGE_INTEGER Timeout);
+static NTSTATUS(*NtReleaseKeyedEvent)(HANDLE Handle, PVOID Key, BOOLEAN Alertable, PLARGE_INTEGER Timeout);
+
+namespace util
+{
+	static const bool keyed_init = []
+	{
+		const auto handle = LoadLibraryA("ntdll.dll");
+		DYNAMIC_IMPORT(handle, NtSetTimerResolution);
+		DYNAMIC_IMPORT(handle, NtWaitForKeyedEvent);
+		DYNAMIC_IMPORT(handle, NtReleaseKeyedEvent);
+		FreeLibrary(handle);
+
+		ULONG res = 100;
+		NtSetTimerResolution(100, TRUE, &res);
+
+		return NtWaitForKeyedEvent && NtReleaseKeyedEvent;
+	}();
+
+	// Wait for specified condition. func() acknowledges success by value modification.
+	template<typename F>
+	inline void keyed_wait(atomic_t<u32>& key, F&& func)
+	{
+		while (true)
+		{
+			NtWaitForKeyedEvent(NULL, &key, FALSE, NULL);
+
+			u32 read = key.load();
+			u32 copy = read;
+
+			while (pred(read), read != copy)
+			{
+				read = key.compare_and_swap(copy, read);
+				
+				if (copy == read)
+				{
+					break;
+				}
+				
+				copy = read;
+			}
+		}
+	}
+
+	// Try to wake up a thread.
+	inline bool keyed_post(atomic_t<u32>& key, u32 acknowledged_value)
+	{
+		LARGE_INTEGER zero;
+		zero.QuadPart = 0;
+
+		while (UNLIKELY(NtReleaseKeyedEvent(NULL, &key, FALSE, &zero) == WAIT_TIMEOUT))
+		{
+			if (key.load() != acknowledged_value)
+				return false;
+
+			//NtReleaseKeyedEvent(NULL, &key, FALSE, NULL);
+			//return true;
+		}
+
+		return true;
+	}
+
+	struct native_rwlock
+	{
+		SRWLOCK rwlock = SRWLOCK_INIT;
+
+		constexpr native_rwlock() = default;
+
+		native_rwlock(const native_rwlock&) = delete;
+
+		void lock()
+		{
+			AcquireSRWLockExclusive(&rwlock);
+		}
+
+		bool try_lock()
+		{
+			return TryAcquireSRWLockExclusive(&rwlock) != 0;
+		}
+
+		void unlock()
+		{
+			ReleaseSRWLockExclusive(&rwlock);
+		}
+
+		void lock_shared()
+		{
+			AcquireSRWLockShared(&rwlock);
+		}
+
+		bool try_lock_shared()
+		{
+			return TryAcquireSRWLockShared(&rwlock) != 0;
+		}
+
+		void unlock_shared()
+		{
+			ReleaseSRWLockShared(&rwlock);
+		}
+	};
+
+	struct native_cond
+	{
+		CONDITION_VARIABLE cond = CONDITION_VARIABLE_INIT;
+
+		constexpr native_cond() = default;
+
+		native_cond(const native_cond&) = delete;
+
+		void notify_one()
+		{
+			WakeConditionVariable(&cond);
+		}
+
+		void notify_all()
+		{
+			WakeAllConditionVariable(&cond);
+		}
+
+		void wait(native_rwlock& rwlock)
+		{
+			SleepConditionVariableSRW(&cond, &rwlock.rwlock, INFINITE, 0);
+		}
+
+		void wait_shared(native_rwlock& rwlock)
+		{
+			SleepConditionVariableSRW(&cond, &rwlock.rwlock, INFINITE, CONDITION_VARIABLE_LOCKMODE_SHARED);
+		}
+	};
+
+	class exclusive_lock
+	{
+		native_rwlock& m_rwlock;
+
+	public:
+		exclusive_lock(native_rwlock& rwlock)
+			: m_rwlock(rwlock)
+		{
+			m_rwlock.lock();
+		}
+
+		~exclusive_lock()
+		{
+			m_rwlock.unlock();
+		}
+	};
+
+	class shared_lock
+	{
+		native_rwlock& m_rwlock;
+
+	public:
+		shared_lock(native_rwlock& rwlock)
+			: m_rwlock(rwlock)
+		{
+			m_rwlock.lock_shared();
+		}
+
+		~shared_lock()
+		{
+			m_rwlock.unlock_shared();
+		}
+	};
+}
+
+#else
+
+namespace util
+{
+	struct native_rwlock;
+	struct native_cond;
+}
+
+#endif
+
+CHECK_SIZE_ALIGN(util::native_rwlock, sizeof(void*), alignof(void*));
+CHECK_SIZE_ALIGN(util::native_cond, sizeof(void*), alignof(void*));

Utilities/types.h

@@ -407,12 +407,22 @@ struct ignore
 	}
 };
 
+// Simplified hash algorithm for pointers. May be used in std::unordered_(map|set).
+template<typename T, std::size_t Align = alignof(T)>
+struct pointer_hash
+{
+	std::size_t operator()(T* ptr) const
+	{
+		return reinterpret_cast<std::uintptr_t>(ptr) / Align;
+	}
+};
+
 // Contains value of any POD type with fixed size and alignment. TT<> is the type converter applied.
 // For example, `simple_t` may be used to remove endianness.
 template<template<typename> class TT, std::size_t S, std::size_t A = S>
 struct alignas(A) any_pod
 {
-	enum class byte : char {} data[S];
+	std::aligned_storage_t<S, A> data;
 
 	any_pod() = default;