From a8da524dd4232ab7399cf1d63b3fc37f4b22f2ff Mon Sep 17 00:00:00 2001 From: Samuliak Date: Wed, 15 Jan 2025 17:40:45 +0100 Subject: [PATCH] use precompiled shaders --- .../HW/Latte/Renderer/Metal/MetalCommon.h | 50 ++++++ .../Renderer/Metal/RendererShaderMtl.cpp | 153 +++++++++++------- .../Latte/Renderer/Metal/RendererShaderMtl.h | 6 +- 3 files changed, 152 insertions(+), 57 deletions(-) diff --git a/src/Cafe/HW/Latte/Renderer/Metal/MetalCommon.h b/src/Cafe/HW/Latte/Renderer/Metal/MetalCommon.h index f3dd1733..28d92225 100644 --- a/src/Cafe/HW/Latte/Renderer/Metal/MetalCommon.h +++ b/src/Cafe/HW/Latte/Renderer/Metal/MetalCommon.h @@ -109,3 +109,53 @@ inline void executeCommand(fmt::format_string fmt, T&&... args) { if (res != 0) cemuLog_log(LogType::Force, "command \"{}\" failed with exit code {}", command, res); } + +class MemoryMappedFile +{ +public: + MemoryMappedFile(const std::string& filePath) + { + // Open the file + m_fd = open(filePath.c_str(), O_RDONLY); + if (m_fd == -1) { + cemuLog_log(LogType::Force, "failed to open file: {}", filePath); + return; + } + + // Get the file size + struct stat fileStat; + if (fstat(m_fd, &fileStat) == -1) + { + close(m_fd); + cemuLog_log(LogType::Force, "failed to get file size: {}", filePath); + return; + } + m_fileSize = fileStat.st_size; + + // Memory map the file + m_data = mmap(nullptr, m_fileSize, PROT_READ, MAP_PRIVATE, m_fd, 0); + if (m_data == MAP_FAILED) + { + close(m_fd); + cemuLog_log(LogType::Force, "failed to memory map file: {}", filePath); + return; + } + } + + ~MemoryMappedFile() + { + if (m_data && m_data != MAP_FAILED) + munmap(m_data, m_fileSize); + + if (m_fd != -1) + close(m_fd); + } + + uint8* data() const { return static_cast(m_data); } + size_t size() const { return m_fileSize; } + +private: + int m_fd = -1; + void* m_data = nullptr; + size_t m_fileSize = 0; +}; diff --git a/src/Cafe/HW/Latte/Renderer/Metal/RendererShaderMtl.cpp b/src/Cafe/HW/Latte/Renderer/Metal/RendererShaderMtl.cpp index dc1256a4..e6aa0cab 100644 --- a/src/Cafe/HW/Latte/Renderer/Metal/RendererShaderMtl.cpp +++ b/src/Cafe/HW/Latte/Renderer/Metal/RendererShaderMtl.cpp @@ -128,7 +128,7 @@ void RendererShaderMtl::ShaderCacheLoading_end() // Close RAM filesystem if (s_hasRAMFilesystem) { - executeCommand("diskutil eject {}", METAL_AIR_CACHE_PATH); + //executeCommand("diskutil eject {}", METAL_AIR_CACHE_PATH); s_hasRAMFilesystem = false; } @@ -211,50 +211,8 @@ bool RendererShaderMtl::ShouldCountCompilation() const return !s_isLoadingShadersMtl && m_isGameShader; } -void RendererShaderMtl::CompileInternal() +MTL::Library* RendererShaderMtl::LibraryFromSource() { - // First, try to retrieve the compiled shader from the AIR cache - if (s_isLoadingShadersMtl && (m_isGameShader && !m_isGfxPackShader) && s_airCache) - { - cemu_assert_debug(m_baseHash != 0); - uint64 h1, h2; - GenerateShaderPrecompiledCacheFilename(m_type, m_baseHash, m_auxHash, h1, h2); - std::vector cacheFileData; - if (s_airCache->GetFile({ h1, h2 }, cacheFileData)) - { - CompileFromAIR(std::span(cacheFileData.data(), cacheFileData.size())); - FinishCompilation(); - } - else - { - // Ensure that RAM filesystem exists - if (!s_hasRAMFilesystem) - { - s_hasRAMFilesystem = true; - executeCommand("diskutil erasevolume HFS+ {} $(hdiutil attach -nomount ram://{})", METAL_AIR_CACHE_NAME, METAL_AIR_CACHE_BLOCK_COUNT); - } - - // The shader is not in the cache, compile it - std::string filename = fmt::format("{}_{}", h1, h2); - // TODO: store the source - executeCommand("xcrun -sdk macosx metal -o {}.ir -c {}.metal", filename, filename); - executeCommand("xcrun -sdk macosx metallib -o {}.metallib {}.ir", filename, filename); - // TODO: clean up - - // Load from the newly Generated AIR - // std::span airData = ; - //CompileFromAIR(std::span((uint8*)cacheFileData.data(), cacheFileData.size() / sizeof(uint8))); - FinishCompilation(); - - // Store in the cache - uint64 h1, h2; - GenerateShaderPrecompiledCacheFilename(m_type, m_baseHash, m_auxHash, h1, h2); - //s_airCache->AddFile({ h1, h2 }, airData.data(), airData.size()); - } - - return; - } - // Compile from source MTL::CompileOptions* options = MTL::CompileOptions::alloc()->init(); // TODO: always disable fast math for problematic shaders @@ -266,28 +224,113 @@ void RendererShaderMtl::CompileInternal() NS::Error* error = nullptr; MTL::Library* library = m_mtlr->GetDevice()->newLibrary(ToNSString(m_mslCode), options, &error); options->release(); + FinishCompilation(); if (error) { - cemuLog_log(LogType::Force, "failed to create library: {} -> {}", error->localizedDescription()->utf8String(), m_mslCode.c_str()); - FinishCompilation(); - return; + cemuLog_log(LogType::Force, "failed to create library from source: {} -> {}", error->localizedDescription()->utf8String(), m_mslCode.c_str()); + return nullptr; } + + return library; +} + +MTL::Library* RendererShaderMtl::LibraryFromAIR(std::span data) +{ + dispatch_data_t dispatchData = dispatch_data_create(data.data(), data.size(), nullptr, DISPATCH_DATA_DESTRUCTOR_DEFAULT); + + NS::Error* error = nullptr; + MTL::Library* library = m_mtlr->GetDevice()->newLibrary(dispatchData, &error); + FinishCompilation(); + printf("AIR size: %zu\n", data.size()); + if (error) + { + cemuLog_log(LogType::Force, "failed to create library from AIR: {}", error->localizedDescription()->utf8String()); + return nullptr; + } + + return library; +} + +void RendererShaderMtl::CompileInternal() +{ + MTL::Library* library; + + // First, try to retrieve the compiled shader from the AIR cache + if (s_isLoadingShadersMtl && (m_isGameShader && !m_isGfxPackShader) && s_airCache) + { + cemu_assert_debug(m_baseHash != 0); + uint64 h1, h2; + GenerateShaderPrecompiledCacheFilename(m_type, m_baseHash, m_auxHash, h1, h2); + std::vector cacheFileData; + if (s_airCache->GetFile({ h1, h2 }, cacheFileData)) + { + library = LibraryFromAIR(std::span(cacheFileData.data(), cacheFileData.size())); + } + else + { + // Ensure that RAM filesystem exists + static std::atomic s_creatingRAMFilesystem{false}; + if (!s_hasRAMFilesystem) + { + if (s_creatingRAMFilesystem) + { + while (!s_hasRAMFilesystem) + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + } + else + { + s_creatingRAMFilesystem = true; + executeCommand("diskutil erasevolume HFS+ {} $(hdiutil attach -nomount ram://{})", METAL_AIR_CACHE_NAME, METAL_AIR_CACHE_BLOCK_COUNT); + s_creatingRAMFilesystem = false; + } + s_hasRAMFilesystem = true; + } + + // The shader is not in the cache, compile it + std::string baseFilename = fmt::format("{}/{}_{}", METAL_AIR_CACHE_PATH, h1, h2); + + // Source + std::ofstream mslFile; + mslFile.open(fmt::format("{}.metal", baseFilename)); + mslFile << m_mslCode; + mslFile.close(); + + // Compile + executeCommand("xcrun -sdk macosx metal -o {}.ir -c {}.metal -Wno-unused-variable -Wno-sign-compare", baseFilename, baseFilename); + executeCommand("xcrun -sdk macosx metallib -o {}.metallib {}.ir", baseFilename, baseFilename); + + // Clean up + executeCommand("rm {}.metal", baseFilename); + executeCommand("rm {}.ir", baseFilename); + + // Load from the newly Generated AIR + MemoryMappedFile airFile(fmt::format("{}.metallib", baseFilename)); + std::span airData = std::span(airFile.data(), airFile.size()); + library = LibraryFromAIR(std::span(cacheFileData.data(), cacheFileData.size())); + + // Store in the cache + uint64 h1, h2; + GenerateShaderPrecompiledCacheFilename(m_type, m_baseHash, m_auxHash, h1, h2); + s_airCache->AddFile({ h1, h2 }, airData.data(), airData.size()); + } + } + else + { + // Compile from source + library = LibraryFromSource(); + } + + if (!library) + return; + m_function = library->newFunction(ToNSString("main0")); library->release(); - FinishCompilation(); - // Count shader compilation if (ShouldCountCompilation()) g_compiled_shaders_total++; } -void RendererShaderMtl::CompileFromAIR(std::span data) -{ - // TODO: implement this - printf("LOADING SHADER FROM AIR CACHE\n"); -} - void RendererShaderMtl::FinishCompilation() { m_mslCode.clear(); diff --git a/src/Cafe/HW/Latte/Renderer/Metal/RendererShaderMtl.h b/src/Cafe/HW/Latte/Renderer/Metal/RendererShaderMtl.h index 98d18687..98973a0e 100644 --- a/src/Cafe/HW/Latte/Renderer/Metal/RendererShaderMtl.h +++ b/src/Cafe/HW/Latte/Renderer/Metal/RendererShaderMtl.h @@ -67,9 +67,11 @@ private: bool ShouldCountCompilation() const; - void CompileInternal(); + MTL::Library* LibraryFromSource(); - void CompileFromAIR(std::span data); + MTL::Library* LibraryFromAIR(std::span data); + + void CompileInternal(); void FinishCompilation(); };