diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 72bbcf52..6ae4b892 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -39,7 +39,7 @@ jobs: - name: "Install system dependencies" run: | sudo apt update -qq - sudo apt install -y clang-15 cmake freeglut3-dev libgcrypt20-dev libglm-dev libgtk-3-dev libpulse-dev libsecret-1-dev libsystemd-dev libudev-dev nasm ninja-build + sudo apt install -y clang-15 cmake freeglut3-dev libgcrypt20-dev libglm-dev libgtk-3-dev libpulse-dev libsecret-1-dev libsystemd-dev libudev-dev nasm ninja-build libbluetooth-dev - name: "Setup cmake" uses: jwlawson/actions-setup-cmake@v2 @@ -96,7 +96,7 @@ jobs: - name: "Install system dependencies" run: | sudo apt update -qq - sudo apt install -y clang-15 cmake freeglut3-dev libgcrypt20-dev libglm-dev libgtk-3-dev libpulse-dev libsecret-1-dev libsystemd-dev nasm ninja-build appstream + sudo apt install -y clang-15 cmake freeglut3-dev libgcrypt20-dev libglm-dev libgtk-3-dev libpulse-dev libsecret-1-dev libsystemd-dev nasm ninja-build appstream libbluetooth-dev - name: "Build AppImage" run: | diff --git a/.github/workflows/deploy_experimental_release.yml b/.github/workflows/deploy_release.yml similarity index 98% rename from .github/workflows/deploy_experimental_release.yml rename to .github/workflows/deploy_release.yml index 97e0c69e..2b9ee491 100644 --- a/.github/workflows/deploy_experimental_release.yml +++ b/.github/workflows/deploy_release.yml @@ -1,4 +1,4 @@ -name: Deploy experimental release +name: Deploy release on: workflow_dispatch: inputs: @@ -54,7 +54,7 @@ jobs: next_version_major: ${{ needs.calculate-version.outputs.next_version_major }} next_version_minor: ${{ needs.calculate-version.outputs.next_version_minor }} deploy: - name: Deploy experimental release + name: Deploy release runs-on: ubuntu-22.04 needs: [call-release-build, calculate-version] steps: diff --git a/.github/workflows/deploy_stable_release.yml b/.github/workflows/deploy_stable_release.yml deleted file mode 100644 index fd339e7d..00000000 --- a/.github/workflows/deploy_stable_release.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: Create new release -on: - workflow_dispatch: - inputs: - PlaceholderInput: - description: PlaceholderInput - required: false -jobs: - call-release-build: - uses: ./.github/workflows/build.yml - with: - deploymode: release - deploy: - name: Deploy release - runs-on: ubuntu-20.04 - needs: call-release-build - steps: - - uses: actions/checkout@v3 - - - uses: actions/download-artifact@v4 - with: - name: cemu-bin-linux-x64 - path: cemu-bin-linux-x64 - - - uses: actions/download-artifact@v4 - with: - name: cemu-appimage-x64 - path: cemu-appimage-x64 - - - uses: actions/download-artifact@v4 - with: - name: cemu-bin-windows-x64 - path: cemu-bin-windows-x64 - - - uses: actions/download-artifact@v4 - with: - name: cemu-bin-macos-x64 - path: cemu-bin-macos-x64 - - - name: Initialize - run: | - mkdir upload - sudo apt update -qq - sudo apt install -y zip - - - name: Get Cemu release version - run: | - gcc -o getversion .github/getversion.cpp - echo "Cemu CI version: $(./getversion)" - echo "CEMU_FOLDER_NAME=Cemu_$(./getversion)" >> $GITHUB_ENV - echo "CEMU_VERSION=$(./getversion)" >> $GITHUB_ENV - - - name: Create release from windows-bin - run: | - ls ./ - ls ./bin/ - cp -R ./bin ./${{ env.CEMU_FOLDER_NAME }} - mv cemu-bin-windows-x64/Cemu.exe ./${{ env.CEMU_FOLDER_NAME }}/Cemu.exe - zip -9 -r upload/cemu-${{ env.CEMU_VERSION }}-windows-x64.zip ${{ env.CEMU_FOLDER_NAME }} - rm -r ./${{ env.CEMU_FOLDER_NAME }} - - - name: Create appimage - run: | - VERSION=${{ env.CEMU_VERSION }} - echo "Cemu Version is $VERSION" - ls cemu-appimage-x64 - mv cemu-appimage-x64/Cemu-*-x86_64.AppImage upload/Cemu-$VERSION-x86_64.AppImage - - - name: Create release from ubuntu-bin - run: | - ls ./ - ls ./bin/ - cp -R ./bin ./${{ env.CEMU_FOLDER_NAME }} - mv cemu-bin-linux-x64/Cemu ./${{ env.CEMU_FOLDER_NAME }}/Cemu - zip -9 -r upload/cemu-${{ env.CEMU_VERSION }}-ubuntu-20.04-x64.zip ${{ env.CEMU_FOLDER_NAME }} - rm -r ./${{ env.CEMU_FOLDER_NAME }} - - - name: Create release from macos-bin - run: cp cemu-bin-macos-x64/Cemu.dmg upload/cemu-${{ env.CEMU_VERSION }}-macos-12-x64.dmg - - - name: Create release - run: | - wget -O ghr.tar.gz https://github.com/tcnksm/ghr/releases/download/v0.15.0/ghr_v0.15.0_linux_amd64.tar.gz - tar xvzf ghr.tar.gz; rm ghr.tar.gz - ghr_v0.15.0_linux_amd64/ghr -t ${{ secrets.GITHUB_TOKEN }} -n "Cemu ${{ env.CEMU_VERSION }}" -b "Changelog:" v${{ env.CEMU_VERSION }} ./upload diff --git a/.github/workflows/generate_pot.yml b/.github/workflows/generate_pot.yml index 7dfa86f8..b057d441 100644 --- a/.github/workflows/generate_pot.yml +++ b/.github/workflows/generate_pot.yml @@ -35,7 +35,7 @@ jobs: -o cemu.pot - name: Upload artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: POT file path: ./cemu.pot diff --git a/BUILD.md b/BUILD.md index 44d69c6c..31c26531 100644 --- a/BUILD.md +++ b/BUILD.md @@ -46,10 +46,10 @@ To compile Cemu, a recent enough compiler and STL with C++20 support is required ### Dependencies #### For Arch and derivatives: -`sudo pacman -S --needed base-devel clang cmake freeglut git glm gtk3 libgcrypt libpulse libsecret linux-headers llvm nasm ninja systemd unzip zip` +`sudo pacman -S --needed base-devel bluez-libs clang cmake freeglut git glm gtk3 libgcrypt libpulse libsecret linux-headers llvm nasm ninja systemd unzip zip` #### For Debian, Ubuntu and derivatives: -`sudo apt install -y cmake curl clang-15 freeglut3-dev git libgcrypt20-dev libglm-dev libgtk-3-dev libpulse-dev libsecret-1-dev libsystemd-dev libtool nasm ninja-build` +`sudo apt install -y cmake curl clang-15 freeglut3-dev git libbluetooth-dev libgcrypt20-dev libglm-dev libgtk-3-dev libpulse-dev libsecret-1-dev libsystemd-dev libtool nasm ninja-build` You may also need to install `libusb-1.0-0-dev` as a workaround for an issue with the vcpkg hidapi package. @@ -57,7 +57,7 @@ At Step 3 in [Build Cemu using cmake and clang](#build-cemu-using-cmake-and-clan `cmake -S . -B build -DCMAKE_BUILD_TYPE=release -DCMAKE_C_COMPILER=/usr/bin/clang-15 -DCMAKE_CXX_COMPILER=/usr/bin/clang++-15 -G Ninja -DCMAKE_MAKE_PROGRAM=/usr/bin/ninja` #### For Fedora and derivatives: -`sudo dnf install clang cmake cubeb-devel freeglut-devel git glm-devel gtk3-devel kernel-headers libgcrypt-devel libsecret-devel libtool libusb1-devel llvm nasm ninja-build perl-core systemd-devel zlib-devel zlib-static` +`sudo dnf install bluez-libs-devel clang cmake cubeb-devel freeglut-devel git glm-devel gtk3-devel kernel-headers libgcrypt-devel libsecret-devel libtool libusb1-devel llvm nasm ninja-build perl-core systemd-devel wayland-protocols-devel zlib-devel zlib-static` ### Build Cemu @@ -120,6 +120,9 @@ This section refers to running `cmake -S...` (truncated). * Compiling failed during rebuild after `git pull` with an error that mentions RPATH * Add the following and try running the command again: * `-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON` +* Environment variable `VCPKG_FORCE_SYSTEM_BINARIES` must be set. + * Execute the folowing and then try running the command again: + * `export VCPKG_FORCE_SYSTEM_BINARIES=1` * If you are getting a random error, read the [package-name-and-platform]-out.log and [package-name-and-platform]-err.log for the actual reason to see if you might be lacking the headers from a dependency. @@ -189,3 +192,41 @@ Then install the dependencies: If CMake complains about Cemu already being compiled or another similar error, try deleting the `CMakeCache.txt` file inside the `build` folder and retry building. +## CMake configure flags +Some flags can be passed during CMake configure to customise which features are enabled on build. + +Example usage: `cmake -S . -B build -DCMAKE_BUILD_TYPE=release -DENABLE_SDL=ON -DENABLE_VULKAN=OFF` + +### All platforms +| Flag | | Description | Default | Note | +|--------------------|:--|-----------------------------------------------------------------------------|---------|--------------------| +| ALLOW_PORTABLE | | Allow Cemu to use the `portable` directory to store configs and data | ON | | +| CEMU_CXX_FLAGS | | Flags passed straight to the compiler, e.g. `-march=native`, `-Wall`, `/W3` | "" | | +| ENABLE_CUBEB | | Enable cubeb audio backend | ON | | +| ENABLE_DISCORD_RPC | | Enable Discord Rich presence support | ON | | +| ENABLE_OPENGL | | Enable OpenGL graphics backend | ON | Currently required | +| ENABLE_HIDAPI | | Enable HIDAPI (used for Wiimote controller API) | ON | | +| ENABLE_SDL | | Enable SDLController controller API | ON | Currently required | +| ENABLE_VCPKG | | Use VCPKG package manager to obtain dependencies | ON | | +| ENABLE_VULKAN | | Enable the Vulkan graphics backend | ON | | +| ENABLE_WXWIDGETS | | Enable wxWidgets UI | ON | Currently required | + +### Windows +| Flag | Description | Default | Note | +|--------------------|-----------------------------------|---------|--------------------| +| ENABLE_DIRECTAUDIO | Enable DirectAudio audio backend | ON | Currently required | +| ENABLE_DIRECTINPUT | Enable DirectInput controller API | ON | Currently required | +| ENABLE_XAUDIO | Enable XAudio audio backend | ON | | +| ENABLE_XINPUT | Enable XInput controller API | ON | | + +### Linux +| Flag | Description | Default | +|-----------------------|----------------------------------------------------|---------| +| ENABLE_BLUEZ | Build with Bluez (used for Wiimote controller API) | ON | +| ENABLE_FERAL_GAMEMODE | Enable Feral Interactive GameMode support | ON | +| ENABLE_WAYLAND | Enable Wayland support | ON | + +### macOS +| Flag | Description | Default | +|--------------|------------------------------------------------|---------| +| MACOS_BUNDLE | MacOS executable will be an application bundle | OFF | diff --git a/CMakeLists.txt b/CMakeLists.txt index 0d1e42db..6d24bd81 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 3.21.1) option(ENABLE_VCPKG "Enable the vcpkg package manager" ON) option(MACOS_BUNDLE "The executable when built on macOS will be created as an application bundle" OFF) +option(ALLOW_PORTABLE "Allow Cemu to be run in portable mode" ON) # used by CI script to set version: set(EMULATOR_VERSION_MAJOR "0" CACHE STRING "") @@ -98,6 +99,7 @@ endif() if (UNIX AND NOT APPLE) option(ENABLE_WAYLAND "Build with Wayland support" ON) option(ENABLE_FERAL_GAMEMODE "Enables Feral Interactive GameMode Support" ON) + option(ENABLE_BLUEZ "Build with Bluez support" ON) endif() option(ENABLE_OPENGL "Enables the OpenGL backend" ON) @@ -122,23 +124,6 @@ if (WIN32) endif() option(ENABLE_CUBEB "Enabled cubeb backend" ON) -# usb hid backends -if (WIN32) - option(ENABLE_NSYSHID_WINDOWS_HID "Enables the native Windows HID backend for nsyshid" ON) -endif () -# libusb and windows hid backends shouldn't be active at the same time; otherwise we'd see all devices twice! -if (NOT ENABLE_NSYSHID_WINDOWS_HID) - option(ENABLE_NSYSHID_LIBUSB "Enables the libusb backend for nsyshid" ON) -else () - set(ENABLE_NSYSHID_LIBUSB OFF CACHE BOOL "" FORCE) -endif () -if (ENABLE_NSYSHID_WINDOWS_HID) - add_compile_definitions(NSYSHID_ENABLE_BACKEND_WINDOWS_HID) -endif () -if (ENABLE_NSYSHID_LIBUSB) - add_compile_definitions(NSYSHID_ENABLE_BACKEND_LIBUSB) -endif () - option(ENABLE_WXWIDGETS "Build with wxWidgets UI (Currently required)" ON) set(THREADS_PREFER_PTHREAD_FLAG true) @@ -179,6 +164,12 @@ if (UNIX AND NOT APPLE) endif() find_package(GTK3 REQUIRED) + if(ENABLE_BLUEZ) + find_package(bluez REQUIRED) + set(ENABLE_WIIMOTE ON) + add_compile_definitions(HAS_BLUEZ) + endif() + endif() if (ENABLE_VULKAN) diff --git a/bin/resources/ar/‏‏cemu.mo b/bin/resources/ar/‏‏cemu.mo new file mode 100644 index 00000000..4062628b Binary files /dev/null and b/bin/resources/ar/‏‏cemu.mo differ diff --git a/bin/resources/de/cemu.mo b/bin/resources/de/cemu.mo index 8dc4e8cc..cd9edd3c 100644 Binary files a/bin/resources/de/cemu.mo and b/bin/resources/de/cemu.mo differ diff --git a/bin/resources/ru/cemu.mo b/bin/resources/ru/cemu.mo index 4ff04e2b..eb8f372f 100644 Binary files a/bin/resources/ru/cemu.mo and b/bin/resources/ru/cemu.mo differ diff --git a/bin/resources/sv/cemu.mo b/bin/resources/sv/cemu.mo index 3e850b36..c8fd68ee 100644 Binary files a/bin/resources/sv/cemu.mo and b/bin/resources/sv/cemu.mo differ diff --git a/cmake/Findbluez.cmake b/cmake/Findbluez.cmake new file mode 100644 index 00000000..007cdac9 --- /dev/null +++ b/cmake/Findbluez.cmake @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: 2022 Andrea Pappacoda +# SPDX-License-Identifier: ISC + +find_package(bluez CONFIG) +if (NOT bluez_FOUND) + find_package(PkgConfig) + if (PKG_CONFIG_FOUND) + pkg_search_module(bluez IMPORTED_TARGET GLOBAL bluez-1.0 bluez) + if (bluez_FOUND) + add_library(bluez::bluez ALIAS PkgConfig::bluez) + endif () + endif () +endif () + +find_package_handle_standard_args(bluez + REQUIRED_VARS + bluez_LINK_LIBRARIES + bluez_FOUND + VERSION_VAR bluez_VERSION +) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c3271790..70db3e3a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -83,8 +83,8 @@ if (MACOS_BUNDLE) set(MACOSX_BUNDLE_ICON_FILE "cemu.icns") set(MACOSX_BUNDLE_GUI_IDENTIFIER "info.cemu.Cemu") set(MACOSX_BUNDLE_BUNDLE_NAME "Cemu") - set(MACOSX_BUNDLE_SHORT_VERSION_STRING ${CMAKE_PROJECT_VERSION}) - set(MACOSX_BUNDLE_BUNDLE_VERSION ${CMAKE_PROJECT_VERSION}) + set(MACOSX_BUNDLE_SHORT_VERSION_STRING "${EMULATOR_VERSION_MAJOR}.${EMULATOR_VERSION_MINOR}.${EMULATOR_VERSION_PATCH}") + set(MACOSX_BUNDLE_BUNDLE_VERSION "${EMULATOR_VERSION_MAJOR}.${EMULATOR_VERSION_MINOR}.${EMULATOR_VERSION_PATCH}") set(MACOSX_BUNDLE_COPYRIGHT "Copyright © 2024 Cemu Project") set(MACOSX_BUNDLE_CATEGORY "public.app-category.games") @@ -102,12 +102,18 @@ if (MACOS_BUNDLE) COMMAND ${CMAKE_COMMAND} ARGS -E copy_directory "${CMAKE_SOURCE_DIR}/bin/${folder}" "${CMAKE_SOURCE_DIR}/bin/${OUTPUT_NAME}.app/Contents/SharedSupport/${folder}") endforeach(folder) + if(CMAKE_BUILD_TYPE STREQUAL "Debug") + set(LIBUSB_PATH "${CMAKE_BINARY_DIR}/vcpkg_installed/x64-osx/debug/lib/libusb-1.0.0.dylib") + else() + set(LIBUSB_PATH "${CMAKE_BINARY_DIR}/vcpkg_installed/x64-osx/lib/libusb-1.0.0.dylib") + endif() + add_custom_command (TARGET CemuBin POST_BUILD COMMAND ${CMAKE_COMMAND} ARGS -E copy "/usr/local/lib/libMoltenVK.dylib" "${CMAKE_SOURCE_DIR}/bin/${OUTPUT_NAME}.app/Contents/Frameworks/libMoltenVK.dylib" - COMMAND ${CMAKE_COMMAND} ARGS -E copy "${CMAKE_BINARY_DIR}/vcpkg_installed/x64-osx/lib/libusb-1.0.0.dylib" "${CMAKE_SOURCE_DIR}/bin/${OUTPUT_NAME}.app/Contents/Frameworks/libusb-1.0.0.dylib" + COMMAND ${CMAKE_COMMAND} ARGS -E copy "${LIBUSB_PATH}" "${CMAKE_SOURCE_DIR}/bin/${OUTPUT_NAME}.app/Contents/Frameworks/libusb-1.0.0.dylib" COMMAND ${CMAKE_COMMAND} ARGS -E copy "${CMAKE_SOURCE_DIR}/src/resource/update.sh" "${CMAKE_SOURCE_DIR}/bin/${OUTPUT_NAME}.app/Contents/MacOS/update.sh" COMMAND bash -c "install_name_tool -add_rpath @executable_path/../Frameworks ${CMAKE_SOURCE_DIR}/bin/${OUTPUT_NAME}.app/Contents/MacOS/${OUTPUT_NAME}" - COMMAND bash -c "install_name_tool -change /Users/runner/work/Cemu/Cemu/build/vcpkg_installed/x64-osx/lib/libusb-1.0.0.dylib @executable_path/../Frameworks/libusb-1.0.0.dylib ${CMAKE_SOURCE_DIR}/bin/${OUTPUT_NAME}.app/Contents/MacOS/${OUTPUT_NAME}") + COMMAND bash -c "install_name_tool -change ${LIBUSB_PATH} @executable_path/../Frameworks/libusb-1.0.0.dylib ${CMAKE_SOURCE_DIR}/bin/${OUTPUT_NAME}.app/Contents/MacOS/${OUTPUT_NAME}") endif() set_target_properties(CemuBin PROPERTIES diff --git a/src/Cafe/CMakeLists.txt b/src/Cafe/CMakeLists.txt index 5f597731..ce84fecc 100644 --- a/src/Cafe/CMakeLists.txt +++ b/src/Cafe/CMakeLists.txt @@ -463,8 +463,8 @@ add_library(CemuCafe OS/libs/nsyshid/BackendEmulated.h OS/libs/nsyshid/BackendLibusb.cpp OS/libs/nsyshid/BackendLibusb.h - OS/libs/nsyshid/BackendWindowsHID.cpp - OS/libs/nsyshid/BackendWindowsHID.h + OS/libs/nsyshid/Dimensions.cpp + OS/libs/nsyshid/Dimensions.h OS/libs/nsyshid/Infinity.cpp OS/libs/nsyshid/Infinity.h OS/libs/nsyshid/Skylander.cpp @@ -530,6 +530,12 @@ set_property(TARGET CemuCafe PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreaded$<$data(), execData->size()); sForegroundTitleId = 0xFFFFFFFF00000000ULL | (uint64)h; cemuLog_log(LogType::Force, "Generated placeholder TitleId: {:016x}", sForegroundTitleId); @@ -834,7 +835,7 @@ namespace CafeSystem // load executable SetupExecutable(); InitVirtualMlcStorage(); - return STATUS_CODE::SUCCESS; + return PREPARE_STATUS_CODE::SUCCESS; } void _LaunchTitleThread() @@ -843,7 +844,7 @@ namespace CafeSystem module->TitleStart(); cemu_initForGame(); // enter scheduler - if (ActiveSettings::GetCPUMode() == CPUMode::MulticoreRecompiler) + if (ActiveSettings::GetCPUMode() == CPUMode::MulticoreRecompiler && !LaunchSettings::ForceInterpreter()) coreinit::OSSchedulerBegin(3); else coreinit::OSSchedulerBegin(1); diff --git a/src/Cafe/CafeSystem.h b/src/Cafe/CafeSystem.h index c4043a59..e9de8d7d 100644 --- a/src/Cafe/CafeSystem.h +++ b/src/Cafe/CafeSystem.h @@ -15,20 +15,19 @@ namespace CafeSystem virtual void CafeRecreateCanvas() = 0; }; - enum class STATUS_CODE + enum class PREPARE_STATUS_CODE { SUCCESS, INVALID_RPX, UNABLE_TO_MOUNT, // failed to mount through TitleInfo (most likely caused by an invalid or outdated path) - //BAD_META_DATA, - the title list only stores titles with valid meta, so this error code is impossible }; void Initialize(); void SetImplementation(SystemImplementation* impl); void Shutdown(); - STATUS_CODE PrepareForegroundTitle(TitleId titleId); - STATUS_CODE PrepareForegroundTitleFromStandaloneRPX(const fs::path& path); + PREPARE_STATUS_CODE PrepareForegroundTitle(TitleId titleId); + PREPARE_STATUS_CODE PrepareForegroundTitleFromStandaloneRPX(const fs::path& path); void LaunchForegroundTitle(); bool IsTitleRunning(); diff --git a/src/Cafe/Filesystem/FST/FST.cpp b/src/Cafe/Filesystem/FST/FST.cpp index 570671d4..f1255778 100644 --- a/src/Cafe/Filesystem/FST/FST.cpp +++ b/src/Cafe/Filesystem/FST/FST.cpp @@ -3,8 +3,7 @@ #include "Cemu/ncrypto/ncrypto.h" #include "Cafe/Filesystem/WUD/wud.h" #include "util/crypto/aes128.h" -#include "openssl/evp.h" /* EVP_Digest */ -#include "openssl/sha.h" /* SHA1 / SHA256_DIGEST_LENGTH */ +#include "openssl/sha.h" /* SHA1 / SHA256 */ #include "fstUtil.h" #include "FST.h" @@ -141,7 +140,7 @@ struct DiscPartitionTableHeader static constexpr uint32 MAGIC_VALUE = 0xCCA6E67B; /* +0x00 */ uint32be magic; - /* +0x04 */ uint32be sectorSize; // must be 0x8000? + /* +0x04 */ uint32be blockSize; // must be 0x8000? /* +0x08 */ uint8 partitionTableHash[20]; // hash of the data range at +0x800 to end of sector (0x8000) /* +0x1C */ uint32be numPartitions; }; @@ -164,10 +163,10 @@ struct DiscPartitionHeader static constexpr uint32 MAGIC_VALUE = 0xCC93A4F5; /* +0x00 */ uint32be magic; - /* +0x04 */ uint32be sectorSize; // must match DISC_SECTOR_SIZE + /* +0x04 */ uint32be sectorSize; // must match DISC_SECTOR_SIZE for hashed blocks /* +0x08 */ uint32be ukn008; - /* +0x0C */ uint32be ukn00C; + /* +0x0C */ uint32be ukn00C; // h3 array size? /* +0x10 */ uint32be h3HashNum; /* +0x14 */ uint32be fstSize; // in bytes /* +0x18 */ uint32be fstSector; // relative to partition start @@ -178,13 +177,15 @@ struct DiscPartitionHeader /* +0x24 */ uint8 fstHashType; /* +0x25 */ uint8 fstEncryptionType; // purpose of this isn't really understood. Maybe it controls which key is being used? (1 -> disc key, 2 -> partition key) - /* +0x26 */ uint8 versionA; - /* +0x27 */ uint8 ukn027; // also a version field? + /* +0x26 */ uint8be versionA; + /* +0x27 */ uint8be ukn027; // also a version field? // there is an array at +0x40 ? Related to H3 list. Also related to value at +0x0C and h3HashNum + /* +0x28 */ uint8be _uknOrPadding028[0x18]; + /* +0x40 */ uint8be h3HashArray[32]; // dynamic size. Only present if fstHashType != 0 }; -static_assert(sizeof(DiscPartitionHeader) == 0x28); +static_assert(sizeof(DiscPartitionHeader) == 0x40+0x20); bool FSTVolume::FindDiscKey(const fs::path& path, NCrypto::AesKey& discTitleKey) { @@ -269,7 +270,7 @@ FSTVolume* FSTVolume::OpenFromDiscImage(const fs::path& path, NCrypto::AesKey& d cemuLog_log(LogType::Force, "Disc image rejected because decryption failed"); return nullptr; } - if (partitionHeader->sectorSize != DISC_SECTOR_SIZE) + if (partitionHeader->blockSize != DISC_SECTOR_SIZE) { cemuLog_log(LogType::Force, "Disc image rejected because partition sector size is invalid"); return nullptr; @@ -336,6 +337,9 @@ FSTVolume* FSTVolume::OpenFromDiscImage(const fs::path& path, NCrypto::AesKey& d cemu_assert_debug(partitionHeaderSI.fstEncryptionType == 1); // todo - check other fields? + if(partitionHeaderSI.fstHashType == 0 && partitionHeaderSI.h3HashNum != 0) + cemuLog_log(LogType::Force, "FST: Partition uses unhashed blocks but stores a non-zero amount of H3 hashes"); + // GM partition DiscPartitionHeader partitionHeaderGM{}; if (!readPartitionHeader(partitionHeaderGM, gmPartitionIndex)) @@ -349,9 +353,10 @@ FSTVolume* FSTVolume::OpenFromDiscImage(const fs::path& path, NCrypto::AesKey& d // if decryption is necessary // load SI FST dataSource->SetBaseOffset((uint64)partitionArray[siPartitionIndex].partitionAddress * DISC_SECTOR_SIZE); - auto siFST = OpenFST(dataSource.get(), (uint64)partitionHeaderSI.fstSector * DISC_SECTOR_SIZE, partitionHeaderSI.fstSize, &discTitleKey, static_cast(partitionHeaderSI.fstHashType)); + auto siFST = OpenFST(dataSource.get(), (uint64)partitionHeaderSI.fstSector * DISC_SECTOR_SIZE, partitionHeaderSI.fstSize, &discTitleKey, static_cast(partitionHeaderSI.fstHashType), nullptr); if (!siFST) return nullptr; + cemu_assert_debug(!(siFST->HashIsDisabled() && partitionHeaderSI.h3HashNum != 0)); // if hash is disabled, no H3 data may be present // load ticket file for partition that we want to decrypt NCrypto::ETicketParser ticketParser; std::vector ticketData = siFST->ExtractFile(fmt::format("{:02x}/title.tik", gmPartitionIndex)); @@ -360,16 +365,32 @@ FSTVolume* FSTVolume::OpenFromDiscImage(const fs::path& path, NCrypto::AesKey& d cemuLog_log(LogType::Force, "Disc image ticket file is invalid"); return nullptr; } +#if 0 + // each SI partition seems to contain a title.tmd that we could parse and which should have information about the associated GM partition + // but the console seems to ignore this file for disc images, at least when mounting, so we shouldn't rely on it either + std::vector tmdData = siFST->ExtractFile(fmt::format("{:02x}/title.tmd", gmPartitionIndex)); + if (tmdData.empty()) + { + cemuLog_log(LogType::Force, "Disc image TMD file is missing"); + return nullptr; + } + // parse TMD + NCrypto::TMDParser tmdParser; + if (!tmdParser.parse(tmdData.data(), tmdData.size())) + { + cemuLog_log(LogType::Force, "Disc image TMD file is invalid"); + return nullptr; + } +#endif delete siFST; - NCrypto::AesKey gmTitleKey; ticketParser.GetTitleKey(gmTitleKey); - // load GM partition dataSource->SetBaseOffset((uint64)partitionArray[gmPartitionIndex].partitionAddress * DISC_SECTOR_SIZE); - FSTVolume* r = OpenFST(std::move(dataSource), (uint64)partitionHeaderGM.fstSector * DISC_SECTOR_SIZE, partitionHeaderGM.fstSize, &gmTitleKey, static_cast(partitionHeaderGM.fstHashType)); + FSTVolume* r = OpenFST(std::move(dataSource), (uint64)partitionHeaderGM.fstSector * DISC_SECTOR_SIZE, partitionHeaderGM.fstSize, &gmTitleKey, static_cast(partitionHeaderGM.fstHashType), nullptr); if (r) SET_FST_ERROR(OK); + cemu_assert_debug(!(r->HashIsDisabled() && partitionHeaderGM.h3HashNum != 0)); // if hash is disabled, no H3 data may be present return r; } @@ -426,15 +447,15 @@ FSTVolume* FSTVolume::OpenFromContentFolder(fs::path folderPath, ErrorCode* erro } // load FST // fstSize = size of first cluster? - FSTVolume* fstVolume = FSTVolume::OpenFST(std::move(dataSource), 0, fstSize, &titleKey, fstHashMode); + FSTVolume* fstVolume = FSTVolume::OpenFST(std::move(dataSource), 0, fstSize, &titleKey, fstHashMode, &tmdParser); if (fstVolume) SET_FST_ERROR(OK); return fstVolume; } -FSTVolume* FSTVolume::OpenFST(FSTDataSource* dataSource, uint64 fstOffset, uint32 fstSize, NCrypto::AesKey* partitionTitleKey, ClusterHashMode fstHashMode) +FSTVolume* FSTVolume::OpenFST(FSTDataSource* dataSource, uint64 fstOffset, uint32 fstSize, NCrypto::AesKey* partitionTitleKey, ClusterHashMode fstHashMode, NCrypto::TMDParser* optionalTMD) { - cemu_assert_debug(fstHashMode != ClusterHashMode::RAW || fstHashMode != ClusterHashMode::RAW2); + cemu_assert_debug(fstHashMode != ClusterHashMode::RAW || fstHashMode != ClusterHashMode::RAW_STREAM); if (fstSize < sizeof(FSTHeader)) return nullptr; constexpr uint64 FST_CLUSTER_OFFSET = 0; @@ -465,6 +486,34 @@ FSTVolume* FSTVolume::OpenFST(FSTDataSource* dataSource, uint64 fstOffset, uint3 clusterTable[i].offset = clusterDataTable[i].offset; clusterTable[i].size = clusterDataTable[i].size; clusterTable[i].hashMode = static_cast((uint8)clusterDataTable[i].hashMode); + clusterTable[i].hasContentHash = false; // from the TMD file (H4?) + } + // if the TMD is available (when opening .app files) we can use the extra info from it to validate unhashed clusters + // each content entry in the TMD corresponds to one cluster used by the FST + if(optionalTMD) + { + if(numCluster != optionalTMD->GetContentList().size()) + { + cemuLog_log(LogType::Force, "FST: Number of clusters does not match TMD content list"); + return nullptr; + } + auto& contentList = optionalTMD->GetContentList(); + for(size_t i=0; im_offsetFactor = fstHeader->offsetFactor; fstVolume->m_sectorSize = DISC_SECTOR_SIZE; fstVolume->m_partitionTitlekey = *partitionTitleKey; - std::swap(fstVolume->m_cluster, clusterTable); - std::swap(fstVolume->m_entries, fstEntries); - std::swap(fstVolume->m_nameStringTable, nameStringTable); + fstVolume->m_hashIsDisabled = fstHeader->hashIsDisabled != 0; + fstVolume->m_cluster = std::move(clusterTable); + fstVolume->m_entries = std::move(fstEntries); + fstVolume->m_nameStringTable = std::move(nameStringTable); return fstVolume; } -FSTVolume* FSTVolume::OpenFST(std::unique_ptr dataSource, uint64 fstOffset, uint32 fstSize, NCrypto::AesKey* partitionTitleKey, ClusterHashMode fstHashMode) +FSTVolume* FSTVolume::OpenFST(std::unique_ptr dataSource, uint64 fstOffset, uint32 fstSize, NCrypto::AesKey* partitionTitleKey, ClusterHashMode fstHashMode, NCrypto::TMDParser* optionalTMD) { FSTDataSource* ds = dataSource.release(); - FSTVolume* fstVolume = OpenFST(ds, fstOffset, fstSize, partitionTitleKey, fstHashMode); + FSTVolume* fstVolume = OpenFST(ds, fstOffset, fstSize, partitionTitleKey, fstHashMode, optionalTMD); if (!fstVolume) { delete ds; @@ -757,7 +807,7 @@ uint32 FSTVolume::ReadFile(FSTFileHandle& fileHandle, uint32 offset, uint32 size return 0; cemu_assert_debug(!HAS_FLAG(entry.GetFlags(), FSTEntry::FLAGS::FLAG_LINK)); FSTCluster& cluster = m_cluster[entry.fileInfo.clusterIndex]; - if (cluster.hashMode == ClusterHashMode::RAW || cluster.hashMode == ClusterHashMode::RAW2) + if (cluster.hashMode == ClusterHashMode::RAW || cluster.hashMode == ClusterHashMode::RAW_STREAM) return ReadFile_HashModeRaw(entry.fileInfo.clusterIndex, entry, offset, size, dataOut); else if (cluster.hashMode == ClusterHashMode::HASH_INTERLEAVED) return ReadFile_HashModeHashed(entry.fileInfo.clusterIndex, entry, offset, size, dataOut); @@ -765,87 +815,15 @@ uint32 FSTVolume::ReadFile(FSTFileHandle& fileHandle, uint32 offset, uint32 size return 0; } -uint32 FSTVolume::ReadFile_HashModeRaw(uint32 clusterIndex, FSTEntry& entry, uint32 readOffset, uint32 readSize, void* dataOut) -{ - const uint32 readSizeInput = readSize; - uint8* dataOutU8 = (uint8*)dataOut; - if (readOffset >= entry.fileInfo.fileSize) - return 0; - else if ((readOffset + readSize) >= entry.fileInfo.fileSize) - readSize = (entry.fileInfo.fileSize - readOffset); - - const FSTCluster& cluster = m_cluster[clusterIndex]; - uint64 clusterOffset = (uint64)cluster.offset * m_sectorSize; - uint64 absFileOffset = entry.fileInfo.fileOffset * m_offsetFactor + readOffset; - - // make sure the raw range we read is aligned to AES block size (16) - uint64 readAddrStart = absFileOffset & ~0xF; - uint64 readAddrEnd = (absFileOffset + readSize + 0xF) & ~0xF; - - bool usesInitialIV = readOffset < 16; - if (!usesInitialIV) - readAddrStart -= 16; // read previous AES block since we require it for the IV - uint32 prePadding = (uint32)(absFileOffset - readAddrStart); // number of extra bytes we read before readOffset (for AES alignment and IV calculation) - uint32 postPadding = (uint32)(readAddrEnd - (absFileOffset + readSize)); - - uint8 readBuffer[64 * 1024]; - // read first chunk - // if file read offset (readOffset) is within the first AES-block then use initial IV calculated from cluster index - // otherwise read previous AES-block is the IV (AES-CBC) - uint64 readAddrCurrent = readAddrStart; - uint32 rawBytesToRead = (uint32)std::min((readAddrEnd - readAddrStart), (uint64)sizeof(readBuffer)); - if (m_dataSource->readData(clusterIndex, clusterOffset, readAddrCurrent, readBuffer, rawBytesToRead) != rawBytesToRead) - { - cemuLog_log(LogType::Force, "FST read error in raw content"); - return 0; - } - readAddrCurrent += rawBytesToRead; - - uint8 iv[16]{}; - if (usesInitialIV) - { - // for the first AES block, the IV is initialized from cluster index - iv[0] = (uint8)(clusterIndex >> 8); - iv[1] = (uint8)(clusterIndex >> 0); - AES128_CBC_decrypt_updateIV(readBuffer, readBuffer, rawBytesToRead, m_partitionTitlekey.b, iv); - std::memcpy(dataOutU8, readBuffer + prePadding, rawBytesToRead - prePadding - postPadding); - dataOutU8 += (rawBytesToRead - prePadding - postPadding); - readSize -= (rawBytesToRead - prePadding - postPadding); - } - else - { - // IV is initialized from previous AES block (AES-CBC) - std::memcpy(iv, readBuffer, 16); - AES128_CBC_decrypt_updateIV(readBuffer + 16, readBuffer + 16, rawBytesToRead - 16, m_partitionTitlekey.b, iv); - std::memcpy(dataOutU8, readBuffer + prePadding, rawBytesToRead - prePadding - postPadding); - dataOutU8 += (rawBytesToRead - prePadding - postPadding); - readSize -= (rawBytesToRead - prePadding - postPadding); - } - - // read remaining chunks - while (readSize > 0) - { - uint32 bytesToRead = (uint32)std::min((uint32)sizeof(readBuffer), readSize); - uint32 alignedBytesToRead = (bytesToRead + 15) & ~0xF; - if (m_dataSource->readData(clusterIndex, clusterOffset, readAddrCurrent, readBuffer, alignedBytesToRead) != alignedBytesToRead) - { - cemuLog_log(LogType::Force, "FST read error in raw content"); - return 0; - } - AES128_CBC_decrypt_updateIV(readBuffer, readBuffer, alignedBytesToRead, m_partitionTitlekey.b, iv); - std::memcpy(dataOutU8, readBuffer, bytesToRead); - dataOutU8 += bytesToRead; - readSize -= bytesToRead; - readAddrCurrent += alignedBytesToRead; - } - - return readSizeInput - readSize; -} - constexpr size_t BLOCK_SIZE = 0x10000; constexpr size_t BLOCK_HASH_SIZE = 0x0400; constexpr size_t BLOCK_FILE_SIZE = 0xFC00; +struct FSTRawBlock +{ + std::vector rawData; // unhashed block size depends on sector size field in partition header +}; + struct FSTHashedBlock { uint8 rawData[BLOCK_SIZE]; @@ -887,12 +865,160 @@ struct FSTHashedBlock static_assert(sizeof(FSTHashedBlock) == BLOCK_SIZE); +struct FSTCachedRawBlock +{ + FSTRawBlock blockData; + uint8 ivForNextBlock[16]; + uint64 lastAccess; +}; + struct FSTCachedHashedBlock { FSTHashedBlock blockData; uint64 lastAccess; }; +// Checks cache fill state and if necessary drops least recently accessed block from the cache. Optionally allows to recycle the released cache entry to cut down cost of memory allocation and clearing +void FSTVolume::TrimCacheIfRequired(FSTCachedRawBlock** droppedRawBlock, FSTCachedHashedBlock** droppedHashedBlock) +{ + // calculate size used by cache + size_t cacheSize = 0; + for (auto& itr : m_cacheDecryptedRawBlocks) + cacheSize += itr.second->blockData.rawData.size(); + for (auto& itr : m_cacheDecryptedHashedBlocks) + cacheSize += sizeof(FSTCachedHashedBlock) + sizeof(FSTHashedBlock); + // only trim if cache is full (larger than 2MB) + if (cacheSize < 2*1024*1024) // 2MB + return; + // scan both cache lists to find least recently accessed block to drop + auto dropRawItr = std::min_element(m_cacheDecryptedRawBlocks.begin(), m_cacheDecryptedRawBlocks.end(), [](const auto& a, const auto& b) -> bool + { return a.second->lastAccess < b.second->lastAccess; }); + auto dropHashedItr = std::min_element(m_cacheDecryptedHashedBlocks.begin(), m_cacheDecryptedHashedBlocks.end(), [](const auto& a, const auto& b) -> bool + { return a.second->lastAccess < b.second->lastAccess; }); + uint64 lastAccess = std::numeric_limits::max(); + if(dropRawItr != m_cacheDecryptedRawBlocks.end()) + lastAccess = dropRawItr->second->lastAccess; + if(dropHashedItr != m_cacheDecryptedHashedBlocks.end()) + lastAccess = std::min(lastAccess, dropHashedItr->second->lastAccess); + if(dropRawItr != m_cacheDecryptedRawBlocks.end() && dropRawItr->second->lastAccess == lastAccess) + { + if (droppedRawBlock) + *droppedRawBlock = dropRawItr->second; + else + delete dropRawItr->second; + m_cacheDecryptedRawBlocks.erase(dropRawItr); + return; + } + else if(dropHashedItr != m_cacheDecryptedHashedBlocks.end() && dropHashedItr->second->lastAccess == lastAccess) + { + if (droppedHashedBlock) + *droppedHashedBlock = dropHashedItr->second; + else + delete dropHashedItr->second; + m_cacheDecryptedHashedBlocks.erase(dropHashedItr); + } +} + +void FSTVolume::DetermineUnhashedBlockIV(uint32 clusterIndex, uint32 blockIndex, uint8 ivOut[16]) +{ + memset(ivOut, 0, sizeof(ivOut)); + if(blockIndex == 0) + { + ivOut[0] = (uint8)(clusterIndex >> 8); + ivOut[1] = (uint8)(clusterIndex >> 0); + } + else + { + // the last 16 encrypted bytes of the previous block are the IV (AES CBC) + // if the previous block is cached we can grab the IV from there. Otherwise we have to read the 16 bytes from the data source + uint32 prevBlockIndex = blockIndex - 1; + uint64 cacheBlockId = ((uint64)clusterIndex << (64 - 16)) | (uint64)prevBlockIndex; + auto itr = m_cacheDecryptedRawBlocks.find(cacheBlockId); + if (itr != m_cacheDecryptedRawBlocks.end()) + { + memcpy(ivOut, itr->second->ivForNextBlock, 16); + } + else + { + cemu_assert(m_sectorSize >= 16); + uint64 clusterOffset = (uint64)m_cluster[clusterIndex].offset * m_sectorSize; + uint8 prevIV[16]; + if (m_dataSource->readData(clusterIndex, clusterOffset, blockIndex * m_sectorSize - 16, prevIV, 16) != 16) + { + cemuLog_log(LogType::Force, "Failed to read IV for raw FST block"); + m_detectedCorruption = true; + return; + } + memcpy(ivOut, prevIV, 16); + } + } +} + +FSTCachedRawBlock* FSTVolume::GetDecryptedRawBlock(uint32 clusterIndex, uint32 blockIndex) +{ + FSTCluster& cluster = m_cluster[clusterIndex]; + uint64 clusterOffset = (uint64)cluster.offset * m_sectorSize; + // generate id for cache + uint64 cacheBlockId = ((uint64)clusterIndex << (64 - 16)) | (uint64)blockIndex; + // lookup block in cache + FSTCachedRawBlock* block = nullptr; + auto itr = m_cacheDecryptedRawBlocks.find(cacheBlockId); + if (itr != m_cacheDecryptedRawBlocks.end()) + { + block = itr->second; + block->lastAccess = ++m_cacheAccessCounter; + return block; + } + // if cache already full, drop least recently accessed block and recycle FSTCachedRawBlock object if possible + TrimCacheIfRequired(&block, nullptr); + if (!block) + block = new FSTCachedRawBlock(); + block->blockData.rawData.resize(m_sectorSize); + // block not cached, read new + block->lastAccess = ++m_cacheAccessCounter; + if (m_dataSource->readData(clusterIndex, clusterOffset, blockIndex * m_sectorSize, block->blockData.rawData.data(), m_sectorSize) != m_sectorSize) + { + cemuLog_log(LogType::Force, "Failed to read raw FST block"); + delete block; + m_detectedCorruption = true; + return nullptr; + } + // decrypt hash data + uint8 iv[16]{}; + DetermineUnhashedBlockIV(clusterIndex, blockIndex, iv); + memcpy(block->ivForNextBlock, block->blockData.rawData.data() + m_sectorSize - 16, 16); + AES128_CBC_decrypt(block->blockData.rawData.data(), block->blockData.rawData.data(), m_sectorSize, m_partitionTitlekey.b, iv); + // if this is the next block, then hash it + if(cluster.hasContentHash) + { + if(cluster.singleHashNumBlocksHashed == blockIndex) + { + cemu_assert_debug(!(cluster.contentSize % m_sectorSize)); // size should be multiple of sector size? Regardless, the hashing code below can handle non-aligned sizes + bool isLastBlock = blockIndex == (std::max(cluster.contentSize / m_sectorSize, 1) - 1); + uint32 hashSize = m_sectorSize; + if(isLastBlock) + hashSize = cluster.contentSize - (uint64)blockIndex*m_sectorSize; + EVP_DigestUpdate(cluster.singleHashCtx.get(), block->blockData.rawData.data(), hashSize); + cluster.singleHashNumBlocksHashed++; + if(isLastBlock) + { + uint8 hash[32]; + EVP_DigestFinal_ex(cluster.singleHashCtx.get(), hash, nullptr); + if(memcmp(hash, cluster.contentHash32, cluster.contentHashIsSHA1 ? 20 : 32) != 0) + { + cemuLog_log(LogType::Force, "FST: Raw section hash mismatch"); + delete block; + m_detectedCorruption = true; + return nullptr; + } + } + } + } + // register in cache + m_cacheDecryptedRawBlocks.emplace(cacheBlockId, block); + return block; +} + FSTCachedHashedBlock* FSTVolume::GetDecryptedHashedBlock(uint32 clusterIndex, uint32 blockIndex) { const FSTCluster& cluster = m_cluster[clusterIndex]; @@ -908,22 +1034,17 @@ FSTCachedHashedBlock* FSTVolume::GetDecryptedHashedBlock(uint32 clusterIndex, ui block->lastAccess = ++m_cacheAccessCounter; return block; } - // if cache already full, drop least recently accessed block (but recycle the FSTHashedBlock* object) - if (m_cacheDecryptedHashedBlocks.size() >= 16) - { - auto dropItr = std::min_element(m_cacheDecryptedHashedBlocks.begin(), m_cacheDecryptedHashedBlocks.end(), [](const auto& a, const auto& b) -> bool - { return a.second->lastAccess < b.second->lastAccess; }); - block = dropItr->second; - m_cacheDecryptedHashedBlocks.erase(dropItr); - } - else + // if cache already full, drop least recently accessed block and recycle FSTCachedHashedBlock object if possible + TrimCacheIfRequired(nullptr, &block); + if (!block) block = new FSTCachedHashedBlock(); // block not cached, read new block->lastAccess = ++m_cacheAccessCounter; if (m_dataSource->readData(clusterIndex, clusterOffset, blockIndex * BLOCK_SIZE, block->blockData.rawData, BLOCK_SIZE) != BLOCK_SIZE) { - cemuLog_log(LogType::Force, "Failed to read FST block"); + cemuLog_log(LogType::Force, "Failed to read hashed FST block"); delete block; + m_detectedCorruption = true; return nullptr; } // decrypt hash data @@ -931,11 +1052,46 @@ FSTCachedHashedBlock* FSTVolume::GetDecryptedHashedBlock(uint32 clusterIndex, ui AES128_CBC_decrypt(block->blockData.getHashData(), block->blockData.getHashData(), BLOCK_HASH_SIZE, m_partitionTitlekey.b, iv); // decrypt file data AES128_CBC_decrypt(block->blockData.getFileData(), block->blockData.getFileData(), BLOCK_FILE_SIZE, m_partitionTitlekey.b, block->blockData.getH0Hash(blockIndex%16)); + // compare with H0 to verify data integrity + NCrypto::CHash160 h0; + SHA1(block->blockData.getFileData(), BLOCK_FILE_SIZE, h0.b); + uint32 h0Index = (blockIndex % 4096); + if (memcmp(h0.b, block->blockData.getH0Hash(h0Index & 0xF), sizeof(h0.b)) != 0) + { + cemuLog_log(LogType::Force, "FST: Hash H0 mismatch in hashed block (section {} index {})", clusterIndex, blockIndex); + delete block; + m_detectedCorruption = true; + return nullptr; + } // register in cache m_cacheDecryptedHashedBlocks.emplace(cacheBlockId, block); return block; } +uint32 FSTVolume::ReadFile_HashModeRaw(uint32 clusterIndex, FSTEntry& entry, uint32 readOffset, uint32 readSize, void* dataOut) +{ + uint8* dataOutU8 = (uint8*)dataOut; + if (readOffset >= entry.fileInfo.fileSize) + return 0; + else if ((readOffset + readSize) >= entry.fileInfo.fileSize) + readSize = (entry.fileInfo.fileSize - readOffset); + uint64 absFileOffset = entry.fileInfo.fileOffset * m_offsetFactor + readOffset; + uint32 remainingReadSize = readSize; + while (remainingReadSize > 0) + { + const FSTCachedRawBlock* rawBlock = this->GetDecryptedRawBlock(clusterIndex, absFileOffset/m_sectorSize); + if (!rawBlock) + break; + uint32 blockOffset = (uint32)(absFileOffset % m_sectorSize); + uint32 bytesToRead = std::min(remainingReadSize, m_sectorSize - blockOffset); + std::memcpy(dataOutU8, rawBlock->blockData.rawData.data() + blockOffset, bytesToRead); + dataOutU8 += bytesToRead; + remainingReadSize -= bytesToRead; + absFileOffset += bytesToRead; + } + return readSize - remainingReadSize; +} + uint32 FSTVolume::ReadFile_HashModeHashed(uint32 clusterIndex, FSTEntry& entry, uint32 readOffset, uint32 readSize, void* dataOut) { /* @@ -966,7 +1122,6 @@ uint32 FSTVolume::ReadFile_HashModeHashed(uint32 clusterIndex, FSTEntry& entry, */ const FSTCluster& cluster = m_cluster[clusterIndex]; - uint64 clusterBaseOffset = (uint64)cluster.offset * m_sectorSize; uint64 fileReadOffset = entry.fileInfo.fileOffset * m_offsetFactor + readOffset; uint32 blockIndex = (uint32)(fileReadOffset / BLOCK_FILE_SIZE); uint32 bytesRemaining = readSize; @@ -1019,6 +1174,8 @@ bool FSTVolume::Next(FSTDirectoryIterator& directoryIterator, FSTFileHandle& fil FSTVolume::~FSTVolume() { + for (auto& itr : m_cacheDecryptedRawBlocks) + delete itr.second; for (auto& itr : m_cacheDecryptedHashedBlocks) delete itr.second; if (m_sourceIsOwned) @@ -1115,4 +1272,4 @@ bool FSTVerifier::VerifyHashedContentFile(FileStream* fileContent, const NCrypto void FSTVolumeTest() { FSTPathUnitTest(); -} \ No newline at end of file +} diff --git a/src/Cafe/Filesystem/FST/FST.h b/src/Cafe/Filesystem/FST/FST.h index 24fc39ea..601799ce 100644 --- a/src/Cafe/Filesystem/FST/FST.h +++ b/src/Cafe/Filesystem/FST/FST.h @@ -1,5 +1,6 @@ #pragma once #include "Cemu/ncrypto/ncrypto.h" +#include "openssl/evp.h" struct FSTFileHandle { @@ -45,6 +46,7 @@ public: ~FSTVolume(); uint32 GetFileCount() const; + bool HasCorruption() const { return m_detectedCorruption; } bool OpenFile(std::string_view path, FSTFileHandle& fileHandleOut, bool openOnlyFiles = false); @@ -86,15 +88,25 @@ private: enum class ClusterHashMode : uint8 { RAW = 0, // raw data + encryption, no hashing? - RAW2 = 1, // raw data + encryption, with hash stored in tmd? + RAW_STREAM = 1, // raw data + encryption, with hash stored in tmd? HASH_INTERLEAVED = 2, // hashes + raw interleaved in 0x10000 blocks (0x400 bytes of hashes at the beginning, followed by 0xFC00 bytes of data) }; struct FSTCluster { + FSTCluster() : singleHashCtx(nullptr, &EVP_MD_CTX_free) {} + uint32 offset; uint32 size; ClusterHashMode hashMode; + // extra data if TMD is available + bool hasContentHash; + uint8 contentHash32[32]; + bool contentHashIsSHA1; // if true then it's SHA1 (with extra bytes zeroed out), otherwise it's SHA256 + uint64 contentSize; // size of the content (in blocks) + // hash context for single hash mode (content hash must be available) + std::unique_ptr singleHashCtx; // unique_ptr to make this move-only + uint32 singleHashNumBlocksHashed{0}; }; struct FSTEntry @@ -164,17 +176,30 @@ private: bool m_sourceIsOwned{}; uint32 m_sectorSize{}; // for cluster offsets uint32 m_offsetFactor{}; // for file offsets + bool m_hashIsDisabled{}; // disables hash verification (for all clusters of this volume?) std::vector m_cluster; std::vector m_entries; std::vector m_nameStringTable; NCrypto::AesKey m_partitionTitlekey; + bool m_detectedCorruption{false}; - /* Cache for decrypted hashed blocks */ + bool HashIsDisabled() const + { + return m_hashIsDisabled; + } + + /* Cache for decrypted raw and hashed blocks */ + std::unordered_map m_cacheDecryptedRawBlocks; std::unordered_map m_cacheDecryptedHashedBlocks; uint64 m_cacheAccessCounter{}; + void DetermineUnhashedBlockIV(uint32 clusterIndex, uint32 blockIndex, uint8 ivOut[16]); + + struct FSTCachedRawBlock* GetDecryptedRawBlock(uint32 clusterIndex, uint32 blockIndex); struct FSTCachedHashedBlock* GetDecryptedHashedBlock(uint32 clusterIndex, uint32 blockIndex); + void TrimCacheIfRequired(struct FSTCachedRawBlock** droppedRawBlock, struct FSTCachedHashedBlock** droppedHashedBlock); + /* File reading */ uint32 ReadFile_HashModeRaw(uint32 clusterIndex, FSTEntry& entry, uint32 readOffset, uint32 readSize, void* dataOut); uint32 ReadFile_HashModeHashed(uint32 clusterIndex, FSTEntry& entry, uint32 readOffset, uint32 readSize, void* dataOut); @@ -185,7 +210,10 @@ private: /* +0x00 */ uint32be magic; /* +0x04 */ uint32be offsetFactor; /* +0x08 */ uint32be numCluster; - /* +0x0C */ uint32be ukn0C; + /* +0x0C */ uint8be hashIsDisabled; + /* +0x0D */ uint8be ukn0D; + /* +0x0E */ uint8be ukn0E; + /* +0x0F */ uint8be ukn0F; /* +0x10 */ uint32be ukn10; /* +0x14 */ uint32be ukn14; /* +0x18 */ uint32be ukn18; @@ -262,8 +290,8 @@ private: static_assert(sizeof(FSTHeader_FileEntry) == 0x10); - static FSTVolume* OpenFST(FSTDataSource* dataSource, uint64 fstOffset, uint32 fstSize, NCrypto::AesKey* partitionTitleKey, ClusterHashMode fstHashMode); - static FSTVolume* OpenFST(std::unique_ptr dataSource, uint64 fstOffset, uint32 fstSize, NCrypto::AesKey* partitionTitleKey, ClusterHashMode fstHashMode); + static FSTVolume* OpenFST(FSTDataSource* dataSource, uint64 fstOffset, uint32 fstSize, NCrypto::AesKey* partitionTitleKey, ClusterHashMode fstHashMode, NCrypto::TMDParser* optionalTMD); + static FSTVolume* OpenFST(std::unique_ptr dataSource, uint64 fstOffset, uint32 fstSize, NCrypto::AesKey* partitionTitleKey, ClusterHashMode fstHashMode, NCrypto::TMDParser* optionalTMD); static bool ProcessFST(FSTHeader_FileEntry* fileTable, uint32 numFileEntries, uint32 numCluster, std::vector& nameStringTable, std::vector& fstEntries); bool MatchFSTEntryName(FSTEntry& entry, std::string_view comparedName) diff --git a/src/Cafe/GameProfile/GameProfile.cpp b/src/Cafe/GameProfile/GameProfile.cpp index ee92107a..ea303226 100644 --- a/src/Cafe/GameProfile/GameProfile.cpp +++ b/src/Cafe/GameProfile/GameProfile.cpp @@ -140,7 +140,7 @@ bool gameProfile_loadEnumOption(IniParser& iniParser, const char* optionName, T& for(const T& v : T()) { // test integer option - if (boost::iequals(fmt::format("{}", static_cast::type>(v)), *option_value)) + if (boost::iequals(fmt::format("{}", fmt::underlying(v)), *option_value)) { option = v; return true; diff --git a/src/Cafe/GraphicPack/GraphicPack2.cpp b/src/Cafe/GraphicPack/GraphicPack2.cpp index c54c31cb..f21bb89d 100644 --- a/src/Cafe/GraphicPack/GraphicPack2.cpp +++ b/src/Cafe/GraphicPack/GraphicPack2.cpp @@ -345,7 +345,7 @@ GraphicPack2::GraphicPack2(fs::path rulesPath, IniParser& rules) const auto preset_name = rules.FindOption("name"); if (!preset_name) { - cemuLog_log(LogType::Force, "Graphic pack \"{}\": Preset in line {} skipped because it has no name option defined", m_name, rules.GetCurrentSectionLineNumber()); + cemuLog_log(LogType::Force, "Graphic pack \"{}\": Preset in line {} skipped because it has no name option defined", GetNormalizedPathString(), rules.GetCurrentSectionLineNumber()); continue; } @@ -369,7 +369,7 @@ GraphicPack2::GraphicPack2(fs::path rulesPath, IniParser& rules) } catch (const std::exception & ex) { - cemuLog_log(LogType::Force, "Graphic pack \"{}\": Can't parse preset \"{}\": {}", m_name, *preset_name, ex.what()); + cemuLog_log(LogType::Force, "Graphic pack \"{}\": Can't parse preset \"{}\": {}", GetNormalizedPathString(), *preset_name, ex.what()); } } else if (boost::iequals(currentSectionName, "RAM")) @@ -383,7 +383,7 @@ GraphicPack2::GraphicPack2(fs::path rulesPath, IniParser& rules) { if (m_version <= 5) { - cemuLog_log(LogType::Force, "Graphic pack \"{}\": [RAM] options are only available for graphic pack version 6 or higher", m_name, optionNameBuf); + cemuLog_log(LogType::Force, "Graphic pack \"{}\": [RAM] options are only available for graphic pack version 6 or higher", GetNormalizedPathString(), optionNameBuf); throw std::exception(); } @@ -393,12 +393,12 @@ GraphicPack2::GraphicPack2(fs::path rulesPath, IniParser& rules) { if (addrEnd <= addrStart) { - cemuLog_log(LogType::Force, "Graphic pack \"{}\": start address (0x{:08x}) must be greater than end address (0x{:08x}) for {}", m_name, addrStart, addrEnd, optionNameBuf); + cemuLog_log(LogType::Force, "Graphic pack \"{}\": start address (0x{:08x}) must be greater than end address (0x{:08x}) for {}", GetNormalizedPathString(), addrStart, addrEnd, optionNameBuf); throw std::exception(); } else if ((addrStart & 0xFFF) != 0 || (addrEnd & 0xFFF) != 0) { - cemuLog_log(LogType::Force, "Graphic pack \"{}\": addresses for %s are not aligned to 0x1000", m_name, optionNameBuf); + cemuLog_log(LogType::Force, "Graphic pack \"{}\": addresses for %s are not aligned to 0x1000", GetNormalizedPathString(), optionNameBuf); throw std::exception(); } else @@ -408,7 +408,7 @@ GraphicPack2::GraphicPack2(fs::path rulesPath, IniParser& rules) } else { - cemuLog_log(LogType::Force, "Graphic pack \"{}\": has invalid syntax for option {}", m_name, optionNameBuf); + cemuLog_log(LogType::Force, "Graphic pack \"{}\": has invalid syntax for option {}", GetNormalizedPathString(), optionNameBuf); throw std::exception(); } } @@ -422,24 +422,32 @@ GraphicPack2::GraphicPack2(fs::path rulesPath, IniParser& rules) std::unordered_map> tmp_map; // all vars must be defined in the default preset vars before - for (const auto& entry : m_presets) + std::vector> mismatchingPresetVars; + for (const auto& presetEntry : m_presets) { - tmp_map[entry->category].emplace_back(entry); + tmp_map[presetEntry->category].emplace_back(presetEntry); - for (auto& kv : entry->variables) + for (auto& presetVar : presetEntry->variables) { - const auto it = m_preset_vars.find(kv.first); + const auto it = m_preset_vars.find(presetVar.first); if (it == m_preset_vars.cend()) { - cemuLog_log(LogType::Force, "Graphic pack: \"{}\" contains preset variables which are not defined in the default section", m_name); - throw std::exception(); + mismatchingPresetVars.emplace_back(presetEntry->name, presetVar.first); + continue; } - // overwrite var type with default var type - kv.second.first = it->second.first; + presetVar.second.first = it->second.first; } } + if(!mismatchingPresetVars.empty()) + { + cemuLog_log(LogType::Force, "Graphic pack \"{}\" contains preset variables which are not defined in the [Default] section:", GetNormalizedPathString()); + for (const auto& [presetName, varName] : mismatchingPresetVars) + cemuLog_log(LogType::Force, "Preset: {} Variable: {}", presetName, varName); + throw std::exception(); + } + // have first entry be default active for every category if no default= is set for(auto entry : get_values(tmp_map)) { @@ -469,7 +477,7 @@ GraphicPack2::GraphicPack2(fs::path rulesPath, IniParser& rules) auto& p2 = kv.second[i + 1]; if (p1->variables.size() != p2->variables.size()) { - cemuLog_log(LogType::Force, "Graphic pack: \"{}\" contains inconsistent preset variables", m_name); + cemuLog_log(LogType::Force, "Graphic pack: \"{}\" contains inconsistent preset variables", GetNormalizedPathString()); throw std::exception(); } @@ -477,14 +485,14 @@ GraphicPack2::GraphicPack2(fs::path rulesPath, IniParser& rules) std::set keys2(get_keys(p2->variables).begin(), get_keys(p2->variables).end()); if (keys1 != keys2) { - cemuLog_log(LogType::Force, "Graphic pack: \"{}\" contains inconsistent preset variables", m_name); + cemuLog_log(LogType::Force, "Graphic pack: \"{}\" contains inconsistent preset variables", GetNormalizedPathString()); throw std::exception(); } if(p1->is_default) { if(has_default) - cemuLog_log(LogType::Force, "Graphic pack: \"{}\" has more than one preset with the default key set for the same category \"{}\"", m_name, p1->name); + cemuLog_log(LogType::Force, "Graphic pack: \"{}\" has more than one preset with the default key set for the same category \"{}\"", GetNormalizedPathString(), p1->name); p1->active = true; has_default = true; } @@ -960,7 +968,7 @@ bool GraphicPack2::Activate() auto option_upscale = rules.FindOption("upscaleMagFilter"); if(option_upscale && boost::iequals(*option_upscale, "NearestNeighbor")) m_output_settings.upscale_filter = LatteTextureView::MagFilter::kNearestNeighbor; - auto option_downscale = rules.FindOption("NearestNeighbor"); + auto option_downscale = rules.FindOption("downscaleMinFilter"); if (option_downscale && boost::iequals(*option_downscale, "NearestNeighbor")) m_output_settings.downscale_filter = LatteTextureView::MagFilter::kNearestNeighbor; } diff --git a/src/Cafe/HW/Espresso/Debugger/Debugger.cpp b/src/Cafe/HW/Espresso/Debugger/Debugger.cpp index e7369af6..e84c9fda 100644 --- a/src/Cafe/HW/Espresso/Debugger/Debugger.cpp +++ b/src/Cafe/HW/Espresso/Debugger/Debugger.cpp @@ -8,6 +8,7 @@ #include "gui/debugger/DebuggerWindow2.h" #include "Cafe/OS/libs/coreinit/coreinit.h" +#include "util/helpers/helpers.h" #if BOOST_OS_WINDOWS #include @@ -136,11 +137,6 @@ void debugger_createCodeBreakpoint(uint32 address, uint8 bpType) debugger_updateExecutionBreakpoint(address); } -void debugger_createExecuteBreakpoint(uint32 address) -{ - debugger_createCodeBreakpoint(address, DEBUGGER_BP_T_NORMAL); -} - namespace coreinit { std::vector& OSGetSchedulerThreads(); @@ -294,8 +290,23 @@ void debugger_toggleExecuteBreakpoint(uint32 address) } else { - // create new breakpoint - debugger_createExecuteBreakpoint(address); + // create new execution breakpoint + debugger_createCodeBreakpoint(address, DEBUGGER_BP_T_NORMAL); + } +} + +void debugger_toggleLoggingBreakpoint(uint32 address) +{ + auto existingBP = debugger_getFirstBP(address, DEBUGGER_BP_T_LOGGING); + if (existingBP) + { + // delete existing breakpoint + debugger_deleteBreakpoint(existingBP); + } + else + { + // create new logging breakpoint + debugger_createCodeBreakpoint(address, DEBUGGER_BP_T_LOGGING); } } @@ -447,6 +458,34 @@ bool debugger_hasPatch(uint32 address) return false; } +void debugger_removePatch(uint32 address) +{ + for (sint32 i = 0; i < debuggerState.patches.size(); i++) + { + auto& patch = debuggerState.patches[i]; + if (address < patch->address || address >= (patch->address + patch->length)) + continue; + MPTR startAddress = patch->address; + MPTR endAddress = patch->address + patch->length; + // remove any breakpoints overlapping with the patch + for (auto& bp : debuggerState.breakpoints) + { + if (bp->address + 4 > startAddress && bp->address < endAddress) + { + bp->enabled = false; + debugger_updateExecutionBreakpoint(bp->address); + } + } + // restore original data + memcpy(MEMPTR(startAddress).GetPtr(), patch->origData.data(), patch->length); + PPCRecompiler_invalidateRange(startAddress, endAddress); + // remove patch + delete patch; + debuggerState.patches.erase(debuggerState.patches.begin() + i); + return; + } +} + void debugger_stepInto(PPCInterpreter_t* hCPU, bool updateDebuggerWindow = true) { bool isRecEnabled = ppcRecompilerEnabled; @@ -510,7 +549,48 @@ void debugger_enterTW(PPCInterpreter_t* hCPU) { if (bp->bpType == DEBUGGER_BP_T_LOGGING && bp->enabled) { - std::string logName = !bp->comment.empty() ? "Breakpoint '"+boost::nowide::narrow(bp->comment)+"'" : fmt::format("Breakpoint at 0x{:08X} (no comment)", bp->address); + std::string comment = !bp->comment.empty() ? boost::nowide::narrow(bp->comment) : fmt::format("Breakpoint at 0x{:08X} (no comment)", bp->address); + + auto replacePlaceholders = [&](const std::string& prefix, const auto& formatFunc) + { + size_t pos = 0; + while ((pos = comment.find(prefix, pos)) != std::string::npos) + { + size_t endPos = comment.find('}', pos); + if (endPos == std::string::npos) + break; + + try + { + if (int regNum = ConvertString(comment.substr(pos + prefix.length(), endPos - pos - prefix.length())); regNum >= 0 && regNum < 32) + { + std::string replacement = formatFunc(regNum); + comment.replace(pos, endPos - pos + 1, replacement); + pos += replacement.length(); + } + else + { + pos = endPos + 1; + } + } + catch (...) + { + pos = endPos + 1; + } + } + }; + + // Replace integer register placeholders {rX} + replacePlaceholders("{r", [&](int regNum) { + return fmt::format("0x{:08X}", hCPU->gpr[regNum]); + }); + + // Replace floating point register placeholders {fX} + replacePlaceholders("{f", [&](int regNum) { + return fmt::format("{}", hCPU->fpr[regNum].fpr); + }); + + std::string logName = "Breakpoint '" + comment + "'"; std::string logContext = fmt::format("Thread: {:08x} LR: 0x{:08x}", MEMPTR(coreinit::OSGetCurrentThread()).GetMPTR(), hCPU->spr.LR, cemuLog_advancedPPCLoggingEnabled() ? " Stack Trace:" : ""); cemuLog_log(LogType::Force, "[Debugger] {} was executed! {}", logName, logContext); if (cemuLog_advancedPPCLoggingEnabled()) @@ -547,7 +627,7 @@ void debugger_enterTW(PPCInterpreter_t* hCPU) debuggerState.debugSession.stepInto = false; debuggerState.debugSession.stepOver = false; debuggerState.debugSession.run = false; - while (true) + while (debuggerState.debugSession.isTrapped) { std::this_thread::sleep_for(std::chrono::milliseconds(1)); // check for step commands diff --git a/src/Cafe/HW/Espresso/Debugger/Debugger.h b/src/Cafe/HW/Espresso/Debugger/Debugger.h index 717df28a..c220eb8a 100644 --- a/src/Cafe/HW/Espresso/Debugger/Debugger.h +++ b/src/Cafe/HW/Espresso/Debugger/Debugger.h @@ -100,8 +100,8 @@ extern debuggerState_t debuggerState; // new API DebuggerBreakpoint* debugger_getFirstBP(uint32 address); void debugger_createCodeBreakpoint(uint32 address, uint8 bpType); -void debugger_createExecuteBreakpoint(uint32 address); void debugger_toggleExecuteBreakpoint(uint32 address); // create/remove execute breakpoint +void debugger_toggleLoggingBreakpoint(uint32 address); // create/remove logging breakpoint void debugger_toggleBreakpoint(uint32 address, bool state, DebuggerBreakpoint* bp); void debugger_createMemoryBreakpoint(uint32 address, bool onRead, bool onWrite); @@ -114,6 +114,7 @@ void debugger_updateExecutionBreakpoint(uint32 address, bool forceRestore = fals void debugger_createPatch(uint32 address, std::span patchData); bool debugger_hasPatch(uint32 address); +void debugger_removePatch(uint32 address); void debugger_forceBreak(); // force breakpoint at the next possible instruction bool debugger_isTrapped(); diff --git a/src/Cafe/HW/Espresso/Recompiler/PPCRecompilerX64.cpp b/src/Cafe/HW/Espresso/Recompiler/PPCRecompilerX64.cpp index a30295b5..97b2c14c 100644 --- a/src/Cafe/HW/Espresso/Recompiler/PPCRecompilerX64.cpp +++ b/src/Cafe/HW/Espresso/Recompiler/PPCRecompilerX64.cpp @@ -114,13 +114,13 @@ void* ATTR_MS_ABI PPCRecompiler_virtualHLE(PPCInterpreter_t* hCPU, uint32 hleFun void ATTR_MS_ABI PPCRecompiler_getTBL(PPCInterpreter_t* hCPU, uint32 gprIndex) { - uint64 coreTime = coreinit::coreinit_getTimerTick(); + uint64 coreTime = coreinit::OSGetSystemTime(); hCPU->gpr[gprIndex] = (uint32)(coreTime&0xFFFFFFFF); } void ATTR_MS_ABI PPCRecompiler_getTBU(PPCInterpreter_t* hCPU, uint32 gprIndex) { - uint64 coreTime = coreinit::coreinit_getTimerTick(); + uint64 coreTime = coreinit::OSGetSystemTime(); hCPU->gpr[gprIndex] = (uint32)((coreTime>>32)&0xFFFFFFFF); } diff --git a/src/Cafe/HW/Latte/Core/Latte.h b/src/Cafe/HW/Latte/Core/Latte.h index e8cb2be4..e5e9dd5c 100644 --- a/src/Cafe/HW/Latte/Core/Latte.h +++ b/src/Cafe/HW/Latte/Core/Latte.h @@ -52,7 +52,7 @@ struct LatteGPUState_t uint32 gx2InitCalled; // incremented every time GX2Init() is called // OpenGL control uint32 glVendor; // GLVENDOR_* - bool alwaysDisplayDRC = false; + bool isDRCPrimary = false; // temporary (replace with proper solution later) bool tvBufferUsesSRGB; bool drcBufferUsesSRGB; diff --git a/src/Cafe/HW/Latte/Core/LatteCommandProcessor.cpp b/src/Cafe/HW/Latte/Core/LatteCommandProcessor.cpp index 167911b6..f592cc9e 100644 --- a/src/Cafe/HW/Latte/Core/LatteCommandProcessor.cpp +++ b/src/Cafe/HW/Latte/Core/LatteCommandProcessor.cpp @@ -141,6 +141,14 @@ private: void LatteCP_processCommandBuffer(DrawPassContext& drawPassCtx); +// called whenever the GPU runs out of commands or hits a wait condition (semaphores, HLE waits) +void LatteCP_signalEnterWait() +{ + // based on the assumption that games won't do a rugpull and swap out buffer data in the middle of an uninterrupted sequence of drawcalls, + // we only flush caches when the GPU goes idle or has to wait for any operation + LatteIndices_invalidateAll(); +} + /* * Read a U32 from the command buffer * If no data is available then wait in a busy loop @@ -466,6 +474,8 @@ LatteCMDPtr LatteCP_itWaitRegMem(LatteCMDPtr cmd, uint32 nWords) const uint32 GPU7_WAIT_MEM_OP_GREATER = 6; const uint32 GPU7_WAIT_MEM_OP_NEVER = 7; + LatteCP_signalEnterWait(); + bool stalls = false; if ((word0 & 0x10) != 0) { @@ -594,6 +604,7 @@ LatteCMDPtr LatteCP_itMemSemaphore(LatteCMDPtr cmd, uint32 nWords) else if(SEM_SIGNAL == 7) { // wait + LatteCP_signalEnterWait(); size_t loopCount = 0; while (true) { @@ -788,7 +799,7 @@ LatteCMDPtr LatteCP_itHLESampleTimer(LatteCMDPtr cmd, uint32 nWords) { cemu_assert_debug(nWords == 1); MPTR timerMPTR = (MPTR)LatteReadCMD(); - memory_writeU64(timerMPTR, coreinit::coreinit_getTimerTick()); + memory_writeU64(timerMPTR, coreinit::OSGetSystemTime()); return cmd; } @@ -1305,11 +1316,13 @@ void LatteCP_processCommandBuffer(DrawPassContext& drawPassCtx) } case IT_HLE_TRIGGER_SCANBUFFER_SWAP: { + LatteCP_signalEnterWait(); LatteCP_itHLESwapScanBuffer(cmdData, nWords); break; } case IT_HLE_WAIT_FOR_FLIP: { + LatteCP_signalEnterWait(); LatteCP_itHLEWaitForFlip(cmdData, nWords); break; } @@ -1594,12 +1607,14 @@ void LatteCP_ProcessRingbuffer() } case IT_HLE_TRIGGER_SCANBUFFER_SWAP: { + LatteCP_signalEnterWait(); LatteCP_itHLESwapScanBuffer(cmd, nWords); timerRecheck += CP_TIMER_RECHECK / 64; break; } case IT_HLE_WAIT_FOR_FLIP: { + LatteCP_signalEnterWait(); LatteCP_itHLEWaitForFlip(cmd, nWords); timerRecheck += CP_TIMER_RECHECK / 1; break; diff --git a/src/Cafe/HW/Latte/Core/LatteIndices.cpp b/src/Cafe/HW/Latte/Core/LatteIndices.cpp index 6e1d7455..aec51725 100644 --- a/src/Cafe/HW/Latte/Core/LatteIndices.cpp +++ b/src/Cafe/HW/Latte/Core/LatteIndices.cpp @@ -1,6 +1,7 @@ #include "Cafe/HW/Latte/Core/LatteConst.h" #include "Cafe/HW/Latte/Renderer/Renderer.h" #include "Cafe/HW/Latte/ISA/RegDefines.h" +#include "Cafe/HW/Latte/Core/LattePerformanceMonitor.h" #include "Common/cpu_features.h" #if defined(ARCH_X86_64) && defined(__GNUC__) @@ -9,32 +10,53 @@ struct { - const void* lastPtr; - uint32 lastCount; - LattePrimitiveMode lastPrimitiveMode; - LatteIndexType lastIndexType; - // output - uint32 indexMin; - uint32 indexMax; - Renderer::INDEX_TYPE renderIndexType; - uint32 outputCount; - uint32 indexBufferOffset; - uint32 indexBufferIndex; + struct CacheEntry + { + // input data + const void* lastPtr; + uint32 lastCount; + LattePrimitiveMode lastPrimitiveMode; + LatteIndexType lastIndexType; + uint64 lastUsed; + // output + uint32 indexMin; + uint32 indexMax; + Renderer::INDEX_TYPE renderIndexType; + uint32 outputCount; + Renderer::IndexAllocation indexAllocation; + }; + std::array entry; + uint64 currentUsageCounter{0}; }LatteIndexCache{}; void LatteIndices_invalidate(const void* memPtr, uint32 size) { - if (LatteIndexCache.lastPtr >= memPtr && (LatteIndexCache.lastPtr < ((uint8*)memPtr + size)) ) + for(auto& entry : LatteIndexCache.entry) { - LatteIndexCache.lastPtr = nullptr; - LatteIndexCache.lastCount = 0; + if (entry.lastPtr >= memPtr && (entry.lastPtr < ((uint8*)memPtr + size)) ) + { + if(entry.lastPtr != nullptr) + g_renderer->indexData_releaseIndexMemory(entry.indexAllocation); + entry.lastPtr = nullptr; + entry.lastCount = 0; + } } } void LatteIndices_invalidateAll() { - LatteIndexCache.lastPtr = nullptr; - LatteIndexCache.lastCount = 0; + for(auto& entry : LatteIndexCache.entry) + { + if (entry.lastPtr != nullptr) + g_renderer->indexData_releaseIndexMemory(entry.indexAllocation); + entry.lastPtr = nullptr; + entry.lastCount = 0; + } +} + +uint64 LatteIndices_GetNextUsageIndex() +{ + return LatteIndexCache.currentUsageCounter++; } uint32 LatteIndices_calculateIndexOutputSize(LattePrimitiveMode primitiveMode, LatteIndexType indexType, uint32 count) @@ -532,7 +554,7 @@ void LatteIndices_alternativeCalculateIndexMinMax(const void* indexData, LatteIn } } -void LatteIndices_decode(const void* indexData, LatteIndexType indexType, uint32 count, LattePrimitiveMode primitiveMode, uint32& indexMin, uint32& indexMax, Renderer::INDEX_TYPE& renderIndexType, uint32& outputCount, uint32& indexBufferOffset, uint32& indexBufferIndex) +void LatteIndices_decode(const void* indexData, LatteIndexType indexType, uint32 count, LattePrimitiveMode primitiveMode, uint32& indexMin, uint32& indexMax, Renderer::INDEX_TYPE& renderIndexType, uint32& outputCount, Renderer::IndexAllocation& indexAllocation) { // what this should do: // [x] use fast SIMD-based index decoding @@ -542,17 +564,18 @@ void LatteIndices_decode(const void* indexData, LatteIndexType indexType, uint32 // [ ] better cache implementation, allow to cache across frames // reuse from cache if data didn't change - if (LatteIndexCache.lastPtr == indexData && - LatteIndexCache.lastCount == count && - LatteIndexCache.lastPrimitiveMode == primitiveMode && - LatteIndexCache.lastIndexType == indexType) + auto cacheEntry = std::find_if(LatteIndexCache.entry.begin(), LatteIndexCache.entry.end(), [indexData, count, primitiveMode, indexType](const auto& entry) { - indexMin = LatteIndexCache.indexMin; - indexMax = LatteIndexCache.indexMax; - renderIndexType = LatteIndexCache.renderIndexType; - outputCount = LatteIndexCache.outputCount; - indexBufferOffset = LatteIndexCache.indexBufferOffset; - indexBufferIndex = LatteIndexCache.indexBufferIndex; + return entry.lastPtr == indexData && entry.lastCount == count && entry.lastPrimitiveMode == primitiveMode && entry.lastIndexType == indexType; + }); + if (cacheEntry != LatteIndexCache.entry.end()) + { + indexMin = cacheEntry->indexMin; + indexMax = cacheEntry->indexMax; + renderIndexType = cacheEntry->renderIndexType; + outputCount = cacheEntry->outputCount; + indexAllocation = cacheEntry->indexAllocation; + cacheEntry->lastUsed = LatteIndices_GetNextUsageIndex(); return; } @@ -576,10 +599,12 @@ void LatteIndices_decode(const void* indexData, LatteIndexType indexType, uint32 indexMin = 0; indexMax = std::max(count, 1u)-1; renderIndexType = Renderer::INDEX_TYPE::NONE; + indexAllocation = {}; return; // no indices } // query index buffer from renderer - void* indexOutputPtr = g_renderer->indexData_reserveIndexMemory(indexOutputSize, indexBufferOffset, indexBufferIndex); + indexAllocation = g_renderer->indexData_reserveIndexMemory(indexOutputSize); + void* indexOutputPtr = indexAllocation.mem; // decode indices indexMin = std::numeric_limits::max(); @@ -704,16 +729,25 @@ void LatteIndices_decode(const void* indexData, LatteIndexType indexType, uint32 // recalculate index range but filter out primitive restart index LatteIndices_alternativeCalculateIndexMinMax(indexData, indexType, count, indexMin, indexMax); } - g_renderer->indexData_uploadIndexMemory(indexBufferOffset, indexOutputSize); + g_renderer->indexData_uploadIndexMemory(indexAllocation); + performanceMonitor.cycle[performanceMonitor.cycleIndex].indexDataUploaded += indexOutputSize; + // get least recently used cache entry + auto lruEntry = std::min_element(LatteIndexCache.entry.begin(), LatteIndexCache.entry.end(), [](const auto& a, const auto& b) + { + return a.lastUsed < b.lastUsed; + }); + // invalidate previous allocation + if(lruEntry->lastPtr != nullptr) + g_renderer->indexData_releaseIndexMemory(lruEntry->indexAllocation); // update cache - LatteIndexCache.lastPtr = indexData; - LatteIndexCache.lastCount = count; - LatteIndexCache.lastPrimitiveMode = primitiveMode; - LatteIndexCache.lastIndexType = indexType; - LatteIndexCache.indexMin = indexMin; - LatteIndexCache.indexMax = indexMax; - LatteIndexCache.renderIndexType = renderIndexType; - LatteIndexCache.outputCount = outputCount; - LatteIndexCache.indexBufferOffset = indexBufferOffset; - LatteIndexCache.indexBufferIndex = indexBufferIndex; + lruEntry->lastPtr = indexData; + lruEntry->lastCount = count; + lruEntry->lastPrimitiveMode = primitiveMode; + lruEntry->lastIndexType = indexType; + lruEntry->indexMin = indexMin; + lruEntry->indexMax = indexMax; + lruEntry->renderIndexType = renderIndexType; + lruEntry->outputCount = outputCount; + lruEntry->indexAllocation = indexAllocation; + lruEntry->lastUsed = LatteIndices_GetNextUsageIndex(); } diff --git a/src/Cafe/HW/Latte/Core/LatteIndices.h b/src/Cafe/HW/Latte/Core/LatteIndices.h index 917d7991..8aace24e 100644 --- a/src/Cafe/HW/Latte/Core/LatteIndices.h +++ b/src/Cafe/HW/Latte/Core/LatteIndices.h @@ -4,4 +4,4 @@ void LatteIndices_invalidate(const void* memPtr, uint32 size); void LatteIndices_invalidateAll(); -void LatteIndices_decode(const void* indexData, LatteIndexType indexType, uint32 count, LattePrimitiveMode primitiveMode, uint32& indexMin, uint32& indexMax, Renderer::INDEX_TYPE& renderIndexType, uint32& outputCount, uint32& indexBufferOffset, uint32& indexBufferIndex); \ No newline at end of file +void LatteIndices_decode(const void* indexData, LatteIndexType indexType, uint32 count, LattePrimitiveMode primitiveMode, uint32& indexMin, uint32& indexMax, Renderer::INDEX_TYPE& renderIndexType, uint32& outputCount, Renderer::IndexAllocation& indexAllocation); \ No newline at end of file diff --git a/src/Cafe/HW/Latte/Core/LatteOverlay.cpp b/src/Cafe/HW/Latte/Core/LatteOverlay.cpp index 238f85e8..e6edb904 100644 --- a/src/Cafe/HW/Latte/Core/LatteOverlay.cpp +++ b/src/Cafe/HW/Latte/Core/LatteOverlay.cpp @@ -107,7 +107,13 @@ void LatteOverlay_renderOverlay(ImVec2& position, ImVec2& pivot, sint32 directio ImGui::Text("VRAM: %dMB / %dMB", g_state.vramUsage, g_state.vramTotal); if (config.overlay.debug) + { + // general debug info + ImGui::Text("--- Debug info ---"); + ImGui::Text("IndexUploadPerFrame: %dKB", (performanceMonitor.stats.indexDataUploadPerFrame+1023)/1024); + // backend specific info g_renderer->AppendOverlayDebugInfo(); + } position.y += (ImGui::GetWindowSize().y + 10.0f) * direction; } diff --git a/src/Cafe/HW/Latte/Core/LattePerformanceMonitor.cpp b/src/Cafe/HW/Latte/Core/LattePerformanceMonitor.cpp index f2767446..14dfe9a9 100644 --- a/src/Cafe/HW/Latte/Core/LattePerformanceMonitor.cpp +++ b/src/Cafe/HW/Latte/Core/LattePerformanceMonitor.cpp @@ -74,7 +74,6 @@ void LattePerformanceMonitor_frameEnd() uniformBankDataUploadedPerFrame /= 1024ULL; uint32 uniformBankCountUploadedPerFrame = (uint32)(uniformBankUploadedCount / (uint64)elapsedFrames); uint64 indexDataUploadPerFrame = (indexDataUploaded / (uint64)elapsedFrames); - indexDataUploadPerFrame /= 1024ULL; double fps = (double)elapsedFrames2S * 1000.0 / (double)totalElapsedTimeFPS; uint32 shaderBindsPerFrame = shaderBindCounter / elapsedFrames; @@ -82,7 +81,7 @@ void LattePerformanceMonitor_frameEnd() uint32 rlps = (uint32)((uint64)recompilerLeaveCount * 1000ULL / (uint64)totalElapsedTime); uint32 tlps = (uint32)((uint64)threadLeaveCount * 1000ULL / (uint64)totalElapsedTime); // set stats - + performanceMonitor.stats.indexDataUploadPerFrame = indexDataUploadPerFrame; // next counter cycle sint32 nextCycleIndex = (performanceMonitor.cycleIndex + 1) % PERFORMANCE_MONITOR_TRACK_CYCLES; performanceMonitor.cycle[nextCycleIndex].drawCallCounter = 0; diff --git a/src/Cafe/HW/Latte/Core/LattePerformanceMonitor.h b/src/Cafe/HW/Latte/Core/LattePerformanceMonitor.h index 713e094e..dbc3cff9 100644 --- a/src/Cafe/HW/Latte/Core/LattePerformanceMonitor.h +++ b/src/Cafe/HW/Latte/Core/LattePerformanceMonitor.h @@ -124,6 +124,7 @@ typedef struct LattePerfStatCounter numGraphicPipelines; LattePerfStatCounter numImages; LattePerfStatCounter numImageViews; + LattePerfStatCounter numSamplers; LattePerfStatCounter numRenderPass; LattePerfStatCounter numFramebuffer; @@ -131,6 +132,12 @@ typedef struct LattePerfStatCounter numDrawBarriersPerFrame; LattePerfStatCounter numBeginRenderpassPerFrame; }vk; + + // calculated stats (per frame) + struct + { + uint32 indexDataUploadPerFrame; + }stats; }performanceMonitor_t; extern performanceMonitor_t performanceMonitor; diff --git a/src/Cafe/HW/Latte/Core/LatteRenderTarget.cpp b/src/Cafe/HW/Latte/Core/LatteRenderTarget.cpp index 60124c02..2efef5bf 100644 --- a/src/Cafe/HW/Latte/Core/LatteRenderTarget.cpp +++ b/src/Cafe/HW/Latte/Core/LatteRenderTarget.cpp @@ -11,7 +11,6 @@ #include "Cafe/HW/Latte/Core/LattePerformanceMonitor.h" #include "Cafe/GraphicPack/GraphicPack2.h" #include "config/ActiveSettings.h" -#include "Cafe/HW/Latte/Renderer/Vulkan/VulkanRenderer.h" #include "gui/guiWrapper.h" #include "Cafe/OS/libs/erreula/erreula.h" #include "input/InputManager.h" @@ -933,13 +932,6 @@ void LatteRenderTarget_copyToBackbuffer(LatteTextureView* textureView, bool isPa if (shader == nullptr) { sint32 scaling_filter = downscaling ? GetConfig().downscale_filter : GetConfig().upscale_filter; - - if (g_renderer->GetType() == RendererAPI::Vulkan) - { - // force linear or nearest neighbor filter - if(scaling_filter != kLinearFilter && scaling_filter != kNearestNeighborFilter) - scaling_filter = kLinearFilter; - } if (scaling_filter == kLinearFilter) { @@ -957,7 +949,7 @@ void LatteRenderTarget_copyToBackbuffer(LatteTextureView* textureView, bool isPa else shader = RendererOutputShader::s_bicubic_shader; - filter = LatteTextureView::MagFilter::kNearestNeighbor; + filter = LatteTextureView::MagFilter::kLinear; } else if (scaling_filter == kBicubicHermiteFilter) { @@ -989,8 +981,6 @@ void LatteRenderTarget_copyToBackbuffer(LatteTextureView* textureView, bool isPa g_renderer->ImguiEnd(); } -bool ctrlTabHotkeyPressed = false; - void LatteRenderTarget_itHLECopyColorBufferToScanBuffer(MPTR colorBufferPtr, uint32 colorBufferWidth, uint32 colorBufferHeight, uint32 colorBufferSliceIndex, uint32 colorBufferFormat, uint32 colorBufferPitch, Latte::E_HWTILEMODE colorBufferTilemode, uint32 colorBufferSwizzle, uint32 renderTarget) { cemu_assert_debug(colorBufferSliceIndex == 0); // todo - support for non-zero slice @@ -1000,38 +990,31 @@ void LatteRenderTarget_itHLECopyColorBufferToScanBuffer(MPTR colorBufferPtr, uin return; } + auto getVPADScreenActive = [](size_t n) -> std::pair { + auto controller = InputManager::instance().get_vpad_controller(n); + if (!controller) + return {false,false}; + auto pressed = controller->is_screen_active(); + auto toggle = controller->is_screen_active_toggle(); + return {pressed && !toggle, pressed && toggle}; + }; + const bool tabPressed = gui_isKeyDown(PlatformKeyCodes::TAB); const bool ctrlPressed = gui_isKeyDown(PlatformKeyCodes::LCONTROL); + const auto [vpad0Active, vpad0Toggle] = getVPADScreenActive(0); + const auto [vpad1Active, vpad1Toggle] = getVPADScreenActive(1); - bool showDRC = swkbd_hasKeyboardInputHook() == false && tabPressed; - bool& alwaysDisplayDRC = LatteGPUState.alwaysDisplayDRC; + const bool altScreenRequested = (!ctrlPressed && tabPressed) || vpad0Active || vpad1Active; + const bool togglePressed = (ctrlPressed && tabPressed) || vpad0Toggle || vpad1Toggle; + static bool togglePressedLast = false; - if (ctrlPressed && tabPressed) - { - if (ctrlTabHotkeyPressed == false) - { - alwaysDisplayDRC = !alwaysDisplayDRC; - ctrlTabHotkeyPressed = true; - } - } - else - ctrlTabHotkeyPressed = false; + bool& isDRCPrimary = LatteGPUState.isDRCPrimary; - if (alwaysDisplayDRC) - showDRC = !tabPressed; + if(togglePressed && !togglePressedLast) + isDRCPrimary = !isDRCPrimary; + togglePressedLast = togglePressed; - if (!showDRC) - { - auto controller = InputManager::instance().get_vpad_controller(0); - if (controller && controller->is_screen_active()) - showDRC = true; - if (!showDRC) - { - controller = InputManager::instance().get_vpad_controller(1); - if (controller && controller->is_screen_active()) - showDRC = true; - } - } + bool showDRC = swkbd_hasKeyboardInputHook() == false && (isDRCPrimary ^ altScreenRequested); if ((renderTarget & RENDER_TARGET_DRC) && g_renderer->IsPadWindowActive()) LatteRenderTarget_copyToBackbuffer(texView, true); diff --git a/src/Cafe/HW/Latte/Core/LatteShader.cpp b/src/Cafe/HW/Latte/Core/LatteShader.cpp index 77f16468..d9f0a5dd 100644 --- a/src/Cafe/HW/Latte/Core/LatteShader.cpp +++ b/src/Cafe/HW/Latte/Core/LatteShader.cpp @@ -451,9 +451,8 @@ void LatteShader_DumpShader(uint64 baseHash, uint64 auxHash, LatteDecompilerShad suffix = "gs"; else if (shader->shaderType == LatteConst::ShaderType::Pixel) suffix = "ps"; - fs::path dumpPath = "dump/shaders"; - dumpPath /= fmt::format("{:016x}_{:016x}_{}.txt", baseHash, auxHash, suffix); - FileStream* fs = FileStream::createFile2(dumpPath); + + FileStream* fs = FileStream::createFile2(ActiveSettings::GetUserDataPath("dump/shaders/{:016x}_{:016x}_{}.txt", baseHash, auxHash, suffix)); if (fs) { if (shader->strBuf_shaderSource) @@ -479,9 +478,8 @@ void LatteShader_DumpRawShader(uint64 baseHash, uint64 auxHash, uint32 type, uin suffix = "copy"; else if (type == SHADER_DUMP_TYPE_COMPUTE) suffix = "compute"; - fs::path dumpPath = "dump/shaders"; - dumpPath /= fmt::format("{:016x}_{:016x}_{}.bin", baseHash, auxHash, suffix); - FileStream* fs = FileStream::createFile2(dumpPath); + + FileStream* fs = FileStream::createFile2(ActiveSettings::GetUserDataPath("dump/shaders/{:016x}_{:016x}_{}.bin", baseHash, auxHash, suffix)); if (fs) { fs->writeData(programCode, programLen); diff --git a/src/Cafe/HW/Latte/Core/LatteShaderCache.cpp b/src/Cafe/HW/Latte/Core/LatteShaderCache.cpp index 9576eb2e..0d427e34 100644 --- a/src/Cafe/HW/Latte/Core/LatteShaderCache.cpp +++ b/src/Cafe/HW/Latte/Core/LatteShaderCache.cpp @@ -25,6 +25,9 @@ #include "util/helpers/Serializer.h" #include +#include