diff --git a/src/Cafe/HW/Latte/Renderer/Metal/LatteTextureReadbackMtl.cpp b/src/Cafe/HW/Latte/Renderer/Metal/LatteTextureReadbackMtl.cpp
index dd2d8aeb..0bcab09f 100644
--- a/src/Cafe/HW/Latte/Renderer/Metal/LatteTextureReadbackMtl.cpp
+++ b/src/Cafe/HW/Latte/Renderer/Metal/LatteTextureReadbackMtl.cpp
@@ -5,6 +5,8 @@
 
 LatteTextureReadbackInfoMtl::~LatteTextureReadbackInfoMtl()
 {
+    if (m_commandBuffer)
+        m_commandBuffer->release();
 }
 
 void LatteTextureReadbackInfoMtl::StartTransfer()
@@ -24,7 +26,7 @@ void LatteTextureReadbackInfoMtl::StartTransfer()
 
 	blitCommandEncoder->copyFromTexture(baseTexture->GetTexture(), 0, 0, MTL::Origin{0, 0, 0}, MTL::Size{(uint32)baseTexture->width, (uint32)baseTexture->height, 1}, m_mtlr->GetTextureReadbackBuffer(), m_bufferOffset, bytesPerRow, bytesPerImage);
 
-	m_commandBuffer = m_mtlr->GetCurrentCommandBuffer();
+	m_commandBuffer = m_mtlr->GetCurrentCommandBuffer()->retain();
 	m_mtlr->RequestSoonCommit();
 }
 
diff --git a/src/Cafe/HW/Latte/Renderer/Metal/MetalBufferAllocator.h b/src/Cafe/HW/Latte/Renderer/Metal/MetalBufferAllocator.h
index 198d9978..ef65c458 100644
--- a/src/Cafe/HW/Latte/Renderer/Metal/MetalBufferAllocator.h
+++ b/src/Cafe/HW/Latte/Renderer/Metal/MetalBufferAllocator.h
@@ -3,6 +3,7 @@
 #include "Cafe/HW/Latte/Renderer/Metal/MetalRenderer.h"
 #include "Common/precompiled.h"
 #include "Metal/MTLResource.hpp"
+#include <utility>
 
 struct MetalBufferRange
 {
@@ -162,7 +163,8 @@ typedef MetalBufferAllocator<Empty> MetalDefaultBufferAllocator;
 
 struct MetalSyncedBuffer
 {
-    std::vector<MTL::CommandBuffer*> m_commandBuffers;
+    uint32 m_commandBufferCount = 0;
+    MTL::CommandBuffer* m_lastCommandBuffer = nullptr;
     uint32 m_lock = 0;
 
     bool IsLocked() const
@@ -191,7 +193,7 @@ public:
 
         // TODO: is this really necessary?
         // Release the buffer if it wasn't released due to the lock
-        if (!buffer.m_data.IsLocked() && buffer.m_data.m_commandBuffers.empty())
+        if (!buffer.m_data.IsLocked() && buffer.m_data.m_commandBufferCount == 0)
             FreeBuffer(bufferIndex);
     }
 
@@ -206,7 +208,7 @@ public:
 
             if (buffer.m_data.m_lock != 0)
             {
-                if (buffer.m_data.m_commandBuffers.empty())
+                if (buffer.m_data.m_commandBufferCount == 0)
                     FreeBuffer(i);
 
                 buffer.m_data.m_lock = 0;
@@ -218,7 +220,7 @@ public:
         if (!m_buffers.empty())
         {
             auto& backBuffer = m_buffers.back();
-            if (backBuffer.m_data.m_commandBuffers.empty())
+            if (backBuffer.m_data.m_commandBufferCount == 0)
             {
                 // Release the back buffer if it hasn't been accessed for a while
                 if (m_framesSinceBackBufferAccess >= BUFFER_RELEASE_FRAME_TRESHOLD)
@@ -246,34 +248,34 @@ public:
     void SetActiveCommandBuffer(MTL::CommandBuffer* commandBuffer)
     {
         m_activeCommandBuffer = commandBuffer;
+        auto result = m_executingCommandBuffers.emplace(std::make_pair(m_activeCommandBuffer, std::vector<uint32>{}));
+        cemu_assert_debug(result.second);
+        m_activeCommandBufferIt = result.first;
+        commandBuffer->retain();
     }
 
     void CheckForCompletedCommandBuffers(/*MTL::CommandBuffer* commandBuffer, bool erase = true*/)
     {
-        for (uint32_t i = 0; i < m_buffers.size(); i++)
+        for (auto it = m_executingCommandBuffers.begin(); it != m_executingCommandBuffers.end();)
         {
-            auto& buffer = m_buffers[i];
-            for (uint32_t j = 0; j < buffer.m_data.m_commandBuffers.size(); j++)
+            if (CommandBufferCompleted(it->first))
             {
-                if (CommandBufferCompleted(buffer.m_data.m_commandBuffers[j]))
+                for (auto bufferIndex : it->second)
                 {
-                    if (buffer.m_data.m_commandBuffers.size() == 1)
-                    {
-                        if (!buffer.m_data.IsLocked())
-                        {
-                            // All command buffers using it have finished execution, we can use it again
-                            FreeBuffer(i);
-                        }
+                    auto& buffer = m_buffers[bufferIndex];
+                    buffer.m_data.m_commandBufferCount--;
 
-                        buffer.m_data.m_commandBuffers.clear();
-                        break;
-                    }
-                    else
-                    {
-                        buffer.m_data.m_commandBuffers.erase(buffer.m_data.m_commandBuffers.begin() + j);
-                        j--;
-                    }
+                    if (!buffer.m_data.IsLocked() && buffer.m_data.m_commandBufferCount == 0)
+                        FreeBuffer(bufferIndex);
                 }
+
+                it->first->release();
+
+                it = m_executingCommandBuffers.erase(it);
+            }
+            else
+            {
+                ++it;
             }
         }
 
@@ -286,8 +288,12 @@ public:
         cemu_assert_debug(m_activeCommandBuffer);
 
         auto& buffer = m_buffers[bufferIndex];
-        if (buffer.m_data.m_commandBuffers.empty() || buffer.m_data.m_commandBuffers.back() != m_activeCommandBuffer/*std::find(buffer.m_commandBuffers.begin(), buffer.m_commandBuffers.end(), m_activeCommandBuffer) == buffer.m_commandBuffers.end()*/)
-            buffer.m_data.m_commandBuffers.push_back(m_activeCommandBuffer);
+        if (buffer.m_data.m_commandBufferCount == 0 || buffer.m_data.m_lastCommandBuffer != m_activeCommandBuffer)
+        {
+            m_activeCommandBufferIt->second.push_back(bufferIndex);
+            buffer.m_data.m_commandBufferCount++;
+            buffer.m_data.m_lastCommandBuffer = m_activeCommandBuffer;
+        }
 
         return buffer.m_buffer;
     }
@@ -348,5 +354,8 @@ public:
 private:
     MTL::CommandBuffer* m_activeCommandBuffer = nullptr;
 
+    std::map<MTL::CommandBuffer*, std::vector<uint32>> m_executingCommandBuffers;
+    std::map<MTL::CommandBuffer*, std::vector<uint32>>::iterator m_activeCommandBufferIt;
+
     uint16 m_framesSinceBackBufferAccess = 0;
 };
diff --git a/src/Cafe/HW/Latte/Renderer/Metal/MetalQuery.cpp b/src/Cafe/HW/Latte/Renderer/Metal/MetalQuery.cpp
index ab24b4db..91f252e8 100644
--- a/src/Cafe/HW/Latte/Renderer/Metal/MetalQuery.cpp
+++ b/src/Cafe/HW/Latte/Renderer/Metal/MetalQuery.cpp
@@ -22,6 +22,9 @@ LatteQueryObjectMtl::~LatteQueryObjectMtl()
 {
     if (m_queryIndex != INVALID_UINT32)
         m_mtlr->ReleaseOcclusionQueryIndex(m_queryIndex);
+
+    if (m_commandBuffer)
+        m_commandBuffer->release();
 }
 
 void LatteQueryObjectMtl::begin()
@@ -35,7 +38,7 @@ void LatteQueryObjectMtl::end()
     m_mtlr->SetActiveOcclusionQueryIndex(INVALID_UINT32);
     if (m_mtlr->IsCommandBufferActive())
     {
-        m_commandBuffer = m_mtlr->GetCurrentCommandBuffer();
+        m_commandBuffer = m_mtlr->GetCurrentCommandBuffer()->retain();
         m_mtlr->RequestSoonCommit();
     }
 }
diff --git a/src/Cafe/HW/Latte/Renderer/Metal/MetalRenderer.cpp b/src/Cafe/HW/Latte/Renderer/Metal/MetalRenderer.cpp
index a063a0b7..f33e527f 100644
--- a/src/Cafe/HW/Latte/Renderer/Metal/MetalRenderer.cpp
+++ b/src/Cafe/HW/Latte/Renderer/Metal/MetalRenderer.cpp
@@ -267,11 +267,8 @@ void MetalRenderer::SwapBuffers(bool swapTV, bool swapDRC)
     if (swapDRC)
         SwapBuffer(false);
 
-    // Release all the command buffers
+    // Reset the command buffers (they are released in)
     CommitCommandBuffer();
-    // TODO: release
-    //for (uint32 i = 0; i < m_commandBuffers.size(); i++)
-    //    m_commandBuffers[i].m_commandBuffer->release();
     m_commandBuffers.clear();
 
     // Release frame persistent buffers
@@ -1581,6 +1578,7 @@ void MetalRenderer::CommitCommandBuffer()
             //});
 
             commandBuffer.m_commandBuffer->commit();
+            commandBuffer.m_commandBuffer->release();
             commandBuffer.m_commited = true;
 
             m_memoryManager->GetTemporaryBufferAllocator().SetActiveCommandBuffer(nullptr);