diff --git a/rpcs3/Emu/RSX/GL/GLTextureCache.h b/rpcs3/Emu/RSX/GL/GLTextureCache.h
index 7de17a47dd..fe4ef07549 100644
--- a/rpcs3/Emu/RSX/GL/GLTextureCache.h
+++ b/rpcs3/Emu/RSX/GL/GLTextureCache.h
@@ -67,14 +67,26 @@ namespace gl
 	public:
 		using baseclass::cached_texture_section;
 
-		void create(u16 w, u16 h, u16 depth, u16 mipmaps, gl::texture* image, u32 rsx_pitch, bool read_only,
+		void create(u16 w, u16 h, u16 depth, u16 mipmaps, gl::texture* image, u32 rsx_pitch, bool managed,
 				gl::texture::format gl_format = gl::texture::format::rgba, gl::texture::type gl_type = gl::texture::type::ubyte, bool swap_bytes = false)
 		{
+			if (vram_texture && !managed_texture && get_protection() == utils::protection::no)
+			{
+				// In-place image swap, still locked. Likely a color buffer that got rebound as depth buffer or vice-versa.
+				gl::as_rtt(vram_texture)->release();
+
+				if (!managed)
+				{
+					// Incoming is also an external resource, reference it immediately
+					gl::as_rtt(image)->add_ref();
+				}
+			}
+
 			auto new_texture = static_cast<gl::viewable_image*>(image);
 			ensure(!exists() || !is_managed() || vram_texture == new_texture);
 			vram_texture = new_texture;
 
-			if (read_only)
+			if (managed)
 			{
 				managed_texture.reset(vram_texture);
 			}
diff --git a/rpcs3/Emu/RSX/VK/VKTextureCache.h b/rpcs3/Emu/RSX/VK/VKTextureCache.h
index 37a79ca320..f63806ec9c 100644
--- a/rpcs3/Emu/RSX/VK/VKTextureCache.h
+++ b/rpcs3/Emu/RSX/VK/VKTextureCache.h
@@ -47,6 +47,18 @@ namespace vk
 
 		void create(u16 w, u16 h, u16 depth, u16 mipmaps, vk::image* image, u32 rsx_pitch, bool managed, u32 gcm_format, bool pack_swap_bytes = false)
 		{
+			if (vram_texture && !managed_texture && get_protection() == utils::protection::no)
+			{
+				// In-place image swap, still locked. Likely a color buffer that got rebound as depth buffer or vice-versa.
+				vk::as_rtt(vram_texture)->release();
+
+				if (!managed)
+				{
+					// Incoming is also an external resource, reference it immediately
+					vk::as_rtt(image)->add_ref();
+				}
+			}
+
 			auto new_texture = static_cast<vk::viewable_image*>(image);
 			ensure(!exists() || !is_managed() || vram_texture == new_texture);
 			vram_texture = new_texture;