gl: Fix problems with framebuffer reuse

- Matching attachments with resource id fails because drivers are reusing
  handles!
- Properly sets up stale fbo ref counting and removal
- Properly sets up resource reference test with subsequent removal to
  avoid using a broken fbo entry

rpcs3/Emu/RSX/GL/GLGSRender.cpp

@@ -1616,19 +1616,17 @@ void GLGSRender::flip(int buffer)
 
 	// Cleanup
 	m_gl_texture_cache.on_frame_end();
-
-	m_rtts.free_invalidated();
 	m_vertex_cache->purge();
 
-	if (m_framebuffer_cache.size() > 32)
+	auto removed_textures = m_rtts.free_invalidated();
+	m_framebuffer_cache.remove_if([&](auto& fbo)
 	{
-		for (auto &fbo : m_framebuffer_cache)
+		if (fbo.deref_count >= 2) return true; // Remove if stale
-		{
+		if (fbo.references_any(removed_textures)) return true; // Remove if any of the attachments is invalid
-			fbo.remove();
-		}
 
-		m_framebuffer_cache.clear();
+		fbo.deref_count++;
-	}
+		return false;
+	});
 
 	//If we are skipping the next frame, do not reset perf counters
 	if (skip_frame) return;

rpcs3/Emu/RSX/GL/GLGSRender.h

@@ -1,4 +1,4 @@
-#pragma once
+#pragma once
 #include "Emu/RSX/GSRender.h"
 #include "GLHelpers.h"
 #include "GLTexture.h"
@@ -330,7 +330,7 @@ private:
 
 	//buffer
 	gl::fbo* m_draw_fbo = nullptr;
-	std::list<gl::fbo> m_framebuffer_cache;
+	std::list<gl::framebuffer_holder> m_framebuffer_cache;
 	gl::fbo m_flip_fbo;
 	std::unique_ptr<gl::texture> m_flip_tex_color;
 

rpcs3/Emu/RSX/GL/GLHelpers.cpp

@@ -1,4 +1,4 @@
-#include "stdafx.h"
+#include "stdafx.h"
 #include "GLHelpers.h"
 #include "Utilities/Log.h"
 
@@ -35,12 +35,16 @@ namespace gl
 		switch (type)
 		{
 		case GL_DEBUG_TYPE_ERROR:
+		{
 			LOG_ERROR(RSX, "%s", message);
 			return;
+		}
 		default:
+		{
 			LOG_WARNING(RSX, "%s", message);
 			return;
 		}
+		}
 	}
 #endif
 
@@ -306,7 +310,7 @@ namespace gl
 		return m_size;
 	}
 
-	bool fbo::matches(std::array<GLuint, 4> color_targets, GLuint depth_stencil_target)
+	bool fbo::matches(const std::array<GLuint, 4>& color_targets, GLuint depth_stencil_target) const
 	{
 		for (u32 index = 0; index < 4; ++index)
 		{
@@ -317,7 +321,18 @@ namespace gl
 		}
 
 		const auto depth_resource = depth.resource_id() | depth_stencil.resource_id();
-		return depth_resource == depth_stencil_target;
+		return (depth_resource == depth_stencil_target);
+	}
+
+	bool fbo::references_any(const std::vector<GLuint>& resources) const
+	{
+		for (const auto &e : m_resource_bindings)
+		{
+			if (std::find(resources.begin(), resources.end(), e.second) != resources.end())
+				return true;
+		}
+
+		return false;
 	}
 
 	bool is_primitive_native(rsx::primitive_type in)

rpcs3/Emu/RSX/GL/GLHelpers.h

@@ -1,4 +1,4 @@
-#pragma once
+#pragma once
 
 #include <exception>
 #include <string>
@@ -2234,7 +2234,8 @@ public:
 		void set_extents(size2i extents);
 		size2i get_extents() const;
 
-		bool matches(std::array<GLuint, 4> color_targets, GLuint depth_stencil_target);
+		bool matches(const std::array<GLuint, 4>& color_targets, GLuint depth_stencil_target) const;
+		bool references_any(const std::vector<GLuint>& resources) const;
 
 		explicit operator bool() const
 		{

rpcs3/Emu/RSX/GL/GLRenderTargets.cpp

@@ -1,4 +1,4 @@
-#include "stdafx.h"
+#include "stdafx.h"
 #include "../rsx_methods.h"
 #include "GLGSRender.h"
 #include "Emu/System.h"
@@ -294,7 +294,12 @@ void GLGSRender::init_buffers(rsx::framebuffer_creation_context context, bool sk
 	{
 		if (fbo.matches(color_targets, depth_stencil_target))
 		{
+			fbo.reset_refs();
+
 			m_draw_fbo = &fbo;
+			m_draw_fbo->bind();
+			m_draw_fbo->set_extents({ (int)layout.width, (int)layout.height });
+
 			framebuffer_status_valid = true;
 			break;
 		}
@@ -327,43 +332,40 @@ void GLGSRender::init_buffers(rsx::framebuffer_creation_context context, bool sk
 				m_draw_fbo->depth = depth_stencil_target;
 			}
 		}
-
-		switch (rsx::method_registers.surface_color_target())
-		{
-		case rsx::surface_target::none: break;
-
-		case rsx::surface_target::surface_a:
-			m_draw_fbo->draw_buffer(m_draw_fbo->color[0]);
-			m_draw_fbo->read_buffer(m_draw_fbo->color[0]);
-			break;
-
-		case rsx::surface_target::surface_b:
-			m_draw_fbo->draw_buffer(m_draw_fbo->color[1]);
-			m_draw_fbo->read_buffer(m_draw_fbo->color[1]);
-			break;
-
-		case rsx::surface_target::surfaces_a_b:
-			m_draw_fbo->draw_buffers({ m_draw_fbo->color[0], m_draw_fbo->color[1] });
-			m_draw_fbo->read_buffer(m_draw_fbo->color[0]);
-			break;
-
-		case rsx::surface_target::surfaces_a_b_c:
-			m_draw_fbo->draw_buffers({ m_draw_fbo->color[0], m_draw_fbo->color[1], m_draw_fbo->color[2] });
-			m_draw_fbo->read_buffer(m_draw_fbo->color[0]);
-			break;
-
-		case rsx::surface_target::surfaces_a_b_c_d:
-			m_draw_fbo->draw_buffers({ m_draw_fbo->color[0], m_draw_fbo->color[1], m_draw_fbo->color[2], m_draw_fbo->color[3] });
-			m_draw_fbo->read_buffer(m_draw_fbo->color[0]);
-			break;
-		}
-
-		framebuffer_status_valid = m_draw_fbo->check();
 	}
-	
-	if (!framebuffer_status_valid) return;
 
-	m_draw_fbo->bind();
+	switch (rsx::method_registers.surface_color_target())
+	{
+	case rsx::surface_target::none: break;
+
+	case rsx::surface_target::surface_a:
+		m_draw_fbo->draw_buffer(m_draw_fbo->color[0]);
+		m_draw_fbo->read_buffer(m_draw_fbo->color[0]);
+		break;
+
+	case rsx::surface_target::surface_b:
+		m_draw_fbo->draw_buffer(m_draw_fbo->color[1]);
+		m_draw_fbo->read_buffer(m_draw_fbo->color[1]);
+		break;
+
+	case rsx::surface_target::surfaces_a_b:
+		m_draw_fbo->draw_buffers({ m_draw_fbo->color[0], m_draw_fbo->color[1] });
+		m_draw_fbo->read_buffer(m_draw_fbo->color[0]);
+		break;
+
+	case rsx::surface_target::surfaces_a_b_c:
+		m_draw_fbo->draw_buffers({ m_draw_fbo->color[0], m_draw_fbo->color[1], m_draw_fbo->color[2] });
+		m_draw_fbo->read_buffer(m_draw_fbo->color[0]);
+		break;
+
+	case rsx::surface_target::surfaces_a_b_c_d:
+		m_draw_fbo->draw_buffers({ m_draw_fbo->color[0], m_draw_fbo->color[1], m_draw_fbo->color[2], m_draw_fbo->color[3] });
+		m_draw_fbo->read_buffer(m_draw_fbo->color[0]);
+		break;
+	}
+
+	framebuffer_status_valid = m_draw_fbo->check();
+	if (!framebuffer_status_valid) return;
 
 	check_zcull_status(true);
 	set_viewport();

rpcs3/Emu/RSX/GL/GLRenderTargets.h

@@ -1,4 +1,4 @@
-#pragma once
+#pragma once
 #include "../Common/surface_store.h"
 #include "GLHelpers.h"
 #include "stdafx.h"
@@ -131,6 +131,11 @@ namespace gl
 			return (rsx::apply_resolution_scale(_width, true) == internal_width) && (rsx::apply_resolution_scale(_height, true) == internal_height);
 		}
 	};
+
+	struct framebuffer_holder : public gl::fbo, public rsx::ref_counted
+	{
+		using gl::fbo::fbo;
+	};
 }
 
 struct gl_render_target_traits
@@ -291,15 +296,21 @@ struct gl_render_target_traits
 
 struct gl_render_targets : public rsx::surface_store<gl_render_target_traits>
 {
-	void free_invalidated()
+	std::vector<GLuint> free_invalidated()
 	{
+		std::vector<GLuint> removed;
 		invalidated_resources.remove_if([&](auto &rtt)
 		{
 			if (rtt->deref_count >= 2)
+			{
+				removed.push_back(rtt->id());
 				return true;
+			}
 
 			rtt->deref_count++;
 			return false;
 		});
+
+		return removed;
 	}
 };