rsx: Rework memory protection management for framebuffer access

- Avoid re-locking memory if there is no reason to do so (no draws issued)
- Actively bound regions should always get written to the backing cache
- Forcefully read memory during download if writes to the target have occured since last sync event

rpcs3/Emu/RSX/Common/texture_cache.h

@@ -70,6 +70,7 @@ namespace rsx
 		u32 gcm_format = 0;
 		bool pack_unpack_swap_bytes = false;
 
+		u64 sync_timestamp = 0;
 		bool synchronized = false;
 		bool flushed = false;
 
@@ -250,6 +251,31 @@ namespace rsx
 
 			return true;
 		}
+
+		void reprotect(utils::protection prot, const std::pair<u32, u32>& range)
+		{
+			//Reset properties and protect again
+			flushed = false;
+			synchronized = false;
+			sync_timestamp = 0ull;
+
+			protect(prot, range);
+		}
+
+		void reprotect(utils::protection prot)
+		{
+			//Reset properties and protect again
+			flushed = false;
+			synchronized = false;
+			sync_timestamp = 0ull;
+
+			protect(prot);
+		}
+
+		u64 get_sync_timestamp() const
+		{
+			return sync_timestamp;
+		}
 	};
 
 	template <typename commandbuffer_type, typename section_storage_type, typename image_resource_type, typename image_view_type, typename image_storage_type, typename texture_format>
@@ -701,6 +727,18 @@ namespace rsx
 							}
 							else
 							{
+								if (obj.first->get_memory_read_flags() == rsx::memory_read_flags::flush_always)
+								{
+									// This region is set to always read from itself (unavoi
+									const auto ROP_timestamp = rsx::get_current_renderer()->ROP_sync_timestamp;
+									if (obj.first->is_synchronized() && ROP_timestamp > obj.first->get_sync_timestamp())
+									{
+										m_num_cache_mispredictions++;
+										m_num_cache_misses++;
+										obj.first->copy_texture(true, std::forward<Args>(extras)...);
+									}
+								}
+
 								if (!obj.first->flush(std::forward<Args>(extras)...))
 								{
 									//Missed address, note this
@@ -1216,6 +1254,18 @@ namespace rsx
 				{
 					if (tex->is_locked())
 					{
+						if (tex->get_memory_read_flags() == rsx::memory_read_flags::flush_always)
+						{
+							// This region is set to always read from itself (unavoi
+							const auto ROP_timestamp = rsx::get_current_renderer()->ROP_sync_timestamp;
+							if (tex->is_synchronized() && ROP_timestamp > tex->get_sync_timestamp())
+							{
+								m_num_cache_mispredictions++;
+								m_num_cache_misses++;
+								tex->copy_texture(true, std::forward<Args>(extras)...);
+							}
+						}
+
 						if (!tex->flush(std::forward<Args>(extras)...))
 						{
 							record_cache_miss(*tex);
@@ -1303,9 +1353,13 @@ namespace rsx
 			// Auto flush if this address keeps missing (not properly synchronized)
 			if (value.misses >= 4)
 			{
-				// TODO: Determine better way of setting threshold
+				// Disable prediction if memory is flagged as flush_always
-				// Allow all types
+				if (m_flush_always_cache.find(memory_address) == m_flush_always_cache.end())
-				flush_mask = 0xFF;
+				{
+					// TODO: Determine better way of setting threshold
+					// Allow all types
+					flush_mask = 0xFF;
+				}
 			}
 
 			if (!flush_memory_to_cache(memory_address, memory_size, true, flush_mask, std::forward<Args>(extras)...) &&

rpcs3/Emu/RSX/D3D12/D3D12GSRender.cpp

@@ -404,7 +404,7 @@ void D3D12GSRender::end()
 			);
 	}
 
-	m_graphics_state = 0;
+	m_graphics_state &= ~rsx::pipeline_state::memory_barrier_bits;
 
 	std::chrono::time_point<steady_clock> constants_duration_end = steady_clock::now();
 	m_timers.constants_duration += std::chrono::duration_cast<std::chrono::microseconds>(constants_duration_end - constants_duration_start).count();

rpcs3/Emu/RSX/GL/GLGSRender.cpp

@@ -1183,7 +1183,7 @@ void GLGSRender::load_program(const gl::vertex_upload_info& upload_info)
 	}
 
 	// Fragment state
-	fill_fragment_state_buffer(buf+fragment_constants_size, current_fragment_program);
+	fill_fragment_state_buffer(buf + fragment_constants_size, current_fragment_program);
 
 	m_vertex_state_buffer->bind_range(0, vertex_state_offset, 512);
 	m_fragment_constants_buffer->bind_range(2, fragment_constants_offset, fragment_buffer_size);
@@ -1198,7 +1198,7 @@ void GLGSRender::load_program(const gl::vertex_upload_info& upload_info)
 		if (update_transform_constants) m_transform_constants_buffer->unmap();
 	}
 
-	m_graphics_state = 0;
+	m_graphics_state &= ~rsx::pipeline_state::memory_barrier_bits;
 }
 
 void GLGSRender::update_draw_state()
@@ -1600,9 +1600,13 @@ void GLGSRender::do_local_task(rsx::FIFO_state state)
 	}
 	else if (!in_begin_end && state != rsx::FIFO_state::lock_wait)
 	{
-		//This will re-engage locks and break the texture cache if another thread is waiting in access violation handler!
+		if (m_graphics_state & rsx::pipeline_state::framebuffer_reads_dirty)
-		//Only call when there are no waiters
+		{
-		m_gl_texture_cache.do_update();
+			//This will re-engage locks and break the texture cache if another thread is waiting in access violation handler!
+			//Only call when there are no waiters
+			m_gl_texture_cache.do_update();
+			m_graphics_state &= ~rsx::pipeline_state::framebuffer_reads_dirty;
+		}
 	}
 
 	rsx::thread::do_local_task(state);

rpcs3/Emu/RSX/GL/GLTextureCache.h

@@ -260,6 +260,7 @@ namespace gl
 
 			flushed = false;
 			synchronized = false;
+			sync_timestamp = 0ull;
 			is_depth = false;
 
 			vram_texture = nullptr;
@@ -291,6 +292,7 @@ namespace gl
 
 			flushed = false;
 			synchronized = false;
+			sync_timestamp = 0ull;
 			is_depth = false;
 
 			this->width = w;
@@ -454,6 +456,7 @@ namespace gl
 
 			m_fence.reset();
 			synchronized = true;
+			sync_timestamp = get_system_time();
 		}
 
 		void fill_texture(gl::texture* tex)
@@ -603,20 +606,6 @@ namespace gl
 			return result;
 		}
 
-		void reprotect(utils::protection prot, const std::pair<u32, u32>& range)
-		{
-			flushed = false;
-			synchronized = false;
-			protect(prot, range);
-		}
-
-		void reprotect(utils::protection prot)
-		{
-			flushed = false;
-			synchronized = false;
-			protect(prot);
-		}
-
 		void destroy()
 		{
 			if (!locked && pbo_id == 0 && vram_texture == 0 && m_fence.is_empty())

rpcs3/Emu/RSX/RSXThread.cpp

@@ -330,6 +330,9 @@ namespace rsx
 
 		in_begin_end = false;
 
+		m_graphics_state |= rsx::pipeline_state::framebuffer_reads_dirty;
+		ROP_sync_timestamp = get_system_time();
+
 		for (u8 index = 0; index < rsx::limits::vertex_count; ++index)
 		{
 			//Disabled, see https://github.com/RPCS3/rpcs3/issues/1932

rpcs3/Emu/RSX/RSXThread.h

@@ -76,8 +76,10 @@ namespace rsx
 		fragment_state_dirty = 4,
 		vertex_state_dirty = 8,
 		transform_constants_dirty = 16,
+		framebuffer_reads_dirty = 32,
 
 		invalidate_pipeline_bits = fragment_program_dirty | vertex_program_dirty,
+		memory_barrier_bits = framebuffer_reads_dirty,
 		all_dirty = 255
 	};
 
@@ -358,6 +360,7 @@ namespace rsx
 		bool m_vertex_textures_dirty[4];
 		bool m_framebuffer_state_contested = false;
 		u32  m_graphics_state = 0;
+		u64  ROP_sync_timestamp = 0;
 
 	protected:
 		std::array<u32, 4> get_color_surface_addresses() const;

rpcs3/Emu/RSX/VK/VKGSRender.cpp

@@ -2065,9 +2065,13 @@ void VKGSRender::do_local_task(rsx::FIFO_state state)
 	}
 	else if (!in_begin_end && state != rsx::FIFO_state::lock_wait)
 	{
-		//This will re-engage locks and break the texture cache if another thread is waiting in access violation handler!
+		if (m_graphics_state & rsx::pipeline_state::framebuffer_reads_dirty)
-		//Only call when there are no waiters
+		{
-		m_texture_cache.do_update();
+			//This will re-engage locks and break the texture cache if another thread is waiting in access violation handler!
+			//Only call when there are no waiters
+			m_texture_cache.do_update();
+			m_graphics_state &= ~rsx::pipeline_state::framebuffer_reads_dirty;
+		}
 	}
 
 	rsx::thread::do_local_task(state);
@@ -2417,7 +2421,7 @@ void VKGSRender::load_program(const vk::vertex_upload_info& vertex_info)
 	}
 
 	//Clear flags
-	m_graphics_state = 0;
+	m_graphics_state &= ~rsx::pipeline_state::memory_barrier_bits;
 }
 
 static const u32 mr_color_offset[rsx::limits::color_buffers_count] =

rpcs3/Emu/RSX/VK/VKTextureCache.h

@@ -20,8 +20,6 @@ namespace vk
 
 		//DMA relevant data
 		VkFence dma_fence = VK_NULL_HANDLE;
-		u64 sync_timestamp = 0;
-		u64 last_use_timestamp = 0;
 		vk::render_device* m_device = nullptr;
 		vk::image *vram_texture = nullptr;
 		std::unique_ptr<vk::buffer> dma_buffer;
@@ -72,7 +70,6 @@ namespace vk
 			synchronized = false;
 			flushed = false;
 			sync_timestamp = 0ull;
-			last_use_timestamp = get_system_time();
 		}
 
 		void release_dma_resources()
@@ -351,41 +348,11 @@ namespace vk
 			pack_unpack_swap_bytes = swap_bytes;
 		}
 
-		void reprotect(utils::protection prot, const std::pair<u32, u32>& range)
-		{
-			//Reset properties and protect again
-			flushed = false;
-			synchronized = false;
-			sync_timestamp = 0ull;
-
-			protect(prot, range);
-		}
-
-		void reprotect(utils::protection prot)
-		{
-			//Reset properties and protect again
-			flushed = false;
-			synchronized = false;
-			sync_timestamp = 0ull;
-
-			protect(prot);
-		}
-
-		void invalidate_cached()
-		{
-			synchronized = false;
-		}
-
 		bool is_synchronized() const
 		{
 			return synchronized;
 		}
 
-		bool sync_valid() const
-		{
-			return (sync_timestamp > last_use_timestamp);
-		}
-
 		bool has_compatible_format(vk::image* tex) const
 		{
 			return vram_texture->info.format == tex->info.format;
@@ -403,11 +370,6 @@ namespace vk
 				return false;
 			}
 		}
-
-		u64 get_sync_timestamp() const
-		{
-			return sync_timestamp;
-		}
 	};
 	
 	struct discarded_storage