rsx: Rebuild shader texture state if we detect a silent mismatch

rpcs3/Emu/RSX/GL/GLDraw.cpp

@@ -341,23 +341,31 @@ void GLGSRender::load_texture_env()
 
 		auto sampler_state = static_cast<gl::texture_cache::sampled_image_descriptor*>(fs_sampler_state[i].get());
 		const auto& tex = rsx::method_registers.fragment_textures[i];
+		const auto previous_format_class = sampler_state->format_class;
 
 		if (m_samplers_dirty || m_textures_dirty[i] || m_gl_texture_cache.test_if_descriptor_expired(cmd, m_rtts, sampler_state, tex))
 		{
 			if (tex.enabled())
 			{
 				*sampler_state = m_gl_texture_cache.upload_texture(cmd, tex, m_rtts);
+
+				if (sampler_state->validate())
+				{
+					if (m_textures_dirty[i])
+					{
+						m_fs_sampler_states[i].apply(tex, fs_sampler_state[i].get());
+					}
+					else if (sampler_state->format_class != previous_format_class)
+					{
+						m_graphics_state |= rsx::fragment_program_state_dirty;
+					}
+				}
 			}
 			else
 			{
 				*sampler_state = {};
 			}
 
-			if (m_textures_dirty[i] && sampler_state->validate())
-			{
-				m_fs_sampler_states[i].apply(tex, fs_sampler_state[i].get());
-			}
-
 			m_textures_dirty[i] = false;
 		}
 	}
@@ -372,23 +380,31 @@ void GLGSRender::load_texture_env()
 
 		auto sampler_state = static_cast<gl::texture_cache::sampled_image_descriptor*>(vs_sampler_state[i].get());
 		const auto& tex = rsx::method_registers.vertex_textures[i];
+		const auto previous_format_class = sampler_state->format_class;
 
 		if (m_samplers_dirty || m_vertex_textures_dirty[i] || m_gl_texture_cache.test_if_descriptor_expired(cmd, m_rtts, sampler_state, tex))
 		{
 			if (rsx::method_registers.vertex_textures[i].enabled())
 			{
 				*sampler_state = m_gl_texture_cache.upload_texture(cmd, rsx::method_registers.vertex_textures[i], m_rtts);
+
+				if (sampler_state->validate())
+				{
+					if (m_vertex_textures_dirty[i])
+					{
+						m_vs_sampler_states[i].apply(tex, vs_sampler_state[i].get());
+					}
+					else if (sampler_state->format_class != previous_format_class)
+					{
+						m_graphics_state |= rsx::vertex_program_state_dirty;
+					}
+				}
 			}
 			else
 			{
 				*sampler_state = {};
 			}
 
-			if (m_vertex_textures_dirty[i] && sampler_state->validate())
-			{
-				m_vs_sampler_states[i].apply(tex, vs_sampler_state[i].get());
-			}
-
 			m_vertex_textures_dirty[i] = false;
 		}
 	}

rpcs3/Emu/RSX/VK/VKDraw.cpp

@@ -256,6 +256,7 @@ void VKGSRender::load_texture_env()
 
 		auto sampler_state = static_cast<vk::texture_cache::sampled_image_descriptor*>(fs_sampler_state[i].get());
 		const auto& tex = rsx::method_registers.fragment_textures[i];
+		const auto previous_format_class = fs_sampler_state[i]->format_class;
 
 		if (m_samplers_dirty || m_textures_dirty[i] || !check_surface_cache_sampler(sampler_state, tex))
 		{
@@ -276,6 +277,12 @@ void VKGSRender::load_texture_env()
 					check_for_cyclic_refs |= true;
 				}
 
+				if (!m_textures_dirty[i] && sampler_state->format_class != previous_format_class)
+				{
+					// Host details changed but RSX is not aware
+					m_graphics_state |= rsx::fragment_program_state_dirty;
+				}
+
 				bool replace = !fs_sampler_handles[i];
 				VkFilter mag_filter;
 				vk::minification_filter min_filter;
@@ -403,6 +410,7 @@ void VKGSRender::load_texture_env()
 
 		auto sampler_state = static_cast<vk::texture_cache::sampled_image_descriptor*>(vs_sampler_state[i].get());
 		const auto& tex = rsx::method_registers.vertex_textures[i];
+		const auto previous_format_class = sampler_state->format_class;
 
 		if (m_samplers_dirty || m_vertex_textures_dirty[i] || !check_surface_cache_sampler(sampler_state, tex))
 		{
@@ -423,6 +431,12 @@ void VKGSRender::load_texture_env()
 					check_for_cyclic_refs |= true;
 				}
 
+				if (!m_vertex_textures_dirty[i] && sampler_state->format_class != previous_format_class)
+				{
+					// Host details changed but RSX is not aware
+					m_graphics_state |= rsx::vertex_program_state_dirty;
+				}
+
 				bool replace = !vs_sampler_handles[i];
 				const VkBool32 unnormalized_coords = !!(tex.format() & CELL_GCM_TEXTURE_UN);
 				const auto min_lod = tex.min_lod();
@@ -1015,10 +1029,17 @@ void VKGSRender::end()
 	// Now bind the shader resources. It is important that this takes place after the barriers so that we don't end up with stale descriptors
 	for (int retry = 0; retry < 3; ++retry)
 	{
-		if (m_samplers_dirty) [[ unlikely ]]
+		if (retry > 0 && m_samplers_dirty) [[ unlikely ]]
 		{
 			// Reload texture env if referenced objects were invalidated during OOM handling.
 			load_texture_env();
+
+			// Do not trust fragment/vertex texture state after a texture state reset.
+			// NOTE: We don't want to change the program - it's too late for that now. We just need to harmonize the state.
+			m_graphics_state |= rsx::vertex_program_state_dirty | rsx::fragment_program_state_dirty;
+			get_current_fragment_program(fs_sampler_state);
+			get_current_vertex_program(vs_sampler_state);
+			m_graphics_state.clear(rsx::pipeline_state::invalidate_pipeline_bits);
 		}
 
 		const bool out_of_memory = m_shader_interpreter.is_interpreter(m_program)