rsx: Support CSAA transparency without multiple rasterization samples enabled

rpcs3/Emu/RSX/Common/GLSLCommon.h

@@ -372,10 +372,10 @@ namespace glsl
 		{
 			// Data is packed into a ubo
 			OS <<
-			"	int block = (location >> 1);\n"
+			"	int block = (location >> 1);\n"
-			"	int sub_block = (location & 1) << 1;\n"
+			"	int sub_block = (location & 1) << 1;\n"
-			"	uvec2 attrib = uvec2(\n"
+			"	uvec2 attrib = uvec2(\n"
-			"		ref(input_attributes_blob[block], sub_block + 0),\n"
+			"		ref(input_attributes_blob[block], sub_block + 0),\n"
 			"		ref(input_attributes_blob[block], sub_block + 1));\n";
 		}
 		else
@@ -571,12 +571,13 @@ namespace glsl
 			// Lowers alpha accuracy down to 2 bits, to mimic A2C banding
 			// Alpha lower than the real threshold (e.g 0.25 for 4 samples) gets a randomized chance to make it to the lowest transparency state
 			// Helps to avoid A2C tested foliage disappearing in the distance
+			// TODO: Fix dithering when mipmap gather is finished to remove muddy appearance. Alpha boost is only present to hide far LOD issues in titles like RDR
 			OS <<
 			"bool coverage_test_passes(/*inout*/in vec4 _sample, uint control)\n"
 			"{\n"
 			"	if ((control & 0x1) == 0) return false;\n"
 			"\n"
-			"	float samples = ((control & 0x2) != 0)? 4.f : 2.f;\n"
+			"	float samples = float(control & 0x6) * 0.5f + 1.f;\n"
 			"	float hash    = _saturate(_rand(gl_FragCoord) + 0.5f) * 0.9f;\n"
 			"	float epsilon = hash / samples;\n"
 			"	float alpha   = trunc((_sample.a + epsilon) * samples) / samples;\n"

rpcs3/Emu/RSX/GL/GLGSRender.cpp

@@ -36,6 +36,7 @@ GLGSRender::GLGSRender() : GSRender()
 	else
 		m_vertex_cache = std::make_unique<gl::weak_vertex_cache>();
 
+	supports_hw_a2c = false;
 	supports_multidraw = true;
 	supports_native_ui = (bool)g_cfg.misc.use_native_interface;
 }
@@ -1517,6 +1518,12 @@ void GLGSRender::update_draw_state()
 
 	gl_state.front_face(front_face(rsx::method_registers.front_face_mode()));
 
+	// Sample control
+	// TODO: MinSampleShading
+	//gl_state.enable(rsx::method_registers.msaa_enabled(), GL_MULTISAMPLE);
+	//gl_state.enable(rsx::method_registers.msaa_alpha_to_coverage_enabled(), GL_SAMPLE_ALPHA_TO_COVERAGE);
+	//gl_state.enable(rsx::method_registers.msaa_alpha_to_one_enabled(), GL_SAMPLE_ALPHA_TO_ONE);
+
 	//TODO
 	//NV4097_SET_ANISO_SPREAD
 	//NV4097_SET_SPECULAR_ENABLE

rpcs3/Emu/RSX/RSXThread.cpp

@@ -691,19 +691,28 @@ namespace rsx
 		auto alpha_ref = rsx::method_registers.alpha_ref() / 255.f;
 		auto rop_control = rsx::method_registers.alpha_test_enabled()? 1u : 0u;
 
-		if (rsx::method_registers.msaa_alpha_to_coverage_enabled() &&
+		if (rsx::method_registers.msaa_alpha_to_coverage_enabled() && !supports_hw_a2c)
-			rsx::method_registers.surface_antialias() != rsx::surface_antialiasing::center_1_sample &&
-			g_cfg.video.antialiasing_level == msaa_level::none)
 		{
 			// Alpha values generate a coverage mask for order independent blending
 			// Requires hardware AA to work properly (or just fragment sample stage in fragment shaders)
 			// Simulated using combined alpha blend and alpha test
 			const u32 mask_bit = rsx::method_registers.msaa_sample_mask() ? 1u : 0u;
-			const u32 samples_bit = rsx::method_registers.surface_antialias() != rsx::surface_antialiasing::diagonal_centered_2_samples ? 1u : 0u;
 
 			rop_control |= (1u << 4);                 // CSAA enable bit
 			rop_control |= (mask_bit << 5);           // MSAA mask enable bit
-			rop_control |= (samples_bit << 6);        // Sample configuration bit
+
+			// Sample configuration bits
+			switch (rsx::method_registers.surface_antialias())
+			{
+				case rsx::surface_antialiasing::center_1_sample:
+					break;
+				case rsx::surface_antialiasing::diagonal_centered_2_samples:
+					rop_control |= 1u << 6;
+					break;
+				default:
+					rop_control |= 3u << 6;
+					break;
+			}
 		}
 
 		const f32 fog0 = rsx::method_registers.fog_params_0();

rpcs3/Emu/RSX/RSXThread.h

@@ -421,8 +421,9 @@ namespace rsx
 		s32 m_skip_frame_ctr = 0;
 		bool skip_frame = false;
 
-		bool supports_multidraw = false;
+		bool supports_multidraw = false;  // Draw call batching
-		bool supports_native_ui = false;
+		bool supports_native_ui = false;  // Native UI rendering
+		bool supports_hw_a2c = false;     // Alpha to coverage
 
 		// FIFO
 		std::unique_ptr<FIFO::FIFO_control> fifo_ctrl;

rpcs3/Emu/RSX/VK/VKGSRender.cpp

@@ -575,6 +575,9 @@ VKGSRender::VKGSRender() : GSRender()
 
 	supports_multidraw = true;
 	supports_native_ui = (bool)g_cfg.misc.use_native_interface;
+	// NOTE: We do not actually need multiple sample support for A2C to work
+	// This is here for visual consistency - will be removed when AA problems due to mipmaps are fixed
+	supports_hw_a2c = (g_cfg.video.antialiasing_level != msaa_level::none);
 }
 
 VKGSRender::~VKGSRender()
@@ -2552,7 +2555,7 @@ bool VKGSRender::load_program()
 	}
 
 	const auto rasterization_samples = u8((m_current_renderpass_key >> 16) & 0xF);
-	if (rasterization_samples > 1)
+	if (supports_hw_a2c || rasterization_samples > 1)
 	{
 		properties.state.set_multisample_state(
 			rasterization_samples,