rsx: Workaround for G8B8 render targets

- Mainly affected are colormasks and read swizzles

NOTES:
- Writes to G write to the second and fourth component (YW)
- Writes to B write to first and third component (XZ)
- This means the actual format layout is BGBG (RGBA) making RG mapping actually GR
- Clear does not seem to have any intended effect on this format (TLOU)

rpcs3/Emu/RSX/GL/GLGSRender.cpp

@@ -976,14 +976,26 @@ void GLGSRender::clear_surface(u32 arg)
 		mask |= GLenum(gl::buffers::stencil);
 	}
 
-	if (arg & 0xf0)
+	if (auto colormask = (arg & 0xf0))
+	{
+		switch (rsx::method_registers.surface_color())
+		{
+		case rsx::surface_color_format::x32:
+		case rsx::surface_color_format::w16z16y16x16:
+		case rsx::surface_color_format::w32z32y32x32:
+		case rsx::surface_color_format::g8b8:
+		{
+			//NOP
+			break;
+		}
+		default:
 		{
 			u8 clear_a = rsx::method_registers.clear_color_a();
 			u8 clear_r = rsx::method_registers.clear_color_r();
 			u8 clear_g = rsx::method_registers.clear_color_g();
 			u8 clear_b = rsx::method_registers.clear_color_b();
 
-		gl_state.color_mask(arg & 0xf0);
+			gl_state.color_mask(colormask);
 			gl_state.clear_color(clear_r, clear_g, clear_b, clear_a);
 
 			mask |= GLenum(gl::buffers::color);
@@ -996,6 +1008,10 @@ void GLGSRender::clear_surface(u32 arg)
 					std::get<1>(rtt)->old_contents = nullptr;
 				}
 			}
+
+			break;
+		}
+		}
 	}
 
 	glClear(mask);
@@ -1153,6 +1169,12 @@ void GLGSRender::update_draw_state()
 	bool color_mask_r = rsx::method_registers.color_mask_r();
 	bool color_mask_a = rsx::method_registers.color_mask_a();
 
+	if (rsx::method_registers.surface_color() == rsx::surface_color_format::g8b8)
+	{
+		//Map GB components onto RG
+		rsx::get_g8b8_r8g8_colormask(color_mask_r, color_mask_g, color_mask_b, color_mask_a);
+	}
+
 	gl_state.color_mask(color_mask_r, color_mask_g, color_mask_b, color_mask_a);
 	gl_state.depth_mask(rsx::method_registers.depth_write_enabled());
 	gl_state.stencil_mask(rsx::method_registers.stencil_mask());

rpcs3/Emu/RSX/GL/GLRenderTargets.cpp

@@ -52,7 +52,8 @@ color_format rsx::internals::surface_color_format_to_gl(rsx::surface_color_forma
 		{ ::gl::texture::channel::one, ::gl::texture::channel::r, ::gl::texture::channel::r, ::gl::texture::channel::r } };
 
 	case rsx::surface_color_format::g8b8:
-		return{ ::gl::texture::type::ubyte, ::gl::texture::format::rg, false, 2, 1 };
+		return{ ::gl::texture::type::ubyte, ::gl::texture::format::rg, false, 2, 1,
+		{ ::gl::texture::channel::g, ::gl::texture::channel::r, ::gl::texture::channel::g, ::gl::texture::channel::r } };
 
 	case rsx::surface_color_format::x32:
 		return{ ::gl::texture::type::f32, ::gl::texture::format::r, true, 1, 4 };

rpcs3/Emu/RSX/VK/VKGSRender.cpp

@@ -100,7 +100,10 @@ namespace vk
 		}
 		
 		case rsx::surface_color_format::g8b8:
-			return std::make_pair(VK_FORMAT_R8G8_UNORM, vk::default_component_map());
+		{
+			VkComponentMapping gb_rg = { VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_R };
+			return std::make_pair(VK_FORMAT_R8G8_UNORM, gb_rg);
+		}
 
 		case rsx::surface_color_format::x32:
 			return std::make_pair(VK_FORMAT_R32_SFLOAT, vk::default_component_map());
@@ -1728,6 +1731,7 @@ void VKGSRender::clear_surface(u32 mask)
 
 	if (clear_descriptors.size() > 0)
 	{
+		//TODO: Implement lw_graphics_pipe objects to manage the color write mask!
 		vk::enter_uninterruptible();
 		begin_render_pass();
 		vkCmdClearAttachments(*m_current_command_buffer, (u32)clear_descriptors.size(), clear_descriptors.data(), 1, &region);
@@ -2167,11 +2171,22 @@ void VKGSRender::load_program(const vk::vertex_upload_info& vertex_info)
 		properties.att_state[i].blendEnable = VK_FALSE;
 	}
 
+	bool color_mask_b = rsx::method_registers.color_mask_b();
+	bool color_mask_g = rsx::method_registers.color_mask_g();
+	bool color_mask_r = rsx::method_registers.color_mask_r();
+	bool color_mask_a = rsx::method_registers.color_mask_a();
+
+	if (rsx::method_registers.surface_color() == rsx::surface_color_format::g8b8)
+	{
+		//Map GB components onto RG
+		rsx::get_g8b8_r8g8_colormask(color_mask_r, color_mask_g, color_mask_b, color_mask_a);
+	}
+
 	VkColorComponentFlags mask = 0;
-	if (rsx::method_registers.color_mask_a()) mask |= VK_COLOR_COMPONENT_A_BIT;
+	if (color_mask_a) mask |= VK_COLOR_COMPONENT_A_BIT;
-	if (rsx::method_registers.color_mask_b()) mask |= VK_COLOR_COMPONENT_B_BIT;
+	if (color_mask_b) mask |= VK_COLOR_COMPONENT_B_BIT;
-	if (rsx::method_registers.color_mask_g()) mask |= VK_COLOR_COMPONENT_G_BIT;
+	if (color_mask_g) mask |= VK_COLOR_COMPONENT_G_BIT;
-	if (rsx::method_registers.color_mask_r()) mask |= VK_COLOR_COMPONENT_R_BIT;
+	if (color_mask_r) mask |= VK_COLOR_COMPONENT_R_BIT;
 
 	for (u8 idx = 0; idx < m_draw_buffers_count; ++idx)
 	{

rpcs3/Emu/RSX/rsx_utils.h

@@ -411,4 +411,21 @@ namespace rsx
 		return ((u64)index + index_base) & 0x000FFFFF;
 	}
 
+	// Convert color write mask for G8B8 to R8G8
+	static inline u32 get_g8b8_r8g8_colormask(u32 mask)
+	{
+		u32 result = 0;
+		if (mask & 0x40) result |= 0x40;
+		if (mask & 0x80) result |= 0x20;
+
+		return result;
+	}
+
+	static inline void get_g8b8_r8g8_colormask(bool &red, bool &green, bool &blue, bool &alpha)
+	{
+		red = blue;
+		green = green;
+		blue = false;
+		alpha = false;
+	}
 }