gl: Implement video-out calibration for gamma and dynamic range

- Seems to be of limited use but if it is determined to be useful, a vulkan implementation can be done

rpcs3/Emu/RSX/GL/GLGSRender.cpp

@@ -788,6 +788,7 @@ void GLGSRender::on_init_thread()
 
 	m_depth_converter.create();
 	m_ui_renderer.create();
+	m_video_output_pass.create();
 
 	m_gl_texture_cache.initialize();
 	m_thread_id = std::this_thread::get_id();
@@ -920,6 +921,7 @@ void GLGSRender::on_exit()
 	m_gl_texture_cache.destroy();
 	m_depth_converter.destroy();
 	m_ui_renderer.destroy();
+	m_video_output_pass.destroy();
 
 	for (u32 i = 0; i < occlusion_query_count; ++i)
 	{
@@ -1304,7 +1306,7 @@ void GLGSRender::flip(int buffer)
 	u32 buffer_height = display_buffers[buffer].height;
 	u32 buffer_pitch = display_buffers[buffer].pitch;
 
-	if ((u32)buffer < display_buffers_count && buffer_width && buffer_height && buffer_pitch)
+	if ((u32)buffer < display_buffers_count && buffer_width && buffer_height)
 	{
 		// Calculate blit coordinates
 		coordi aspect_ratio;
@@ -1335,29 +1337,26 @@ void GLGSRender::flip(int buffer)
 		rsx::tiled_region buffer_region = get_tiled_address(display_buffers[buffer].offset, CELL_GCM_LOCATION_LOCAL);
 		u32 absolute_address = buffer_region.address + buffer_region.base;
 
-		m_flip_fbo.recreate();
+		GLuint image = GL_NONE;
-		m_flip_fbo.bind();
 
-		const u32 size = buffer_pitch * buffer_height;
 		if (auto render_target_texture = m_rtts.get_texture_from_render_target_if_applicable(absolute_address))
 		{
 			buffer_width = render_target_texture->width();
 			buffer_height = render_target_texture->height();
 
-			m_flip_fbo.color = *render_target_texture;
+			image = render_target_texture->id();
-			m_flip_fbo.read_buffer(m_flip_fbo.color);
 		}
 		else if (auto surface = m_gl_texture_cache.find_texture_from_dimensions(absolute_address))
 		{
 			//Hack - this should be the first location to check for output
 			//The render might have been done offscreen or in software and a blit used to display
-			m_flip_fbo.color = surface->get_raw_view();
+			image = surface->get_raw_view();
-			m_flip_fbo.read_buffer(m_flip_fbo.color);
 		}
 		else
 		{
 			LOG_WARNING(RSX, "Flip texture was not found in cache. Uploading surface from CPU");
 
+			if (!buffer_pitch) buffer_pitch = buffer_width * 4;
 			if (!m_flip_tex_color || m_flip_tex_color.size() != sizei{ (int)buffer_width, (int)buffer_height })
 			{
 				m_flip_tex_color.recreate(gl::texture::target::texture2D);
@@ -1381,17 +1380,34 @@ void GLGSRender::flip(int buffer)
 				m_flip_tex_color.copy_from(buffer_region.ptr, gl::texture::format::bgra, gl::texture::type::uint_8_8_8_8);
 			}
 
-			m_flip_fbo.color = m_flip_tex_color;
+			image = m_flip_tex_color.id();
-			m_flip_fbo.read_buffer(m_flip_fbo.color);
 		}
 
+		areai screen_area = coordi({}, { (int)buffer_width, (int)buffer_height });
+		auto avconfig = fxm::get<rsx::avconf>();
+
+		if (g_cfg.video.full_rgb_range_output && (!avconfig || avconfig->gamma == 1.f))
+		{
 			// Blit source image to the screen
 			// Disable scissor test (affects blit)
 			glDisable(GL_SCISSOR_TEST);
 
-		areai screen_area = coordi({}, { (int)buffer_width, (int)buffer_height });
+			m_flip_fbo.recreate();
+			m_flip_fbo.bind();
+			m_flip_fbo.color = image;
+			m_flip_fbo.read_buffer(m_flip_fbo.color);
 			m_flip_fbo.blit(gl::screen, screen_area, areai(aspect_ratio).flipped_vertical(), gl::buffers::color, gl::filter::linear);
 		}
+		else
+		{
+			const f32 gamma = avconfig ? avconfig->gamma : 1.f;
+			const bool limited_range = !g_cfg.video.full_rgb_range_output;
+
+			gl::screen.bind();
+			glViewport(0, 0, m_frame->client_width(), m_frame->client_height());
+			m_video_output_pass.run(m_frame->client_width(), m_frame->client_height(), image, areai(aspect_ratio), gamma, limited_range);
+		}
+	}
 
 	if (m_custom_ui)
 	{

rpcs3/Emu/RSX/GL/GLGSRender.h

@@ -308,6 +308,7 @@ private:
 	gl::text_writer m_text_printer;
 	gl::depth_convert_pass m_depth_converter;
 	gl::ui_overlay_renderer m_ui_renderer;
+	gl::video_out_calibration_pass m_video_output_pass;
 
 	std::vector<u64> m_overlay_cleanup_requests;
 

rpcs3/Emu/RSX/GL/GLOverlays.h

@@ -569,4 +569,68 @@ namespace gl
 			ui.update();
 		}
 	};
+
+	struct video_out_calibration_pass : public overlay_pass
+	{
+		video_out_calibration_pass()
+		{
+			vs_src =
+			{
+				"#version 420\n\n"
+				"layout(location=0) out vec2 tc0;\n"
+				"uniform float x_scale;\n"
+				"uniform float y_scale;\n"
+				"uniform float x_offset;\n"
+				"uniform float y_offset;\n"
+				"\n"
+				"void main()\n"
+				"{\n"
+				"	vec2 positions[] = {vec2(-1., -1.), vec2(1., -1.), vec2(-1., 1.), vec2(1., 1.)};\n"
+				"	vec2 coords[] = {vec2(0., 1.), vec2(1., 1.), vec2(0., 0.), vec2(1., 0.)};\n"
+				"	tc0 = coords[gl_VertexID % 4];\n"
+				"	vec2 pos = positions[gl_VertexID % 4] * vec2(x_scale, y_scale) + (2. * vec2(x_offset, y_offset));\n"
+				"	gl_Position = vec4(pos, 0., 1.);\n"
+				"}\n"
+			};
+
+			fs_src =
+			{
+				"#version 420\n\n"
+				"layout(binding=31) uniform sampler2D fs0;\n"
+				"layout(location=0) in vec2 tc0;\n"
+				"layout(location=0) out vec4 ocol;\n"
+				"\n"
+				"uniform float gamma;\n"
+				"uniform int limit_range;\n"
+				"\n"
+				"void main()\n"
+				"{\n"
+				"	vec4 color = texture(fs0, tc0);\n"
+				"	color.rgb = pow(color.rgb, vec3(gamma));\n"
+				"	if (limit_range > 0)\n"
+				"		ocol = ((color * 220.) + 16.) / 255.;\n"
+				"	else\n"
+				"		ocol = color;\n"
+				"}\n"
+			};
+		}
+
+		void run(u16 w, u16 h, GLuint source, const areai& region, f32 gamma, bool limited_rgb)
+		{
+			const f32 x_scale = (f32)(region.x2 - region.x1) / w;
+			const f32 y_scale = (f32)(region.y2 - region.y1) / h;
+			const f32 x_offset = (f32)(region.x1) / w;
+			const f32 y_offset = (f32)(region.y1) / h;
+
+			program_handle.uniforms["x_scale"] = x_scale;
+			program_handle.uniforms["y_scale"] = y_scale;
+			program_handle.uniforms["x_offset"] = x_offset;
+			program_handle.uniforms["y_offset"] = y_offset;
+			program_handle.uniforms["gamma"] = gamma;
+
+			glActiveTexture(GL_TEXTURE31);
+			glBindTexture(GL_TEXTURE_2D, source);
+			overlay_pass::run(w, h, GL_NONE, false, false);
+		}
+	};
 }

rpcs3/Emu/System.h

@@ -353,6 +353,7 @@ struct cfg_root : cfg::node
 		cfg::_bool frame_skip_enabled{this, "Enable Frame Skip", false};
 		cfg::_bool force_cpu_blit_processing{this, "Force CPU Blit", false}; // Debugging option
 		cfg::_bool disable_on_disk_shader_cache{this, "Disable On-Disk Shader Cache", false};
+		cfg::_bool full_rgb_range_output{this, "Use full RGB output range", true}; // Video out dynamic range
 		cfg::_int<1, 8> consequtive_frames_to_draw{this, "Consecutive Frames To Draw", 1};
 		cfg::_int<1, 8> consequtive_frames_to_skip{this, "Consecutive Frames To Skip", 1};
 		cfg::_int<50, 800> resolution_scale_percent{this, "Resolution Scale", 100};