rsx: Program cache fixes

- Reorganize storage hash vs ucode hash
- Scan for actual fragment program start in case leading NOPed code precedes the actual instructions
-- e.g FEAR2 Demo has over 32k of padding before actual program code that messes up hashes

rpcs3/Emu/RSX/Common/ProgramStateCache.cpp

@@ -3,7 +3,7 @@
 
 using namespace program_hash_util;
 
-size_t vertex_program_hash::operator()(const RSXVertexProgram &program) const
+size_t vertex_program_utils::get_vertex_program_ucode_hash(const RSXVertexProgram &program)
 {
 	// 64-bit Fowler/Noll/Vo FNV-1a hash code
 	size_t hash = 0xCBF29CE484222325ULL;
@@ -22,6 +22,13 @@ size_t vertex_program_hash::operator()(const RSXVertexProgram &program) const
 	return hash;
 }
 
+size_t vertex_program_storage_hash::operator()(const RSXVertexProgram &program) const
+{
+	size_t hash = vertex_program_utils::get_vertex_program_ucode_hash(program);
+	hash ^= program.output_mask;
+	return hash;
+}
+
 bool vertex_program_compare::operator()(const RSXVertexProgram &binary1, const RSXVertexProgram &binary2) const
 {
 	if (binary1.output_mask != binary2.output_mask)
@@ -77,7 +84,24 @@ size_t fragment_program_utils::get_fragment_program_ucode_size(void *ptr)
 	}
 }
 
-size_t fragment_program_hash::operator()(const RSXFragmentProgram& program) const
+u32 fragment_program_utils::get_fragment_program_start(void *ptr)
+{
+	const qword *instBuffer = (const qword*)ptr;
+	size_t instIndex = 0;
+	while (true)
+	{
+		const qword& inst = instBuffer[instIndex];
+		u32 opcode = inst.word[0] >> 16 & 0x3F;
+		if (opcode)
+			break;
+
+		instIndex++;
+	}
+
+	return instIndex * 16;
+}
+
+size_t fragment_program_utils::get_fragment_program_ucode_hash(const RSXFragmentProgram& program)
 {
 	// 64-bit Fowler/Noll/Vo FNV-1a hash code
 	size_t hash = 0xCBF29CE484222325ULL;
@@ -104,6 +128,21 @@ size_t fragment_program_hash::operator()(const RSXFragmentProgram& program) cons
 	return 0;
 }
 
+size_t fragment_program_storage_hash::operator()(const RSXFragmentProgram& program) const
+{
+	size_t hash = fragment_program_utils::get_fragment_program_ucode_hash(program);
+	hash ^= program.ctrl;
+	hash ^= program.texture_dimensions;
+	hash ^= program.unnormalized_coords;
+	hash ^= program.back_color_diffuse_output;
+	hash ^= program.back_color_specular_output;
+	hash ^= program.front_back_color_enabled;
+	hash ^= program.shadow_textures;
+	hash ^= program.redirected_textures;
+
+	return hash;
+}
+
 bool fragment_program_compare::operator()(const RSXFragmentProgram& binary1, const RSXFragmentProgram& binary2) const
 {
 	if (binary1.ctrl != binary2.ctrl || binary1.texture_dimensions != binary2.texture_dimensions || binary1.unnormalized_coords != binary2.unnormalized_coords ||

rpcs3/Emu/RSX/Common/ProgramStateCache.h

@@ -24,7 +24,12 @@ namespace program_hash_util
 		u32 word[4];
 	};
 
-	struct vertex_program_hash
+	struct vertex_program_utils
+	{
+		static size_t get_vertex_program_ucode_hash(const RSXVertexProgram &program);
+	};
+
+	struct vertex_program_storage_hash
 	{
 		size_t operator()(const RSXVertexProgram &program) const;
 	};
@@ -42,9 +47,13 @@ namespace program_hash_util
 		static bool is_constant(u32 sourceOperand);
 
 		static size_t get_fragment_program_ucode_size(void *ptr);
+
+		static u32 get_fragment_program_start(void *ptr);
+
+		static size_t get_fragment_program_ucode_hash(const RSXFragmentProgram &program);
 	};
 
-	struct fragment_program_hash
+	struct fragment_program_storage_hash
 	{
 		size_t operator()(const RSXFragmentProgram &program) const;
 	};
@@ -78,8 +87,8 @@ class program_state_cache
 	using vertex_program_type = typename backend_traits::vertex_program_type;
 	using fragment_program_type = typename backend_traits::fragment_program_type;
 
-	using binary_to_vertex_program = std::unordered_map<RSXVertexProgram, vertex_program_type, program_hash_util::vertex_program_hash, program_hash_util::vertex_program_compare> ;
+	using binary_to_vertex_program = std::unordered_map<RSXVertexProgram, vertex_program_type, program_hash_util::vertex_program_storage_hash, program_hash_util::vertex_program_compare> ;
-	using binary_to_fragment_program = std::unordered_map<RSXFragmentProgram, fragment_program_type, program_hash_util::fragment_program_hash, program_hash_util::fragment_program_compare>;
+	using binary_to_fragment_program = std::unordered_map<RSXFragmentProgram, fragment_program_type, program_hash_util::fragment_program_storage_hash, program_hash_util::fragment_program_compare>;
 
 
 	struct pipeline_key

rpcs3/Emu/RSX/GL/GLProgramBuffer.h

@@ -82,12 +82,12 @@ public:
 
 	u64 get_hash(RSXVertexProgram &prog)
 	{
-		return program_hash_util::vertex_program_hash()(prog);
+		return program_hash_util::vertex_program_utils::get_vertex_program_ucode_hash(prog);
 	}
 
 	u64 get_hash(RSXFragmentProgram &prog)
 	{
-		return program_hash_util::fragment_program_hash()(prog);
+		return program_hash_util::fragment_program_utils::get_fragment_program_ucode_hash(prog);
 	}
 
 	template <typename... Args>

rpcs3/Emu/RSX/RSXThread.cpp

@@ -1348,8 +1348,11 @@ namespace rsx
 		const u32 program_location = (shader_program & 0x3) - 1;
 		const u32 program_offset = (shader_program & ~0x3);
 
-		result.offset = program_offset;
 		result.addr = vm::base(rsx::get_address(program_offset, program_location));
+		auto program_start = program_hash_util::fragment_program_utils::get_fragment_program_start(result.addr);
+
+		result.addr = ((u8*)result.addr + program_start);
+		result.offset = program_offset + program_start;
 		result.valid = true;
 		result.ctrl = rsx::method_registers.shader_control() & (CELL_GCM_SHADER_CONTROL_32_BITS_EXPORTS | CELL_GCM_SHADER_CONTROL_DEPTH_EXPORT);
 		result.unnormalized_coords = 0;
@@ -1454,8 +1457,11 @@ namespace rsx
 		const u32 program_location = (shader_program & 0x3) - 1;
 		const u32 program_offset = (shader_program & ~0x3);
 
-		result.offset = program_offset;
 		result.addr = vm::base(rsx::get_address(program_offset, program_location));
+		auto program_start = program_hash_util::fragment_program_utils::get_fragment_program_start(result.addr);
+
+		result.addr = ((u8*)result.addr + program_start);
+		result.offset = program_offset + program_start;
 		result.valid = true;
 		result.ctrl = rsx::method_registers.shader_control() & (CELL_GCM_SHADER_CONTROL_32_BITS_EXPORTS | CELL_GCM_SHADER_CONTROL_DEPTH_EXPORT);
 		result.unnormalized_coords = 0;

rpcs3/Emu/RSX/VK/VKProgramBuffer.h

@@ -185,12 +185,12 @@ public:
 
 	u64 get_hash(RSXVertexProgram &prog)
 	{
-		return program_hash_util::vertex_program_hash()(prog);
+		return program_hash_util::vertex_program_utils::get_vertex_program_ucode_hash(prog);
 	}
 
 	u64 get_hash(RSXFragmentProgram &prog)
 	{
-		return program_hash_util::fragment_program_hash()(prog);
+		return program_hash_util::fragment_program_utils::get_fragment_program_ucode_hash(prog);
 	}
 
 	template <typename... Args>