rsx: Improve framebuffer check heuristics for contested memory buffers

rpcs3/Emu/RSX/GL/GLRenderTargets.cpp

@@ -169,6 +169,7 @@ void GLGSRender::init_buffers(bool skip_reading)
 	const u16 clip_y = rsx::method_registers.surface_clip_origin_y();
 
 	framebuffer_status_valid = false;
+	m_framebuffer_state_contested = false;
 
 	if (clip_horizontal == 0 || clip_vertical == 0)
 	{
@@ -207,12 +208,24 @@ void GLGSRender::init_buffers(bool skip_reading)
 		if (surface_addresses[index] == depth_address &&
 			zeta_pitch >= required_z_pitch)
 		{
-			//LOG_ERROR(RSX, "Some game dev set up the MRT to write to the same address as depth and color attachment. Not sure how to deal with that so the draw is discarded.");
+			LOG_TRACE(RSX, "Framebuffer at 0x%X has aliasing color/depth targets, zeta_pitch = %d, color_pitch=%d", depth_address, zeta_pitch, pitchs[index]);
-			//framebuffer_status_valid = false;
+			m_framebuffer_state_contested = true;
+
+			if (rsx::method_registers.depth_test_enabled() ||
+				(!rsx::method_registers.color_write_enabled() && rsx::method_registers.depth_write_enabled()) ||
+				!!(rsx::method_registers.shader_control() & CELL_GCM_SHADER_CONTROL_DEPTH_EXPORT))
+			{
+				// Use address for depth data
+				surface_addresses[index] = 0;
+			}
+			else
+			{
+				// Use address for color data
 				depth_address = 0;
 				break;
 			}
 		}
+	}
 
 	if (!framebuffer_status_valid && !depth_address)
 		return;

rpcs3/Emu/RSX/RSXThread.h

@@ -188,6 +188,7 @@ namespace rsx
 		bool m_transform_constants_dirty;
 		bool m_textures_dirty[16];
 		bool m_vertex_textures_dirty[4];
+		bool m_framebuffer_state_contested = false;
 
 	protected:
 		std::array<u32, 4> get_color_surface_addresses() const;

rpcs3/Emu/RSX/VK/VKGSRender.cpp

@@ -2315,6 +2315,7 @@ void VKGSRender::prepare_rtts()
 	u32 clip_y = rsx::method_registers.surface_clip_origin_y();
 
 	framebuffer_status_valid = false;
+	m_framebuffer_state_contested = false;
 
 	if (clip_width == 0 || clip_height == 0)
 	{
@@ -2354,11 +2355,24 @@ void VKGSRender::prepare_rtts()
 		if (surface_addresses[index] == zeta_address &&
 			zeta_pitch >= required_z_pitch)
 		{
-			//LOG_ERROR(RSX, "Some game dev set up the MRT to write to the same address as depth and color attachment. Not sure how to deal with that so the draw is discarded.");
+			LOG_TRACE(RSX, "Framebuffer at 0x%X has aliasing color/depth targets, zeta_pitch = %d, color_pitch=%d", zeta_address, zeta_pitch, surface_pitchs[index]);
-			//framebuffer_status_valid = false;
+			m_framebuffer_state_contested = true;
+
+			if (rsx::method_registers.depth_test_enabled() ||
+				(!rsx::method_registers.color_write_enabled() && rsx::method_registers.depth_write_enabled()) ||
+				!!(rsx::method_registers.shader_control() & CELL_GCM_SHADER_CONTROL_DEPTH_EXPORT))
+			{
+				// Use address for color data
 				zeta_address = 0;
 				break;
 			}
+			else
+			{
+				// Use address for depth data
+				// TODO: create a temporary render buffer for this to keep MRT outputs aligned
+				surface_addresses[index] = 0;
+			}
+		}
 	}
 
 	if (!framebuffer_status_valid && !zeta_address)

rpcs3/Emu/RSX/rsx_decode.h

@@ -3196,6 +3196,11 @@ struct registers_decoder<NV4097_SET_COLOR_MASK>
 		{
 			return bool(m_data.color_a);
 		}
+
+		bool color_write_enabled() const
+		{
+			return m_data.raw_data != 0;
+		}
 	};
 
 	static std::string dump(decoded_type &&decoded_values)

rpcs3/Emu/RSX/rsx_methods.cpp

@@ -471,6 +471,12 @@ namespace rsx
 			rsx->m_rtts_dirty = true;
 		}
 
+		void set_surface_options_dirty_bit(thread* rsx, u32, u32)
+		{
+			if (rsx->m_framebuffer_state_contested)
+				rsx->m_rtts_dirty = true;
+		}
+
 		template<u32 index>
 		struct set_texture_dirty_bit
 		{
@@ -1598,6 +1604,9 @@ namespace rsx
 		bind<NV4097_SET_ZCULL_STATS_ENABLE, nv4097::set_zcull_stats_enable>();
 		bind<NV4097_SET_ZPASS_PIXEL_COUNT_ENABLE, nv4097::set_zcull_pixel_count_enable>();
 		bind<NV4097_CLEAR_ZCULL_SURFACE, nv4097::clear_zcull>();
+		bind<NV4097_SET_DEPTH_TEST_ENABLE, nv4097::set_surface_options_dirty_bit>();
+		bind<NV4097_SET_DEPTH_MASK, nv4097::set_surface_options_dirty_bit>();
+		bind<NV4097_SET_COLOR_MASK, nv4097::set_surface_options_dirty_bit>();
 
 		//NV308A
 		bind_range<NV308A_COLOR, 1, 256, nv308a::color>();

rpcs3/Emu/RSX/rsx_methods.h

@@ -345,6 +345,11 @@ namespace rsx
 			return decode<NV4097_SET_COLOR_MASK>().color_a();
 		}
 
+		bool color_write_enabled() const
+		{
+			return decode<NV4097_SET_COLOR_MASK>().color_write_enabled();
+		}
+
 		u8 clear_color_b() const
 		{
 			return decode<NV4097_SET_COLOR_CLEAR_VALUE>().blue();