rsx: Check av configuration when selecting display buffers!

- Some applications have mismatch between video output configuration and display buffer sizes

rpcs3/Emu/Cell/Modules/cellVideoOut.cpp

@@ -1,4 +1,4 @@
-#include "stdafx.h"
+#include "stdafx.h"
 #include "Emu/System.h"
 #include "Emu/Cell/PPUModule.h"
 #include "Emu/IdManager.h"
@@ -147,10 +147,15 @@ error_code cellVideoOutConfigure(u32 videoOut, vm::ptr<CellVideoOutConfiguration
 		return CELL_VIDEO_OUT_ERROR_ILLEGAL_CONFIGURATION;
 	}
 
+	verify(HERE), config->resolutionId > 0;
+	auto& res_info = g_video_out_resolution_map.at(static_cast<video_resolution>(config->resolutionId - 1));
+
 	auto conf = fxm::get_always<rsx::avconf>();
 	conf->aspect = config->aspect;
 	conf->format = config->format;
 	conf->scanline_pitch = config->pitch;
+	conf->resolution_x = u32(res_info.first);
+	conf->resolution_y = u32(res_info.second);
 
 	return CELL_OK;
 }

rpcs3/Emu/RSX/GL/GLGSRender.cpp

@@ -1585,6 +1585,13 @@ void GLGSRender::flip(int buffer)
 
 	if (!buffer_pitch) buffer_pitch = buffer_width * 4;
 
+	auto avconfig = fxm::get<rsx::avconf>();
+	if (avconfig)
+	{
+		buffer_width = std::min(buffer_width, avconfig->resolution_x);
+		buffer_height = std::min(buffer_height, avconfig->resolution_y);
+	}
+
 	// Disable scissor test (affects blit, clear, etc)
 	glDisable(GL_SCISSOR_TEST);
 
@@ -1648,6 +1655,19 @@ void GLGSRender::flip(int buffer)
 			{
 				buffer_width = rsx::apply_resolution_scale(buffer_width, true);
 				buffer_height = rsx::apply_resolution_scale(buffer_height, true);
+
+				if (buffer_width < render_target_texture->width() ||
+					buffer_height < render_target_texture->height())
+				{
+					// TODO: Should emit only once to avoid flooding the log file
+					// TODO: Take AA scaling into account
+					LOG_WARNING(RSX, "Selected output image does not satisfy the video configuration. Display buffer resolution=%dx%d, avconf resolution=%dx%d, surface=%dx%d",
+						display_buffers[buffer].width, display_buffers[buffer].height, avconfig ? avconfig->resolution_x : 0, avconfig ? avconfig->resolution_y : 0,
+						render_target_texture->get_surface_width(), render_target_texture->get_surface_height());
+
+					buffer_width = render_target_texture->width();
+					buffer_height = render_target_texture->height();
+				}
 			}
 		}
 		else if (auto surface = m_gl_texture_cache.find_texture_from_dimensions(absolute_address, buffer_width, buffer_height))
@@ -1684,7 +1704,6 @@ void GLGSRender::flip(int buffer)
 		}
 
 		areai screen_area = coordi({}, { (int)buffer_width, (int)buffer_height });
-		auto avconfig = fxm::get<rsx::avconf>();
 
 		if (g_cfg.video.full_rgb_range_output && (!avconfig || avconfig->gamma == 1.f))
 		{

rpcs3/Emu/RSX/VK/VKGSRender.cpp

@@ -3196,6 +3196,13 @@ void VKGSRender::flip(int buffer)
 
 	if (!buffer_pitch) buffer_pitch = buffer_width * 4; // TODO: Check avconf
 
+	auto avconfig = fxm::get<rsx::avconf>();
+	if (avconfig)
+	{
+		buffer_width = std::min(buffer_width, avconfig->resolution_x);
+		buffer_height = std::min(buffer_height, avconfig->resolution_y);
+	}
+
 	coordi aspect_ratio;
 
 	sizei csize = { (s32)m_client_width, (s32)m_client_height };
@@ -3298,6 +3305,19 @@ void VKGSRender::flip(int buffer)
 			{
 				buffer_width = rsx::apply_resolution_scale(buffer_width, true);
 				buffer_height = rsx::apply_resolution_scale(buffer_height, true);
+
+				if (buffer_width < render_target_texture->width() ||
+					buffer_height < render_target_texture->height())
+				{
+					// TODO: Should emit only once to avoid flooding the log file
+					// TODO: Take AA scaling into account
+					LOG_WARNING(RSX, "Selected output image does not satisfy the video configuration. Display buffer resolution=%dx%d, avconf resolution=%dx%d, surface=%dx%d",
+						display_buffers[buffer].width, display_buffers[buffer].height, avconfig? avconfig->resolution_x : 0, avconfig? avconfig->resolution_y : 0,
+						render_target_texture->get_surface_width(), render_target_texture->get_surface_height());
+
+					buffer_width = render_target_texture->width();
+					buffer_height = render_target_texture->height();
+				}
 			}
 		}
 		else if (auto surface = m_texture_cache.find_texture_from_dimensions(absolute_address, buffer_width, buffer_height))

rpcs3/Emu/RSX/rsx_utils.h

@@ -73,10 +73,12 @@ namespace rsx
 
 	struct avconf
 	{
-		u8 format = 0; //XRGB
+		u8 format = 0;             // XRGB
-		u8 aspect = 0; //AUTO
+		u8 aspect = 0;             // AUTO
-		u32 scanline_pitch = 0; //PACKED
+		u32 scanline_pitch = 0;    // PACKED
-		f32 gamma = 1.f; //NO GAMMA CORRECTION
+		f32 gamma = 1.f;           // NO GAMMA CORRECTION
+		u32 resolution_x = 1280;   // X RES
+		u32 resolution_y = 720;    // Y RES
 	};
 
 	struct blit_src_info