Integer overscale GPU screenshot crash fixes (#17118)

gfx/common/d3d9_common.c

@@ -872,18 +872,39 @@ void d3d9_set_viewport(void *data,
       bool force_full,
       bool allow_rotate)
 {
+   d3d9_video_t *d3d   = (d3d9_video_t*)data;
+   float translate_x   = d3d->translate_x;
+   float translate_y   = d3d->translate_y;
    int x               = 0;
    int y               = 0;
-   d3d9_video_t *d3d   = (d3d9_video_t*)data;
 
    d3d9_calculate_rect(d3d, &width, &height, &x, &y,
          force_full, allow_rotate);
 
    /* D3D doesn't support negative X/Y viewports ... */
    if (x < 0)
+   {
+      if (!force_full)
+         d3d->translate_x = x * 2;
       x = 0;
+   }
+   else if (!force_full)
+      d3d->translate_x = 0;
+
    if (y < 0)
+   {
+      if (!force_full)
+         d3d->translate_y = y * 2;
       y = 0;
+   }
+   else if (!force_full)
+      d3d->translate_y = 0;
+
+   if (!force_full)
+   {
+      if (translate_x != d3d->translate_x || translate_y != d3d->translate_y)
+         d3d->needs_restore = true;
+   }
 
    d3d->final_viewport.X      = x;
    d3d->final_viewport.Y      = y;
@@ -1073,6 +1094,7 @@ void d3d9_set_menu_texture_frame(void *data,
             || (d3d->menu->tex_w != width)
             || (d3d->menu->tex_h != height))
    {
+      if (d3d->menu->tex)
          IDirect3DTexture9_Release((LPDIRECT3DTEXTURE9)d3d->menu->tex);
 
       d3d->menu->tex = d3d9_texture_new(d3d->dev,
@@ -1096,6 +1118,7 @@ void d3d9_set_menu_texture_frame(void *data,
          0, &d3dlr, NULL, D3DLOCK_NOSYSLOCK);
    {
       unsigned h, w;
+
       if (rgb32)
       {
          uint8_t        *dst = (uint8_t*)d3dlr.pBits;
@@ -1281,16 +1304,18 @@ bool d3d9_read_viewport(void *data, uint8_t *buffer, bool is_idle)
 
    {
       unsigned x, y;
+      unsigned vp_width       = (d3d->final_viewport.Width  > width)  ? width  : d3d->final_viewport.Width;
+      unsigned vp_height      = (d3d->final_viewport.Height > height) ? height : d3d->final_viewport.Height;
       unsigned pitchpix       = rect.Pitch / 4;
       const uint32_t *pixels  = (const uint32_t*)rect.pBits;
 
       pixels                 += d3d->final_viewport.X;
-      pixels                 += (d3d->final_viewport.Height - 1) * pitchpix;
+      pixels                 += (vp_height - 1) * pitchpix;
       pixels                 -= d3d->final_viewport.Y * pitchpix;
 
-      for (y = 0; y < d3d->final_viewport.Height; y++, pixels -= pitchpix)
+      for (y = 0; y < vp_height; y++, pixels -= pitchpix)
       {
-         for (x = 0; x < d3d->final_viewport.Width; x++)
+         for (x = 0; x < vp_width; x++)
          {
             *buffer++ = (pixels[x] >>  0) & 0xff;
             *buffer++ = (pixels[x] >>  8) & 0xff;

gfx/common/d3d9_common.h

@@ -78,6 +78,8 @@ typedef struct d3d9_video
 #endif
    LPDIRECT3DDEVICE9 dev;
    D3DVIEWPORT9 final_viewport;
+   float translate_x;
+   float translate_y;
 
    char *shader_path;
 

gfx/common/d3d_common.c

@@ -66,7 +66,7 @@ void d3d_matrix_ortho_off_center_lh(void *_pout,
    pout->m[0][0] = 2.0f / (r - l);
    pout->m[1][1] = 2.0f / (t - b);
    pout->m[2][2] = 1.0f / (zf -zn);
-   pout->m[3][0] = -1.0f -2.0f *l / (r - l);
+   pout->m[3][0] = -1.0f - 2.0f * l / (r - l);
    pout->m[3][1] =  1.0f + 2.0f * t / (b - t);
    pout->m[3][2] = zn / (zn -zf);
 }

gfx/drivers/d3d11.c

@@ -3694,12 +3694,17 @@ static bool d3d11_gfx_read_viewport(void* data, uint8_t* buffer, bool is_idle)
    /* Assuming format is DXGI_FORMAT_R8G8B8A8_UNORM */
    if (StagingDesc.Format == DXGI_FORMAT_R8G8B8A8_UNORM)
    {
-      BackBufferData += Map.RowPitch * d3d11->vp.y;
+      unsigned vp_y      = (d3d11->vp.y > 0) ? d3d11->vp.y : 0;
-      for (y = 0; y < d3d11->vp.height; y++, BackBufferData += Map.RowPitch)
+      unsigned vp_width  = (d3d11->vp.width  > d3d11->vp.full_width)  ? d3d11->vp.full_width  : d3d11->vp.width;
-      {
+      unsigned vp_height = (d3d11->vp.height > d3d11->vp.full_height) ? d3d11->vp.full_height : d3d11->vp.height;
-         bufferRow = buffer + 3 * (d3d11->vp.height - y - 1) * d3d11->vp.width;
 
-         for (x = 0; x < d3d11->vp.width; x++)
+      BackBufferData += Map.RowPitch * vp_y;
+
+      for (y = 0; y < vp_height; y++, BackBufferData += Map.RowPitch)
+      {
+         bufferRow = buffer + 3 * (vp_height - y - 1) * vp_width;
+
+         for (x = 0; x < vp_width; x++)
          {
             bufferRow[3 * x + 2] = BackBufferData[4 * (x + d3d11->vp.x) + 0];
             bufferRow[3 * x + 1] = BackBufferData[4 * (x + d3d11->vp.x) + 1];

gfx/drivers/d3d9cg.c

@@ -1755,7 +1755,21 @@ static bool d3d9_cg_initialize(d3d9_video_t *d3d, const video_info_t *info)
 
    d3d_matrix_identity(&d3d->mvp_transposed);
    d3d_matrix_ortho_off_center_lh(&d3d->mvp_transposed, 0, 1, 0, 1, 0, 1);
-   d3d_matrix_transpose(&d3d->mvp, &d3d->mvp_transposed);
+   d3d->mvp = d3d->mvp_transposed;
+
+   if (d3d->translate_x)
+   {
+      struct d3d_matrix *pout = (struct d3d_matrix*)&d3d->mvp;
+      float vp_x = -(d3d->translate_x/(float)d3d->final_viewport.Width);
+      pout->m[3][0] = -1.0f + vp_x - 2.0f * 1 / (0 - 1);
+   }
+
+   if (d3d->translate_y)
+   {
+      struct d3d_matrix *pout = (struct d3d_matrix*)&d3d->mvp;
+      float vp_y = -(d3d->translate_y/(float)d3d->final_viewport.Height);
+      pout->m[3][1] = 1.0f + vp_y + 2.0f * 1 / (0 - 1);
+   }
 
    IDirect3DDevice9_SetRenderState(d3d->dev, D3DRS_CULLMODE, D3DCULL_NONE);
    IDirect3DDevice9_SetRenderState(d3d->dev, D3DRS_SCISSORTESTENABLE, TRUE);
@@ -2103,7 +2117,7 @@ static bool d3d9_cg_frame(void *data, const void *frame,
    if (d3d->overlays_enabled && overlay_behind_menu)
    {
       IDirect3DDevice9_SetVertexShaderConstantF(d3d->dev,
-         0, (const float*)&d3d->mvp, 4);
+         0, (const float*)&d3d->mvp_transposed, 4);
       for (i = 0; i < d3d->overlays_size; i++)
          d3d9_overlay_render(d3d, width, height, &d3d->overlays[i], true);
    }
@@ -2121,7 +2135,6 @@ static bool d3d9_cg_frame(void *data, const void *frame,
       IDirect3DDevice9_SetStreamSource(d3d->dev, 0,
             (LPDIRECT3DVERTEXBUFFER9)d3d->menu_display.buffer,
             0, sizeof(Vertex));
-
       IDirect3DDevice9_SetViewport(d3d->dev, (D3DVIEWPORT9*)&screen_vp);
       menu_driver_frame(menu_is_alive, video_info);
    }
@@ -2142,7 +2155,7 @@ static bool d3d9_cg_frame(void *data, const void *frame,
    if (d3d->overlays_enabled && !overlay_behind_menu)
    {
       IDirect3DDevice9_SetVertexShaderConstantF(d3d->dev,
-         0, (const float*)&d3d->mvp, 4);
+         0, (const float*)&d3d->mvp_transposed, 4);
       for (i = 0; i < d3d->overlays_size; i++)
          d3d9_overlay_render(d3d, width, height, &d3d->overlays[i], true);
    }

gfx/drivers/d3d9hlsl.c

@@ -1343,7 +1343,21 @@ static bool d3d9_hlsl_initialize(
 
    d3d_matrix_identity(&d3d->mvp_transposed);
    d3d_matrix_ortho_off_center_lh(&d3d->mvp_transposed, 0, 1, 0, 1, 0, 1);
-   d3d_matrix_transpose(&d3d->mvp, &d3d->mvp_transposed);
+   d3d->mvp = d3d->mvp_transposed;
+
+   if (d3d->translate_x)
+   {
+      struct d3d_matrix *pout = (struct d3d_matrix*)&d3d->mvp;
+      float vp_x = -(d3d->translate_x/(float)d3d->final_viewport.Width);
+      pout->m[3][0] = -1.0f + vp_x - 2.0f * 1 / (0 - 1);
+   }
+
+   if (d3d->translate_y)
+   {
+      struct d3d_matrix *pout = (struct d3d_matrix*)&d3d->mvp;
+      float vp_y = -(d3d->translate_y/(float)d3d->final_viewport.Height);
+      pout->m[3][1] = 1.0f + vp_y + 2.0f * 1 / (0 - 1);
+   }
 
    IDirect3DDevice9_SetRenderState(d3d->dev, D3DRS_CULLMODE, D3DCULL_NONE);
    IDirect3DDevice9_SetRenderState(d3d->dev, D3DRS_SCISSORTESTENABLE, TRUE);
@@ -1675,7 +1689,7 @@ static bool d3d9_hlsl_frame(void *data, const void *frame,
          0, 1, 0);
 
    IDirect3DDevice9_SetVertexShaderConstantF(d3d->dev, 0,
-         (const float*)&d3d->mvp_transposed, 4);
+         (const float*)&d3d->mvp, 4);
    hlsl_d3d9_renderchain_render(
          d3d, frame, frame_width, frame_height,
          pitch, d3d->dev_rotation);
@@ -1713,7 +1727,7 @@ static bool d3d9_hlsl_frame(void *data, const void *frame,
    if (d3d->menu && d3d->menu->enabled)
    {
       IDirect3DDevice9_SetVertexShaderConstantF(d3d->dev, 0,
-            (const float*)&d3d->mvp_transposed, 4);
+            (const float*)&d3d->mvp, 4);
       d3d9_overlay_render(d3d, width, height, d3d->menu, false);
 
       d3d->menu_display.offset = 0;

gfx/drivers/gl1.c

@@ -1408,6 +1408,7 @@ static void gl1_draw_tex(gl1_t *gl1, int pot_width, int pot_height, int width, i
 
 static void gl1_readback(gl1_t *gl1,
       unsigned alignment, unsigned fmt, unsigned type,
+      unsigned video_width, unsigned video_height,
       void *src)
 {
 #ifndef VITA
@@ -1415,8 +1416,12 @@ static void gl1_readback(gl1_t *gl1,
    glPixelStorei(GL_PACK_ROW_LENGTH, 0);
    glReadBuffer(GL_BACK);
 #endif
-   glReadPixels(gl1->vp.x, gl1->vp.y,
+
-         gl1->vp.width, gl1->vp.height,
+   glReadPixels(
+         (gl1->vp.x > 0) ? gl1->vp.x : 0,
+         (gl1->vp.y > 0) ? gl1->vp.y : 0,
+         (gl1->vp.width  > video_width)  ? video_width  : gl1->vp.width,
+         (gl1->vp.height > video_height) ? video_height : gl1->vp.height,
          (GLenum)fmt, (GLenum)type, (GLvoid*)src);
 }
 
@@ -1656,6 +1661,7 @@ static bool gl1_frame(void *data, const void *frame,
 #else
             GL_UNSIGNED_BYTE,
 #endif
+            video_width, video_height,
             gl1->readback_buffer_screenshot);
 
 

gfx/drivers/gl2.c

@@ -2391,8 +2391,11 @@ static void gl2_renderchain_readback(
    glReadBuffer(GL_BACK);
 #endif
 
-   glReadPixels(gl->vp.x, gl->vp.y,
+   glReadPixels(
-         gl->vp.width, gl->vp.height,
+         (gl->vp.x > 0) ? gl->vp.x : 0,
+         (gl->vp.y > 0) ? gl->vp.y : 0,
+         (gl->vp.width  > gl->video_width)  ? gl->video_width  : gl->vp.width,
+         (gl->vp.height > gl->video_height) ? gl->video_height : gl->vp.height,
          (GLenum)fmt, (GLenum)type, (GLvoid*)src);
 }
 

gfx/drivers/gl3.c

@@ -2469,7 +2469,6 @@ static void gl3_viewport_info(void *data, struct video_viewport *vp)
    unsigned width  = gl->video_width;
    unsigned height = gl->video_height;
 
-
    *vp             = gl->vp;
    vp->full_width  = width;
    vp->full_height = height;
@@ -2490,6 +2489,7 @@ static bool gl3_read_viewport(void *data, uint8_t *buffer, bool is_idle)
 
    if (gl->flags & GL3_FLAG_USE_SHARED_CONTEXT)
       gl->ctx_driver->bind_hw_render(gl->ctx_data, false);
+
    num_pixels = gl->vp.width * gl->vp.height;
 
    if (gl->flags & GL3_FLAG_PBO_READBACK_ENABLE)
@@ -2843,8 +2843,11 @@ static bool gl3_frame(void *data, const void *frame,
 #ifndef HAVE_OPENGLES
       glReadBuffer(GL_BACK);
 #endif
-      glReadPixels(gl->vp.x, gl->vp.y,
+      glReadPixels(
-            gl->vp.width, gl->vp.height,
+            (gl->vp.x > 0) ? gl->vp.x : 0,
+            (gl->vp.y > 0) ? gl->vp.y : 0,
+            (gl->vp.width  > gl->video_width)  ? gl->video_width  : gl->vp.width,
+            (gl->vp.height > gl->video_height) ? gl->video_height : gl->vp.height,
             GL_RGBA, GL_UNSIGNED_BYTE,
             gl->readback_buffer_screenshot);
    }
@@ -2857,7 +2860,6 @@ static bool gl3_frame(void *data, const void *frame,
          gl3_pbo_async_readback(gl);
    }
 
-
    if (gl->ctx_driver->swap_buffers)
       gl->ctx_driver->swap_buffers(gl->ctx_data);
 

gfx/drivers/vulkan.c

@@ -3798,6 +3798,9 @@ static void vulkan_set_projection(vk_t *vk,
       matrix_4x4_multiply(tmp, rot, vk->mvp_no_rot);
    }
    matrix_4x4_multiply(vk->mvp, trn, tmp);
+
+   /* Required for translate_x+y / negative offsets to also work in RGUI */
+   matrix_4x4_multiply(vk->mvp_no_rot, trn, tmp);
 }
 
 static void vulkan_set_rotation(void *data, unsigned rotation)
@@ -3867,6 +3870,7 @@ static void vulkan_set_viewport(void *data, unsigned viewport_width,
    }
    else
       vk->translate_x = 0.0;
+
    if (vk->vp.y < 0)
    {
       vk->translate_y = (float)vk->vp.y * 2;
@@ -3920,8 +3924,8 @@ static void vulkan_readback(vk_t *vk, struct vk_image *readback_image)
    region.imageOffset.x                   = vp.x;
    region.imageOffset.y                   = vp.y;
    region.imageOffset.z                   = 0;
-   region.imageExtent.width               = vp.width;
+   region.imageExtent.width               = vp.width + vk->translate_x;
-   region.imageExtent.height              = vp.height;
+   region.imageExtent.height              = vp.height + vk->translate_y;
    region.imageExtent.depth               = 1;
 
    staging  = &vk->readback.staging[vk->context->current_frame_index];
@@ -5633,17 +5637,20 @@ static bool vulkan_read_viewport(void *data, uint8_t *buffer, bool is_idle)
 
       {
          int y;
+         unsigned vp_width  = (vk->vp.width  > vk->video_width)  ? vk->video_width  : vk->vp.width;
+         unsigned vp_height = (vk->vp.height > vk->video_height) ? vk->video_height : vk->vp.height;
          const uint8_t *src = (const uint8_t*)staging->mapped;
-         buffer            += 3 * (vk->vp.height - 1) * vk->vp.width;
+
+         buffer            += 3 * (vp_height - 1) * vp_width;
 
          switch (format)
          {
             case VK_FORMAT_B8G8R8A8_UNORM:
-               for (y = 0; y < (int) vk->vp.height; y++,
+               for (y = 0; y < (int) vp_height; y++,
-                     src += staging->stride, buffer -= 3 * vk->vp.width)
+                     src += staging->stride, buffer -= 3 * vp_width)
                {
                   int x;
-                  for (x = 0; x < (int) vk->vp.width; x++)
+                  for (x = 0; x < (int) vp_width; x++)
                   {
                      buffer[3 * x + 0] = src[4 * x + 0];
                      buffer[3 * x + 1] = src[4 * x + 1];
@@ -5654,11 +5661,11 @@ static bool vulkan_read_viewport(void *data, uint8_t *buffer, bool is_idle)
 
             case VK_FORMAT_R8G8B8A8_UNORM:
             case VK_FORMAT_A8B8G8R8_UNORM_PACK32:
-               for (y = 0; y < (int) vk->vp.height; y++,
+               for (y = 0; y < (int) vp_height; y++,
-                     src += staging->stride, buffer -= 3 * vk->vp.width)
+                     src += staging->stride, buffer -= 3 * vp_width)
                {
                   int x;
-                  for (x = 0; x < (int) vk->vp.width; x++)
+                  for (x = 0; x < (int) vp_width; x++)
                   {
                      buffer[3 * x + 2] = src[4 * x + 0];
                      buffer[3 * x + 1] = src[4 * x + 1];

tasks/task_screenshot.c

@@ -476,6 +476,12 @@ static bool take_screenshot_viewport(
             video_st->data, buffer, runloop_flags & RUNLOOP_FLAG_IDLE)))
       goto error;
 
+   /* Limit image to screen size */
+   if (vp.width > video_st->width)
+      vp.width = video_st->width;
+   if (vp.height > video_st->height)
+      vp.height = video_st->height;
+
    /* Data read from viewport is in bottom-up order, suitable for BMP. */
    if (!screenshot_dump(screenshot_dir,
             name_base,