Attempt to fix validation errors with HDR swapchain.

Always use final render pass type equal to swapchain format. Use more
direct logic to expose if filter chain emits HDR10 color space or not.

gfx/drivers/vulkan.c

@@ -2090,9 +2090,8 @@ static bool vulkan_frame(void *data, const void *frame,
    bool overlay_behind_menu                      = video_info->overlay_behind_menu;
 
 #ifdef VULKAN_HDR_SWAPCHAIN
-   struct video_shader* shader_preset           = vulkan_filter_chain_get_preset(vk->filter_chain); 
+   bool use_main_buffer                          = vk->context->hdr_enable &&
-   VkFormat main_buffer_format                  = shader_preset && shader_preset->passes ? vulkan_filter_chain_get_pass_rt_format(vk->filter_chain, shader_preset->passes - 1) : VK_FORMAT_R8G8B8A8_UNORM;
+      (!vk->filter_chain || !vulkan_filter_chain_emits_hdr10(vk->filter_chain));
-   bool use_main_buffer                         = main_buffer_format != vk->context->swapchain_format; 
 #endif /* VULKAN_HDR_SWAPCHAIN */
 
    /* Bookkeeping on start of frame. */
@@ -2330,7 +2329,7 @@ static bool vulkan_frame(void *data, const void *frame,
 #endif
 
 #ifdef VULKAN_HDR_SWAPCHAIN
-   if(vk->context->hdr_enable && use_main_buffer)
+   if (use_main_buffer)
       backbuffer = &vk->main_buffer;
 #endif /* VULKAN_HDR_SWAPCHAIN */
 
@@ -2440,7 +2439,7 @@ static bool vulkan_frame(void *data, const void *frame,
 
 #ifdef VULKAN_HDR_SWAPCHAIN
       /* Copy over back buffer to swap chain render targets */
-      if (vk->context->hdr_enable && use_main_buffer)
+      if (use_main_buffer)
       {
          backbuffer = &vk->backbuffers[swapchain_index];
 
@@ -2789,7 +2788,7 @@ static bool vulkan_frame(void *data, const void *frame,
       if (!(vk->hdr.support = vk->context->swapchain_colour_space == VK_COLOR_SPACE_HDR10_ST2084_EXT))
          vk->context->hdr_enable                = false;
 
-      if(vk->context->hdr_enable)
+      if (vk->context->hdr_enable)
       {
          VkMemoryRequirements mem_reqs;
          VkImageCreateInfo image_info;
@@ -2804,7 +2803,7 @@ static bool vulkan_frame(void *data, const void *frame,
          image_info.pNext                = NULL;
          image_info.flags                = 0;
          image_info.imageType            = VK_IMAGE_TYPE_2D;
-         image_info.format               = main_buffer_format;
+         image_info.format               = vk->context->swapchain_format;
          image_info.extent.width         = video_width;
          image_info.extent.height        = video_height;
          image_info.extent.depth         = 1;
@@ -2843,7 +2842,7 @@ static bool vulkan_frame(void *data, const void *frame,
          view.flags                           = 0;
          view.image                           = vk->main_buffer.image;
          view.viewType                        = VK_IMAGE_VIEW_TYPE_2D;
-         view.format                          = main_buffer_format;
+         view.format                          = image_info.format;
          view.components.r                    = VK_COMPONENT_SWIZZLE_R;
          view.components.g                    = VK_COMPONENT_SWIZZLE_G;
          view.components.b                    = VK_COMPONENT_SWIZZLE_B;

gfx/drivers_shader/shader_vulkan.cpp

@@ -685,6 +685,9 @@ struct vulkan_filter_chain
 
       VkFormat get_pass_rt_format(unsigned pass);
 
+      bool emits_hdr10() const;
+      void set_hdr10();
+
    private:
       VkDevice device;
       VkPhysicalDevice gpu;
@@ -719,6 +722,7 @@ struct vulkan_filter_chain
       void clear_history_and_feedback(VkCommandBuffer cmd);
       void update_feedback_info();
       void update_history_info();
+      bool emits_hdr_colorspace = false;
 };
 
 static uint32_t find_memory_type_fallback(
@@ -1488,10 +1492,20 @@ void vulkan_filter_chain::set_pass_info(unsigned pass,
    pass_info[pass] = info;
 }
 
- VkFormat vulkan_filter_chain::get_pass_rt_format(unsigned pass)
+VkFormat vulkan_filter_chain::get_pass_rt_format(unsigned pass)
- {
+{
-    return pass_info[pass].rt_format;
+   return pass_info[pass].rt_format;
- }
+}
+
+bool vulkan_filter_chain::emits_hdr10() const
+{
+   return emits_hdr_colorspace;
+}
+
+void vulkan_filter_chain::set_hdr10()
+{
+   emits_hdr_colorspace = true;
+}
 
 void vulkan_filter_chain::set_num_passes(unsigned num_passes)
 {
@@ -3026,22 +3040,19 @@ vulkan_filter_chain_t *vulkan_filter_chain_create_from_preset(
          {
             pass_info.scale_type_x = GLSLANG_FILTER_CHAIN_SCALE_VIEWPORT;
             pass_info.scale_type_y = GLSLANG_FILTER_CHAIN_SCALE_VIEWPORT;
-#ifdef VULKAN_HDR_SWAPCHAIN
-            if (tmpinfo.swapchain.format == VK_FORMAT_A2B10G10R10_UNORM_PACK32)
-            {
-               pass_info.rt_format    = glslang_format_to_vk( output.meta.rt_format);
 
-               RARCH_LOG("[slang]: Using render target format %s for pass output #%u.\n",
+            /* Always inherit swapchain format. */
-                     glslang_format_to_string(output.meta.rt_format), i);
+            pass_info.rt_format  = tmpinfo.swapchain.format;
-            }
+            VkFormat pass_format = glslang_format_to_vk(output.meta.rt_format);
-            else
-#endif /* VULKAN_HDR_SWAPCHAIN */
-            {
-               pass_info.rt_format    = tmpinfo.swapchain.format;
 
-               if (explicit_format)
+            /* If final pass explicitly emits RGB10, consider it HDR color space. */
-                  RARCH_WARN("[slang]: Using explicit format for last pass in chain,"
+            if (explicit_format && pass_format == VK_FORMAT_A2B10G10R10_UNORM_PACK32)
-                        " but it is not rendered to framebuffer, using swapchain format instead.\n");
+               chain->set_hdr10();
+
+            if (explicit_format && pass_format != pass_info.rt_format)
+            {
+               RARCH_WARN("[slang]: Using explicit format for last pass in chain,"
+                     " but it is not rendered to framebuffer, using swapchain format instead.\n");
             }
          }
          else
@@ -3260,3 +3271,8 @@ void vulkan_filter_chain_end_frame(
 {
    chain->end_frame(cmd);
 }
+
+bool vulkan_filter_chain_emits_hdr10(vulkan_filter_chain_t *chain)
+{
+   return chain->emits_hdr10();
+}

gfx/drivers_shader/shader_vulkan.h

@@ -141,6 +141,8 @@ vulkan_filter_chain_t *vulkan_filter_chain_create_from_preset(
 struct video_shader *vulkan_filter_chain_get_preset(
       vulkan_filter_chain_t *chain);
 
+bool vulkan_filter_chain_emits_hdr10(vulkan_filter_chain_t *chain);
+
 RETRO_END_DECLS
 
 #endif