vk: Fix border color swizzling behavior

rpcs3/Emu/RSX/RSXTexture.h

@@ -142,4 +142,7 @@ namespace rsx
 		rsx::texture_dimension_extended get_extended_texture_dimension() const;
 		u16 get_exact_mipmap_count() const;
 	};
+
+	template<typename T>
+	concept Texture = std::is_same_v<T, fragment_texture> || std::is_same_v<T, vertex_texture>;
 }

rpcs3/Emu/RSX/VK/VKDraw.cpp

@@ -244,6 +244,14 @@ void VKGSRender::load_texture_env()
 		return false;
 	};
 
+	auto get_border_color = [&](const rsx::Texture auto& tex)
+	{
+		const auto require_border_color_remap = m_device->get_custom_border_color_support().require_border_color_remap;
+		return require_border_color_remap
+			? tex.remapped_border_color()
+			: rsx::decode_border_color(tex.border_color());
+	};
+
 	std::lock_guard lock(m_sampler_mutex);
 
 	for (u32 textures_ref = current_fp_metadata.referenced_textures_mask, i = 0; textures_ref; textures_ref >>= 1, ++i)
@@ -304,10 +312,10 @@ void VKGSRender::load_texture_env()
 				const auto wrap_t = vk::vk_wrap_mode(tex.wrap_t());
 				const auto wrap_r = vk::vk_wrap_mode(tex.wrap_r());
 
-				// NOTE: In vulkan, the border color bypasses the swizzling defined in the image view.
+				// NOTE: In vulkan, the border color can bypass the sample swizzle stage.
-				// It is a direct texel replacement and must be remapped before attaching to the sampler.
+				// Check the device properties to determine whether to pre-swizzle the colors or not.
 				const auto border_color = rsx::is_border_clamped_texture(tex)
-					? vk::border_color_t(tex.remapped_border_color())
+					? vk::border_color_t(get_border_color(tex))
 					: vk::border_color_t(VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK);
 
 				// Check if non-point filtering can even be used on this format
@@ -449,10 +457,10 @@ void VKGSRender::load_texture_env()
 				const auto wrap_s = vk::vk_wrap_mode(tex.wrap_s());
 				const auto wrap_t = vk::vk_wrap_mode(tex.wrap_t());
 
-				// NOTE: In vulkan, the border color bypasses the swizzling defined in the image view.
+				// NOTE: In vulkan, the border color can bypass the sample swizzle stage.
-				// It is a direct texel replacement and must be remapped before attaching to the sampler.
+				// Check the device properties to determine whether to pre-swizzle the colors or not.
 				const auto border_color = is_border_clamped_texture(tex)
-					? vk::border_color_t(tex.remapped_border_color())
+					? vk::border_color_t(get_border_color(tex))
 					: vk::border_color_t(VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK);
 
 				if (vs_sampler_handles[i])

rpcs3/Emu/RSX/VK/vkutils/device.cpp

@@ -34,6 +34,7 @@ namespace vk
 			VkPhysicalDeviceAttachmentFeedbackLoopLayoutFeaturesEXT fbo_loops_info{};
 			VkPhysicalDeviceFragmentShaderBarycentricFeaturesKHR shader_barycentric_info{};
 			VkPhysicalDeviceCustomBorderColorFeaturesEXT custom_border_color_info{};
+			VkPhysicalDeviceBorderColorSwizzleFeaturesEXT border_color_swizzle_info{};
 
 			if (device_extensions.is_supported(VK_KHR_SHADER_FLOAT16_INT8_EXTENSION_NAME))
 			{
@@ -70,6 +71,13 @@ namespace vk
 				features2.pNext                = &custom_border_color_info;
 			}
 
+			if (device_extensions.is_supported(VK_EXT_BORDER_COLOR_SWIZZLE_EXTENSION_NAME))
+			{
+				border_color_swizzle_info.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_BORDER_COLOR_SWIZZLE_FEATURES_EXT;
+				border_color_swizzle_info.pNext = features2.pNext;
+				features2.pNext                 = &border_color_swizzle_info;
+			}
+
 			auto _vkGetPhysicalDeviceFeatures2KHR = reinterpret_cast<PFN_vkGetPhysicalDeviceFeatures2KHR>(vkGetInstanceProcAddr(parent, "vkGetPhysicalDeviceFeatures2KHR"));
 			ensure(_vkGetPhysicalDeviceFeatures2KHR); // "vkGetInstanceProcAddress failed to find entry point!"
 			_vkGetPhysicalDeviceFeatures2KHR(dev, &features2);
@@ -78,7 +86,10 @@ namespace vk
 			shader_types_support.allow_float16 = !!shader_support_info.shaderFloat16;
 			shader_types_support.allow_int8    = !!shader_support_info.shaderInt8;
 
-			optional_features_support.custom_border_color = !!custom_border_color_info.customBorderColors && !!custom_border_color_info.customBorderColorWithoutFormat;
+			custom_border_color_support.supported = !!custom_border_color_info.customBorderColors && !!custom_border_color_info.customBorderColorWithoutFormat;
+			custom_border_color_support.swizzle_extension_supported = border_color_swizzle_info.sType == VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_BORDER_COLOR_SWIZZLE_FEATURES_EXT;
+			custom_border_color_support.require_border_color_remap = !border_color_swizzle_info.borderColorSwizzleFromImage;
+
 			optional_features_support.barycentric_coords  = !!shader_barycentric_info.fragmentShaderBarycentric;
 			optional_features_support.framebuffer_loops   = !!fbo_loops_info.attachmentFeedbackLoopLayout;
 
@@ -107,6 +118,13 @@ namespace vk
 
 		optional_features_support.debug_utils              = instance_extensions.is_supported(VK_EXT_DEBUG_UTILS_EXTENSION_NAME);
 		optional_features_support.surface_capabilities_2   = instance_extensions.is_supported(VK_KHR_GET_PHYSICAL_DEVICE_PROPERTIES_2_EXTENSION_NAME);
+
+		// Post-initialization checks
+		if (!custom_border_color_support.swizzle_extension_supported)
+		{
+			// So far only AMD is known to remap image view and border color together. Mark as not required.
+			custom_border_color_support.require_border_color_remap = get_driver_vendor() != driver_vendor::AMD;
+		}
 	}
 
 	void physical_device::get_physical_device_properties(bool allow_extensions)
@@ -438,6 +456,11 @@ namespace vk
 			requested_extensions.push_back(VK_KHR_SHADER_FLOAT16_INT8_EXTENSION_NAME);
 		}
 
+		if (pgpu->custom_border_color_support)
+		{
+			requested_extensions.push_back(VK_EXT_CUSTOM_BORDER_COLOR_EXTENSION_NAME);
+		}
+
 		if (pgpu->optional_features_support.conditional_rendering)
 		{
 			requested_extensions.push_back(VK_EXT_CONDITIONAL_RENDERING_EXTENSION_NAME);
@@ -480,11 +503,6 @@ namespace vk
 			requested_extensions.push_back(VK_KHR_FRAGMENT_SHADER_BARYCENTRIC_EXTENSION_NAME);
 		}
 
-		if (pgpu->optional_features_support.custom_border_color)
-		{
-			requested_extensions.push_back(VK_EXT_CUSTOM_BORDER_COLOR_EXTENSION_NAME);
-		}
-
 		if (pgpu->optional_features_support.synchronization_2)
 		{
 			requested_extensions.push_back(VK_KHR_SYNCHRONIZATION_2_EXTENSION_NAME);
@@ -665,6 +683,16 @@ namespace vk
 			device.pNext = &indexing_features;
 		}
 
+		VkPhysicalDeviceCustomBorderColorFeaturesEXT custom_border_color_features{};
+		if (pgpu->custom_border_color_support)
+		{
+			custom_border_color_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CUSTOM_BORDER_COLOR_FEATURES_EXT;
+			custom_border_color_features.customBorderColors = VK_TRUE;
+			custom_border_color_features.customBorderColorWithoutFormat = VK_TRUE;
+			custom_border_color_features.pNext = const_cast<void*>(device.pNext);
+			device.pNext = &custom_border_color_features;
+		}
+
 		VkPhysicalDeviceAttachmentFeedbackLoopLayoutFeaturesEXT fbo_loop_features{};
 		if (pgpu->optional_features_support.framebuffer_loops)
 		{
@@ -674,16 +702,6 @@ namespace vk
 			device.pNext = &fbo_loop_features;
 		}
 
-		VkPhysicalDeviceCustomBorderColorFeaturesEXT custom_border_color_features{};
-		if (pgpu->optional_features_support.custom_border_color)
-		{
-			custom_border_color_features.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_CUSTOM_BORDER_COLOR_FEATURES_EXT;
-			custom_border_color_features.customBorderColors = VK_TRUE;
-			custom_border_color_features.customBorderColorWithoutFormat = VK_TRUE;
-			custom_border_color_features.pNext = const_cast<void*>(device.pNext);
-			device.pNext = &custom_border_color_features;
-		}
-
 		VkPhysicalDeviceSynchronization2FeaturesKHR synchronization2_info{};
 		if (pgpu->optional_features_support.synchronization_2)
 		{

rpcs3/Emu/RSX/VK/vkutils/device.h

@@ -49,7 +49,16 @@ namespace vk
 		descriptor_indexing_features(bool supported = false)
 			: supported(supported) {}
 
-		operator bool() { return supported; }
+		operator bool() const { return supported; }
+	};
+
+	struct custom_border_color_features
+	{
+		bool supported = false;
+		bool swizzle_extension_supported = false;
+		bool require_border_color_remap = true;  // Assume that without the swizzle extension and explicit remap the device just samples border color as replacement.
+
+		operator bool() const { return supported; }
 	};
 
 	class physical_device
@@ -68,11 +77,12 @@ namespace vk
 		u32 descriptor_max_draw_calls = DESCRIPTOR_MAX_DRAW_CALLS;
 		descriptor_indexing_features descriptor_indexing_support{};
 
+		custom_border_color_features custom_border_color_support{};
+
 		struct
 		{
 			bool barycentric_coords = false;
 			bool conditional_rendering = false;
-			bool custom_border_color = false;
 			bool debug_utils = false;
 			bool external_memory_host = false;
 			bool framebuffer_loops = false;
@@ -161,6 +171,7 @@ namespace vk
 		const gpu_formats_support& get_formats_support() const { return m_formats_support; }
 		const pipeline_binding_table& get_pipeline_binding_table() const { return m_pipeline_binding_table; }
 		const gpu_shader_types_support& get_shader_types_support() const { return pgpu->shader_types_support; }
+		const custom_border_color_features& get_custom_border_color_support() const { return pgpu->custom_border_color_support; }
 
 		bool get_shader_stencil_export_support() const { return pgpu->optional_features_support.shader_stencil_export; }
 		bool get_depth_bounds_support() const { return pgpu->features.depthBounds != VK_FALSE; }
@@ -175,7 +186,6 @@ namespace vk
 		bool get_descriptor_indexing_support() const { return pgpu->descriptor_indexing_support; }
 		bool get_framebuffer_loops_support() const { return pgpu->optional_features_support.framebuffer_loops; }
 		bool get_barycoords_support() const { return pgpu->optional_features_support.barycentric_coords; }
-		bool get_custom_border_color_support() const { return pgpu->optional_features_support.custom_border_color; }
 		bool get_synchronization2_support() const { return pgpu->optional_features_support.synchronization_2; }
 
 		u64 get_descriptor_update_after_bind_support() const { return pgpu->descriptor_indexing_support.update_after_bind_mask; }