diff --git a/src/Backends/DRMBackend.cpp b/src/Backends/DRMBackend.cpp index 1f87effb3..c62635f19 100644 --- a/src/Backends/DRMBackend.cpp +++ b/src/Backends/DRMBackend.cpp @@ -3174,10 +3174,10 @@ namespace gamescope // thus: newLayout is ignored. return VK_IMAGE_LAYOUT_GENERAL; } - virtual void GetPreferredOutputFormat( VkFormat *pPrimaryPlaneFormat, VkFormat *pOverlayPlaneFormat ) const override + virtual void GetPreferredOutputFormat( uint32_t *pPrimaryPlaneFormat, uint32_t *pOverlayPlaneFormat ) const override { - *pPrimaryPlaneFormat = DRMFormatToVulkan( g_nDRMFormat, false ); - *pOverlayPlaneFormat = DRMFormatToVulkan( g_nDRMFormatOverlay, false ); + *pPrimaryPlaneFormat = g_nDRMFormat; + *pOverlayPlaneFormat = g_nDRMFormatOverlay; } virtual bool ValidPhysicalDevice( VkPhysicalDevice pVkPhysicalDevice ) const override { diff --git a/src/Backends/HeadlessBackend.cpp b/src/Backends/HeadlessBackend.cpp index ddf8c1052..49987f69f 100644 --- a/src/Backends/HeadlessBackend.cpp +++ b/src/Backends/HeadlessBackend.cpp @@ -147,10 +147,10 @@ namespace gamescope { return VK_IMAGE_LAYOUT_GENERAL; } - virtual void GetPreferredOutputFormat( VkFormat *pPrimaryPlaneFormat, VkFormat *pOverlayPlaneFormat ) const override + virtual void GetPreferredOutputFormat( uint32_t *pPrimaryPlaneFormat, uint32_t *pOverlayPlaneFormat ) const override { - *pPrimaryPlaneFormat = VK_FORMAT_A2B10G10R10_UNORM_PACK32; - *pOverlayPlaneFormat = VK_FORMAT_B8G8R8A8_UNORM; + *pPrimaryPlaneFormat = VulkanFormatToDRM( VK_FORMAT_A2B10G10R10_UNORM_PACK32 ); + *pOverlayPlaneFormat = VulkanFormatToDRM( VK_FORMAT_B8G8R8A8_UNORM ); } virtual bool ValidPhysicalDevice( VkPhysicalDevice pVkPhysicalDevice ) const override { diff --git a/src/Backends/OpenVRBackend.cpp b/src/Backends/OpenVRBackend.cpp index c4de4aca8..f03a65d2b 100644 --- a/src/Backends/OpenVRBackend.cpp +++ b/src/Backends/OpenVRBackend.cpp @@ -532,10 +532,10 @@ namespace gamescope { return VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; } - virtual void GetPreferredOutputFormat( VkFormat *pPrimaryPlaneFormat, VkFormat *pOverlayPlaneFormat ) const override + virtual void GetPreferredOutputFormat( uint32_t *pPrimaryPlaneFormat, uint32_t *pOverlayPlaneFormat ) const override { - *pPrimaryPlaneFormat = VK_FORMAT_A2B10G10R10_UNORM_PACK32; - *pOverlayPlaneFormat = VK_FORMAT_B8G8R8A8_UNORM; + *pPrimaryPlaneFormat = VulkanFormatToDRM( VK_FORMAT_A2B10G10R10_UNORM_PACK32 ); + *pOverlayPlaneFormat = VulkanFormatToDRM( VK_FORMAT_B8G8R8A8_UNORM ); } virtual bool ValidPhysicalDevice( VkPhysicalDevice pVkPhysicalDevice ) const override { diff --git a/src/Backends/SDLBackend.cpp b/src/Backends/SDLBackend.cpp index c2821a123..8b3646420 100644 --- a/src/Backends/SDLBackend.cpp +++ b/src/Backends/SDLBackend.cpp @@ -122,7 +122,7 @@ namespace gamescope virtual std::span GetInstanceExtensions() const override; virtual std::span GetDeviceExtensions( VkPhysicalDevice pVkPhysicalDevice ) const override; virtual VkImageLayout GetPresentLayout() const override; - virtual void GetPreferredOutputFormat( VkFormat *pPrimaryPlaneFormat, VkFormat *pOverlayPlaneFormat ) const override; + virtual void GetPreferredOutputFormat( uint32_t *pPrimaryPlaneFormat, uint32_t *pOverlayPlaneFormat ) const override; virtual bool ValidPhysicalDevice( VkPhysicalDevice pVkPhysicalDevice ) const override; virtual int Present( const FrameInfo_t *pFrameInfo, bool bAsync ) override; @@ -351,10 +351,10 @@ namespace gamescope return VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; } - void CSDLBackend::GetPreferredOutputFormat( VkFormat *pPrimaryPlaneFormat, VkFormat *pOverlayPlaneFormat ) const + void CSDLBackend::GetPreferredOutputFormat( uint32_t *pPrimaryPlaneFormat, uint32_t *pOverlayPlaneFormat ) const { - *pPrimaryPlaneFormat = VK_FORMAT_A2B10G10R10_UNORM_PACK32; - *pOverlayPlaneFormat = VK_FORMAT_B8G8R8A8_UNORM; + *pPrimaryPlaneFormat = VulkanFormatToDRM( VK_FORMAT_A2B10G10R10_UNORM_PACK32 ); + *pOverlayPlaneFormat = VulkanFormatToDRM( VK_FORMAT_B8G8R8A8_UNORM ); } bool CSDLBackend::ValidPhysicalDevice( VkPhysicalDevice pVkPhysicalDevice ) const diff --git a/src/Backends/WaylandBackend.cpp b/src/Backends/WaylandBackend.cpp index 6498f3b49..bb8bd6f64 100644 --- a/src/Backends/WaylandBackend.cpp +++ b/src/Backends/WaylandBackend.cpp @@ -504,7 +504,7 @@ namespace gamescope virtual std::span GetInstanceExtensions() const override; virtual std::span GetDeviceExtensions( VkPhysicalDevice pVkPhysicalDevice ) const override; virtual VkImageLayout GetPresentLayout() const override; - virtual void GetPreferredOutputFormat( VkFormat *pPrimaryPlaneFormat, VkFormat *pOverlayPlaneFormat ) const override; + virtual void GetPreferredOutputFormat( uint32_t *pPrimaryPlaneFormat, uint32_t *pOverlayPlaneFormat ) const override; virtual bool ValidPhysicalDevice( VkPhysicalDevice pVkPhysicalDevice ) const override; virtual int Present( const FrameInfo_t *pFrameInfo, bool bAsync ) override; @@ -1542,23 +1542,33 @@ namespace gamescope return VK_IMAGE_LAYOUT_GENERAL; } - void CWaylandBackend::GetPreferredOutputFormat( VkFormat *pPrimaryPlaneFormat, VkFormat *pOverlayPlaneFormat ) const + void CWaylandBackend::GetPreferredOutputFormat( uint32_t *pPrimaryPlaneFormat, uint32_t *pOverlayPlaneFormat ) const { - VkFormat u8BitFormat = VK_FORMAT_UNDEFINED; - if ( SupportsFormat( DRM_FORMAT_ARGB8888 ) ) - u8BitFormat = VK_FORMAT_B8G8R8A8_UNORM; - else if ( SupportsFormat( DRM_FORMAT_ABGR8888 ) ) - u8BitFormat = VK_FORMAT_R8G8B8A8_UNORM; + // Prefer opaque for composition on the Wayland backend. - VkFormat u10BitFormat = VK_FORMAT_UNDEFINED; - if ( SupportsFormat( DRM_FORMAT_ABGR2101010 ) ) - u10BitFormat = VK_FORMAT_A2B10G10R10_UNORM_PACK32; + uint32_t u8BitFormat = DRM_FORMAT_INVALID; + if ( SupportsFormat( DRM_FORMAT_XRGB8888 ) ) + u8BitFormat = DRM_FORMAT_XRGB8888; + else if ( SupportsFormat( DRM_FORMAT_XBGR8888 ) ) + u8BitFormat = DRM_FORMAT_XBGR8888; + else if ( SupportsFormat( DRM_FORMAT_ARGB8888 ) ) + u8BitFormat = DRM_FORMAT_ARGB8888; + else if ( SupportsFormat( DRM_FORMAT_ABGR8888 ) ) + u8BitFormat = DRM_FORMAT_ABGR8888; + + uint32_t u10BitFormat = DRM_FORMAT_INVALID; + if ( SupportsFormat( DRM_FORMAT_XBGR2101010 ) ) + u10BitFormat = DRM_FORMAT_XBGR2101010; + else if ( SupportsFormat( DRM_FORMAT_XRGB2101010 ) ) + u10BitFormat = DRM_FORMAT_XRGB2101010; + else if ( SupportsFormat( DRM_FORMAT_ABGR2101010 ) ) + u10BitFormat = DRM_FORMAT_ABGR2101010; else if ( SupportsFormat( DRM_FORMAT_ARGB2101010 ) ) - u10BitFormat = VK_FORMAT_A2R10G10B10_UNORM_PACK32; + u10BitFormat = DRM_FORMAT_ARGB2101010; - assert( u8BitFormat != VK_FORMAT_UNDEFINED ); + assert( u8BitFormat != DRM_FORMAT_INVALID ); - *pPrimaryPlaneFormat = u10BitFormat != VK_FORMAT_UNDEFINED ? u10BitFormat : u8BitFormat; + *pPrimaryPlaneFormat = u10BitFormat != DRM_FORMAT_INVALID ? u10BitFormat : u8BitFormat; *pOverlayPlaneFormat = u8BitFormat; } diff --git a/src/backend.h b/src/backend.h index 7d9fb465d..341e9d783 100644 --- a/src/backend.h +++ b/src/backend.h @@ -178,7 +178,7 @@ namespace gamescope virtual std::span GetInstanceExtensions() const = 0; virtual std::span GetDeviceExtensions( VkPhysicalDevice pVkPhysicalDevice ) const = 0; virtual VkImageLayout GetPresentLayout() const = 0; - virtual void GetPreferredOutputFormat( VkFormat *pPrimaryPlaneFormat, VkFormat *pOverlayPlaneFormat ) const = 0; + virtual void GetPreferredOutputFormat( uint32_t *pPrimaryPlaneFormat, uint32_t *pOverlayPlaneFormat ) const = 0; virtual bool ValidPhysicalDevice( VkPhysicalDevice pVkPhysicalDevice ) const = 0; virtual int Present( const FrameInfo_t *pFrameInfo, bool bAsync ) = 0; diff --git a/src/rendervulkan.cpp b/src/rendervulkan.cpp index 63d34b3ca..97e6fbe22 100644 --- a/src/rendervulkan.cpp +++ b/src/rendervulkan.cpp @@ -3112,12 +3112,13 @@ bool vulkan_make_swapchain( VulkanOutput_t *pOutput ) if ( surfaceFormat == formatCount ) return false; - pOutput->outputFormat = pOutput->surfaceFormats[ surfaceFormat ].format; + VkFormat eVkFormat = pOutput->surfaceFormats[ surfaceFormat ].format; + pOutput->uOutputFormat = VulkanFormatToDRM( pOutput->surfaceFormats[ surfaceFormat ].format ); VkFormat formats[2] = { - ToSrgbVulkanFormat( pOutput->outputFormat ), - ToLinearVulkanFormat( pOutput->outputFormat ), + ToSrgbVulkanFormat( eVkFormat ), + ToLinearVulkanFormat( eVkFormat ), }; VkImageFormatListCreateInfo usageListInfo = { @@ -3126,7 +3127,7 @@ bool vulkan_make_swapchain( VulkanOutput_t *pOutput ) .pViewFormats = formats, }; - vk_log.infof("Creating Gamescope nested swapchain with format %u and colorspace %u", pOutput->outputFormat, pOutput->surfaceFormats[surfaceFormat].colorSpace); + vk_log.infof("Creating Gamescope nested swapchain with format %u and colorspace %u", eVkFormat, pOutput->surfaceFormats[surfaceFormat].colorSpace); VkSwapchainCreateInfoKHR createInfo = { .sType = VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR, @@ -3134,7 +3135,7 @@ bool vulkan_make_swapchain( VulkanOutput_t *pOutput ) .flags = formats[0] != formats[1] ? VK_SWAPCHAIN_CREATE_MUTABLE_FORMAT_BIT_KHR : (VkSwapchainCreateFlagBitsKHR )0, .surface = pOutput->surface, .minImageCount = imageCount, - .imageFormat = pOutput->outputFormat, + .imageFormat = eVkFormat, .imageColorSpace = pOutput->surfaceFormats[surfaceFormat].colorSpace, .imageExtent = { .width = g_nOutputWidth, @@ -3163,7 +3164,7 @@ bool vulkan_make_swapchain( VulkanOutput_t *pOutput ) { pOutput->outputImages[i] = new CVulkanTexture(); - if ( !pOutput->outputImages[i]->BInitFromSwapchain(swapchainImages[i], g_nOutputWidth, g_nOutputHeight, pOutput->outputFormat)) + if ( !pOutput->outputImages[i]->BInitFromSwapchain(swapchainImages[i], g_nOutputWidth, g_nOutputHeight, eVkFormat)) return false; } @@ -3220,10 +3221,10 @@ static bool vulkan_make_output_images( VulkanOutput_t *pOutput ) pOutput->outputImagesPartialOverlay[1] = nullptr; pOutput->outputImagesPartialOverlay[2] = nullptr; - VkFormat format = pOutput->outputFormat; + uint32_t uDRMFormat = pOutput->uOutputFormat; pOutput->outputImages[0] = new CVulkanTexture(); - bool bSuccess = pOutput->outputImages[0]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, VulkanFormatToDRM(format, false), outputImageflags ); + bool bSuccess = pOutput->outputImages[0]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, uDRMFormat, outputImageflags ); if ( bSuccess != true ) { vk_log.errorf( "failed to allocate buffer for KMS" ); @@ -3231,7 +3232,7 @@ static bool vulkan_make_output_images( VulkanOutput_t *pOutput ) } pOutput->outputImages[1] = new CVulkanTexture(); - bSuccess = pOutput->outputImages[1]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, VulkanFormatToDRM(format, false), outputImageflags ); + bSuccess = pOutput->outputImages[1]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, uDRMFormat, outputImageflags ); if ( bSuccess != true ) { vk_log.errorf( "failed to allocate buffer for KMS" ); @@ -3239,7 +3240,7 @@ static bool vulkan_make_output_images( VulkanOutput_t *pOutput ) } pOutput->outputImages[2] = new CVulkanTexture(); - bSuccess = pOutput->outputImages[2]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, VulkanFormatToDRM(format, false), outputImageflags ); + bSuccess = pOutput->outputImages[2]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, uDRMFormat, outputImageflags ); if ( bSuccess != true ) { vk_log.errorf( "failed to allocate buffer for KMS" ); @@ -3249,12 +3250,12 @@ static bool vulkan_make_output_images( VulkanOutput_t *pOutput ) // Oh no. pOutput->temporaryHackyBlankImage = vulkan_create_debug_blank_texture(); - if ( pOutput->outputFormatOverlay != VK_FORMAT_UNDEFINED && !kDisablePartialComposition ) + if ( pOutput->uOutputFormatOverlay != VK_FORMAT_UNDEFINED && !kDisablePartialComposition ) { - VkFormat partialFormat = pOutput->outputFormatOverlay; + uint32_t uPartialDRMFormat = pOutput->uOutputFormatOverlay; pOutput->outputImagesPartialOverlay[0] = new CVulkanTexture(); - bool bSuccess = pOutput->outputImagesPartialOverlay[0]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, VulkanFormatToDRM(partialFormat), outputImageflags, nullptr, 0, 0, pOutput->outputImages[0].get() ); + bool bSuccess = pOutput->outputImagesPartialOverlay[0]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, uPartialDRMFormat, outputImageflags, nullptr, 0, 0, pOutput->outputImages[0].get() ); if ( bSuccess != true ) { vk_log.errorf( "failed to allocate buffer for KMS" ); @@ -3262,7 +3263,7 @@ static bool vulkan_make_output_images( VulkanOutput_t *pOutput ) } pOutput->outputImagesPartialOverlay[1] = new CVulkanTexture(); - bSuccess = pOutput->outputImagesPartialOverlay[1]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, VulkanFormatToDRM(partialFormat), outputImageflags, nullptr, 0, 0, pOutput->outputImages[1].get() ); + bSuccess = pOutput->outputImagesPartialOverlay[1]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, uPartialDRMFormat, outputImageflags, nullptr, 0, 0, pOutput->outputImages[1].get() ); if ( bSuccess != true ) { vk_log.errorf( "failed to allocate buffer for KMS" ); @@ -3270,7 +3271,7 @@ static bool vulkan_make_output_images( VulkanOutput_t *pOutput ) } pOutput->outputImagesPartialOverlay[2] = new CVulkanTexture(); - bSuccess = pOutput->outputImagesPartialOverlay[2]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, VulkanFormatToDRM(partialFormat), outputImageflags, nullptr, 0, 0, pOutput->outputImages[2].get() ); + bSuccess = pOutput->outputImagesPartialOverlay[2]->BInit( g_nOutputWidth, g_nOutputHeight, 1u, uPartialDRMFormat, outputImageflags, nullptr, 0, 0, pOutput->outputImages[2].get() ); if ( bSuccess != true ) { vk_log.errorf( "failed to allocate buffer for KMS" ); @@ -3356,15 +3357,15 @@ bool vulkan_make_output() } else { - GetBackend()->GetPreferredOutputFormat( &pOutput->outputFormat, &pOutput->outputFormatOverlay ); + GetBackend()->GetPreferredOutputFormat( &pOutput->uOutputFormat, &pOutput->uOutputFormatOverlay ); - if ( pOutput->outputFormat == VK_FORMAT_UNDEFINED ) + if ( pOutput->uOutputFormat == DRM_FORMAT_INVALID ) { vk_log.errorf( "failed to find Vulkan format suitable for KMS" ); return false; } - if ( pOutput->outputFormatOverlay == VK_FORMAT_UNDEFINED ) + if ( pOutput->uOutputFormatOverlay == DRM_FORMAT_INVALID ) { vk_log.errorf( "failed to find Vulkan format suitable for KMS partial overlays" ); return false; diff --git a/src/rendervulkan.hpp b/src/rendervulkan.hpp index f5fef302b..b967e849f 100644 --- a/src/rendervulkan.hpp +++ b/src/rendervulkan.hpp @@ -508,8 +508,8 @@ struct VulkanOutput_t std::vector> outputImagesPartialOverlay; gamescope::OwningRc temporaryHackyBlankImage; - VkFormat outputFormat = VK_FORMAT_UNDEFINED; - VkFormat outputFormatOverlay = VK_FORMAT_UNDEFINED; + uint32_t uOutputFormat = DRM_FORMAT_INVALID; + uint32_t uOutputFormatOverlay = DRM_FORMAT_INVALID; std::array, 2> pScreenshotImages;