diff --git a/layer/VkLayer_FROG_gamescope_wsi.cpp b/layer/VkLayer_FROG_gamescope_wsi.cpp index af0c200993..8f706eed12 100644 --- a/layer/VkLayer_FROG_gamescope_wsi.cpp +++ b/layer/VkLayer_FROG_gamescope_wsi.cpp @@ -500,6 +500,7 @@ namespace GamescopeWSILayer { VkSurfaceKHR surface; // Always the Gamescope Surface surface -- so the Wayland one. bool isWayland; bool isBypassingXWayland; + bool forcedBypass; bool forceFifo; VkPresentModeKHR presentMode; VkExtent2D extent; @@ -1111,20 +1112,10 @@ namespace GamescopeWSILayer { return pDispatch->CreateSwapchainKHR(device, pCreateInfo, pAllocator, pSwapchain); } - const bool canBypass = gamescopeSurface->canBypassXWayland(); + bool canBypass = gamescopeSurface->canBypassXWayland(); VkSwapchainCreateInfoKHR swapchainInfo = *pCreateInfo; - if (pCreateInfo->oldSwapchain) { - if (auto gamescopeSwapchain = GamescopeSwapchain::get(pCreateInfo->oldSwapchain)) { - gamescopeSwapchain->retired = true; - // If we are going to/from being able to bypass XWayland, make sure - // we NULL out oldSwapchain, as they'll be for different surfaces and swapchain types. - if (gamescopeSwapchain->isBypassingXWayland != canBypass) - swapchainInfo.oldSwapchain = VK_NULL_HANDLE; - } - } - if (gamescopeSurface->flags & GamescopeLayerClient::Flag::ForceSwapchainExtent) { if (!gamescopeSurface->isWayland()) { auto rect = xcb::getWindowRect(gamescopeSurface->connection, gamescopeSurface->window); @@ -1135,6 +1126,42 @@ namespace GamescopeWSILayer { } } + auto isFormatSupportedOnSurface = [&](VkSurfaceKHR vkSurface, VkFormat format) { + std::vector formats; + vkroots::helpers::enumerate( + pDispatch->pPhysicalDeviceDispatch->pInstanceDispatch->GetPhysicalDeviceSurfaceFormatsKHR, + formats, + pDispatch->PhysicalDevice, + vkSurface); + return std::ranges::any_of( + formats, + std::bind_front(std::equal_to{}, format), + &VkSurfaceFormatKHR::format); + }; + + // If bypass supports the format but the XCB fallback doesn't, force bypass. + // Refusing here would leave the app with no recovery path. + bool forcedBypass = false; + if (!canBypass + && !isFormatSupportedOnSurface(gamescopeSurface->fallbackSurface, pCreateInfo->imageFormat) + && isFormatSupportedOnSurface(pCreateInfo->surface, pCreateInfo->imageFormat)) { + fprintf(stderr, "[Gamescope WSI] Forcing bypass (format unsupported on fallback) for xid: 0x%0x - format: %s\n", + gamescopeSurface->window, + vkroots::helpers::enumString(pCreateInfo->imageFormat)); + canBypass = true; + forcedBypass = true; + } + + if (pCreateInfo->oldSwapchain) { + if (auto gamescopeSwapchain = GamescopeSwapchain::get(pCreateInfo->oldSwapchain)) { + gamescopeSwapchain->retired = true; + // If we are going to/from being able to bypass XWayland, make sure + // we NULL out oldSwapchain, as they'll be for different surfaces and swapchain types. + if (gamescopeSwapchain->isBypassingXWayland != canBypass) + swapchainInfo.oldSwapchain = VK_NULL_HANDLE; + } + } + // If we can't flip, fallback to the regular XCB surface on the XCB window. if (!canBypass) swapchainInfo.surface = gamescopeSurface->fallbackSurface; @@ -1175,28 +1202,14 @@ namespace GamescopeWSILayer { // Check for VkFormat support and return VK_ERROR_INITIALIZATION_FAILED // if that VkFormat is unsupported for the underlying surface. - { - std::vector supportedSurfaceFormats; - vkroots::helpers::enumerate( - pDispatch->pPhysicalDeviceDispatch->pInstanceDispatch->GetPhysicalDeviceSurfaceFormatsKHR, - supportedSurfaceFormats, - pDispatch->PhysicalDevice, - swapchainInfo.surface); - - bool supportedSwapchainFormat = std::ranges::any_of( - supportedSurfaceFormats, - std::bind_front(std::equal_to{}, swapchainInfo.imageFormat), - &VkSurfaceFormatKHR::format) ; - - if (!supportedSwapchainFormat) { - fprintf(stderr, "[Gamescope WSI] Refusing to make swapchain (unsupported VkFormat) for xid: 0x%0x - format: %s - colorspace: %s - flip: %s\n", - gamescopeSurface->window, - vkroots::helpers::enumString(pCreateInfo->imageFormat), - vkroots::helpers::enumString(pCreateInfo->imageColorSpace), - canBypass ? "true" : "false"); - - return VK_ERROR_INITIALIZATION_FAILED; - } + if (!isFormatSupportedOnSurface(swapchainInfo.surface, swapchainInfo.imageFormat)) { + fprintf(stderr, "[Gamescope WSI] Refusing to make swapchain (unsupported VkFormat) for xid: 0x%0x - format: %s - colorspace: %s - flip: %s\n", + gamescopeSurface->window, + vkroots::helpers::enumString(pCreateInfo->imageFormat), + vkroots::helpers::enumString(pCreateInfo->imageColorSpace), + canBypass ? "true" : "false"); + + return VK_ERROR_INITIALIZATION_FAILED; } uint32_t serverId = ~0u; @@ -1232,6 +1245,7 @@ namespace GamescopeWSILayer { .surface = pCreateInfo->surface, // Always the Wayland side surface. .isWayland = gamescopeSurface->isWayland(), .isBypassingXWayland = canBypass, + .forcedBypass = forcedBypass, .forceFifo = gamescopeIsForcingFifo(), // Were we forcing fifo when this swapchain was made? .presentMode = pCreateInfo->presentMode, // The new present mode. .extent = pCreateInfo->imageExtent, @@ -1431,8 +1445,10 @@ namespace GamescopeWSILayer { if (canBypass) { if (!(gamescopeSurface->flags & GamescopeLayerClient::Flag::NoSuboptimal)) UpdateSwapchainResult(VK_SUBOPTIMAL_KHR); - } else { - UpdateSwapchainResult(VK_ERROR_OUT_OF_DATE_KHR); + } else if (!gamescopeSwapchain->forcedBypass) { + // A forced-bypass swapchain has no fallback path; recreating + // would re-force bypass and loop. + UpdateSwapchainResult(VK_ERROR_OUT_OF_DATE_KHR); } }