main: Pick correct Vulkan device for DRM

Closes: #1080

src/main.cpp

@@ -765,9 +765,6 @@ int main(int argc, char **argv)
 		}
 	}
 
-	VkInstance instance = vulkan_create_instance();
-	VkSurfaceKHR surface = VK_NULL_HANDLE;
-
 	if ( !BIsNested() )
 	{
 		g_bForceRelativeMouse = false;
@@ -790,30 +787,15 @@ int main(int argc, char **argv)
 		return 1;
 	}
 
-	if ( BIsSDLSession() )
-	{
-		if ( !SDL_Vulkan_CreateSurface( g_SDLWindow, instance, &surface ) )
-		{
-			fprintf(stderr, "SDL_Vulkan_CreateSurface failed: %s", SDL_GetError() );
-			return 1;
-		}
-	}
-
 	g_ForcedNV12ColorSpace = parse_colorspace_string( getenv( "GAMESCOPE_NV12_COLORSPACE" ) );
 
-	if ( !vulkan_init(instance, surface) )
-	{
-		fprintf( stderr, "Failed to initialize Vulkan\n" );
-		return 1;
-	}
-
 	if ( !vulkan_init_formats() )
 	{
 		fprintf( stderr, "vulkan_init_formats failed\n" );
 		return 1;
 	}
 
-	if ( !vulkan_make_output(surface) )
+	if ( !vulkan_make_output() )
 	{
 		fprintf( stderr, "vulkan_make_output failed\n" );
 		return 1;
@@ -925,6 +907,8 @@ static void steamCompMgrThreadRun(int argc, char **argv)
 
 static bool initOutput( int preferredWidth, int preferredHeight, int preferredRefresh )
 {
+	VkInstance instance = vulkan_create_instance();
+
 	if ( BIsNested() )
 	{
 		g_nOutputWidth = preferredWidth;
@@ -936,7 +920,7 @@ static bool initOutput( int preferredWidth, int preferredHeight, int preferredRe
 			if ( g_nOutputWidth != 0 )
 			{
 				fprintf( stderr, "Cannot specify -W without -H\n" );
-				return 1;
+				return false;
 			}
 			g_nOutputHeight = 720;
 		}
@@ -948,16 +932,45 @@ static bool initOutput( int preferredWidth, int preferredHeight, int preferredRe
 		if ( BIsVRSession() )
 		{
 #if HAVE_OPENVR
-			return vrsession_init();
+			if ( !vrsession_init() )
+				return false;
 #else
 			return false;
 #endif
 		}
 		else if ( BIsSDLSession() )
 		{
-			return sdlwindow_init();
+			if ( !sdlwindow_init() )
+				return false;
+		}
+
+		VkSurfaceKHR surface = VK_NULL_HANDLE;
+
+		if ( BIsSDLSession() )
+		{
+			if ( !SDL_Vulkan_CreateSurface( g_SDLWindow, instance, &surface ) )
+			{
+				fprintf(stderr, "SDL_Vulkan_CreateSurface failed: %s", SDL_GetError() );
+				return false;
+			}
+		}
+
+		if ( !vulkan_init( instance, surface ) )
+		{
+			fprintf( stderr, "Failed to initialize Vulkan\n" );
+			return false;
 		}
+
 		return true;
 	}
-	return init_drm( &g_DRM, preferredWidth, preferredHeight, preferredRefresh, s_bInitialWantsVRREnabled );
+	else
+	{
+		if ( !vulkan_init( instance, VK_NULL_HANDLE ) )
+		{
+			fprintf( stderr, "Failed to initialize Vulkan\n" );
+			return false;
+		}
+
+		return init_drm( &g_DRM, preferredWidth, preferredHeight, preferredRefresh, s_bInitialWantsVRREnabled );
+	}
 }

src/rendervulkan.cpp

@@ -253,6 +253,8 @@ bool CVulkanDevice::BInit(VkInstance instance, VkSurfaceKHR surface)
 	assert(instance);
 	assert(!m_bInitialized);
 
+	g_output.surface = surface;
+
 	m_instance = instance;
 	#define VK_FUNC(x) vk.x = (PFN_vk##x) vkGetInstanceProcAddr(instance, "vk"#x);
 	VULKAN_INSTANCE_FUNCTIONS
@@ -3129,7 +3131,7 @@ bool vulkan_remake_output_images()
 	return bRet;
 }
 
-bool vulkan_make_output( VkSurfaceKHR surface )
+bool vulkan_make_output()
 {
 	VulkanOutput_t *pOutput = &g_output;
 
@@ -3142,9 +3144,6 @@ bool vulkan_make_output( VkSurfaceKHR surface )
 	}
 	else if ( BIsSDLSession() )
 	{
-		assert(surface);
-		pOutput->surface = surface;
-
 		result = g_device.vk.GetPhysicalDeviceSurfaceCapabilitiesKHR( g_device.physDev(), pOutput->surface, &pOutput->surfaceCaps );
 		if ( result != VK_SUCCESS )
 		{

src/rendervulkan.hpp

@@ -368,7 +368,7 @@ namespace CompositeDebugFlag
 VkInstance vulkan_create_instance(void);
 bool vulkan_init(VkInstance instance, VkSurfaceKHR surface);
 bool vulkan_init_formats(void);
-bool vulkan_make_output(VkSurfaceKHR surface);
+bool vulkan_make_output();
 
 std::shared_ptr<CVulkanTexture> vulkan_create_texture_from_dmabuf( struct wlr_dmabuf_attributes *pDMA );
 std::shared_ptr<CVulkanTexture> vulkan_create_texture_from_bits( uint32_t width, uint32_t height, uint32_t contentWidth, uint32_t contentHeight, uint32_t drmFormat, CVulkanTexture::createFlags texCreateFlags, void *bits );