Gracefully abort headless mode with unsupported video renderers

Also fix no_return bug

rpcs3/headless_application.cpp

@@ -1,6 +1,7 @@
 #include "headless_application.h"
 
 #include "Emu/RSX/GSRender.h"
+#include "Emu/RSX/Null/NullGSRender.h"
 #include "Emu/Cell/Modules/cellMsgDialog.h"
 #include "Emu/Cell/Modules/cellOskDialog.h"
 #include "Emu/Cell/Modules/cellSaveData.h"
@@ -54,7 +55,38 @@ void headless_application::InitializeCallbacks()
 		RequestCallAfter(std::move(func));
 	};
 
-	callbacks.get_gs_frame                   = []() -> std::unique_ptr<GSFrameBase> { return std::unique_ptr<GSFrameBase>(); };
+	callbacks.init_gs_render = []()
+	{
+		switch (video_renderer type = g_cfg.video.renderer)
+		{
+		case video_renderer::null:
+		{
+			g_fxo->init<rsx::thread, named_thread<NullGSRender>>();
+			break;
+		}
+		case video_renderer::opengl:
+#if defined(_WIN32) || defined(HAVE_VULKAN)
+		case video_renderer::vulkan:
+#endif
+		{
+			fmt::throw_exception("Headless mode can only be used with the %s video renderer. Current renderer: %s", video_renderer::null, type);
+		}
+		default:
+		{
+			fmt::throw_exception("Invalid video renderer: %s" HERE, type);
+		}
+		}
+	};
+
+	callbacks.get_gs_frame = []() -> std::unique_ptr<GSFrameBase>
+	{
+		if (g_cfg.video.renderer != video_renderer::null)
+		{
+			fmt::throw_exception("Headless mode can only be used with the %s video renderer. Current renderer: %s", video_renderer::null, g_cfg.video.renderer.get());
+		}
+		return std::unique_ptr<GSFrameBase>();
+	};
+
 	callbacks.get_msg_dialog                 = []() -> std::shared_ptr<MsgDialogBase> { return std::shared_ptr<MsgDialogBase>(); };
 	callbacks.get_osk_dialog                 = []() -> std::shared_ptr<OskDialogBase> { return std::shared_ptr<OskDialogBase>(); };
 	callbacks.get_save_dialog                = []() -> std::unique_ptr<SaveDialogBase> { return std::unique_ptr<SaveDialogBase>(); };

rpcs3/main.cpp

@@ -62,7 +62,7 @@ LOG_CHANNEL(sys_log, "SYS");
 	if (s_headless)
 	{
 		fprintf(stderr, "RPCS3: %s\n", text.c_str());
-		return;
+		std::abort();
 	}
 
 	const bool local = s_qt_init.try_lock();

rpcs3/main_application.cpp

@@ -2,6 +2,8 @@
 
 #include "Input/pad_thread.h"
 #include "Emu/System.h"
+#include "Emu/system_config.h"
+#include "Emu/IdManager.h"
 #include "Emu/Io/Null/NullKeyboardHandler.h"
 #include "Emu/Io/Null/NullMouseHandler.h"
 #include "Emu/Io/KeyboardHandler.h"
@@ -25,14 +27,6 @@
 #include "Emu/Audio/FAudio/FAudioBackend.h"
 #endif
 
-#include "Emu/RSX/GSRender.h"
-#include "Emu/RSX/Null/NullGSRender.h"
-#include "Emu/RSX/GL/GLGSRender.h"
-
-#if defined(_WIN32) || defined(HAVE_VULKAN)
-#include "Emu/RSX/VK/VKGSRender.h"
-#endif
-
 LOG_CHANNEL(sys_log, "SYS");
 
 /** Emu.Init() wrapper for user manager */
@@ -110,31 +104,6 @@ EmuCallbacks main_application::CreateCallbacks()
 		g_fxo->init<pad_thread>(get_thread(), m_game_window, title_id);
 	};
 
-	callbacks.init_gs_render = []()
-	{
-		switch (video_renderer type = g_cfg.video.renderer)
-		{
-		case video_renderer::null:
-		{
-			g_fxo->init<rsx::thread, named_thread<NullGSRender>>();
-			break;
-		}
-		case video_renderer::opengl:
-		{
-			g_fxo->init<rsx::thread, named_thread<GLGSRender>>();
-			break;
-		}
-#if defined(_WIN32) || defined(HAVE_VULKAN)
-		case video_renderer::vulkan:
-		{
-			g_fxo->init<rsx::thread, named_thread<VKGSRender>>();
-			break;
-		}
-#endif
-		default: fmt::throw_exception("Invalid video renderer: %s" HERE, type);
-		}
-	};
-
 	callbacks.get_audio = []() -> std::shared_ptr<AudioBackend>
 	{
 		std::shared_ptr<AudioBackend> result;

rpcs3/rpcs3qt/gui_application.cpp

@@ -29,6 +29,14 @@
 
 #include <clocale>
 
+#include "Emu/RSX/GSRender.h"
+#include "Emu/RSX/Null/NullGSRender.h"
+#include "Emu/RSX/GL/GLGSRender.h"
+
+#if defined(_WIN32) || defined(HAVE_VULKAN)
+#include "Emu/RSX/VK/VKGSRender.h"
+#endif
+
 LOG_CHANNEL(gui_log, "GUI");
 
 gui_application::gui_application(int& argc, char** argv) : QApplication(argc, argv)
@@ -290,6 +298,34 @@ void gui_application::InitializeCallbacks()
 		RequestCallAfter(std::move(func));
 	};
 
+	callbacks.init_gs_render = []()
+	{
+		switch (video_renderer type = g_cfg.video.renderer)
+		{
+		case video_renderer::null:
+		{
+			g_fxo->init<rsx::thread, named_thread<NullGSRender>>();
+			break;
+		}
+		case video_renderer::opengl:
+		{
+			g_fxo->init<rsx::thread, named_thread<GLGSRender>>();
+			break;
+		}
+#if defined(_WIN32) || defined(HAVE_VULKAN)
+		case video_renderer::vulkan:
+		{
+			g_fxo->init<rsx::thread, named_thread<VKGSRender>>();
+			break;
+		}
+#endif
+		default:
+		{
+			fmt::throw_exception("Invalid video renderer: %s" HERE, type);
+		}
+		}
+	};
+
 	callbacks.get_gs_frame    = [this]() -> std::unique_ptr<GSFrameBase> { return get_gs_frame(); };
 	callbacks.get_msg_dialog  = [this]() -> std::shared_ptr<MsgDialogBase> { return m_show_gui ? std::make_shared<msg_dialog_frame>() : nullptr; };
 	callbacks.get_osk_dialog  = [this]() -> std::shared_ptr<OskDialogBase> { return m_show_gui ? std::make_shared<osk_dialog_frame>() : nullptr; };