diff --git a/rpcs3/Input/pad_thread.cpp b/rpcs3/Input/pad_thread.cpp
index 6a22185f92..b622f4c29a 100644
--- a/rpcs3/Input/pad_thread.cpp
+++ b/rpcs3/Input/pad_thread.cpp
@@ -90,7 +90,25 @@ void pad_thread::Init()
 	handlers.clear();
 
 	g_cfg_profile.load();
-	g_cfg_input.load(pad::g_title_id, g_cfg_profile.active_profiles.get_value(pad::g_title_id));
+
+	// Load in order to get the pad handlers
+	if (!g_cfg_input.load(pad::g_title_id, g_cfg_profile.active_profiles.get_value(pad::g_title_id)))
+	{
+		input_log.notice("Loaded empty pad config");
+	}
+
+	// Adjust to the different pad handlers
+	for (usz i = 0; i < g_cfg_input.player.size(); i++)
+	{
+		std::shared_ptr<PadHandlerBase> handler;
+		pad_thread::InitPadConfig(g_cfg_input.player[i]->config, g_cfg_input.player[i]->handler, handler);
+	}
+
+	// Reload with proper defaults
+	if (!g_cfg_input.load(pad::g_title_id, g_cfg_profile.active_profiles.get_value(pad::g_title_id)))
+	{
+		input_log.notice("Reloaded empty pad config");
+	}
 
 	std::shared_ptr<keyboard_pad_handler> keyptr;
 
@@ -336,3 +354,72 @@ void pad_thread::UnregisterLddPad(u32 handle)
 
 	num_ldd_pad--;
 }
+
+std::shared_ptr<PadHandlerBase> pad_thread::GetHandler(pad_handler type)
+{
+	switch (type)
+	{
+	case pad_handler::null:
+		return std::make_unique<NullPadHandler>();
+	case pad_handler::keyboard:
+		return std::make_unique<keyboard_pad_handler>();
+	case pad_handler::ds3:
+		return std::make_unique<ds3_pad_handler>();
+	case pad_handler::ds4:
+		return std::make_unique<ds4_pad_handler>();
+	case pad_handler::dualsense:
+		return std::make_unique<dualsense_pad_handler>();
+#ifdef _WIN32
+	case pad_handler::xinput:
+		return std::make_unique<xinput_pad_handler>();
+	case pad_handler::mm:
+		return std::make_unique<mm_joystick_handler>();
+#endif
+#ifdef HAVE_LIBEVDEV
+	case pad_handler::evdev:
+		return std::make_unique<evdev_joystick_handler>();
+#endif
+	}
+
+	return nullptr;
+}
+
+void pad_thread::InitPadConfig(cfg_pad& cfg, pad_handler type, std::shared_ptr<PadHandlerBase>& handler)
+{
+	if (!handler)
+	{
+		handler = GetHandler(type);
+	}
+
+	switch (handler->m_type)
+	{
+	case pad_handler::null:
+		static_cast<NullPadHandler*>(handler.get())->init_config(&cfg);
+		break;
+	case pad_handler::keyboard:
+		static_cast<keyboard_pad_handler*>(handler.get())->init_config(&cfg);
+		break;
+	case pad_handler::ds3:
+		static_cast<ds3_pad_handler*>(handler.get())->init_config(&cfg);
+		break;
+	case pad_handler::ds4:
+		static_cast<ds4_pad_handler*>(handler.get())->init_config(&cfg);
+		break;
+	case pad_handler::dualsense:
+		static_cast<dualsense_pad_handler*>(handler.get())->init_config(&cfg);
+		break;
+#ifdef _WIN32
+	case pad_handler::xinput:
+		static_cast<xinput_pad_handler*>(handler.get())->init_config(&cfg);
+		break;
+	case pad_handler::mm:
+		static_cast<mm_joystick_handler*>(handler.get())->init_config(&cfg);
+		break;
+#endif
+#ifdef HAVE_LIBEVDEV
+	case pad_handler::evdev:
+		static_cast<evdev_joystick_handler*>(handler.get())->init_config(&cfg);
+		break;
+#endif
+	}
+}
diff --git a/rpcs3/Input/pad_thread.h b/rpcs3/Input/pad_thread.h
index ccb0914524..f6ead8aac8 100644
--- a/rpcs3/Input/pad_thread.h
+++ b/rpcs3/Input/pad_thread.h
@@ -3,6 +3,7 @@
 #include "util/types.hpp"
 #include "util/atomic.hpp"
 #include "Emu/Io/pad_types.h"
+#include "Emu/Io/pad_config.h"
 #include "Emu/Io/pad_config_types.h"
 #include "Utilities/mutex.h"
 
@@ -31,6 +32,9 @@ public:
 	s32 AddLddPad();
 	void UnregisterLddPad(u32 handle);
 
+	static std::shared_ptr<PadHandlerBase> GetHandler(pad_handler type);
+	static void InitPadConfig(cfg_pad& cfg, pad_handler type, std::shared_ptr<PadHandlerBase>& handler);
+
 protected:
 	void InitLddPad(u32 handle);
 	void ThreadFunc();
diff --git a/rpcs3/rpcs3qt/pad_settings_dialog.cpp b/rpcs3/rpcs3qt/pad_settings_dialog.cpp
index c3ef64bb45..66ce4c807a 100644
--- a/rpcs3/rpcs3qt/pad_settings_dialog.cpp
+++ b/rpcs3/rpcs3qt/pad_settings_dialog.cpp
@@ -1120,35 +1120,6 @@ void pad_settings_dialog::OnTabChanged(int index)
 	RefreshHandlers();
 }
 
-std::shared_ptr<PadHandlerBase> pad_settings_dialog::GetHandler(pad_handler type) const
-{
-	switch (type)
-	{
-	case pad_handler::null:
-		return std::make_unique<NullPadHandler>();
-	case pad_handler::keyboard:
-		return std::make_unique<keyboard_pad_handler>();
-	case pad_handler::ds3:
-		return std::make_unique<ds3_pad_handler>();
-	case pad_handler::ds4:
-		return std::make_unique<ds4_pad_handler>();
-	case pad_handler::dualsense:
-		return std::make_unique<dualsense_pad_handler>();
-#ifdef _WIN32
-	case pad_handler::xinput:
-		return std::make_unique<xinput_pad_handler>();
-	case pad_handler::mm:
-		return std::make_unique<mm_joystick_handler>();
-#endif
-#ifdef HAVE_LIBEVDEV
-	case pad_handler::evdev:
-		return std::make_unique<evdev_joystick_handler>();
-#endif
-	}
-
-	return nullptr;
-}
-
 void pad_settings_dialog::ChangeHandler()
 {
 	bool force_enable = false; // enable configs even with disconnected devices
@@ -1180,11 +1151,12 @@ void pad_settings_dialog::ChangeHandler()
 		}
 
 		// Initialize the new pad config's defaults
-		InitPadConfig(cfg, cfg_handler);
+		m_handler = pad_thread::GetHandler(g_cfg_input.player[player]->handler);
+		pad_thread::InitPadConfig(cfg, cfg_handler, m_handler);
 	}
 	else
 	{
-		m_handler = GetHandler(g_cfg_input.player[player]->handler);
+		m_handler = pad_thread::GetHandler(g_cfg_input.player[player]->handler);
 	}
 
 	ensure(m_handler);
@@ -1357,43 +1329,6 @@ void pad_settings_dialog::ChangeHandler()
 	}
 }
 
-void pad_settings_dialog::InitPadConfig(cfg_pad& cfg, pad_handler type)
-{
-	m_handler = GetHandler(type);
-
-	switch (type)
-	{
-	case pad_handler::null:
-		static_cast<NullPadHandler*>(m_handler.get())->init_config(&cfg);
-		break;
-	case pad_handler::keyboard:
-		static_cast<keyboard_pad_handler*>(m_handler.get())->init_config(&cfg);
-		break;
-	case pad_handler::ds3:
-		static_cast<ds3_pad_handler*>(m_handler.get())->init_config(&cfg);
-		break;
-	case pad_handler::ds4:
-		static_cast<ds4_pad_handler*>(m_handler.get())->init_config(&cfg);
-		break;
-	case pad_handler::dualsense:
-		static_cast<dualsense_pad_handler*>(m_handler.get())->init_config(&cfg);
-		break;
-#ifdef _WIN32
-	case pad_handler::xinput:
-		static_cast<xinput_pad_handler*>(m_handler.get())->init_config(&cfg);
-		break;
-	case pad_handler::mm:
-		static_cast<mm_joystick_handler*>(m_handler.get())->init_config(&cfg);
-		break;
-#endif
-#ifdef HAVE_LIBEVDEV
-	case pad_handler::evdev:
-		static_cast<evdev_joystick_handler*>(m_handler.get())->init_config(&cfg);
-		break;
-#endif
-	}
-}
-
 void pad_settings_dialog::ChangeProfile(const QString& profile)
 {
 	if (profile.isEmpty())
@@ -1401,25 +1336,25 @@ void pad_settings_dialog::ChangeProfile(const QString& profile)
 
 	m_profile = sstr(profile);
 
-	std::unique_ptr<cfg_input> tmp = std::make_unique<cfg_input>();
-
-	if (!tmp->load(m_title_id, m_profile, true))
-	{
-		cfg_log.notice("Loaded empty temporary pad config");
-	}
-
-	// Adjust to the different pad handlers
-	for (usz i = 0; i < tmp->player.size(); i++)
-	{
-		auto& cfg = g_cfg_input.player[i]->config;
-		InitPadConfig(cfg, tmp->player[i]->handler);
-	}
-
+	// Load in order to get the pad handlers
 	if (!g_cfg_input.load(m_title_id, m_profile, true))
 	{
 		cfg_log.notice("Loaded empty pad config");
 	}
 
+	// Adjust to the different pad handlers
+	for (usz i = 0; i < g_cfg_input.player.size(); i++)
+	{
+		std::shared_ptr<PadHandlerBase> handler;
+		pad_thread::InitPadConfig(g_cfg_input.player[i]->config, g_cfg_input.player[i]->handler, handler);
+	}
+
+	// Reload with proper defaults
+	if (!g_cfg_input.load(m_title_id, m_profile, true))
+	{
+		cfg_log.notice("Reloaded empty pad config");
+	}
+
 	const u32 player_id = GetPlayerIndex();
 	const std::string handler = fmt::format("%s", g_cfg_input.player[player_id]->handler.get());
 
diff --git a/rpcs3/rpcs3qt/pad_settings_dialog.h b/rpcs3/rpcs3qt/pad_settings_dialog.h
index 30264e25b4..74b38d5206 100644
--- a/rpcs3/rpcs3qt/pad_settings_dialog.h
+++ b/rpcs3/rpcs3qt/pad_settings_dialog.h
@@ -179,13 +179,9 @@ private:
 	void InitButtons();
 	void ReloadButtons();
 
-	void InitPadConfig(cfg_pad& cfg, pad_handler type);
-
 	/** Repaints a stick deadzone preview label */
 	void RepaintPreviewLabel(QLabel* l, int deadzone, int desired_width, int x, int y, int squircle, double multiplier) const;
 
-	std::shared_ptr<PadHandlerBase> GetHandler(pad_handler type) const;
-
 	QString GetLocalizedPadHandler(const QString& original, pad_handler handler);
 
 	/** Checks if the port at the given index is already reserved by the application as custom controller (ldd pad) */