diff --git a/Source/Core/Common/LibusbContext.cpp b/Source/Core/Common/LibusbContext.cpp
index a2ed128200..1eb729c4a0 100644
--- a/Source/Core/Common/LibusbContext.cpp
+++ b/Source/Core/Common/LibusbContext.cpp
@@ -8,6 +8,7 @@
 
 #include "Common/LibusbContext.h"
 #include "Common/MsgHandler.h"
+#include "Common/StringUtil.h"
 
 namespace LibusbContext
 {
@@ -24,10 +25,12 @@ static libusb_context* Create()
 #ifdef _WIN32
     is_windows = true;
 #endif
-    if (is_windows && ret == LIBUSB_ERROR_NOT_FOUND)
-      PanicAlertT("Failed to initialize libusb because usbdk is not installed.");
-    else
-      PanicAlertT("Failed to initialize libusb: %s", libusb_error_name(ret));
+    const std::string reason =
+        is_windows && ret == LIBUSB_ERROR_NOT_FOUND ?
+            GetStringT("Failed to initialize libusb because usbdk is not installed.") :
+            StringFromFormat(GetStringT("Failed to initialize libusb (%s).").c_str(),
+                             libusb_error_name(ret));
+    PanicAlertT("%s\nSome USB features will not work.", reason.c_str());
     return nullptr;
   }
   return context;
diff --git a/Source/Core/Core/IOS/USB/Bluetooth/BTReal.cpp b/Source/Core/Core/IOS/USB/Bluetooth/BTReal.cpp
index 0e7267b614..72d536fd3a 100644
--- a/Source/Core/Core/IOS/USB/Bluetooth/BTReal.cpp
+++ b/Source/Core/Core/IOS/USB/Bluetooth/BTReal.cpp
@@ -61,8 +61,6 @@ BluetoothReal::BluetoothReal(u32 device_id, const std::string& device_name)
     : BluetoothBase(device_id, device_name)
 {
   m_libusb_context = LibusbContext::Get();
-  _assert_msg_(IOS_WIIMOTE, m_libusb_context, "Failed to init libusb.");
-
   LoadLinkKeys();
 }
 
@@ -84,6 +82,9 @@ BluetoothReal::~BluetoothReal()
 
 ReturnCode BluetoothReal::Open(const OpenRequest& request)
 {
+  if (!m_libusb_context)
+    return IPC_EACCES;
+
   libusb_device** list;
   const ssize_t cnt = libusb_get_device_list(m_libusb_context.get(), &list);
   _dbg_assert_msg_(IOS, cnt > 0, "Couldn't get device list");
diff --git a/Source/Core/Core/IOS/USB/Host.cpp b/Source/Core/Core/IOS/USB/Host.cpp
index 875dc451f1..226c107eb0 100644
--- a/Source/Core/Core/IOS/USB/Host.cpp
+++ b/Source/Core/Core/IOS/USB/Host.cpp
@@ -30,10 +30,6 @@ namespace Device
 {
 USBHost::USBHost(u32 device_id, const std::string& device_name) : Device(device_id, device_name)
 {
-#ifdef __LIBUSB__
-  m_libusb_context = LibusbContext::Get();
-  _assert_msg_(IOS_USB, m_libusb_context, "Failed to init libusb.");
-#endif
 }
 
 ReturnCode USBHost::Open(const OpenRequest& request)
@@ -117,31 +113,40 @@ bool USBHost::AddNewDevices(std::set<u64>& new_devices, DeviceChangeHooks& hooks
                             const bool always_add_hooks)
 {
 #ifdef __LIBUSB__
-  libusb_device** list;
-  const ssize_t count = libusb_get_device_list(m_libusb_context.get(), &list);
-  if (count < 0)
-  {
-    WARN_LOG(IOS_USB, "Failed to get device list: %s", libusb_error_name(static_cast<int>(count)));
-    return false;
-  }
+  if (SConfig::GetInstance().m_usb_passthrough_devices.empty())
+    return true;
 
-  for (ssize_t i = 0; i < count; ++i)
+  auto libusb_context = LibusbContext::Get();
+  if (libusb_context)
   {
-    libusb_device* device = list[i];
-    libusb_device_descriptor descriptor;
-    libusb_get_device_descriptor(device, &descriptor);
-    if (!SConfig::GetInstance().IsUSBDeviceWhitelisted({descriptor.idVendor, descriptor.idProduct}))
-      continue;
+    libusb_device** list;
+    const ssize_t count = libusb_get_device_list(libusb_context.get(), &list);
+    if (count < 0)
+    {
+      WARN_LOG(IOS_USB, "Failed to get device list: %s",
+               libusb_error_name(static_cast<int>(count)));
+      return false;
+    }
 
-    auto usb_device = std::make_unique<USB::LibusbDevice>(device, descriptor);
-    if (!ShouldAddDevice(*usb_device))
-      continue;
-    const u64 id = usb_device->GetId();
-    new_devices.insert(id);
-    if (AddDevice(std::move(usb_device)) || always_add_hooks)
-      hooks.emplace(GetDeviceById(id), ChangeEvent::Inserted);
+    for (ssize_t i = 0; i < count; ++i)
+    {
+      libusb_device* device = list[i];
+      libusb_device_descriptor descriptor;
+      libusb_get_device_descriptor(device, &descriptor);
+      if (!SConfig::GetInstance().IsUSBDeviceWhitelisted(
+              {descriptor.idVendor, descriptor.idProduct}))
+        continue;
+
+      auto usb_device = std::make_unique<USB::LibusbDevice>(device, descriptor);
+      if (!ShouldAddDevice(*usb_device))
+        continue;
+      const u64 id = usb_device->GetId();
+      new_devices.insert(id);
+      if (AddDevice(std::move(usb_device)) || always_add_hooks)
+        hooks.emplace(GetDeviceById(id), ChangeEvent::Inserted);
+    }
+    libusb_free_device_list(list, 1);
   }
-  libusb_free_device_list(list, 1);
 #endif
   return true;
 }
@@ -200,10 +205,24 @@ void USBHost::StartThreads()
     m_event_thread_running.Set();
     m_event_thread = std::thread([this] {
       Common::SetCurrentThreadName("USB Passthrough Thread");
+      std::shared_ptr<libusb_context> context;
       while (m_event_thread_running.IsSet())
       {
+        if (SConfig::GetInstance().m_usb_passthrough_devices.empty())
+        {
+          Common::SleepCurrentThread(50);
+          continue;
+        }
+
+        if (!context)
+          context = LibusbContext::Get();
+
+        // If we failed to get a libusb context, stop the thread.
+        if (!context)
+          return;
+
         static timeval tv = {0, 50000};
-        libusb_handle_events_timeout_completed(m_libusb_context.get(), &tv, nullptr);
+        libusb_handle_events_timeout_completed(context.get(), &tv, nullptr);
       }
     });
   }
diff --git a/Source/Core/Core/IOS/USB/Host.h b/Source/Core/Core/IOS/USB/Host.h
index 40c09ca709..27d94d805f 100644
--- a/Source/Core/Core/IOS/USB/Host.h
+++ b/Source/Core/Core/IOS/USB/Host.h
@@ -71,7 +71,6 @@ private:
   void DispatchHooks(const DeviceChangeHooks& hooks);
 
 #ifdef __LIBUSB__
-  std::shared_ptr<libusb_context> m_libusb_context;
   // Event thread for libusb
   Common::Flag m_event_thread_running;
   std::thread m_event_thread;
diff --git a/Source/Core/InputCommon/GCAdapter.cpp b/Source/Core/InputCommon/GCAdapter.cpp
index c16b1ce635..1baaceee12 100644
--- a/Source/Core/InputCommon/GCAdapter.cpp
+++ b/Source/Core/InputCommon/GCAdapter.cpp
@@ -170,17 +170,8 @@ void Init()
 
   s_libusb_driver_not_supported = false;
 
-  s_libusb_context = LibusbContext::Get();
-  if (!s_libusb_context)
-  {
-    s_libusb_driver_not_supported = true;
-    Shutdown();
-  }
-  else
-  {
-    if (UseAdapter())
-      StartScanThread();
-  }
+  if (UseAdapter())
+    StartScanThread();
 }
 
 void StartScanThread()
@@ -188,6 +179,9 @@ void StartScanThread()
   if (s_adapter_detect_thread_running.IsSet())
     return;
 
+  s_libusb_context = LibusbContext::Get();
+  if (!s_libusb_context)
+    return;
   s_adapter_detect_thread_running.Set(true);
   s_adapter_detect_thread = std::thread(ScanThreadFunc);
 }
@@ -334,7 +328,7 @@ void Shutdown()
 {
   StopScanThread();
 #if defined(LIBUSB_API_VERSION) && LIBUSB_API_VERSION >= 0x01000102
-  if (s_libusb_hotplug_enabled)
+  if (s_libusb_context && s_libusb_hotplug_enabled)
     libusb_hotplug_deregister_callback(s_libusb_context.get(), s_hotplug_handle);
 #endif
   Reset();
diff --git a/Source/Core/UICommon/UICommon.cpp b/Source/Core/UICommon/UICommon.cpp
index 8de73bd151..0842d511bf 100644
--- a/Source/Core/UICommon/UICommon.cpp
+++ b/Source/Core/UICommon/UICommon.cpp
@@ -30,7 +30,6 @@ void Init()
   VideoBackendBase::PopulateList();
   WiimoteReal::LoadSettings();
   GCAdapter::Init();
-  USBUtils::Init();
   VideoBackendBase::ActivateBackend(SConfig::GetInstance().m_strVideoBackend);
 
   SetEnableAlert(SConfig::GetInstance().bUsePanicHandlers);
@@ -42,7 +41,6 @@ void Shutdown()
   WiimoteReal::Shutdown();
   VideoBackendBase::ClearList();
   SConfig::Shutdown();
-  USBUtils::Shutdown();
   LogManager::Shutdown();
 }
 
diff --git a/Source/Core/UICommon/USBUtils.cpp b/Source/Core/UICommon/USBUtils.cpp
index da7c603580..4fbabda856 100644
--- a/Source/Core/UICommon/USBUtils.cpp
+++ b/Source/Core/UICommon/USBUtils.cpp
@@ -3,6 +3,7 @@
 // Refer to the license.txt file included.
 
 #include <memory>
+#include <mutex>
 
 #ifdef __LIBUSB__
 #include <libusb.h>
@@ -14,6 +15,7 @@
 #include "UICommon/USBUtils.h"
 
 #ifdef __LIBUSB__
+static std::once_flag s_tried_libusb_init;
 static std::shared_ptr<libusb_context> s_libusb_context;
 #endif
 
@@ -36,25 +38,12 @@ static const std::map<std::pair<u16, u16>, std::string> s_wii_peripherals = {{
 
 namespace USBUtils
 {
-void Init()
-{
-#ifdef __LIBUSB__
-  s_libusb_context = LibusbContext::Get();
-#endif
-}
-
-void Shutdown()
-{
-#ifdef __LIBUSB__
-  s_libusb_context = nullptr;
-#endif
-}
-
 std::map<std::pair<u16, u16>, std::string> GetInsertedDevices()
 {
   std::map<std::pair<u16, u16>, std::string> devices;
 
 #ifdef __LIBUSB__
+  std::call_once(s_tried_libusb_init, []() { s_libusb_context = LibusbContext::Get(); });
   if (!s_libusb_context)
     return devices;
 
diff --git a/Source/Core/UICommon/USBUtils.h b/Source/Core/UICommon/USBUtils.h
index 6ae778e8a4..84d54a5381 100644
--- a/Source/Core/UICommon/USBUtils.h
+++ b/Source/Core/UICommon/USBUtils.h
@@ -12,9 +12,6 @@
 
 namespace USBUtils
 {
-void Init();
-void Shutdown();
-
 std::map<std::pair<u16, u16>, std::string> GetInsertedDevices();
 std::string GetDeviceName(std::pair<u16, u16> vid_pid);
 }  // namespace USBUtils