diff --git a/Source/Core/DolphinQt/GameList/GameList.cpp b/Source/Core/DolphinQt/GameList/GameList.cpp
index c8412b4902..e3056b4ef4 100644
--- a/Source/Core/DolphinQt/GameList/GameList.cpp
+++ b/Source/Core/DolphinQt/GameList/GameList.cpp
@@ -203,9 +203,13 @@ void GameList::UpdateColumnVisibility()
 
 void GameList::MakeEmptyView()
 {
+  const QString refreshing_msg = tr("Refreshing...");
+  const QString empty_msg = tr("Dolphin could not find any GameCube/Wii ISOs or WADs.\n"
+                               "Double-click here to set a games directory...");
+
   m_empty = new QLabel(this);
-  m_empty->setText(tr("Dolphin could not find any GameCube/Wii ISOs or WADs.\n"
-                      "Double-click here to set a games directory..."));
+  m_empty->setText(refreshing_msg);
+  m_empty->setEnabled(false);
   m_empty->setAlignment(Qt::AlignHCenter | Qt::AlignVCenter);
 
   auto event_filter = new DoubleClickEventFilter{m_empty};
@@ -216,6 +220,21 @@ void GameList::MakeEmptyView()
     if (!dir.isEmpty())
       Settings::Instance().AddPath(dir);
   });
+
+  QSizePolicy size_policy{m_empty->sizePolicy()};
+  size_policy.setRetainSizeWhenHidden(true);
+  m_empty->setSizePolicy(size_policy);
+
+  connect(&Settings::Instance(), &Settings::GameListRefreshRequested, this,
+          [this, refreshing_msg = refreshing_msg] {
+            m_empty->setText(refreshing_msg);
+            m_empty->setEnabled(false);
+          });
+  connect(&Settings::Instance(), &Settings::GameListRefreshCompleted, this,
+          [this, empty_msg = empty_msg] {
+            m_empty->setText(empty_msg);
+            m_empty->setEnabled(true);
+          });
 }
 
 void GameList::resizeEvent(QResizeEvent* event)
diff --git a/Source/Core/DolphinQt/GameList/GameTracker.cpp b/Source/Core/DolphinQt/GameList/GameTracker.cpp
index 5d8e5d20a2..a641640d6d 100644
--- a/Source/Core/DolphinQt/GameList/GameTracker.cpp
+++ b/Source/Core/DolphinQt/GameList/GameTracker.cpp
@@ -35,7 +35,10 @@ GameTracker::GameTracker(QObject* parent) : QFileSystemWatcher(parent)
   qRegisterMetaType<std::shared_ptr<const UICommon::GameFile>>();
   qRegisterMetaType<std::string>();
 
-  connect(qApp, &QApplication::aboutToQuit, this, [this] { m_load_thread.Cancel(); });
+  connect(qApp, &QApplication::aboutToQuit, this, [this] {
+    m_processing_halted = true;
+    m_load_thread.Cancel();
+  });
   connect(this, &QFileSystemWatcher::directoryChanged, this, &GameTracker::UpdateDirectory);
   connect(this, &QFileSystemWatcher::fileChanged, this, &GameTracker::UpdateFile);
   connect(&Settings::Instance(), &Settings::AutoRefreshToggled, this, [] {
@@ -75,27 +78,24 @@ GameTracker::GameTracker(QObject* parent) : QFileSystemWatcher(parent)
       break;
     case CommandType::UpdateMetadata:
       m_cache.UpdateAdditionalMetadata(
-          [this](const std::shared_ptr<const UICommon::GameFile>& game) {
-            emit GameUpdated(game);
-          });
+          [this](const std::shared_ptr<const UICommon::GameFile>& game) { emit GameUpdated(game); },
+          m_processing_halted);
       QueueOnObject(this, [] { Settings::Instance().NotifyMetadataRefreshComplete(); });
       break;
+    case CommandType::ResumeProcessing:
+      m_processing_halted = false;
+      break;
     case CommandType::PurgeCache:
       m_cache.Clear(UICommon::GameFileCache::DeleteOnDisk::Yes);
       break;
     case CommandType::BeginRefresh:
-      if (m_busy_count++ == 0)
-      {
-        for (auto& file : m_tracked_files.keys())
-          emit GameRemoved(file.toStdString());
-        m_tracked_files.clear();
-      }
+      QueueOnObject(this, [] { Settings::Instance().NotifyRefreshGameListStarted(); });
+      for (auto& file : m_tracked_files.keys())
+        emit GameRemoved(file.toStdString());
+      m_tracked_files.clear();
       break;
     case CommandType::EndRefresh:
-      if (--m_busy_count == 0)
-      {
-        QueueOnObject(this, [] { Settings::Instance().NotifyRefreshGameListComplete(); });
-      }
+      QueueOnObject(this, [] { Settings::Instance().NotifyRefreshGameListComplete(); });
       break;
     }
   });
@@ -135,6 +135,8 @@ void GameTracker::StartInternal()
 
   m_started = true;
 
+  QueueOnObject(this, [] { Settings::Instance().NotifyRefreshGameListStarted(); });
+
   std::vector<std::string> paths;
   paths.reserve(m_tracked_files.size());
   for (const QString& path : m_tracked_files.keys())
@@ -150,8 +152,9 @@ void GameTracker::StartInternal()
 
   m_initial_games_emitted_event.Wait();
 
-  bool cache_updated = m_cache.Update(paths, emit_game_loaded, emit_game_removed);
-  cache_updated |= m_cache.UpdateAdditionalMetadata(emit_game_updated);
+  bool cache_updated =
+      m_cache.Update(paths, emit_game_loaded, emit_game_removed, m_processing_halted);
+  cache_updated |= m_cache.UpdateAdditionalMetadata(emit_game_updated, m_processing_halted);
   if (cache_updated)
     m_cache.Save();
 
@@ -196,6 +199,16 @@ void GameTracker::RemoveDirectory(const QString& dir)
 
 void GameTracker::RefreshAll()
 {
+  m_processing_halted = true;
+  m_load_thread.Clear();
+  m_load_thread.EmplaceItem(Command{CommandType::ResumeProcessing, {}});
+
+  if (m_needs_purge)
+  {
+    m_load_thread.EmplaceItem(Command{CommandType::PurgeCache, {}});
+    m_needs_purge = false;
+  }
+
   m_load_thread.EmplaceItem(Command{CommandType::BeginRefresh});
 
   for (const QString& dir : Settings::Instance().GetPaths())
@@ -257,7 +270,7 @@ void GameTracker::RemoveDirectoryInternal(const QString& dir)
 void GameTracker::UpdateDirectoryInternal(const QString& dir)
 {
   auto it = GetIterator(dir);
-  while (it->hasNext() && !m_load_thread.IsCancelled())
+  while (it->hasNext() && !m_processing_halted)
   {
     QString path = QFileInfo(it->next()).canonicalFilePath();
 
@@ -277,6 +290,9 @@ void GameTracker::UpdateDirectoryInternal(const QString& dir)
 
   for (const auto& missing : FindMissingFiles(dir))
   {
+    if (m_processing_halted)
+      break;
+
     auto& tracked_file = m_tracked_files[missing];
 
     tracked_file.remove(dir);
@@ -347,6 +363,6 @@ void GameTracker::LoadGame(const QString& path)
 
 void GameTracker::PurgeCache()
 {
-  m_load_thread.EmplaceItem(Command{CommandType::PurgeCache, {}});
+  m_needs_purge = true;
   Settings::Instance().RefreshGameList();
 }
diff --git a/Source/Core/DolphinQt/GameList/GameTracker.h b/Source/Core/DolphinQt/GameList/GameTracker.h
index 93720614b3..135ee43fd0 100644
--- a/Source/Core/DolphinQt/GameList/GameTracker.h
+++ b/Source/Core/DolphinQt/GameList/GameTracker.h
@@ -4,6 +4,7 @@
 
 #pragma once
 
+#include <atomic>
 #include <memory>
 #include <string>
 
@@ -72,6 +73,7 @@ private:
     UpdateDirectory,
     UpdateFile,
     UpdateMetadata,
+    ResumeProcessing,
     PurgeCache,
     BeginRefresh,
     EndRefresh,
@@ -92,8 +94,8 @@ private:
   Common::Event m_initial_games_emitted_event;
   bool m_initial_games_emitted = false;
   bool m_started = false;
-  // Count of currently running refresh jobs
-  u32 m_busy_count = 0;
+  bool m_needs_purge = false;
+  std::atomic_bool m_processing_halted = false;
 };
 
 Q_DECLARE_METATYPE(std::shared_ptr<const UICommon::GameFile>)
diff --git a/Source/Core/DolphinQt/MenuBar.cpp b/Source/Core/DolphinQt/MenuBar.cpp
index 680e6c86cb..6b6617cff8 100644
--- a/Source/Core/DolphinQt/MenuBar.cpp
+++ b/Source/Core/DolphinQt/MenuBar.cpp
@@ -511,7 +511,7 @@ void MenuBar::AddViewMenu()
   purge_action->setEnabled(false);
   connect(&Settings::Instance(), &Settings::GameListRefreshRequested, purge_action,
           [purge_action] { purge_action->setEnabled(false); });
-  connect(&Settings::Instance(), &Settings::GameListRefreshCompleted, purge_action,
+  connect(&Settings::Instance(), &Settings::GameListRefreshStarted, purge_action,
           [purge_action] { purge_action->setEnabled(true); });
   view_menu->addSeparator();
   view_menu->addAction(tr("Search"), this, &MenuBar::ShowSearch, QKeySequence::Find);
diff --git a/Source/Core/DolphinQt/Settings.cpp b/Source/Core/DolphinQt/Settings.cpp
index 20ab451342..c7c156bfd5 100644
--- a/Source/Core/DolphinQt/Settings.cpp
+++ b/Source/Core/DolphinQt/Settings.cpp
@@ -147,6 +147,11 @@ void Settings::RefreshGameList()
   emit GameListRefreshRequested();
 }
 
+void Settings::NotifyRefreshGameListStarted()
+{
+  emit GameListRefreshStarted();
+}
+
 void Settings::NotifyRefreshGameListComplete()
 {
   emit GameListRefreshCompleted();
diff --git a/Source/Core/DolphinQt/Settings.h b/Source/Core/DolphinQt/Settings.h
index d1c2f38a20..2e493feddd 100644
--- a/Source/Core/DolphinQt/Settings.h
+++ b/Source/Core/DolphinQt/Settings.h
@@ -72,6 +72,7 @@ public:
   QString GetDefaultGame() const;
   void SetDefaultGame(QString path);
   void RefreshGameList();
+  void NotifyRefreshGameListStarted();
   void NotifyRefreshGameListComplete();
   void RefreshMetadata();
   void NotifyMetadataRefreshComplete();
@@ -150,6 +151,7 @@ signals:
   void PathRemoved(const QString&);
   void DefaultGameChanged(const QString&);
   void GameListRefreshRequested();
+  void GameListRefreshStarted();
   void GameListRefreshCompleted();
   void TitleDBReloadRequested();
   void MetadataRefreshRequested();
diff --git a/Source/Core/DolphinQt/ToolBar.cpp b/Source/Core/DolphinQt/ToolBar.cpp
index 5205bff5a1..e36bcd4f72 100644
--- a/Source/Core/DolphinQt/ToolBar.cpp
+++ b/Source/Core/DolphinQt/ToolBar.cpp
@@ -48,7 +48,7 @@ ToolBar::ToolBar(QWidget* parent) : QToolBar(parent)
 
   connect(&Settings::Instance(), &Settings::GameListRefreshRequested, this,
           [this] { m_refresh_action->setEnabled(false); });
-  connect(&Settings::Instance(), &Settings::GameListRefreshCompleted, this,
+  connect(&Settings::Instance(), &Settings::GameListRefreshStarted, this,
           [this] { m_refresh_action->setEnabled(true); });
 
   OnEmulationStateChanged(Core::GetState());
diff --git a/Source/Core/UICommon/GameFileCache.cpp b/Source/Core/UICommon/GameFileCache.cpp
index c551016d7f..71c73a5f9f 100644
--- a/Source/Core/UICommon/GameFileCache.cpp
+++ b/Source/Core/UICommon/GameFileCache.cpp
@@ -90,7 +90,8 @@ std::shared_ptr<const GameFile> GameFileCache::AddOrGet(const std::string& path,
 bool GameFileCache::Update(
     const std::vector<std::string>& all_game_paths,
     std::function<void(const std::shared_ptr<const GameFile>&)> game_added_to_cache,
-    std::function<void(const std::string&)> game_removed_from_cache)
+    std::function<void(const std::string&)> game_removed_from_cache,
+    const std::atomic_bool& processing_halted)
 {
   // Copy game paths into a set, except ones that match DiscIO::ShouldHideFromGameList.
   // TODO: Prevent DoFileSearch from looking inside /files/ directories of DirectoryBlobs at all?
@@ -113,6 +114,9 @@ bool GameFileCache::Update(
     auto end = m_cached_files.end();
     while (it != end)
     {
+      if (processing_halted)
+        break;
+
       if (game_paths.erase((*it)->GetFilePath()))
       {
         ++it;
@@ -134,6 +138,9 @@ bool GameFileCache::Update(
   // aren't in m_cached_files, so we simply add all of them to m_cached_files.
   for (const std::string& path : game_paths)
   {
+    if (processing_halted)
+      break;
+
     auto file = std::make_shared<GameFile>(path);
     if (file->IsValid())
     {
@@ -149,12 +156,16 @@ bool GameFileCache::Update(
 }
 
 bool GameFileCache::UpdateAdditionalMetadata(
-    std::function<void(const std::shared_ptr<const GameFile>&)> game_updated)
+    std::function<void(const std::shared_ptr<const GameFile>&)> game_updated,
+    const std::atomic_bool& processing_halted)
 {
   bool cache_changed = false;
 
   for (std::shared_ptr<GameFile>& file : m_cached_files)
   {
+    if (processing_halted)
+      break;
+
     const bool updated = UpdateAdditionalMetadata(&file);
     cache_changed |= updated;
     if (game_updated && updated)
diff --git a/Source/Core/UICommon/GameFileCache.h b/Source/Core/UICommon/GameFileCache.h
index 2e09c6da16..2c3fd39492 100644
--- a/Source/Core/UICommon/GameFileCache.h
+++ b/Source/Core/UICommon/GameFileCache.h
@@ -4,6 +4,7 @@
 
 #pragma once
 
+#include <atomic>
 #include <cstddef>
 #include <functional>
 #include <memory>
@@ -44,9 +45,11 @@ public:
   // These functions return true if the call modified the cache.
   bool Update(const std::vector<std::string>& all_game_paths,
               std::function<void(const std::shared_ptr<const GameFile>&)> game_added_to_cache = {},
-              std::function<void(const std::string&)> game_removed_from_cache = {});
+              std::function<void(const std::string&)> game_removed_from_cache = {},
+              const std::atomic_bool& processing_halted = false);
   bool UpdateAdditionalMetadata(
-      std::function<void(const std::shared_ptr<const GameFile>&)> game_updated = {});
+      std::function<void(const std::shared_ptr<const GameFile>&)> game_updated = {},
+      const std::atomic_bool& processing_halted = false);
 
   bool Load();
   bool Save();