Skip encoder reprobing if no GPU changes have occurred

src/platform/common.h

@@ -558,6 +558,13 @@ namespace platf {
   std::vector<std::string>
   display_names(mem_type_e hwdevice_type);
 
+  /**
+   * @brief Returns if GPUs/drivers have changed since the last call to this function.
+   * @return `true` if a change has occurred or if it is unknown whether a change occurred.
+   */
+  bool
+  needs_encoder_reenumeration();
+
   boost::process::child
   run_command(bool elevated, bool interactive, const std::string &cmd, boost::filesystem::path &working_dir, const boost::process::environment &env, FILE *file, std::error_code &ec, boost::process::group *group);
 

src/platform/linux/misc.cpp

@@ -773,6 +773,16 @@ namespace platf {
     return {};
   }
 
+  /**
+   * @brief Returns if GPUs/drivers have changed since the last call to this function.
+   * @return `true` if a change has occurred or if it is unknown whether a change occurred.
+   */
+  bool
+  needs_encoder_reenumeration() {
+    // We don't track GPU state, so we will always reenumerate. Fortunately, it is fast on Linux.
+    return true;
+  }
+
   std::shared_ptr<display_t>
   display(mem_type_e hwdevice_type, const std::string &display_name, const video::config_t &config) {
 #ifdef SUNSHINE_BUILD_CUDA

src/platform/macos/display.mm

@@ -181,4 +181,14 @@ namespace platf {
 
     return display_names;
   }
+
+  /**
+   * @brief Returns if GPUs/drivers have changed since the last call to this function.
+   * @return `true` if a change has occurred or if it is unknown whether a change occurred.
+   */
+  bool
+  needs_encoder_reenumeration() {
+    // We don't track GPU state, so we will always reenumerate. Fortunately, it is fast on macOS.
+    return true;
+  }
 }  // namespace platf

src/platform/windows/display_base.cpp

@@ -1124,4 +1124,35 @@ namespace platf {
     return display_names;
   }
 
+  /**
+   * @brief Returns if GPUs/drivers have changed since the last call to this function.
+   * @return `true` if a change has occurred or if it is unknown whether a change occurred.
+   */
+  bool
+  needs_encoder_reenumeration() {
+    // Serialize access to the static DXGI factory
+    static std::mutex reenumeration_state_lock;
+    auto lg = std::lock_guard(reenumeration_state_lock);
+
+    // Keep a reference to the DXGI factory, which will keep track of changes internally.
+    static dxgi::factory1_t factory;
+    if (!factory || !factory->IsCurrent()) {
+      factory.reset();
+
+      auto status = CreateDXGIFactory1(IID_IDXGIFactory1, (void **) &factory);
+      if (FAILED(status)) {
+        BOOST_LOG(error) << "Failed to create DXGIFactory1 [0x"sv << util::hex(status).to_string_view() << ']';
+        factory.release();
+      }
+
+      // Always request reenumeration on the first streaming session just to ensure we
+      // can deal with any initialization races that may occur when the system is booting.
+      BOOST_LOG(info) << "Encoder reenumeration is required"sv;
+      return true;
+    }
+    else {
+      // The DXGI factory from last time is still current, so no encoder changes have occurred.
+      return false;
+    }
+  }
 }  // namespace platf

src/video.cpp

@@ -276,7 +276,7 @@ namespace video {
 
   enum flag_e : uint32_t {
     DEFAULT = 0,
-    PARALLEL_ENCODING = 1 << 1,
+    PARALLEL_ENCODING = 1 << 1,  // Capture and encoding can run concurrently on separate threads
     H264_ONLY = 1 << 2,  // When HEVC is too heavy
     LIMITED_GOP_SIZE = 1 << 3,  // Some encoders don't like it when you have an infinite GOP_SIZE. *cough* VAAPI *cough*
     SINGLE_SLICE_ONLY = 1 << 4,  // Never use multiple slices <-- Older intel iGPU's ruin it for everyone else :P
@@ -284,6 +284,7 @@ namespace video {
     RELAXED_COMPLIANCE = 1 << 6,  // Use FF_COMPLIANCE_UNOFFICIAL compliance mode
     NO_RC_BUF_LIMIT = 1 << 7,  // Don't set rc_buffer_size
     REF_FRAMES_INVALIDATION = 1 << 8,  // Support reference frames invalidation
+    ALWAYS_REPROBE = 1 << 9,  // This is an encoder of last resort and we want to aggressively probe for a better one
   };
 
   struct encoder_platform_formats_t {
@@ -922,7 +923,7 @@ namespace video {
       std::make_optional<encoder_t::option_t>("qp"s, &config::video.qp),
       "libx264"s,
     },
-    H264_ONLY | PARALLEL_ENCODING
+    H264_ONLY | PARALLEL_ENCODING | ALWAYS_REPROBE
   };
 
 #ifdef __linux__
@@ -2579,6 +2580,11 @@ namespace video {
   probe_encoders() {
     auto encoder_list = encoders;
 
+    // If we already have a good encoder, check to see if another probe is required
+    if (chosen_encoder && !(chosen_encoder->flags & ALWAYS_REPROBE) && !platf::needs_encoder_reenumeration()) {
+      return 0;
+    }
+
     // Restart encoder selection
     auto previous_encoder = chosen_encoder;
     chosen_encoder = nullptr;