Fix decoder switching and context re-usage.

The re-usage of the decoder context is not properly implemented.
Restart options will only be applied when the core is started.

cores/libretro-ffmpeg/ffmpeg_core.c

@@ -84,7 +84,6 @@ static int video_stream_index;
 static enum AVColorSpace colorspace;
 
 static enum AVPixelFormat pix_fmt;
-#define PIX_FMT_NOT_CONFIGURED -2
 
 static enum AVHWDeviceType hw_decoder;
 static bool force_sw_decoder;
@@ -330,7 +329,7 @@ void CORE_PREFIX(retro_reset)(void)
    reset_triggered = true;
 }
 
-static void check_variables(void)
+static void check_variables(bool firststart)
 {
    struct retro_variable hw_var  = {0};
    struct retro_variable sw_threads_var = {0};
@@ -395,6 +394,8 @@ static void check_variables(void)
       slock_unlock(decode_thread_lock);
    }
 
+   if (firststart)
+   {
       hw_var.key = "ffmpeg_hw_decoder";
 
       force_sw_decoder = false;
@@ -425,6 +426,7 @@ static void check_variables(void)
          else if (string_is_equal(hw_var.value, "videotoolbox"))
             hw_decoder = AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
       }
+   }
 
    sw_threads_var.key = "ffmpeg_sw_decoder_threads";
    if (CORE_PREFIX(environ_cb)(RETRO_ENVIRONMENT_GET_VARIABLE, &sw_threads_var) && sw_threads_var.value)
@@ -495,7 +497,7 @@ void CORE_PREFIX(retro_run)(void)
 #endif
 
    if (CORE_PREFIX(environ_cb)(RETRO_ENVIRONMENT_GET_VARIABLE_UPDATE, &updated) && updated)
-      check_variables();
+      check_variables(false);
 
 #ifdef HAVE_GL_FFT
    if (fft_width != old_fft_width || fft_height != old_fft_height)
@@ -829,15 +831,13 @@ static enum AVPixelFormat init_hw_decoder(struct AVCodecContext *ctx,
          enum AVPixelFormat device_pix_fmt = config->pix_fmt;
 
          /* Look if codec can supports the pix format of the device */
-         const enum AVPixelFormat *p;
+         for (size_t i = 0; pix_fmts[i] != AV_PIX_FMT_NONE; i++)
-         for (p = pix_fmts; *p != -1; p++)
+            if (pix_fmts[i] == device_pix_fmt)
             {
-            if (*p == device_pix_fmt)
+               decoder_pix_fmt = pix_fmts[i];
-            {
-               decoder_pix_fmt =  *p;
                goto exit;
             }
-         }
+
          log_cb(RETRO_LOG_ERROR, "[FFMPEG] Codec %s does not support device pixel format %s.\n",
                codec->name, av_get_pix_fmt_name(config->pix_fmt));
       }
@@ -852,9 +852,9 @@ exit:
       {
          log_cb(RETRO_LOG_ERROR, "[FFMPEG] Failed to create specified HW device: %s\n", av_err2str(ret));
          decoder_pix_fmt = AV_PIX_FMT_NONE;
-      } else {
-         ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
       }
+      else
+         ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
    }
 
    return decoder_pix_fmt;
@@ -878,20 +878,19 @@ static enum AVPixelFormat auto_hw_decoder(AVCodecContext *ctx,
    return decoder_pix_fmt;
 }
 
-/* Callback used by ffmpeg to configure the pixelformat to use.
+/*
+ * Callback used by ffmpeg to configure the pixelformat to use.
  * Used to initialize hw decoding if configured and accessible.
  */
 static enum AVPixelFormat get_format(AVCodecContext *ctx,
                                      const enum AVPixelFormat *pix_fmts)
 {
-   /* 
+   /* Look if we can reuse the current context */
-    * We only need to update the pixel format once new content has been loaded.
+   for (size_t i = 0; pix_fmts[i] != AV_PIX_FMT_NONE; i++)
-    * This also means that you need to reload the core for changing the sw thread
+      if (pix_fmts[i] == pix_fmt)
-    * core count and decoder backend.
-    */
-   if (pix_fmt == PIX_FMT_NOT_CONFIGURED)
       {
-      pix_fmt = AV_PIX_FMT_NONE;
+         return pix_fmt;
+      }
 
    if (!force_sw_decoder)
    {
@@ -900,10 +899,8 @@ static enum AVPixelFormat get_format(AVCodecContext *ctx,
          pix_fmt = auto_hw_decoder(ctx, pix_fmts);
       }
       else
-         {
          pix_fmt = init_hw_decoder(ctx, pix_fmts, hw_decoder);
    }
-      }
 
    /* Fallback to SW rendering */
    if (pix_fmt == AV_PIX_FMT_NONE)
@@ -918,7 +915,6 @@ static enum AVPixelFormat get_format(AVCodecContext *ctx,
    }
    else
       hw_decoding_enabled = true;
-   }
 
    return pix_fmt;
 }
@@ -1040,7 +1036,7 @@ static bool open_codecs(void)
             {
                if (!open_codec(&vctx, i))
                   return false;
-               pix_fmt = PIX_FMT_NOT_CONFIGURED;
+               pix_fmt = AV_PIX_FMT_NONE;
                vctx->get_format  = get_format;
                video_stream_index = i;
             }
@@ -1871,7 +1867,7 @@ bool CORE_PREFIX(retro_load_game)(const struct retro_game_info *info)
    decode_thread_dead = false;
    slock_unlock(fifo_lock);
 
-   check_variables();
+   check_variables(true);
 
    decode_thread_handle = sthread_create(decode_thread, NULL);