Add amd decoder support

.gitignore

@@ -5,4 +5,4 @@ cmake-build-*
 *.swp
 *.kdev4
 
-.idea
+.idea

sunshine/config.cpp

@@ -85,6 +85,60 @@ int coder_from_view(const std::string_view &coder) {
 }
 }
 
+namespace amd {
+enum quality_e : int {
+  _default = 0,
+  speed,
+  balanced,
+  //quality2,
+};
+
+enum rc_e : int {
+  constqp   = 0x0,       /**< Constant QP mode */
+  vbr       = 0x1,       /**< Variable bitrate mode */
+  cbr       = 0x2,       /**< Constant bitrate mode */
+  cbr_ld_hq = 0x8,       /**< low-delay CBR, high quality */
+  cbr_hq    = 0x10,      /**< CBR, high quality (slower) */
+  vbr_hq    = 0x20       /**< VBR, high quality (slower) */
+};
+
+enum coder_e : int {
+  _auto = 0,
+  cabac,
+  cavlc
+};
+
+std::optional<quality_e> quality_from_view(const std::string_view &quality) {
+#define _CONVERT_(x) if(quality == #x##sv) return x
+  _CONVERT_(speed);
+  _CONVERT_(balanced);
+  //_CONVERT_(quality2);
+  if(quality == "default"sv) return _default;
+#undef _CONVERT_
+  return std::nullopt;
+}
+
+std::optional<rc_e> rc_from_view(const std::string_view &rc) {
+#define _CONVERT_(x) if(rc == #x##sv) return x
+  _CONVERT_(constqp);
+  _CONVERT_(vbr);
+  _CONVERT_(cbr);
+  _CONVERT_(cbr_hq);
+  _CONVERT_(vbr_hq);
+  _CONVERT_(cbr_ld_hq);
+#undef _CONVERT_
+  return std::nullopt;
+}
+
+int coder_from_view(const std::string_view &coder) {
+  if(coder == "auto"sv) return _auto;
+  if(coder == "cabac"sv  || coder == "ac"sv) return cabac;
+  if(coder == "cavlc"sv  || coder == "vlc"sv) return cavlc;
+
+  return -1;
+}
+}
+
 video_t video {
   0, // crf
   28, // qp
@@ -103,6 +157,12 @@ video_t video {
     -1
   }, // nv
 
+  {
+    amd::balanced,
+    std::nullopt,
+    -1
+  }, // amd  
+
   {}, // encoder
   {}, // adapter_name
   {}  // output_name
@@ -357,8 +417,13 @@ void apply_config(std::unordered_map<std::string, std::string> &&vars) {
   string_f(vars, "sw_preset", video.sw.preset);
   string_f(vars, "sw_tune", video.sw.tune);
   int_f(vars, "nv_preset", video.nv.preset, nv::preset_from_view);
-  int_f(vars, "nv_rc", video.nv.preset, nv::rc_from_view);
+  int_f(vars, "nv_rc", video.nv.rc, nv::rc_from_view);
   int_f(vars, "nv_coder", video.nv.coder, nv::coder_from_view);
+
+  int_f(vars, "amd_quality", video.amd.quality, amd::quality_from_view);
+  int_f(vars, "amd_rc", video.amd.rc, amd::rc_from_view);
+  int_f(vars, "amd_coder", video.amd.coder, amd::coder_from_view);
+
   string_f(vars, "encoder", video.encoder);
   string_f(vars, "adapter_name", video.adapter_name);
   string_f(vars, "output_name", video.output_name);

sunshine/config.h

@@ -26,6 +26,12 @@ struct video_t {
     int coder;
   } nv;
 
+  struct {
+    std::optional<int> quality;
+    std::optional<int> rc;
+    int coder;
+  } amd;
+
   std::string encoder;
   std::string adapter_name;
   std::string output_name;

sunshine/video.cpp

@@ -58,6 +58,20 @@ enum class profile_hevc_e : int {
 };
 }
 
+namespace amd {
+
+enum class profile_h264_e : int {
+  main,
+  high,
+  constrained_baseline,
+  constrained_high,
+};
+
+enum class profile_hevc_e : int {
+  main,
+};
+}
+
 using ctx_t       = util::safe_ptr<AVCodecContext, free_ctx>;
 using frame_t     = util::safe_ptr<AVFrame, free_frame>;
 using buffer_t    = util::safe_ptr<AVBufferRef, free_buffer>;
@@ -70,6 +84,8 @@ platf::pix_fmt_e map_pix_fmt(AVPixelFormat fmt);
 void sw_img_to_frame(const platf::img_t &img, frame_t &frame);
 void nv_d3d_img_to_frame(const platf::img_t &img, frame_t &frame);
 util::Either<buffer_t, int> nv_d3d_make_hwdevice_ctx(platf::hwdevice_t *hwdevice_ctx);
+void amd_d3d_img_to_frame(const platf::img_t &img, frame_t &frame);
+util::Either<buffer_t, int> amd_d3d_make_hwdevice_ctx(platf::hwdevice_t *hwdevice_ctx);
 
 util::Either<buffer_t, int> make_hwdevice_ctx(AVHWDeviceType type, void *hwdevice_ctx);
 int hwframe_ctx(ctx_t &ctx, buffer_t &hwdevice, AVPixelFormat format);
@@ -284,6 +300,38 @@ static encoder_t nvenc {
   nv_d3d_img_to_frame,
   nv_d3d_make_hwdevice_ctx
 };
+
+static encoder_t amdvce {
+  "amdvce"sv,
+  { (int)amd::profile_h264_e::high, (int)amd::profile_hevc_e::main },
+  //AV_HWDEVICE_TYPE_D3D11VA,
+  //AV_PIX_FMT_NONE,
+  //AV_PIX_FMT_YUV420P,
+  AV_HWDEVICE_TYPE_D3D11VA,
+  AV_PIX_FMT_D3D11,
+  AV_PIX_FMT_NV12, AV_PIX_FMT_YUV420P,
+  {
+    {
+      { "quality"s, &config::video.amd.quality },
+      { "rc"s, &config::video.amd.rc }
+    },
+    std::nullopt, std::nullopt,
+    "hevc_amf"s,
+  },
+  {
+    {
+      { "quality"s, &config::video.amd.quality },
+      { "rc"s, &config::video.amd.rc }
+    },
+    std::nullopt, std::make_optional<encoder_t::option_t>({"qp"s, &config::video.qp}),
+    "h264_amf"s
+  },
+  false,
+  true,
+
+  amd_d3d_img_to_frame,
+  amd_d3d_make_hwdevice_ctx
+};
 #endif
 
 static encoder_t software {
@@ -323,6 +371,7 @@ static encoder_t software {
 static std::vector<encoder_t> encoders {
 #ifdef _WIN32
   nvenc,
+  amdvce,
 #endif
   software
 };
@@ -1249,6 +1298,30 @@ void nv_d3d_img_to_frame(const platf::img_t &img, frame_t &frame) {
   frame->width = img.width;
 }
 
+void amd_d3d_img_to_frame(const platf::img_t &img, frame_t &frame) {
+  if(img.data == frame->data[0]) {
+    return;
+  }
+  
+  // Need to have something refcounted
+  if(!frame->buf[0]) {
+    frame->buf[0] = av_buffer_allocz(sizeof(AVD3D11FrameDescriptor));
+  }
+
+  auto desc = (AVD3D11FrameDescriptor*)frame->buf[0]->data;
+  desc->texture = (ID3D11Texture2D*)img.data;
+  desc->index = 0;
+
+  frame->data[0] = img.data;
+  frame->data[1] = 0;
+
+  frame->linesize[0] = img.row_pitch;
+
+  frame->height = img.height;
+  frame->width = img.width;
+}
+
+
 util::Either<buffer_t, int> nv_d3d_make_hwdevice_ctx(platf::hwdevice_t *hwdevice_ctx) {
   buffer_t ctx_buf { av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_D3D11VA) };
   auto ctx = (AVD3D11VADeviceContext*)((AVHWDeviceContext*)ctx_buf->data)->hwctx;
@@ -1269,6 +1342,27 @@ util::Either<buffer_t, int> nv_d3d_make_hwdevice_ctx(platf::hwdevice_t *hwdevice
 
   return ctx_buf;
 }
+
+util::Either<buffer_t, int> amd_d3d_make_hwdevice_ctx(platf::hwdevice_t *hwdevice_ctx) {
+  buffer_t ctx_buf { av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_D3D11VA) };
+  auto ctx = (AVD3D11VADeviceContext*)((AVHWDeviceContext*)ctx_buf->data)->hwctx;
+  
+  std::fill_n((std::uint8_t*)ctx, sizeof(AVD3D11VADeviceContext), 0);
+
+  auto device = (ID3D11Device*)hwdevice_ctx->data;
+  device->AddRef();
+  ctx->device = device;
+
+  auto err = av_hwdevice_ctx_init(ctx_buf.get());
+  if(err) {
+    char err_str[AV_ERROR_MAX_STRING_SIZE] {0};
+    BOOST_LOG(error) << "Failed to create FFMpeg amddech: "sv << av_make_error_string(err_str, AV_ERROR_MAX_STRING_SIZE, err);
+
+    return err;
+  }
+
+  return ctx_buf;
+}
 #endif
 
 int start_capture_async(capture_thread_async_ctx_t &capture_thread_ctx) {