Sunshine/src/platform/linux/vaapi.cpp

/**
 * @file src/platform/linux/vaapi.cpp
 * @brief todo
 */
#include <sstream>
#include <string>

#include <fcntl.h>

extern "C" {
#include <libavcodec/avcodec.h>
#include <va/va.h>
#if !VA_CHECK_VERSION(1, 9, 0)
// vaSyncBuffer stub allows Sunshine built against libva <2.9.0 to link against ffmpeg on libva 2.9.0 or later
VAStatus
vaSyncBuffer(
  VADisplay dpy,
  VABufferID buf_id,
  uint64_t timeout_ns) {
  return VA_STATUS_ERROR_UNIMPLEMENTED;
}
#endif
}

#include "graphics.h"
#include "misc.h"
#include "src/config.h"
#include "src/main.h"
#include "src/platform/common.h"
#include "src/utility.h"
#include "src/video.h"

using namespace std::literals;

extern "C" struct AVBufferRef;

namespace va {
  constexpr auto SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2 = 0x40000000;
  constexpr auto EXPORT_SURFACE_WRITE_ONLY = 0x0002;
  constexpr auto EXPORT_SURFACE_COMPOSED_LAYERS = 0x0008;

  using VADisplay = void *;
  using VAStatus = int;
  using VAGenericID = unsigned int;
  using VASurfaceID = VAGenericID;

  struct DRMPRIMESurfaceDescriptor {
    // VA Pixel format fourcc of the whole surface (VA_FOURCC_*).
    uint32_t fourcc;

    uint32_t width;
    uint32_t height;

    // Number of distinct DRM objects making up the surface.
    uint32_t num_objects;

    struct {
      // DRM PRIME file descriptor for this object.
      // Needs to be closed manually
      int fd;

      // Total size of this object (may include regions which are not part of the surface)
      uint32_t size;
      // Format modifier applied to this object, not sure what that means
      uint64_t drm_format_modifier;
    } objects[4];

    // Number of layers making up the surface.
    uint32_t num_layers;
    struct {
      // DRM format fourcc of this layer (DRM_FOURCC_*).
      uint32_t drm_format;

      // Number of planes in this layer.
      uint32_t num_planes;

      // references objects --> DRMPRIMESurfaceDescriptor.objects[object_index[0]]
      uint32_t object_index[4];

      // Offset within the object of each plane.
      uint32_t offset[4];

      // Pitch of each plane.
      uint32_t pitch[4];
    } layers[4];
  };

  /**
   * @brief Defined profiles
   */
  enum class profile_e {
    // Profile ID used for video processing.
    ProfileNone = -1,
    MPEG2Simple = 0,
    MPEG2Main = 1,
    MPEG4Simple = 2,
    MPEG4AdvancedSimple = 3,
    MPEG4Main = 4,
    H264Baseline = 5,
    H264Main = 6,
    H264High = 7,
    VC1Simple = 8,
    VC1Main = 9,
    VC1Advanced = 10,
    H263Baseline = 11,
    JPEGBaseline = 12,
    H264ConstrainedBaseline = 13,
    VP8Version0_3 = 14,
    H264MultiviewHigh = 15,
    H264StereoHigh = 16,
    HEVCMain = 17,
    HEVCMain10 = 18,
    VP9Profile0 = 19,
    VP9Profile1 = 20,
    VP9Profile2 = 21,
    VP9Profile3 = 22,
    HEVCMain12 = 23,
    HEVCMain422_10 = 24,
    HEVCMain422_12 = 25,
    HEVCMain444 = 26,
    HEVCMain444_10 = 27,
    HEVCMain444_12 = 28,
    HEVCSccMain = 29,
    HEVCSccMain10 = 30,
    HEVCSccMain444 = 31,
    AV1Profile0 = 32,
    AV1Profile1 = 33,
    HEVCSccMain444_10 = 34,

    // Profile ID used for protected video playback.
    Protected = 35
  };

  enum class entry_e {
    VLD = 1,
    IZZ = 2,
    IDCT = 3,
    MoComp = 4,
    Deblocking = 5,
    EncSlice = 6, /** slice level encode */
    EncPicture = 7, /** picture encode, JPEG, etc */
    /**
     * For an implementation that supports a low power/high performance variant
     * for slice level encode, it can choose to expose the
     * VAEntrypointEncSliceLP entrypoint. Certain encoding tools may not be
     * available with this entrypoint (e.g. interlace, MBAFF) and the
     * application can query the encoding configuration attributes to find
     * out more details if this entrypoint is supported.
     */
    EncSliceLP = 8,
    VideoProc = 10, /**< Video pre/post-processing. */
    /**
     * @brief FEI
     *
     * The purpose of FEI (Flexible Encoding Infrastructure) is to allow applications to
     * have more controls and trade off quality for speed with their own IPs.
     * The application can optionally provide input to ENC for extra encode control
     * and get the output from ENC. Application can chose to modify the ENC
     * output/PAK input during encoding, but the performance impact is significant.
     *
     * On top of the existing buffers for normal encode, there will be
     * one extra input buffer (VAEncMiscParameterFEIFrameControl) and
     * three extra output buffers (VAEncFEIMVBufferType, VAEncFEIMBModeBufferType
     * and VAEncFEIDistortionBufferType) for FEI entry function.
     * If separate PAK is set, two extra input buffers
     * (VAEncFEIMVBufferType, VAEncFEIMBModeBufferType) are needed for PAK input.
     */
    FEI = 11,
    /**
     * @brief Stats
     *
     * A pre-processing function for getting some statistics and motion vectors is added,
     * and some extra controls for Encode pipeline are provided. The application can
     * optionally call the statistics function to get motion vectors and statistics like
     * variances, distortions before calling Encode function via this entry point.
     *
     * Checking whether Statistics is supported can be performed with vaQueryConfigEntrypoints().
     * If Statistics entry point is supported, then the list of returned entry-points will
     * include #Stats. Supported pixel format, maximum resolution and statistics
     * specific attributes can be obtained via normal attribute query. One input buffer
     * (VAStatsStatisticsParameterBufferType) and one or two output buffers
     * (VAStatsStatisticsBufferType, VAStatsStatisticsBottomFieldBufferType (for interlace only)
     * and VAStatsMVBufferType) are needed for this entry point.
     */
    Stats = 12,
    /**
     * @brief ProtectedTEEComm
     *
     * A function for communicating with TEE (Trusted Execution Environment).
     */
    ProtectedTEEComm = 13,
    /**
     * @brief ProtectedContent
     *
     * A function for protected content to decrypt encrypted content.
     */
    ProtectedContent = 14,
  };

  typedef VAStatus (*queryConfigEntrypoints_fn)(VADisplay dpy, profile_e profile, entry_e *entrypoint_list, int *num_entrypoints);
  typedef int (*maxNumEntrypoints_fn)(VADisplay dpy);
  typedef VADisplay (*getDisplayDRM_fn)(int fd);
  typedef VAStatus (*terminate_fn)(VADisplay dpy);
  typedef VAStatus (*initialize_fn)(VADisplay dpy, int *major_version, int *minor_version);
  typedef const char *(*errorStr_fn)(VAStatus error_status);
  typedef void (*VAMessageCallback)(void *user_context, const char *message);
  typedef VAMessageCallback (*setErrorCallback_fn)(VADisplay dpy, VAMessageCallback callback, void *user_context);
  typedef VAMessageCallback (*setInfoCallback_fn)(VADisplay dpy, VAMessageCallback callback, void *user_context);
  typedef const char *(*queryVendorString_fn)(VADisplay dpy);
  typedef VAStatus (*exportSurfaceHandle_fn)(
    VADisplay dpy, VASurfaceID surface_id,
    uint32_t mem_type, uint32_t flags,
    void *descriptor);

  static maxNumEntrypoints_fn maxNumEntrypoints;
  static queryConfigEntrypoints_fn queryConfigEntrypoints;
  static getDisplayDRM_fn getDisplayDRM;
  static terminate_fn terminate;
  static initialize_fn initialize;
  static errorStr_fn errorStr;
  static setErrorCallback_fn setErrorCallback;
  static setInfoCallback_fn setInfoCallback;
  static queryVendorString_fn queryVendorString;
  static exportSurfaceHandle_fn exportSurfaceHandle;

  using display_t = util::dyn_safe_ptr_v2<void, VAStatus, &terminate>;

  int
  init_main_va() {
    static void *handle { nullptr };
    static bool funcs_loaded = false;

    if (funcs_loaded) return 0;

    if (!handle) {
      handle = dyn::handle({ "libva.so.2", "libva.so" });
      if (!handle) {
        return -1;
      }
    }

    std::vector<std::tuple<dyn::apiproc *, const char *>> funcs {
      { (dyn::apiproc *) &maxNumEntrypoints, "vaMaxNumEntrypoints" },
      { (dyn::apiproc *) &queryConfigEntrypoints, "vaQueryConfigEntrypoints" },
      { (dyn::apiproc *) &terminate, "vaTerminate" },
      { (dyn::apiproc *) &initialize, "vaInitialize" },
      { (dyn::apiproc *) &errorStr, "vaErrorStr" },
      { (dyn::apiproc *) &setErrorCallback, "vaSetErrorCallback" },
      { (dyn::apiproc *) &setInfoCallback, "vaSetInfoCallback" },
      { (dyn::apiproc *) &queryVendorString, "vaQueryVendorString" },
      { (dyn::apiproc *) &exportSurfaceHandle, "vaExportSurfaceHandle" },
    };

    if (dyn::load(handle, funcs)) {
      return -1;
    }

    funcs_loaded = true;
    return 0;
  }

  int
  init() {
    if (init_main_va()) {
      return -1;
    }

    static void *handle { nullptr };
    static bool funcs_loaded = false;

    if (funcs_loaded) return 0;

    if (!handle) {
      handle = dyn::handle({ "libva-drm.so.2", "libva-drm.so" });
      if (!handle) {
        return -1;
      }
    }

    std::vector<std::tuple<dyn::apiproc *, const char *>> funcs {
      { (dyn::apiproc *) &getDisplayDRM, "vaGetDisplayDRM" },
    };

    if (dyn::load(handle, funcs)) {
      return -1;
    }

    funcs_loaded = true;
    return 0;
  }

  int
  vaapi_init_avcodec_hardware_input_buffer(platf::avcodec_encode_device_t *encode_device, AVBufferRef **hw_device_buf);

  class va_t: public platf::avcodec_encode_device_t {
  public:
    int
    init(int in_width, int in_height, file_t &&render_device) {
      file = std::move(render_device);

      if (!va::initialize || !gbm::create_device) {
        if (!va::initialize) BOOST_LOG(warning) << "libva not initialized"sv;
        if (!gbm::create_device) BOOST_LOG(warning) << "libgbm not initialized"sv;
        return -1;
      }

      this->data = (void *) vaapi_init_avcodec_hardware_input_buffer;

      gbm.reset(gbm::create_device(file.el));
      if (!gbm) {
        char string[1024];
        BOOST_LOG(error) << "Couldn't create GBM device: ["sv << strerror_r(errno, string, sizeof(string)) << ']';
        return -1;
      }

      display = egl::make_display(gbm.get());
      if (!display) {
        return -1;
      }

      auto ctx_opt = egl::make_ctx(display.get());
      if (!ctx_opt) {
        return -1;
      }

      ctx = std::move(*ctx_opt);

      width = in_width;
      height = in_height;

      return 0;
    }

    int
    set_frame(AVFrame *frame, AVBufferRef *hw_frames_ctx) override {
      this->hwframe.reset(frame);
      this->frame = frame;

      if (!frame->buf[0]) {
        if (av_hwframe_get_buffer(hw_frames_ctx, frame, 0)) {
          BOOST_LOG(error) << "Couldn't get hwframe for VAAPI"sv;
          return -1;
        }
      }

      va::DRMPRIMESurfaceDescriptor prime;
      va::VASurfaceID surface = (std::uintptr_t) frame->data[3];

      auto status = va::exportSurfaceHandle(
        this->va_display,
        surface,
        va::SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2,
        va::EXPORT_SURFACE_WRITE_ONLY | va::EXPORT_SURFACE_COMPOSED_LAYERS,
        &prime);
      if (status) {
        BOOST_LOG(error) << "Couldn't export va surface handle: ["sv << (int) surface << "]: "sv << va::errorStr(status);

        return -1;
      }

      // Keep track of file descriptors
      std::array<file_t, egl::nv12_img_t::num_fds> fds;
      for (int x = 0; x < prime.num_objects; ++x) {
        fds[x] = prime.objects[x].fd;
      }

      auto nv12_opt = egl::import_target(
        display.get(),
        std::move(fds),
        { (int) prime.width,
          (int) prime.height,
          { prime.objects[prime.layers[0].object_index[0]].fd, -1, -1, -1 },
          0,
          0,
          { prime.layers[0].pitch[0] },
          { prime.layers[0].offset[0] } },
        { (int) prime.width / 2,
          (int) prime.height / 2,
          { prime.objects[prime.layers[0].object_index[1]].fd, -1, -1, -1 },
          0,
          0,
          { prime.layers[0].pitch[1] },
          { prime.layers[0].offset[1] } });

      if (!nv12_opt) {
        return -1;
      }

      auto sws_opt = egl::sws_t::make(width, height, frame->width, frame->height);
      if (!sws_opt) {
        return -1;
      }

      this->sws = std::move(*sws_opt);
      this->nv12 = std::move(*nv12_opt);

      return 0;
    }

    void
    apply_colorspace() override {
      sws.apply_colorspace(colorspace);
    }

    va::display_t::pointer va_display;
    file_t file;

    gbm::gbm_t gbm;
    egl::display_t display;
    egl::ctx_t ctx;

    // This must be destroyed before display_t to ensure the GPU
    // driver is still loaded when vaDestroySurfaces() is called.
    frame_t hwframe;

    egl::sws_t sws;
    egl::nv12_t nv12;

    int width, height;
  };

  class va_ram_t: public va_t {
  public:
    int
    convert(platf::img_t &img) override {
      sws.load_ram(img);

      sws.convert(nv12->buf);
      return 0;
    }
  };

  class va_vram_t: public va_t {
  public:
    int
    convert(platf::img_t &img) override {
      auto &descriptor = (egl::img_descriptor_t &) img;

      if (descriptor.sequence > sequence) {
        sequence = descriptor.sequence;

        rgb = egl::rgb_t {};

        auto rgb_opt = egl::import_source(display.get(), descriptor.sd);

        if (!rgb_opt) {
          return -1;
        }

        rgb = std::move(*rgb_opt);
      }

      sws.load_vram(descriptor, offset_x, offset_y, rgb->tex[0]);

      sws.convert(nv12->buf);
      return 0;
    }

    int
    init(int in_width, int in_height, file_t &&render_device, int offset_x, int offset_y) {
      if (va_t::init(in_width, in_height, std::move(render_device))) {
        return -1;
      }

      sequence = 0;

      this->offset_x = offset_x;
      this->offset_y = offset_y;

      return 0;
    }

    std::uint64_t sequence;
    egl::rgb_t rgb;

    int offset_x, offset_y;
  };

  /**
   * This is a private structure of FFmpeg, I need this to manually create
   * a VAAPI hardware context
   *
   * xdisplay will not be used internally by FFmpeg
   */
  typedef struct VAAPIDevicePriv {
    union {
      void *xdisplay;
      int fd;
    } drm;
    int drm_fd;
  } VAAPIDevicePriv;

  /**
   * VAAPI connection details.
   *
   * Allocated as AVHWDeviceContext.hwctx
   */
  typedef struct AVVAAPIDeviceContext {
    /**
     * The VADisplay handle, to be filled by the user.
     */
    va::VADisplay display;
    /**
     * Driver quirks to apply - this is filled by av_hwdevice_ctx_init(),
     * with reference to a table of known drivers, unless the
     * AV_VAAPI_DRIVER_QUIRK_USER_SET bit is already present.  The user
     * may need to refer to this field when performing any later
     * operations using VAAPI with the same VADisplay.
     */
    unsigned int driver_quirks;
  } AVVAAPIDeviceContext;

  static void
  __log(void *level, const char *msg) {
    BOOST_LOG(*(boost::log::sources::severity_logger<int> *) level) << msg;
  }

  static void
  vaapi_hwdevice_ctx_free(AVHWDeviceContext *ctx) {
    auto hwctx = (AVVAAPIDeviceContext *) ctx->hwctx;
    auto priv = (VAAPIDevicePriv *) ctx->user_opaque;

    vaTerminate(hwctx->display);
    close(priv->drm_fd);
    av_freep(&priv);
  }

  int
  vaapi_init_avcodec_hardware_input_buffer(platf::avcodec_encode_device_t *base, AVBufferRef **hw_device_buf) {
    if (!va::initialize) {
      BOOST_LOG(warning) << "libva not loaded"sv;
      return -1;
    }

    if (!va::getDisplayDRM) {
      BOOST_LOG(warning) << "libva-drm not loaded"sv;
      return -1;
    }

    auto va = (va::va_t *) base;
    auto fd = dup(va->file.el);

    auto *priv = (VAAPIDevicePriv *) av_mallocz(sizeof(VAAPIDevicePriv));
    priv->drm_fd = fd;

    auto fg = util::fail_guard([fd, priv]() {
      close(fd);
      av_free(priv);
    });

    va::display_t display { va::getDisplayDRM(fd) };
    if (!display) {
      auto render_device = config::video.adapter_name.empty() ? "/dev/dri/renderD128" : config::video.adapter_name.c_str();

      BOOST_LOG(error) << "Couldn't open a va display from DRM with device: "sv << render_device;
      return -1;
    }

    va->va_display = display.get();

    va::setErrorCallback(display.get(), __log, &error);
    va::setErrorCallback(display.get(), __log, &info);

    int major, minor;
    auto status = va::initialize(display.get(), &major, &minor);
    if (status) {
      BOOST_LOG(error) << "Couldn't initialize va display: "sv << va::errorStr(status);
      return -1;
    }

    BOOST_LOG(debug) << "vaapi vendor: "sv << va::queryVendorString(display.get());

    *hw_device_buf = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_VAAPI);
    auto ctx = (AVHWDeviceContext *) (*hw_device_buf)->data;
    auto hwctx = (AVVAAPIDeviceContext *) ctx->hwctx;

    // Ownership of the VADisplay and DRM fd is now ours to manage via the free() function
    hwctx->display = display.release();
    ctx->user_opaque = priv;
    ctx->free = vaapi_hwdevice_ctx_free;
    fg.disable();

    auto err = av_hwdevice_ctx_init(*hw_device_buf);
    if (err) {
      char err_str[AV_ERROR_MAX_STRING_SIZE] { 0 };
      BOOST_LOG(error) << "Failed to create FFMpeg hardware device context: "sv << av_make_error_string(err_str, AV_ERROR_MAX_STRING_SIZE, err);

      return err;
    }

    return 0;
  }

  static bool
  query(display_t::pointer display, profile_e profile) {
    std::vector<entry_e> entrypoints;
    entrypoints.resize(maxNumEntrypoints(display));

    int count;
    auto status = queryConfigEntrypoints(display, profile, entrypoints.data(), &count);
    if (status) {
      BOOST_LOG(error) << "Couldn't query entrypoints: "sv << va::errorStr(status);
      return false;
    }
    entrypoints.resize(count);

    for (auto entrypoint : entrypoints) {
      if (entrypoint == entry_e::EncSlice || entrypoint == entry_e::EncSliceLP) {
        return true;
      }
    }

    return false;
  }

  bool
  validate(int fd) {
    if (init()) {
      return false;
    }

    va::display_t display { va::getDisplayDRM(fd) };
    if (!display) {
      char string[1024];

      auto bytes = readlink(("/proc/self/fd/" + std::to_string(fd)).c_str(), string, sizeof(string));

      std::string_view render_device { string, (std::size_t) bytes };

      BOOST_LOG(error) << "Couldn't open a va display from DRM with device: "sv << render_device;
      return false;
    }

    int major, minor;
    auto status = initialize(display.get(), &major, &minor);
    if (status) {
      BOOST_LOG(error) << "Couldn't initialize va display: "sv << va::errorStr(status);
      return false;
    }

    if (!query(display.get(), profile_e::H264Main)) {
      return false;
    }

    if (video::active_hevc_mode > 1 && !query(display.get(), profile_e::HEVCMain)) {
      return false;
    }

    if (video::active_hevc_mode > 2 && !query(display.get(), profile_e::HEVCMain10)) {
      return false;
    }

    return true;
  }

  std::unique_ptr<platf::avcodec_encode_device_t>
  make_avcodec_encode_device(int width, int height, file_t &&card, int offset_x, int offset_y, bool vram) {
    if (vram) {
      auto egl = std::make_unique<va::va_vram_t>();
      if (egl->init(width, height, std::move(card), offset_x, offset_y)) {
        return nullptr;
      }

      return egl;
    }

    else {
      auto egl = std::make_unique<va::va_ram_t>();
      if (egl->init(width, height, std::move(card))) {
        return nullptr;
      }

      return egl;
    }
  }

  std::unique_ptr<platf::avcodec_encode_device_t>
  make_avcodec_encode_device(int width, int height, int offset_x, int offset_y, bool vram) {
    auto render_device = config::video.adapter_name.empty() ? "/dev/dri/renderD128" : config::video.adapter_name.c_str();

    file_t file = open(render_device, O_RDWR);
    if (file.el < 0) {
      char string[1024];
      BOOST_LOG(error) << "Couldn't open "sv << render_device << ": " << strerror_r(errno, string, sizeof(string));

      return nullptr;
    }

    return make_avcodec_encode_device(width, height, std::move(file), offset_x, offset_y, vram);
  }

  std::unique_ptr<platf::avcodec_encode_device_t>
  make_avcodec_encode_device(int width, int height, bool vram) {
    return make_avcodec_encode_device(width, height, 0, 0, vram);
  }
}  // namespace va