From 3ceb9b37a07b7a343da5a2500f664f0ed79818f3 Mon Sep 17 00:00:00 2001 From: loki Date: Tue, 31 Mar 2020 21:18:33 +0200 Subject: [PATCH] Reinitialize the video encoder in addition to the capturing device --- sunshine/main.cpp | 3 +- sunshine/platform/common.h | 16 +- sunshine/platform/linux.cpp | 8 +- sunshine/platform/windows_dxgi.cpp | 81 ++++---- sunshine/sync.h | 9 + sunshine/video.cpp | 321 +++++++++++++++++++++-------- sunshine/video.h | 2 + 7 files changed, 310 insertions(+), 130 deletions(-) diff --git a/sunshine/main.cpp b/sunshine/main.cpp index 1405008a..bc64ce25 100644 --- a/sunshine/main.cpp +++ b/sunshine/main.cpp @@ -5,7 +5,6 @@ #include "process.h" #include -#include #include #include @@ -14,6 +13,7 @@ #include #include +#include "video.h" #include "input.h" #include "nvhttp.h" #include "rtsp.h" @@ -128,6 +128,7 @@ int main(int argc, char *argv[]) { auto deinit_guard = platf::init(); input::init(); reed_solomon_init(); + video::init(); task_pool.start(1); diff --git a/sunshine/platform/common.h b/sunshine/platform/common.h index 0c0a5255..fc35ab75 100644 --- a/sunshine/platform/common.h +++ b/sunshine/platform/common.h @@ -70,6 +70,20 @@ public: virtual capture_e snapshot(img_t *img, bool cursor) = 0; virtual std::shared_ptr alloc_img() = 0; + virtual int dummy_img(img_t *img, int &dummy_data_p) { + img->row_pitch = 4; + img->height = 1; + img->width = 1; + img->pixel_pitch = 4; + img->data = (std::uint8_t*)&dummy_data_p; + + return 0; + } + + virtual std::shared_ptr get_hwdevice() { + return nullptr; + } + virtual ~display_t() = default; }; @@ -91,7 +105,7 @@ std::string from_sockaddr(const sockaddr *const); std::pair from_sockaddr_ex(const sockaddr *const); std::unique_ptr microphone(std::uint32_t sample_rate); -std::shared_ptr display(); +std::shared_ptr display(int hwdevice_type); input_t input(); void move_mouse(input_t &input, int deltaX, int deltaY); diff --git a/sunshine/platform/linux.cpp b/sunshine/platform/linux.cpp index 407cd6d3..46f043e2 100644 --- a/sunshine/platform/linux.cpp +++ b/sunshine/platform/linux.cpp @@ -255,6 +255,12 @@ struct shm_attr_t : public x11_attr_t { return std::make_shared(); } + int dummy_img(platf::img_t *img, int &) override { + auto dummy_data_p = new int[1]; + + return platf::display_t::dummy_img(img, *dummy_data_p); + } + int init() { shm_xdisplay.reset(XOpenDisplay(nullptr)); xcb.reset(xcb_connect(nullptr, nullptr)); @@ -325,7 +331,7 @@ std::shared_ptr shm_display() { return shm; } -std::shared_ptr display() { +std::shared_ptr display(int hwdevice_type) { auto shm_disp = shm_display(); if(!shm_disp) { diff --git a/sunshine/platform/windows_dxgi.cpp b/sunshine/platform/windows_dxgi.cpp index 98d779a4..bf6021e2 100644 --- a/sunshine/platform/windows_dxgi.cpp +++ b/sunshine/platform/windows_dxgi.cpp @@ -2,14 +2,18 @@ // Created by loki on 1/12/20. // +extern "C" { +#include +} + #include #include #include #include #include -#include +#include "sunshine/config.h" #include "sunshine/main.h" #include "common.h" @@ -115,39 +119,6 @@ struct img_t : public ::platf::img_t { } } - int reset(int width, int height, DXGI_FORMAT format, device_t::pointer device, device_ctx_t::pointer device_ctx_p, const std::shared_ptr &display) { - unmap(); - - D3D11_TEXTURE2D_DESC t {}; - t.Width = width; - t.Height = height; - t.MipLevels = 1; - t.ArraySize = 1; - t.SampleDesc.Count = 1; - t.Usage = D3D11_USAGE_STAGING; - t.Format = format; - t.CPUAccessFlags = D3D11_CPU_ACCESS_READ; - - dxgi::texture2d_t::pointer tex_p {}; - auto status = device->CreateTexture2D(&t, nullptr, &tex_p); - texture.reset(tex_p); - - if(FAILED(status)) { - BOOST_LOG(error) << "Failed to create texture [0x"sv << util::hex(status).to_string_view() << ']'; - return -1; - } - - this->display = display; - this->device_ctx_p = device_ctx_p; - this->data = nullptr; - this->row_pitch = 0; - this->pixel_pitch = 4; - this->width = width; - this->height = height; - - return 0; - } - std::shared_ptr display; texture2d_t texture; @@ -299,11 +270,6 @@ class display_t : public ::platf::display_t, public std::enable_shared_from_this public: capture_e snapshot(::platf::img_t *img_base, bool cursor_visible) override { auto img = (img_t*)img_base; - if(img->display.get() != this) { - if(img->reset(width, height, format, device.get(), device_ctx.get(), shared_from_this())) { - return capture_e::error; - } - } HRESULT status; @@ -391,10 +357,39 @@ public: std::shared_ptr<::platf::img_t> alloc_img() override { auto img = std::make_shared(); - if(img->reset(width, height, format, device.get(), device_ctx.get(), shared_from_this())) { + D3D11_TEXTURE2D_DESC t {}; + t.Width = width; + t.Height = height; + t.MipLevels = 1; + t.ArraySize = 1; + t.SampleDesc.Count = 1; + t.Format = format; + + if(true) { + t.Usage = D3D11_USAGE_STAGING; + t.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + } + else /* gpu memory */ { + t.Usage = D3D11_USAGE_DEFAULT; + } + + dxgi::texture2d_t::pointer tex_p {}; + auto status = device->CreateTexture2D(&t, nullptr, &tex_p); + img->texture.reset(tex_p); + + if(FAILED(status)) { + BOOST_LOG(error) << "Failed to create texture [0x"sv << util::hex(status).to_string_view() << ']'; return nullptr; } + img->display = shared_from_this(); + img->device_ctx_p = device_ctx.get(); + img->data = nullptr; + img->row_pitch = 0; + img->pixel_pitch = 4; + img->width = width; + img->height = height; + return img; } @@ -738,7 +733,11 @@ const char *format_str[] = { } namespace platf { -std::shared_ptr display() { +std::shared_ptr display(int hwdevice_type) { + if(hwdevice_type != AV_HWDEVICE_TYPE_NONE) { + return nullptr; + } + auto disp = std::make_shared(); if (disp->init()) { diff --git a/sunshine/sync.h b/sunshine/sync.h index a0613980..4689e595 100644 --- a/sunshine/sync.h +++ b/sunshine/sync.h @@ -46,6 +46,15 @@ public: return *this; } + template + sync_t &operator=(V &&val) { + auto lg = lock(); + + raw = val; + + return *this; + } + sync_t &operator=(const value_t &val) noexcept { auto lg = lock(); diff --git a/sunshine/video.cpp b/sunshine/video.cpp index 103eccc7..080ce1d1 100644 --- a/sunshine/video.cpp +++ b/sunshine/video.cpp @@ -10,8 +10,8 @@ extern "C" { } #include "platform/common.h" -#include "thread_pool.h" #include "round_robin.h" +#include "sync.h" #include "config.h" #include "video.h" #include "main.h" @@ -43,6 +43,7 @@ using sws_t = util::safe_ptr; using img_event_t = std::shared_ptr>>; void sw_img_to_frame(sws_t &sws, platf::img_t &img, frame_t &frame); +void nv_d3d_img_to_frame(sws_t &sws, platf::img_t &img, frame_t &frame); struct encoder_t { struct option_t { @@ -78,6 +79,7 @@ struct session_t { frame_t frame; + AVPixelFormat sw_format; int sws_color_format; }; @@ -93,7 +95,7 @@ static encoder_t nvenc { }, false, - nullptr + nv_d3d_img_to_frame // D3D11Device }; @@ -131,6 +133,10 @@ struct capture_ctx_t { struct capture_thread_ctx_t { std::shared_ptr> capture_ctx_queue; std::thread capture_thread; + + safe::signal_t reinit_event; + const encoder_t *encoder_p; + util::sync_t> display_wp; }; [[nodiscard]] codec_t open_codec(ctx_t &ctx, AVCodec *codec, AVDictionary **options) { @@ -139,7 +145,12 @@ struct capture_thread_ctx_t { return codec_t { ctx.get() }; } -void captureThread(std::shared_ptr> capture_ctx_queue) { +void captureThread( + std::shared_ptr> capture_ctx_queue, + util::sync_t> &display_wp, + safe::signal_t &reinit_event, + const encoder_t &encoder + ) { std::vector capture_ctxs; auto fg = util::fail_guard([&]() { @@ -156,14 +167,16 @@ void captureThread(std::shared_ptr> capture_ctx_que std::chrono::nanoseconds delay = 1s; - auto disp = platf::display(); + auto disp = platf::display(encoder.dev_type); if(!disp) { return; } + display_wp = disp; std::vector> imgs(12); - auto round_robin = util::make_round_robin>(std::begin(imgs), std::end(imgs)); + auto round_robin = util::make_round_robin>(std::begin(imgs) +1, std::end(imgs)); + int dummy_data = 0; for(auto &img : imgs) { img = disp->alloc_img(); if(!img) { @@ -171,12 +184,17 @@ void captureThread(std::shared_ptr> capture_ctx_que return; } } + auto &dummy_img = imgs.front(); + disp->dummy_img(dummy_img.get(), dummy_data); auto next_frame = std::chrono::steady_clock::now(); while(capture_ctx_queue->running()) { while(capture_ctx_queue->peek()) { capture_ctxs.emplace_back(std::move(*capture_ctx_queue->pop())); + // Temporary image to ensure something is send to Moonlight even if no frame has been captured yet. + capture_ctxs.back().images->raise(dummy_img); + delay = std::min(delay, capture_ctxs.back().delay); } @@ -190,21 +208,24 @@ void captureThread(std::shared_ptr> capture_ctx_que auto status = disp->snapshot(img.get(), display_cursor); switch (status) { case platf::capture_e::reinit: { + reinit_event.raise(true); + // Some classes of images contain references to the display --> display won't delete unless img is deleted for(auto &img : imgs) { img.reset(); } - while(disp.use_count() > 1) { - std::this_thread::sleep_for(100ms); - } - // We try this twice, in case we still get an error on reinitialization for(int x = 0; x < 2; ++x) { // Some classes of display cannot have multiple instances at once disp.reset(); - disp = platf::display(); + // display_wp is modified in this thread only + while(!display_wp->expired()) { + std::this_thread::sleep_for(100ms); + } + + disp = platf::display(encoder.dev_type); if(disp) { break; } @@ -216,6 +237,7 @@ void captureThread(std::shared_ptr> capture_ctx_que return; } + display_wp = disp; // Re-allocate images for(auto &img : imgs) { img = disp->alloc_img(); @@ -224,7 +246,9 @@ void captureThread(std::shared_ptr> capture_ctx_que return; } } + disp->dummy_img(dummy_img.get(), dummy_data); + reinit_event.reset(); continue; } case platf::capture_e::error: @@ -291,31 +315,14 @@ int hwframe_ctx(ctx_t &ctx, buffer_t &hwdevice, AVPixelFormat format) { return 0; } -void sw_img_to_frame(sws_t &sws, platf::img_t &img, frame_t &frame) { - av_frame_make_writable(frame.get()); - - const int linesizes[2] { - img.row_pitch, 0 - }; - - int ret = sws_scale(sws.get(), (std::uint8_t*const*)&img.data, linesizes, 0, img.height, frame->data, frame->linesize); - if(ret <= 0) { - BOOST_LOG(fatal) << "Couldn't convert image to required format and/or size"sv; - - log_flush(); - std::abort(); - } -} - -void encode(int64_t frame_nr, ctx_t &ctx, frame_t &frame, packet_queue_t &packets, void *channel_data) { +int encode(int64_t frame_nr, ctx_t &ctx, frame_t &frame, packet_queue_t &packets, void *channel_data) { frame->pts = frame_nr; /* send the frame to the encoder */ auto ret = avcodec_send_frame(ctx.get(), frame.get()); if (ret < 0) { - BOOST_LOG(fatal) << "Could not send a frame for encoding"sv; - log_flush(); - std::abort(); + BOOST_LOG(error) << "Could not send a frame for encoding"sv; + return -1; } while (ret >= 0) { @@ -323,24 +330,31 @@ void encode(int64_t frame_nr, ctx_t &ctx, frame_t &frame, packet_queue_t &packet ret = avcodec_receive_packet(ctx.get(), packet.get()); if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { - return; + return 0; } else if (ret < 0) { - BOOST_LOG(fatal) << "Could not encode video packet"sv; - log_flush(); - std::abort(); + return ret; } packet->channel_data = channel_data; packets->raise(std::move(packet)); } + + return 0; } int start_capture(capture_thread_ctx_t &capture_thread_ctx) { + capture_thread_ctx.encoder_p = &software; + capture_thread_ctx.reinit_event.reset(); + capture_thread_ctx.capture_ctx_queue = std::make_shared>(); capture_thread_ctx.capture_thread = std::thread { - captureThread, capture_thread_ctx.capture_ctx_queue + captureThread, + capture_thread_ctx.capture_ctx_queue, + std::ref(capture_thread_ctx.display_wp), + std::ref(capture_thread_ctx.reinit_event), + std::ref(*capture_thread_ctx.encoder_p) }; return 0; @@ -351,8 +365,8 @@ void end_capture(capture_thread_ctx_t &capture_thread_ctx) { capture_thread_ctx.capture_thread.join(); } -std::optional make_session(const encoder_t &encoder, const config_t &config, void *device_ctx) { - bool hardware = device_ctx; +std::optional make_session(const encoder_t &encoder, const config_t &config, void *device_ctx) { + bool hardware = encoder.dev_type != AV_HWDEVICE_TYPE_NONE; auto &video_format = config.videoFormat == 0 ? encoder.h264 : encoder.hevc; @@ -437,12 +451,12 @@ std::optional make_session(const encoder_t &encoder, const config_t & break; } - AVPixelFormat src_fmt; + AVPixelFormat sw_fmt; if(config.dynamicRange == 0) { - src_fmt = AV_PIX_FMT_YUV420P; + sw_fmt = AV_PIX_FMT_YUV420P; } else { - src_fmt = AV_PIX_FMT_YUV420P10; + sw_fmt = AV_PIX_FMT_YUV420P10; } if(hardware) { @@ -450,7 +464,7 @@ std::optional make_session(const encoder_t &encoder, const config_t & ((AVHWFramesContext *)ctx->hw_frames_ctx->data)->device_ctx = (AVHWDeviceContext*)device_ctx; - if(auto err = hwframe_ctx(ctx, hwdevice, src_fmt); err < 0) { + if(auto err = hwframe_ctx(ctx, hwdevice, sw_fmt); err < 0) { char err_str[AV_ERROR_MAX_STRING_SIZE] {0}; BOOST_LOG(error) << "Failed to initialize hardware frame: "sv << av_make_error_string(err_str, AV_ERROR_MAX_STRING_SIZE, err) << std::endl; @@ -458,7 +472,7 @@ std::optional make_session(const encoder_t &encoder, const config_t & } } else /* software */ { - ctx->pix_fmt = src_fmt; + ctx->pix_fmt = sw_fmt; // Clients will request for the fewest slices per frame to get the // most efficient encode, but we may want to provide more slices than @@ -503,7 +517,7 @@ std::optional make_session(const encoder_t &encoder, const config_t & frame->height = ctx->height; - if(config.videoFormat == 1) { + if(hardware) { auto err = av_hwframe_get_buffer(ctx->hw_frames_ctx, frame.get(), 0); if(err < 0) { char err_str[AV_ERROR_MAX_STRING_SIZE] {0}; @@ -521,57 +535,30 @@ std::optional make_session(const encoder_t &encoder, const config_t & std::move(ctx), std::move(codec_handle), std::move(frame), + sw_fmt, sws_color_space }); } -void capture( - safe::signal_t *shutdown_event, +void encode_run( + int &frame_nr, int &key_frame_nr, // Store progress of the frame number + safe::signal_t* shutdown_event, // Signal for shutdown event of the session packet_queue_t packets, idr_event_t idr_events, + img_event_t images, config_t config, + platf::display_t &display, + safe::signal_t &reinit_event, + const encoder_t &encoder, void *channel_data) { - auto session = make_session(software, config, nullptr); + auto hwdevice = display.get_hwdevice(); + auto session = make_session(encoder, config, hwdevice.get()); if(!session) { return; } - int framerate = config.framerate; - - auto images = std::make_shared(); - - // Temporary image to ensure something is send to Moonlight even if no frame has been captured yet. - int dummy_data = 0; - { - auto img = std::make_shared(); - img->row_pitch = 4; - img->height = 1; - img->width = 1; - img->pixel_pitch = 4; - img->data = (std::uint8_t*)&dummy_data; - - images->raise(std::move(img)); - } - - // Keep a reference counter to ensure the capture thread only runs when other threads have a reference to the capture thread - static auto capture_thread = safe::make_shared(start_capture, end_capture); - auto ref = capture_thread.ref(); - if(!ref) { - return; - } - - auto delay = std::chrono::floor(1s) / framerate; - ref->capture_ctx_queue->raise(capture_ctx_t { - images, delay - }); - - if(!ref->capture_ctx_queue->running()) { - return; - } - - int64_t frame_nr = 1; - int64_t key_frame_nr = 1; + auto delay = std::chrono::floor(1s) / config.framerate; auto img_width = 0; auto img_height = 0; @@ -581,7 +568,7 @@ void capture( auto next_frame = std::chrono::steady_clock::now(); while(true) { - if(shutdown_event->peek() || !images->running()) { + if(shutdown_event->peek() || reinit_event.peek() || !images->running()) { break; } @@ -604,7 +591,7 @@ void capture( // When Moonlight request an IDR frame, send frames even if there is no new captured frame if(frame_nr > (key_frame_nr + config.framerate) || images->peek()) { if(auto img = images->pop(delay)) { - if(software.system_memory) { + if(encoder.system_memory) { auto new_width = img->width; auto new_height = img->height; @@ -615,7 +602,7 @@ void capture( sws.reset( sws_getContext( img_width, img_height, AV_PIX_FMT_BGR0, - session->ctx->width, session->ctx->height, session->ctx->pix_fmt, + session->ctx->width, session->ctx->height, session->sw_format, SWS_LANCZOS | SWS_ACCURATE_RND, nullptr, nullptr, nullptr)); @@ -625,7 +612,7 @@ void capture( } } - software.img_to_frame(sws, *img, session->frame); + encoder.img_to_frame(sws, *img, session->frame); } else if(images->running()) { continue; @@ -635,11 +622,173 @@ void capture( } } - encode(frame_nr++, session->ctx, session->frame, packets, channel_data); + if(encode(frame_nr++, session->ctx, session->frame, packets, channel_data)) { + BOOST_LOG(fatal) << "Could not encode video packet"sv; + log_flush(); + std::abort(); + } session->frame->pict_type = AV_PICTURE_TYPE_NONE; } +} + +void capture( + safe::signal_t *shutdown_event, + packet_queue_t packets, + idr_event_t idr_events, + config_t config, + void *channel_data) { + + auto images = std::make_shared(); + + // Keep a reference counter to ensure the Fcapture thread only runs when other threads have a reference to the capture thread + static auto capture_thread = safe::make_shared(start_capture, end_capture); + auto ref = capture_thread.ref(); + if(!ref) { + return; + } + + auto delay = std::chrono::floor(1s) / config.framerate; + ref->capture_ctx_queue->raise(capture_ctx_t { + images, delay + }); + + if(!ref->capture_ctx_queue->running()) { + return; + } + + int frame_nr = 1; + int key_frame_nr = 1; + while(!shutdown_event->peek() && images->running()) { + // Wait for the display to be ready + std::shared_ptr display; + { + auto lg = ref->display_wp.lock(); + if(ref->display_wp->expired()) { + continue; + } + + display = ref->display_wp->lock(); + } + + encode_run(frame_nr, key_frame_nr, shutdown_event, packets, idr_events, images, config, *display, ref->reinit_event, *ref->encoder_p, channel_data); + } images->stop(); } + +bool validate_config(const encoder_t &encoder, const config_t &config, platf::display_t &disp) { + auto hwdevice = disp.get_hwdevice(); + + auto session = make_session(encoder, config, hwdevice.get()); + if(!session) { + return false; + } + + int dummy_data; + auto img = disp.alloc_img(); + disp.dummy_img(img.get(), dummy_data); + + sws_t sws; + if(encoder.system_memory) { + sws.reset(sws_getContext( + img->width, img->height, AV_PIX_FMT_BGR0, + session->ctx->width, session->ctx->height, session->sw_format, + SWS_LANCZOS | SWS_ACCURATE_RND, + nullptr, nullptr, nullptr)); + + sws_setColorspaceDetails(sws.get(), sws_getCoefficients(SWS_CS_DEFAULT), 0, + sws_getCoefficients(session->sws_color_format), config.encoderCscMode & 0x1, + 0, 1 << 16, 1 << 16); + + + } + + encoder.img_to_frame(sws, *img, session->frame); + + session->frame->pict_type = AV_PICTURE_TYPE_I; + + auto packets = std::make_shared(); + if(encode(1, session->ctx, session->frame, packets, nullptr)) { + return false; + } + + return true; +} + +bool validate_encoder(const encoder_t &encoder) { + config_t config_h264 { + 1920, 1080, + 60, + 1000, + 1, + 1, + 1, + 0, + 0 + }; + + config_t config_hevc { + 1920, 1080, + 60, + 1000, + 1, + 1, + 1, + 1, + 1 + }; + + auto disp = platf::display(encoder.dev_type); + if(!disp) { + return false; + } + + return + validate_config(encoder, config_h264, *disp) && + validate_config(encoder, config_hevc, *disp); +} + +void init() { + KITTY_WHILE_LOOP(auto pos = std::begin(encoders), pos != std::end(encoders), { + if(!validate_encoder(*pos)) { + pos = encoders.erase(pos); + + continue; + } + + ++pos; + }) + + for(auto &encoder : encoders) { + BOOST_LOG(info) << "Found encoder ["sv << encoder.h264.name << ", "sv << encoder.hevc.name << ']'; + } +} + +void sw_img_to_frame(sws_t &sws, platf::img_t &img, frame_t &frame) { + av_frame_make_writable(frame.get()); + + const int linesizes[2] { + img.row_pitch, 0 + }; + + int ret = sws_scale(sws.get(), (std::uint8_t*const*)&img.data, linesizes, 0, img.height, frame->data, frame->linesize); + if(ret <= 0) { + BOOST_LOG(fatal) << "Couldn't convert image to required format and/or size"sv; + + log_flush(); + std::abort(); + } +} + +void nv_d3d_img_to_frame(sws_t &sws, platf::img_t &img, frame_t &frame) { + frame->data[0] = img.data; + frame->data[1] = 0; + + frame->linesize[0] = img.row_pitch; + frame->linesize[1] = 0; + + frame->height = img.height; + frame->width = img.width; +} } diff --git a/sunshine/video.h b/sunshine/video.h index 70f0f9e3..fc3c1426 100644 --- a/sunshine/video.h +++ b/sunshine/video.h @@ -56,6 +56,8 @@ void capture( idr_event_t idr_events, config_t config, void *channel_data); + +void init(); } #endif //SUNSHINE_VIDEO_H