mirror/Sunshine
1
0
Fork 0
mirror of https://github.com/LizardByte/Sunshine.git synced 2025-01-30 03:32:43 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Decrease normal capture buffer to single image

Browse Source
This commit is contained in:
ns6089 2023-03-29 18:07:20 +03:00 committed by Cameron Gutman
parent 191b929ab0
commit eed27d3c0a
12 changed files with 333 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
src/platform/common.h
View File

@ -169,7 +169,7 @@ namespace platf {
virtual ~deinit_t() = default;
};
struct img_t {
struct img_t: std::enable_shared_from_this<img_t> {
public:
img_t() = default;
@ -245,6 +245,7 @@ namespace platf {
ok,
reinit,
timeout,
interrupted,
error
};
@ -255,20 +256,33 @@ namespace platf {
* If a frame was captured, frame_captured will be true. If a timeout occurred, it will be false.
*
* On Break Request -->
* Returns nullptr
* Returns false
*
* On Success -->
* Returns the image object that should be filled next.
* This may or may not be the image send with the callback
* Returns true
*/
using snapshot_cb_t = std::function<std::shared_ptr<img_t>(std::shared_ptr<img_t> &img, bool frame_captured)>;
using push_captured_image_cb_t = std::function<bool(std::shared_ptr<img_t> &&img, bool frame_captured)>;
/**
* Use to get free image from the pool. Calls must be synchronized.
* Blocks until there is free image in the pool or capture is interrupted.
*
* Returns:
* 'true' on success, img_out contains free image
* 'false' when capture has been interrupted, img_out contains nullptr
*/
using pull_free_image_cb_t = std::function<bool(std::shared_ptr<img_t> &img_out)>;
display_t() noexcept:
offset_x { 0 }, offset_y { 0 } {}
/**
* snapshot_cb --> the callback
* std::shared_ptr<img_t> img --> The first image to use
* push_captured_image_cb --> The callback that is called with captured image,
* must be called from the same thread as capture()
* pull_free_image_cb --> Capture backends call this callback to get empty image
* from the pool. If backend uses multiple threads, calls to this
* callback must be synchronized. Calls to this callback and
* push_captured_image_cb must be synchronized as well.
* bool *cursor --> A pointer to the flag that indicates wether the cursor should be captured as well
*
* Returns either:
@ -277,7 +291,7 @@ namespace platf {
* capture_e::reinit when need of reinitialization
*/
virtual capture_e
capture(snapshot_cb_t &&snapshot_cb, std::shared_ptr<img_t> img, bool *cursor) = 0;
capture(const push_captured_image_cb_t &push_captured_image_cb, const pull_free_image_cb_t &pull_free_image_cb, bool *cursor) = 0;
virtual std::shared_ptr<img_t>
alloc_img() = 0;

32
src/platform/linux/cuda.cpp
View File

@ -504,9 +504,16 @@ namespace cuda {
}
platf::capture_e
capture(snapshot_cb_t &&snapshot_cb, std::shared_ptr<platf::img_t> img, bool *cursor) override {
capture(const push_captured_image_cb_t &push_captured_image_cb, const pull_free_image_cb_t &pull_free_image_cb, bool *cursor) override {
auto next_frame = std::chrono::steady_clock::now();
{
// We must create at least one texture on this thread before calling NvFBCToCudaSetUp()
// Otherwise it fails with "Unable to register an OpenGL buffer to a CUDA resource (result: 201)" message
std::shared_ptr<platf::img_t> img_dummy;
pull_free_image_cb(img_dummy);
}
// Force display_t::capture to initialize handle_t::capture
cursor_visible = !*cursor;
@ -515,7 +522,7 @@ namespace cuda {
handle.reset();
});
while (img) {
while (true) {
auto now = std::chrono::steady_clock::now();
if (next_frame > now) {
std::this_thread::sleep_for((next_frame - now) / 3 * 2);
@ -526,16 +533,22 @@ namespace cuda {
}
next_frame = now + delay;
auto status = snapshot(img.get(), 150ms, *cursor);
std::shared_ptr<platf::img_t> img_out;
auto status = snapshot(pull_free_image_cb, img_out, 150ms, *cursor);
switch (status) {
case platf::capture_e::reinit:
case platf::capture_e::error:
case platf::capture_e::interrupted:
return status;
case platf::capture_e::timeout:
img = snapshot_cb(img, false);
if (!push_captured_image_cb(std::move(img_out), false)) {
return platf::capture_e::ok;
}
break;
case platf::capture_e::ok:
img = snapshot_cb(img, true);
if (!push_captured_image_cb(std::move(img_out), true)) {
return platf::capture_e::ok;
}
break;
default:
BOOST_LOG(error) << "Unrecognized capture status ["sv << (int) status << ']';
@ -618,7 +631,7 @@ namespace cuda {
}
platf::capture_e
snapshot(platf::img_t *img, std::chrono::milliseconds timeout, bool cursor) {
snapshot(const pull_free_image_cb_t &pull_free_image_cb, std::shared_ptr<platf::img_t> &img_out, std::chrono::milliseconds timeout, bool cursor) {
if (cursor != cursor_visible) {
auto status = reinit(cursor);
if (status != platf::capture_e::ok) {
@ -646,7 +659,12 @@ namespace cuda {
return platf::capture_e::error;
}
if (((img_t *) img)->tex.copy((std::uint8_t *) device_ptr, img->height, img->row_pitch)) {
if (!pull_free_image_cb(img_out)) {
return platf::capture_e::interrupted;
}
auto img = (img_t *) img_out.get();
if (img->tex.copy((std::uint8_t *) device_ptr, img->height, img->row_pitch)) {
return platf::capture_e::error;
}

4
src/platform/linux/cuda.cu
View File

@ -35,7 +35,7 @@ using namespace std::literals;
* Therefore, some declarations need to be added explicitely
*/
namespace platf {
struct img_t {
struct img_t: std::enable_shared_from_this<img_t> {
public:
std::uint8_t *data {};
std::int32_t width {};
@ -70,7 +70,7 @@ struct alignas(16) color_extern_t {
static_assert(sizeof(video::color_t) == sizeof(video::color_extern_t), "color matrix struct mismatch");
extern color_t colors[4];
extern color_t colors[6];
} // namespace video
//////////////////// End special declarations

62
src/platform/linux/kmsgrab.cpp
View File

@ -723,10 +723,10 @@ namespace platf {
}
capture_e
capture(snapshot_cb_t &&snapshot_cb, std::shared_ptr<img_t> img, bool *cursor) override {
capture(const push_captured_image_cb_t &push_captured_image_cb, const pull_free_image_cb_t &pull_free_image_cb, bool *cursor) override {
auto next_frame = std::chrono::steady_clock::now();
while (img) {
while (true) {
auto now = std::chrono::steady_clock::now();
if (next_frame > now) {
@ -738,16 +738,22 @@ namespace platf {
}
next_frame = now + delay;
auto status = snapshot(img.get(), 1000ms, *cursor);
std::shared_ptr<platf::img_t> img_out;
auto status = snapshot(pull_free_image_cb, img_out, 1000ms, *cursor);
switch (status) {
case platf::capture_e::reinit:
case platf::capture_e::error:
case platf::capture_e::interrupted:
return status;
case platf::capture_e::timeout:
img = snapshot_cb(img, false);
if (!push_captured_image_cb(std::move(img_out), false)) {
return platf::capture_e::ok;
}
break;
case platf::capture_e::ok:
img = snapshot_cb(img, true);
if (!push_captured_image_cb(std::move(img_out), true)) {
return platf::capture_e::ok;
}
break;
default:
BOOST_LOG(error) << "Unrecognized capture status ["sv << (int) status << ']';
@ -768,7 +774,7 @@ namespace platf {
}
capture_e
snapshot(img_t *img_out_base, std::chrono::milliseconds timeout, bool cursor) {
snapshot(const pull_free_image_cb_t &pull_free_image_cb, std::shared_ptr<platf::img_t> &img_out, std::chrono::milliseconds timeout, bool cursor) {
file_t fb_fd[4];
egl::surface_descriptor_t sd;
@ -793,10 +799,14 @@ namespace platf {
gl::ctx.GetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_HEIGHT, &h);
BOOST_LOG(debug) << "width and height: w "sv << w << " h "sv << h;
gl::ctx.GetTextureSubImage(rgb->tex[0], 0, img_offset_x, img_offset_y, 0, width, height, 1, GL_BGRA, GL_UNSIGNED_BYTE, img_out_base->height * img_out_base->row_pitch, img_out_base->data);
if (!pull_free_image_cb(img_out)) {
return platf::capture_e::interrupted;
}
gl::ctx.GetTextureSubImage(rgb->tex[0], 0, img_offset_x, img_offset_y, 0, width, height, 1, GL_BGRA, GL_UNSIGNED_BYTE, img_out->height * img_out->row_pitch, img_out->data);
if (cursor_opt && cursor) {
cursor_opt->blend(*img_out_base, img_offset_x, img_offset_y);
cursor_opt->blend(*img_out, img_offset_x, img_offset_y);
}
return capture_e::ok;
@ -855,14 +865,23 @@ namespace platf {
int
dummy_img(platf::img_t *img) override {
return snapshot(img, 1s, false) != capture_e::ok;
// TODO: stop cheating and give black image
if (!img) {
return -1;
};
auto pull_dummy_img_callback = [&img](std::shared_ptr<platf::img_t> &img_out) -> bool {
img_out = img->shared_from_this();
return true;
};
std::shared_ptr<platf::img_t> img_out;
return snapshot(pull_dummy_img_callback, img_out, 1s, false) != platf::capture_e::ok;
}
capture_e
capture(snapshot_cb_t &&snapshot_cb, std::shared_ptr<img_t> img, bool *cursor) {
capture(const push_captured_image_cb_t &push_captured_image_cb, const pull_free_image_cb_t &pull_free_image_cb, bool *cursor) {
auto next_frame = std::chrono::steady_clock::now();
while (img) {
while (true) {
auto now = std::chrono::steady_clock::now();
if (next_frame > now) {
@ -874,16 +893,22 @@ namespace platf {
}
next_frame = now + delay;
auto status = snapshot(img.get(), 1000ms, *cursor);
std::shared_ptr<platf::img_t> img_out;
auto status = snapshot(pull_free_image_cb, img_out, 1000ms, *cursor);
switch (status) {
case platf::capture_e::reinit:
case platf::capture_e::error:
case platf::capture_e::interrupted:
return status;
case platf::capture_e::timeout:
img = snapshot_cb(img, false);
if (!push_captured_image_cb(std::move(img_out), false)) {
return platf::capture_e::ok;
}
break;
case platf::capture_e::ok:
img = snapshot_cb(img, true);
if (!push_captured_image_cb(std::move(img_out), true)) {
return platf::capture_e::ok;
}
break;
default:
BOOST_LOG(error) << "Unrecognized capture status ["sv << (int) status << ']';
@ -895,10 +920,13 @@ namespace platf {
}
capture_e
snapshot(img_t *img_out_base, std::chrono::milliseconds /* timeout */, bool cursor) {
snapshot(const pull_free_image_cb_t &pull_free_image_cb, std::shared_ptr<platf::img_t> &img_out, std::chrono::milliseconds /* timeout */, bool cursor) {
file_t fb_fd[4];
auto img = (egl::img_descriptor_t *) img_out_base;
if (!pull_free_image_cb(img_out)) {
return platf::capture_e::interrupted;
}
auto img = (egl::img_descriptor_t *) img_out.get();
img->reset();
auto status = refresh(fb_fd, &img->sd);
@ -909,7 +937,7 @@ namespace platf {
img->sequence = ++sequence;
if (!cursor || !cursor_opt) {
img_out_base->data = nullptr;
img->data = nullptr;
for (auto x = 0; x < 4; ++x) {
fb_fd[x].release();

64
src/platform/linux/wlgrab.cpp
View File

@ -80,7 +80,7 @@ namespace wl {
}
inline platf::capture_e
snapshot(platf::img_t *img_out_base, std::chrono::milliseconds timeout, bool cursor) {
snapshot(const pull_free_image_cb_t &pull_free_image_cb, std::shared_ptr<platf::img_t> &img_out, std::chrono::milliseconds timeout, bool cursor) {
auto to = std::chrono::steady_clock::now() + timeout;
dmabuf.listen(interface.dmabuf_manager, output, cursor);
@ -118,10 +118,10 @@ namespace wl {
class wlr_ram_t: public wlr_t {
public:
platf::capture_e
capture(snapshot_cb_t &&snapshot_cb, std::shared_ptr<platf::img_t> img, bool *cursor) override {
capture(const push_captured_image_cb_t &push_captured_image_cb, const pull_free_image_cb_t &pull_free_image_cb, bool *cursor) override {
auto next_frame = std::chrono::steady_clock::now();
while (img) {
while (true) {
auto now = std::chrono::steady_clock::now();
if (next_frame > now) {
@ -132,16 +132,22 @@ namespace wl {
}
next_frame = now + delay;
auto status = snapshot(img.get(), 1000ms, *cursor);
std::shared_ptr<platf::img_t> img_out;
auto status = snapshot(pull_free_image_cb, img_out, 1000ms, *cursor);
switch (status) {
case platf::capture_e::reinit:
case platf::capture_e::error:
case platf::capture_e::interrupted:
return status;
case platf::capture_e::timeout:
img = snapshot_cb(img, false);
if (!push_captured_image_cb(std::move(img_out), false)) {
return platf::capture_e::ok;
}
break;
case platf::capture_e::ok:
img = snapshot_cb(img, true);
if (!push_captured_image_cb(std::move(img_out), true)) {
return platf::capture_e::ok;
}
break;
default:
BOOST_LOG(error) << "Unrecognized capture status ["sv << (int) status << ']';
@ -153,8 +159,8 @@ namespace wl {
}
platf::capture_e
snapshot(platf::img_t *img_out_base, std::chrono::milliseconds timeout, bool cursor) {
auto status = wlr_t::snapshot(img_out_base, timeout, cursor);
snapshot(const pull_free_image_cb_t &pull_free_image_cb, std::shared_ptr<platf::img_t> &img_out, std::chrono::milliseconds timeout, bool cursor) {
auto status = wlr_t::snapshot(pull_free_image_cb, img_out, timeout, cursor);
if (status != platf::capture_e::ok) {
return status;
}
@ -167,6 +173,10 @@ namespace wl {
return platf::capture_e::reinit;
}
if (!pull_free_image_cb(img_out)) {
return platf::capture_e::interrupted;
}
gl::ctx.BindTexture(GL_TEXTURE_2D, (*rgb_opt)->tex[0]);
int w, h;
@ -174,7 +184,7 @@ namespace wl {
gl::ctx.GetTexLevelParameteriv(GL_TEXTURE_2D, 0, GL_TEXTURE_HEIGHT, &h);
BOOST_LOG(debug) << "width and height: w "sv << w << " h "sv << h;
gl::ctx.GetTextureSubImage((*rgb_opt)->tex[0], 0, 0, 0, 0, width, height, 1, GL_BGRA, GL_UNSIGNED_BYTE, img_out_base->height * img_out_base->row_pitch, img_out_base->data);
gl::ctx.GetTextureSubImage((*rgb_opt)->tex[0], 0, 0, 0, 0, width, height, 1, GL_BGRA, GL_UNSIGNED_BYTE, img_out->height * img_out->row_pitch, img_out->data);
gl::ctx.BindTexture(GL_TEXTURE_2D, 0);
return platf::capture_e::ok;
@ -229,10 +239,10 @@ namespace wl {
class wlr_vram_t: public wlr_t {
public:
platf::capture_e
capture(snapshot_cb_t &&snapshot_cb, std::shared_ptr<platf::img_t> img, bool *cursor) override {
capture(const push_captured_image_cb_t &push_captured_image_cb, const pull_free_image_cb_t &pull_free_image_cb, bool *cursor) override {
auto next_frame = std::chrono::steady_clock::now();
while (img) {
while (true) {
auto now = std::chrono::steady_clock::now();
if (next_frame > now) {
@ -243,16 +253,22 @@ namespace wl {
}
next_frame = now + delay;
auto status = snapshot(img.get(), 1000ms, *cursor);
std::shared_ptr<platf::img_t> img_out;
auto status = snapshot(pull_free_image_cb, img_out, 1000ms, *cursor);
switch (status) {
case platf::capture_e::reinit:
case platf::capture_e::error:
case platf::capture_e::interrupted:
return status;
case platf::capture_e::timeout:
img = snapshot_cb(img, false);
if (!push_captured_image_cb(std::move(img_out), false)) {
return platf::capture_e::ok;
}
break;
case platf::capture_e::ok:
img = snapshot_cb(img, true);
if (!push_captured_image_cb(std::move(img_out), true)) {
return platf::capture_e::ok;
}
break;
default:
BOOST_LOG(error) << "Unrecognized capture status ["sv << (int) status << ']';
@ -264,13 +280,16 @@ namespace wl {
}
platf::capture_e
snapshot(platf::img_t *img_out_base, std::chrono::milliseconds timeout, bool cursor) {
auto status = wlr_t::snapshot(img_out_base, timeout, cursor);
snapshot(const pull_free_image_cb_t &pull_free_image_cb, std::shared_ptr<platf::img_t> &img_out, std::chrono::milliseconds timeout, bool cursor) {
auto status = wlr_t::snapshot(pull_free_image_cb, img_out, timeout, cursor);
if (status != platf::capture_e::ok) {
return status;
}
auto img = (egl::img_descriptor_t *) img_out_base;
if (!pull_free_image_cb(img_out)) {
return platf::capture_e::interrupted;
}
auto img = (egl::img_descriptor_t *) img_out.get();
img->reset();
auto current_frame = dmabuf.current_frame;
@ -311,7 +330,16 @@ namespace wl {
int
dummy_img(platf::img_t *img) override {
return snapshot(img, 1000ms, false) != platf::capture_e::ok;
// TODO: stop cheating and give black image
if (!img) {
return -1;
};
auto pull_dummy_img_callback = [&img](std::shared_ptr<platf::img_t> &img_out) -> bool {
img_out = img->shared_from_this();
return true;
};
std::shared_ptr<platf::img_t> img_out;
return snapshot(pull_dummy_img_callback, img_out, 1000ms, false) != platf::capture_e::ok;
}
std::uint64_t sequence {};

78
src/platform/linux/x11grab.cpp
View File

@ -476,10 +476,10 @@ namespace platf {
}
capture_e
capture(snapshot_cb_t &&snapshot_cb, std::shared_ptr<img_t> img, bool *cursor) override {
capture(const push_captured_image_cb_t &push_captured_image_cb, const pull_free_image_cb_t &pull_free_image_cb, bool *cursor) override {
auto next_frame = std::chrono::steady_clock::now();
while (img) {
while (true) {
auto now = std::chrono::steady_clock::now();
if (next_frame > now) {
@ -491,16 +491,22 @@ namespace platf {
}
next_frame = now + delay;
auto status = snapshot(img.get(), 1000ms, *cursor);
std::shared_ptr<platf::img_t> img_out;
auto status = snapshot(pull_free_image_cb, img_out, 1000ms, *cursor);
switch (status) {
case platf::capture_e::reinit:
case platf::capture_e::error:
case platf::capture_e::interrupted:
return status;
case platf::capture_e::timeout:
img = snapshot_cb(img, false);
if (!push_captured_image_cb(std::move(img_out), false)) {
return platf::capture_e::ok;
}
break;
case platf::capture_e::ok:
img = snapshot_cb(img, true);
if (!push_captured_image_cb(std::move(img_out), true)) {
return platf::capture_e::ok;
}
break;
default:
BOOST_LOG(error) << "Unrecognized capture status ["sv << (int) status << ']';
@ -512,7 +518,7 @@ namespace platf {
}
capture_e
snapshot(img_t *img_out_base, std::chrono::milliseconds timeout, bool cursor) {
snapshot(const pull_free_image_cb_t &pull_free_image_cb, std::shared_ptr<platf::img_t> &img_out, std::chrono::milliseconds timeout, bool cursor) {
refresh();
//The whole X server changed, so we must reinit everything
@ -520,18 +526,23 @@ namespace platf {
BOOST_LOG(warning) << "X dimensions changed in non-SHM mode, request reinit"sv;
return capture_e::reinit;
}
XImage *img { x11::GetImage(xdisplay.get(), xwindow, offset_x, offset_y, width, height, AllPlanes, ZPixmap) };
auto img_out = (x11_img_t *) img_out_base;
img_out->width = img->width;
img_out->height = img->height;
img_out->data = (uint8_t *) img->data;
img_out->row_pitch = img->bytes_per_line;
img_out->pixel_pitch = img->bits_per_pixel / 8;
img_out->img.reset(img);
if (!pull_free_image_cb(img_out)) {
return platf::capture_e::interrupted;
}
auto img = (x11_img_t *) img_out.get();
XImage *x_img { x11::GetImage(xdisplay.get(), xwindow, offset_x, offset_y, width, height, AllPlanes, ZPixmap) };
img->width = x_img->width;
img->height = x_img->height;
img->data = (uint8_t *) x_img->data;
img->row_pitch = x_img->bytes_per_line;
img->pixel_pitch = x_img->bits_per_pixel / 8;
img->img.reset(x_img);
if (cursor) {
blend_cursor(xdisplay.get(), *img_out_base, offset_x, offset_y);
blend_cursor(xdisplay.get(), *img, offset_x, offset_y);
}
return capture_e::ok;
@ -559,7 +570,16 @@ namespace platf {
int
dummy_img(img_t *img) override {
snapshot(img, 0s, true);
// TODO: stop cheating and give black image
if (!img) {
return -1;
};
auto pull_dummy_img_callback = [&img](std::shared_ptr<platf::img_t> &img_out) -> bool {
img_out = img->shared_from_this();
return true;
};
std::shared_ptr<platf::img_t> img_out;
snapshot(pull_dummy_img_callback, img_out, 0s, true);
return 0;
}
};
@ -594,10 +614,10 @@ namespace platf {
}
capture_e
capture(snapshot_cb_t &&snapshot_cb, std::shared_ptr<img_t> img, bool *cursor) override {
capture(const push_captured_image_cb_t &push_captured_image_cb, const pull_free_image_cb_t &pull_free_image_cb, bool *cursor) override {
auto next_frame = std::chrono::steady_clock::now();
while (img) {
while (true) {
auto now = std::chrono::steady_clock::now();
if (next_frame > now) {
@ -609,16 +629,22 @@ namespace platf {
}
next_frame = now + delay;
auto status = snapshot(img.get(), 1000ms, *cursor);
std::shared_ptr<platf::img_t> img_out;
auto status = snapshot(pull_free_image_cb, img_out, 1000ms, *cursor);
switch (status) {
case platf::capture_e::reinit:
case platf::capture_e::error:
case platf::capture_e::interrupted:
return status;
case platf::capture_e::timeout:
img = snapshot_cb(img, false);
if (!push_captured_image_cb(std::move(img_out), false)) {
return platf::capture_e::ok;
}
break;
case platf::capture_e::ok:
img = snapshot_cb(img, true);
if (!push_captured_image_cb(std::move(img_out), true)) {
return platf::capture_e::ok;
}
break;
default:
BOOST_LOG(error) << "Unrecognized capture status ["sv << (int) status << ']';
@ -630,7 +656,7 @@ namespace platf {
}
capture_e
snapshot(img_t *img, std::chrono::milliseconds timeout, bool cursor) {
snapshot(const pull_free_image_cb_t &pull_free_image_cb, std::shared_ptr<platf::img_t> &img_out, std::chrono::milliseconds timeout, bool cursor) {
//The whole X server changed, so we must reinit everything
if (xattr.width != env_width || xattr.height != env_height) {
BOOST_LOG(warning) << "X dimensions changed in SHM mode, request reinit"sv;
@ -645,10 +671,14 @@ namespace platf {
return capture_e::reinit;
}
std::copy_n((std::uint8_t *) data.data, frame_size(), img->data);
if (!pull_free_image_cb(img_out)) {
return platf::capture_e::interrupted;
}
std::copy_n((std::uint8_t *) data.data, frame_size(), img_out->data);
if (cursor) {
blend_cursor(shm_xdisplay.get(), *img, offset_x, offset_y);
blend_cursor(shm_xdisplay.get(), *img_out, offset_x, offset_y);
}
return capture_e::ok;

45
src/platform/macos/display.mm
View File

@ -37,38 +37,46 @@ namespace platf {
}
capture_e
capture(snapshot_cb_t &&snapshot_cb, std::shared_ptr<img_t> img, bool *cursor) override {
__block auto img_next = std::move(img);
capture(const push_captured_image_cb_t &push_captured_image_cb, const pull_free_image_cb_t &pull_free_image_cb, bool *cursor) override {
auto signal = [av_capture capture:^(CMSampleBufferRef sampleBuffer) {
auto av_img_next = std::static_pointer_cast<av_img_t>(img_next);
CFRetain(sampleBuffer);
CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
if (av_img_next->pixel_buffer != nullptr)
CVPixelBufferUnlockBaseAddress(av_img_next->pixel_buffer, 0);
std::shared_ptr<img_t> img_out;
if (!pull_free_image_cb(img_out)) {
// got interrupt signal
// returning false here stops capture backend
return false;
}
auto av_img = std::static_pointer_cast<av_img_t>(img_out);
if (av_img_next->sample_buffer != nullptr)
CFRelease(av_img_next->sample_buffer);
if (av_img->pixel_buffer != nullptr)
CVPixelBufferUnlockBaseAddress(av_img->pixel_buffer, 0);
av_img_next->sample_buffer = sampleBuffer;
av_img_next->pixel_buffer = pixelBuffer;
img_next->data = (uint8_t *) CVPixelBufferGetBaseAddress(pixelBuffer);
if (av_img->sample_buffer != nullptr)
CFRelease(av_img->sample_buffer);
av_img->sample_buffer = sampleBuffer;
av_img->pixel_buffer = pixelBuffer;
img_out->data = (uint8_t *) CVPixelBufferGetBaseAddress(pixelBuffer);
size_t extraPixels[4];
CVPixelBufferGetExtendedPixels(pixelBuffer, &extraPixels[0], &extraPixels[1], &extraPixels[2], &extraPixels[3]);
img_next->width = CVPixelBufferGetWidth(pixelBuffer) + extraPixels[0] + extraPixels[1];
img_next->height = CVPixelBufferGetHeight(pixelBuffer) + extraPixels[2] + extraPixels[3];
img_next->row_pitch = CVPixelBufferGetBytesPerRow(pixelBuffer);
img_next->pixel_pitch = img_next->row_pitch / img_next->width;
img_out->width = CVPixelBufferGetWidth(pixelBuffer) + extraPixels[0] + extraPixels[1];
img_out->height = CVPixelBufferGetHeight(pixelBuffer) + extraPixels[2] + extraPixels[3];
img_out->row_pitch = CVPixelBufferGetBytesPerRow(pixelBuffer);
img_out->pixel_pitch = img_out->row_pitch / img_out->width;
img_next = snapshot_cb(img_next, true);
if (!push_captured_image_cb(std::move(img_out), false)) {
// got interrupt signal
// returning false here stops capture backend
return false;
}
return img_next != nullptr;
return true;
}];
// FIXME: We should time out if an image isn't returned for a while
@ -132,6 +140,7 @@ namespace platf {
img->row_pitch = CVPixelBufferGetBytesPerRow(pixelBuffer);
img->pixel_pitch = img->row_pitch / img->width;
// returning false here stops capture backend
return false;
}];

9
src/platform/windows/display.h
View File

@ -125,8 +125,9 @@ namespace platf::dxgi {
public:
int
init(const ::video::config_t &config, const std::string &display_name);
capture_e
capture(snapshot_cb_t &&snapshot_cb, std::shared_ptr<img_t> img, bool *cursor) override;
capture(const push_captured_image_cb_t &push_captured_image_cb, const pull_free_image_cb_t &pull_free_image_cb, bool *cursor) override;
std::chrono::nanoseconds delay;
@ -168,7 +169,7 @@ namespace platf::dxgi {
colorspace_to_string(DXGI_COLOR_SPACE_TYPE type);
virtual capture_e
snapshot(img_t *img, std::chrono::milliseconds timeout, bool cursor_visible) = 0;
snapshot(const pull_free_image_cb_t &pull_free_image_cb, std::shared_ptr<platf::img_t> &img_out, std::chrono::milliseconds timeout, bool cursor_visible) = 0;
virtual int
complete_img(img_t *img, bool dummy) = 0;
virtual std::vector<DXGI_FORMAT>
@ -178,7 +179,7 @@ namespace platf::dxgi {
class display_ram_t: public display_base_t {
public:
virtual capture_e
snapshot(img_t *img, std::chrono::milliseconds timeout, bool cursor_visible) override;
snapshot(const pull_free_image_cb_t &pull_free_image_cb, std::shared_ptr<platf::img_t> &img_out, std::chrono::milliseconds timeout, bool cursor_visible) override;
std::shared_ptr<img_t>
alloc_img() override;
@ -200,7 +201,7 @@ namespace platf::dxgi {
class display_vram_t: public display_base_t, public std::enable_shared_from_this<display_vram_t> {
public:
virtual capture_e
snapshot(img_t *img, std::chrono::milliseconds timeout, bool cursor_visible) override;
snapshot(const pull_free_image_cb_t &pull_free_image_cb, std::shared_ptr<platf::img_t> &img_out, std::chrono::milliseconds timeout, bool cursor_visible) override;
std::shared_ptr<img_t>
alloc_img() override;

16
src/platform/windows/display_base.cpp
View File

@ -92,7 +92,7 @@ namespace platf::dxgi {
}
capture_e
display_base_t::capture(snapshot_cb_t &&snapshot_cb, std::shared_ptr<::platf::img_t> img, bool *cursor) {
display_base_t::capture(const push_captured_image_cb_t &push_captured_image_cb, const pull_free_image_cb_t &pull_free_image_cb, bool *cursor) {
auto next_frame = std::chrono::steady_clock::now();
// Use CREATE_WAITABLE_TIMER_HIGH_RESOLUTION if supported (Windows 10 1809+)
@ -110,7 +110,7 @@ namespace platf::dxgi {
CloseHandle(timer);
});
while (img) {
while (true) {
// This will return false if the HDR state changes or for any number of other
// display or GPU changes. We should reinit to examine the updated state of
// the display subsystem. It is recommended to call this once per frame.
@ -135,16 +135,22 @@ namespace platf::dxgi {
next_frame = std::chrono::steady_clock::now() + delay;
}
auto status = snapshot(img.get(), 1000ms, *cursor);
std::shared_ptr<img_t> img_out;
auto status = snapshot(pull_free_image_cb, img_out, 1000ms, *cursor);
switch (status) {
case platf::capture_e::reinit:
case platf::capture_e::error:
case platf::capture_e::interrupted:
return status;
case platf::capture_e::timeout:
img = snapshot_cb(img, false);
if (!push_captured_image_cb(std::move(img_out), false)) {
return capture_e::ok;
}
break;
case platf::capture_e::ok:
img = snapshot_cb(img, true);
if (!push_captured_image_cb(std::move(img_out), true)) {
return capture_e::ok;
}
break;
default:
BOOST_LOG(error) << "Unrecognized capture status ["sv << (int) status << ']';

9
src/platform/windows/display_ram.cpp
View File

@ -171,9 +171,7 @@ namespace platf::dxgi {
}
capture_e
display_ram_t::snapshot(::platf::img_t *img_base, std::chrono::milliseconds timeout, bool cursor_visible) {
auto img = (img_t *) img_base;
display_ram_t::snapshot(const pull_free_image_cb_t &pull_free_image_cb, std::shared_ptr<platf::img_t> &img_out, std::chrono::milliseconds timeout, bool cursor_visible) {
HRESULT status;
DXGI_OUTDUPL_FRAME_INFO frame_info;
@ -269,6 +267,11 @@ namespace platf::dxgi {
}
}
if (!pull_free_image_cb(img_out)) {
return capture_e::interrupted;
}
auto img = (img_t *) img_out.get();
// If we don't know the final capture format yet, encode a dummy image
if (capture_format == DXGI_FORMAT_UNKNOWN) {
BOOST_LOG(debug) << "Capture format is still unknown. Encoding a blank image"sv;

34
src/platform/windows/display_vram.cpp
View File

@ -311,6 +311,16 @@ namespace platf::dxgi {
public:
int
convert(platf::img_t &img_base) override {
// Garbage collect mapped capture images whose weak references have expired
for (auto it = img_ctx_map.begin(); it != img_ctx_map.end();) {
if (it->second.img_weak.expired()) {
it = img_ctx_map.erase(it);
}
else {
it++;
}
}
auto &img = (img_d3d_t &) img_base;
auto &img_ctx = img_ctx_map[img.id];
@ -342,6 +352,9 @@ namespace platf::dxgi {
// Release encoder mutex to allow capture code to reuse this image
img_ctx.encoder_mutex->ReleaseSync(0);
ID3D11ShaderResourceView *emptyShaderResourceView = nullptr;
device_ctx->PSSetShaderResources(0, 1, &emptyShaderResourceView);
return 0;
}
@ -650,12 +663,15 @@ namespace platf::dxgi {
shader_res_t encoder_input_res;
keyed_mutex_t encoder_mutex;
std::weak_ptr<const platf::img_t> img_weak;
void
reset() {
capture_texture_p = nullptr;
encoder_texture.reset();
encoder_input_res.reset();
encoder_mutex.reset();
img_weak.reset();
}
};
@ -699,6 +715,9 @@ namespace platf::dxgi {
}
img_ctx.capture_texture_p = img.capture_texture.get();
img_ctx.img_weak = img.weak_from_this();
return 0;
}
@ -789,9 +808,7 @@ namespace platf::dxgi {
}
capture_e
display_vram_t::snapshot(platf::img_t *img_base, std::chrono::milliseconds timeout, bool cursor_visible) {
auto img = (img_d3d_t *) img_base;
display_vram_t::snapshot(const pull_free_image_cb_t &pull_free_image_cb, std::shared_ptr<platf::img_t> &img_out, std::chrono::milliseconds timeout, bool cursor_visible) {
HRESULT status;
DXGI_OUTDUPL_FRAME_INFO frame_info;
@ -839,6 +856,11 @@ namespace platf::dxgi {
cursor_xor.set_pos(frame_info.PointerPosition.Position.x, frame_info.PointerPosition.Position.y, frame_info.PointerPosition.Visible);
}
if (!pull_free_image_cb(img_out)) {
return capture_e::interrupted;
}
auto img = (img_d3d_t *) img_out.get();
if (frame_update_flag) {
texture2d_t src {};
@ -969,6 +991,12 @@ namespace platf::dxgi {
}
device_ctx->OMSetBlendState(blend_disable.get(), nullptr, 0xFFFFFFFFu);
ID3D11RenderTargetView *emptyRenderTarget = nullptr;
device_ctx->OMSetRenderTargets(1, &emptyRenderTarget, nullptr);
device_ctx->RSSetViewports(0, nullptr);
ID3D11ShaderResourceView *emptyShaderResourceView = nullptr;
device_ctx->PSSetShaderResources(0, 1, &emptyShaderResourceView);
}
// Release the mutex to allow encoding of this frame

95
src/video.cpp
View File

@ -2,6 +2,7 @@
#include <atomic>
#include <bitset>
#include <list>
#include <thread>
extern "C" {
@ -14,7 +15,6 @@ extern "C" {
#include "input.h"
#include "main.h"
#include "platform/common.h"
#include "round_robin.h"
#include "sync.h"
#include "video.h"
@ -791,16 +791,45 @@ namespace video {
}
display_wp = disp;
std::vector<std::shared_ptr<platf::img_t>> imgs(12);
auto round_robin = round_robin_util::make_round_robin<std::shared_ptr<platf::img_t>>(std::begin(imgs), std::end(imgs));
constexpr auto capture_buffer_normal_size = 1;
constexpr auto capture_buffer_size = 12;
std::list<std::shared_ptr<platf::img_t>> imgs(capture_buffer_size);
for (auto &img : imgs) {
img = disp->alloc_img();
if (!img) {
BOOST_LOG(error) << "Couldn't initialize an image"sv;
return;
auto pull_free_image_callback = [&](std::shared_ptr<platf::img_t> &img_out) -> bool {
img_out.reset();
while (capture_ctx_queue->running()) {
for (auto it = imgs.begin(); it != imgs.end(); it++) {
// pick first unallocated or unused
if (!*it || it->use_count() == 1) {
if (!*it) {
// allocate if unallocated
*it = disp->alloc_img();
}
img_out = *it;
if (it != imgs.begin()) {
// move freshly allocated or unused img to the front of the list to prioritize its reusal
imgs.erase(it);
imgs.push_front(img_out);
}
break;
}
}
if (img_out) {
// unallocate unused above normal buffer size
size_t index = 0;
for (auto &img : imgs) {
if (index >= capture_buffer_normal_size && img && img.use_count() == 1) img.reset();
index++;
}
return true;
}
else {
// sleep and retry if image pool is full
std::this_thread::sleep_for(1ms);
}
}
}
return false;
};
// Capture takes place on this thread
platf::adjust_thread_priority(platf::thread_priority_e::critical);
@ -808,7 +837,7 @@ namespace video {
while (capture_ctx_queue->running()) {
bool artificial_reinit = false;
auto status = disp->capture([&](std::shared_ptr<platf::img_t> &img, bool frame_captured) -> std::shared_ptr<platf::img_t> {
auto push_captured_image_callback = [&](std::shared_ptr<platf::img_t> &&img, bool frame_captured) -> bool {
KITTY_WHILE_LOOP(auto capture_ctx = std::begin(capture_ctxs), capture_ctx != std::end(capture_ctxs), {
if (!capture_ctx->images->running()) {
capture_ctx = capture_ctxs.erase(capture_ctx);
@ -824,8 +853,9 @@ namespace video {
})
if (!capture_ctx_queue->running()) {
return nullptr;
return false;
}
while (capture_ctx_queue->peek()) {
capture_ctxs.emplace_back(std::move(*capture_ctx_queue->pop()));
}
@ -834,18 +864,13 @@ namespace video {
artificial_reinit = true;
display_p = std::clamp(*switch_display_event->pop(), 0, (int) display_names.size() - 1);
return nullptr;
return false;
}
auto &next_img = *round_robin++;
while (next_img.use_count() > 1) {
// Sleep a bit to avoid starving the encoder threads
std::this_thread::sleep_for(2ms);
}
return true;
};
return next_img;
},
*round_robin++, &display_cursor);
auto status = disp->capture(push_captured_image_callback, pull_free_image_callback, &display_cursor);
if (artificial_reinit && status != platf::capture_e::error) {
status = platf::capture_e::reinit;
@ -899,21 +924,13 @@ namespace video {
display_wp = disp;
// Re-allocate images
for (auto &img : imgs) {
img = disp->alloc_img();
if (!img) {
BOOST_LOG(error) << "Couldn't initialize an image"sv;
return;
}
}
reinit_event.reset();
continue;
}
case platf::capture_e::error:
case platf::capture_e::ok:
case platf::capture_e::timeout:
case platf::capture_e::interrupted:
return;
default:
BOOST_LOG(error) << "Unrecognized capture status ["sv << (int) status << ']';
@ -1487,11 +1504,11 @@ namespace video {
auto ec = platf::capture_e::ok;
while (encode_session_ctx_queue.running()) {
auto snapshot_cb = [&](std::shared_ptr<platf::img_t> &img, bool frame_captured) -> std::shared_ptr<platf::img_t> {
auto push_captured_image_callback = [&](std::shared_ptr<platf::img_t> &&img, bool frame_captured) -> bool {
while (encode_session_ctx_queue.peek()) {
auto encode_session_ctx = encode_session_ctx_queue.pop();
if (!encode_session_ctx) {
return nullptr;
return false;
}
synced_session_ctxs.emplace_back(std::make_unique<sync_session_ctx_t>(std::move(*encode_session_ctx)));
@ -1499,7 +1516,7 @@ namespace video {
auto encode_session = make_synced_session(disp.get(), encoder, *img, *synced_session_ctxs.back());
if (!encode_session) {
ec = platf::capture_e::error;
return nullptr;
return false;
}
synced_sessions.emplace_back(std::move(*encode_session));
@ -1518,7 +1535,7 @@ namespace video {
}));
if (synced_sessions.empty()) {
return nullptr;
return false;
}
continue;
@ -1555,18 +1572,24 @@ namespace video {
ec = platf::capture_e::reinit;
display_p = std::clamp(*switch_display_event->pop(), 0, (int) display_names.size() - 1);
return nullptr;
return false;
}
return img;
return true;
};
auto status = disp->capture(std::move(snapshot_cb), img, &display_cursor);
auto pull_free_image_callback = [&img](std::shared_ptr<platf::img_t> &img_out) -> bool {
img_out = img;
return true;
};
auto status = disp->capture(push_captured_image_callback, pull_free_image_callback, &display_cursor);
switch (status) {
case platf::capture_e::reinit:
case platf::capture_e::error:
case platf::capture_e::ok:
case platf::capture_e::timeout:
case platf::capture_e::interrupted:
return ec != platf::capture_e::ok ? ec : status;
}
}