Use SHM module if available for capturing display

This commit is contained in:
loki 2019-12-30 11:09:35 +01:00
parent d812d7b889
commit 0289662017
5 changed files with 324 additions and 193 deletions

View File

@ -15,6 +15,9 @@ set(PLATFORM_TARGET_FILES
set(PLATFORM_LIBRARIES
Xfixes
Xtst
xcb
xcb-shm
xcb-xfixes
${X11_LIBRARIES}
evdev
pulse

View File

@ -10,31 +10,36 @@
namespace platf {
void freeDisplay(void*);
void freeImage(void*);
struct img_t {
public:
std::uint8_t *data;
std::int32_t width;
std::int32_t height;
virtual ~img_t() {};
};
class display_t {
public:
virtual std::unique_ptr<img_t> snapshot(bool cursor) = 0;
virtual ~display_t() {};
};
void freeAudio(void*);
void freeMic(void*);
void freeInput(void*);
using display_t = util::safe_ptr<void, freeDisplay>;
using img_t = util::safe_ptr<void, freeImage>;
using mic_t = util::safe_ptr<void, freeMic>;
using audio_t = util::safe_ptr<void, freeAudio>;
using input_t = util::safe_ptr<void, freeInput>;
std::string get_local_ip();
void terminate_process(std::uint64_t handle);
mic_t microphone();
audio_t audio(mic_t &mic, std::uint32_t sample_size);
display_t display();
img_t snapshot(display_t &display_void, bool cursor);
int32_t img_width(img_t &);
int32_t img_height(img_t &);
std::unique_ptr<display_t> display();
uint8_t *img_data(img_t &);
int16_t *audio_data(audio_t &);
input_t input();

View File

@ -11,8 +11,11 @@
#include <X11/X.h>
#include <X11/Xlib.h>
#include <X11/Xutil.h>
#include <X11/XKBlib.h>
#include <X11/extensions/Xfixes.h>
#include <xcb/shm.h>
#include <xcb/xfixes.h>
#include <sys/ipc.h>
#include <sys/shm.h>
#include <pulse/simple.h>
#include <pulse/error.h>
@ -23,7 +26,285 @@
namespace platf {
using namespace std::literals;
void freeImage(XImage *);
using ifaddr_t = util::safe_ptr<ifaddrs, freeifaddrs>;
using xcb_connect_t = util::safe_ptr<xcb_connection_t, xcb_disconnect>;
using xcb_img_t = util::c_ptr<xcb_shm_get_image_reply_t>;
using xcb_cursor_img = util::c_ptr<xcb_xfixes_get_cursor_image_reply_t>;
using xdisplay_t = util::safe_ptr_v2<Display, int, XCloseDisplay>;
using ximg_t = util::safe_ptr<XImage, freeImage>;
class shm_id_t {
public:
shm_id_t() : id { -1 } {}
shm_id_t(int id) : id {id } {}
shm_id_t(shm_id_t &&other) noexcept : id(other.id) {
other.id = -1;
}
~shm_id_t() {
if(id != -1) {
shmctl(id, IPC_RMID, nullptr);
id = -1;
}
}
int id;
};
class shm_data_t {
public:
shm_data_t() : data {(void*)-1 } {}
shm_data_t(void *data) : data {data } {}
shm_data_t(shm_data_t &&other) noexcept : data(other.data) {
other.data = (void*)-1;
}
~shm_data_t() {
if((std::uintptr_t)data != -1) {
shmdt(data);
data = (void*)-1;
}
}
void *data;
};
struct x11_img_t : public img_t {
x11_img_t(std::uint8_t *data, std::int32_t width, std::int32_t height, XImage *img) : img { img } {
this->data = data;
this->width = width;
this->height = height;
}
ximg_t img;
};
struct shm_img_t : public img_t {
~shm_img_t() override {
if(data) {
delete(data);
}
}
};
void blend_cursor(Display *display, std::uint8_t *img_data, int width, int height) {
XFixesCursorImage *overlay = XFixesGetCursorImage(display);
overlay->x -= overlay->xhot;
overlay->y -= overlay->yhot;
overlay->x = std::max((short)0, overlay->x);
overlay->y = std::max((short)0, overlay->y);
auto pixels = (int*)img_data;
auto screen_height = height;
auto screen_width = width;
auto delta_height = std::min<uint16_t>(overlay->height, std::max(0, screen_height - overlay->y));
auto delta_width = std::min<uint16_t>(overlay->width, std::max(0, screen_width - overlay->x));
for(auto y = 0; y < delta_height; ++y) {
auto overlay_begin = &overlay->pixels[y * overlay->width];
auto overlay_end = &overlay->pixels[y * overlay->width + delta_width];
auto pixels_begin = &pixels[(y + overlay->y) * screen_width + overlay->x];
std::for_each(overlay_begin, overlay_end, [&](long pixel) {
int *pixel_p = (int*)&pixel;
auto colors_in = (uint8_t*)pixels_begin;
auto alpha = (*(uint*)pixel_p) >> 24u;
if(alpha == 255) {
*pixels_begin = *pixel_p;
}
else {
auto colors_out = (uint8_t*)pixel_p;
colors_in[0] = colors_out[0] + (colors_in[0] * (255 - alpha) + 255/2) / 255;
colors_in[1] = colors_out[1] + (colors_in[1] * (255 - alpha) + 255/2) / 255;
colors_in[2] = colors_out[2] + (colors_in[2] * (255 - alpha) + 255/2) / 255;
}
++pixels_begin;
});
}
}
struct x11_attr_t : public display_t {
x11_attr_t() : display {XOpenDisplay(nullptr) }, window {DefaultRootWindow(display.get()) }, attr {} {
refresh();
}
void refresh() {
XGetWindowAttributes(display.get(), window, &attr);
}
std::unique_ptr<img_t> snapshot(bool cursor) {
refresh();
XImage *img { XGetImage(
display.get(),
window,
0, 0,
attr.width, attr.height,
AllPlanes, ZPixmap)
};
if(!cursor) {
return std::make_unique<x11_img_t>((std::uint8_t*)img->data, img->width, img->height, img);
}
blend_cursor(display.get(), (std::uint8_t*)img->data, img->width, img->height);
return std::make_unique<x11_img_t>((std::uint8_t*)img->data, img->width, img->height, img);
}
xdisplay_t display;
Window window;
XWindowAttributes attr;
};
struct shm_attr_t : display_t {
xdisplay_t xdisplay;
xcb_connect_t xcb;
xcb_screen_t *display;
std::uint32_t seg;
shm_id_t shm_id;
shm_data_t data;
std::unique_ptr<img_t> snapshot(bool cursor) override {
auto img_cookie = xcb_shm_get_image_unchecked(
xcb.get(),
display->root,
0, 0,
display->width_in_pixels, display->height_in_pixels,
~0,
XCB_IMAGE_FORMAT_Z_PIXMAP,
seg,
0
);
xcb_img_t img_reply { xcb_shm_get_image_reply(xcb.get(), img_cookie, nullptr) };
if(!img_reply) {
std::cout << "FATAL ERROR: Could not get image reply"sv << std::endl;
std::abort();
}
auto img = std::make_unique<shm_img_t>();
img->data = new std::uint8_t[frame_size()];
img->width = display->width_in_pixels;
img->height = display->height_in_pixels;
std::copy((std::uint8_t*)data.data, (std::uint8_t*)data.data + frame_size(), img->data);
if(!cursor) {
return img;
}
blend_cursor(xdisplay.get(), img->data, img->width, img->height);
return img;
}
std::uint32_t frame_size() {
return display->height_in_pixels * display->width_in_pixels * 4;
}
};
struct mic_attr_t {
pa_sample_spec ss;
util::safe_ptr<pa_simple, pa_simple_free> mic;
};
std::unique_ptr<display_t> shm_display() {
auto shm = std::make_unique<shm_attr_t>();
shm->xdisplay.reset(XOpenDisplay(nullptr));
shm->xcb.reset(xcb_connect(nullptr, nullptr));
if(xcb_connection_has_error(shm->xcb.get())) {
return nullptr;
}
if(!xcb_get_extension_data(shm->xcb.get(), &xcb_shm_id)->present) {
std::cout << "Missing SHM extension"sv << std::endl;
return nullptr;
}
auto iter = xcb_setup_roots_iterator(xcb_get_setup(shm->xcb.get()));
shm->display = iter.data;
shm->seg = xcb_generate_id(shm->xcb.get());
shm->shm_id.id = shmget(IPC_PRIVATE, shm->frame_size(), IPC_CREAT | 0777);
if(shm->shm_id.id == -1) {
std::cout << "shmget failed"sv << std::endl;
return nullptr;
}
xcb_shm_attach(shm->xcb.get(), shm->seg, shm->shm_id.id, false);
shm->data.data = shmat(shm->shm_id.id, nullptr, 0);
if ((uintptr_t)shm->data.data == -1) {
std::cout << "shmat failed"sv << std::endl;
return nullptr;
}
return shm;
}
std::unique_ptr<display_t> display() {
auto shm_disp = shm_display();
if(!shm_disp) {
return std::unique_ptr<display_t> { new x11_attr_t {} };
}
return shm_disp;
}
//FIXME: Pass frame_rate instead of hard coding it
mic_t microphone() {
mic_t mic {
new mic_attr_t {
{ PA_SAMPLE_S16LE, 48000, 2 },
{ }
}
};
int error;
mic_attr_t *mic_attr = (mic_attr_t*)mic.get();
mic_attr->mic.reset(
pa_simple_new(nullptr, "sunshine", pa_stream_direction_t::PA_STREAM_RECORD, nullptr, "sunshine_record", &mic_attr->ss, nullptr, nullptr, &error)
);
if(!mic_attr->mic) {
auto err_str = pa_strerror(error);
std::cout << "pa_simple_new() failed: "sv << err_str << std::endl;
exit(1);
}
return mic;
}
audio_t audio(mic_t &mic, std::uint32_t buf_size) {
auto mic_attr = (mic_attr_t*)mic.get();
audio_t result { new std::uint8_t[buf_size] };
auto buf = (std::uint8_t*)result.get();
int error;
if(pa_simple_read(mic_attr->mic.get(), buf, buf_size, &error)) {
std::cout << "pa_simple_read() failed: "sv << pa_strerror(error) << std::endl;
}
return result;
}
std::int16_t *audio_data(audio_t &audio) {
return (int16_t*)audio.get();
}
ifaddr_t get_ifaddrs() {
ifaddrs *p { nullptr };
@ -74,7 +355,7 @@ std::string get_local_ip(int family) {
(family_f[1] && pos->ifa_addr->sa_family == AF_INET6)
){
ip_addr = from_sockaddr(pos->ifa_addr);
break;
break;
}
}
}
@ -84,156 +365,8 @@ std::string get_local_ip(int family) {
std::string get_local_ip() { return get_local_ip(AF_INET); }
void terminate_process(std::uint64_t handle) {
kill((pid_t)handle, SIGTERM);
}
struct display_attr_t {
display_attr_t() : display { XOpenDisplay(nullptr) }, window { DefaultRootWindow(display) }, attr {} {
refresh();
}
~display_attr_t() {
XCloseDisplay(display);
}
void refresh() {
XGetWindowAttributes(display, window, &attr);
}
Display *display;
Window window;
XWindowAttributes attr;
};
struct mic_attr_t {
pa_sample_spec ss;
util::safe_ptr<pa_simple, pa_simple_free> mic;
};
display_t display() {
return display_t { new display_attr_t {} };
}
img_t snapshot(display_t &display_void, bool cursor) {
auto &display = *((display_attr_t*)display_void.get());
display.refresh();
XImage *img { XGetImage(
display.display,
display.window,
0, 0,
display.attr.width, display.attr.height,
AllPlanes, ZPixmap)
};
if(!cursor) {
return img_t { img };
}
XFixesCursorImage *overlay = XFixesGetCursorImage(display.display);
overlay->x -= overlay->xhot;
overlay->y -= overlay->yhot;
overlay->x = std::max((short)0, overlay->x);
overlay->y = std::max((short)0, overlay->y);
auto pixels = (int*)img->data;
auto screen_height = display.attr.height;
auto screen_width = display.attr.width;
auto delta_height = std::min<uint16_t>(overlay->height, std::max(0, screen_height - overlay->y));
auto delta_width = std::min<uint16_t>(overlay->width, std::max(0, screen_width - overlay->x));
for(auto y = 0; y < delta_height; ++y) {
auto overlay_begin = &overlay->pixels[y * overlay->width];
auto overlay_end = &overlay->pixels[y * overlay->width + delta_width];
auto pixels_begin = &pixels[(y + overlay->y) * screen_width + overlay->x];
std::for_each(overlay_begin, overlay_end, [&](long pixel) {
int *pixel_p = (int*)&pixel;
auto colors_in = (uint8_t*)pixels_begin;
auto alpha = (*(uint*)pixel_p) >> 24u;
if(alpha == 255) {
*pixels_begin = *pixel_p;
}
else {
auto colors_out = (uint8_t*)pixel_p;
colors_in[0] = colors_out[0] + (colors_in[0] * (255 - alpha) + 255/2) / 255;
colors_in[1] = colors_out[1] + (colors_in[1] * (255 - alpha) + 255/2) / 255;
colors_in[2] = colors_out[2] + (colors_in[2] * (255 - alpha) + 255/2) / 255;
}
++pixels_begin;
});
}
return img_t { img };
}
uint8_t *img_data(img_t &img) {
return (uint8_t*)((XImage*)img.get())->data;
}
int32_t img_width(img_t &img) {
return ((XImage*)img.get())->width;
}
int32_t img_height(img_t &img) {
return ((XImage*)img.get())->height;
}
//FIXME: Pass frame_rate instead of hard coding it
mic_t microphone() {
mic_t mic {
new mic_attr_t {
{ PA_SAMPLE_S16LE, 48000, 2 },
{ }
}
};
int error;
mic_attr_t *mic_attr = (mic_attr_t*)mic.get();
mic_attr->mic.reset(
pa_simple_new(nullptr, "sunshine", pa_stream_direction_t::PA_STREAM_RECORD, nullptr, "sunshine_record", &mic_attr->ss, nullptr, nullptr, &error)
);
if(!mic_attr->mic) {
auto err_str = pa_strerror(error);
std::cout << "pa_simple_new() failed: "sv << err_str << std::endl;
exit(1);
}
return mic;
}
audio_t audio(mic_t &mic, std::uint32_t buf_size) {
auto mic_attr = (mic_attr_t*)mic.get();
audio_t result { new std::uint8_t[buf_size] };
auto buf = (std::uint8_t*)result.get();
int error;
if(pa_simple_read(mic_attr->mic.get(), buf, buf_size, &error)) {
std::cout << "pa_simple_read() failed: "sv << pa_strerror(error) << std::endl;
}
return result;
}
std::int16_t *audio_data(audio_t &audio) {
return (int16_t*)audio.get();
}
void freeDisplay(void*p) {
delete (display_attr_t*)p;
}
void freeImage(void*p) {
XDestroyImage((XImage*)p);
void freeImage(XImage *p) {
XDestroyImage(p);
}
void freeMic(void*p) {

View File

@ -18,6 +18,7 @@ using namespace std::literals;
using evdev_t = util::safe_ptr<libevdev, libevdev_free>;
using uinput_t = util::safe_ptr<libevdev_uinput, libevdev_uinput_destroy>;
using keyboard_t = util::safe_ptr_v2<Display, int, XCloseDisplay>;
struct input_raw_t {
evdev_t gamepad_dev;
uinput_t gamepad_input;
@ -25,22 +26,7 @@ struct input_raw_t {
evdev_t mouse_dev;
uinput_t mouse_input;
display_t display;
};
//TODO: Use libevdev for keyboard and mouse, then any mention of X11 can be removed from linux_evdev.cpp
struct display_attr_t {
display_attr_t() : display { XOpenDisplay(nullptr) }, window { DefaultRootWindow(display) }, attr {} {
XGetWindowAttributes(display, window, &attr);
}
~display_attr_t() {
XCloseDisplay(display);
}
Display *display;
Window window;
XWindowAttributes attr;
keyboard_t keyboard;
};
void move_mouse(input_t &input, int deltaX, int deltaY) {
@ -203,17 +189,17 @@ uint16_t keysym(uint16_t modcode) {
}
void keyboard(input_t &input, uint16_t modcode, bool release) {
auto &disp = *((display_attr_t *) ((input_raw_t*)input.get())->display.get());
KeyCode kc = XKeysymToKeycode(disp.display, keysym(modcode));
auto &keyboard = ((input_raw_t*)input.get())->keyboard;
KeyCode kc = XKeysymToKeycode(keyboard.get(), keysym(modcode));
if(!kc) {
return;
}
XTestFakeKeyEvent(disp.display, kc, !release, 0);
XTestFakeKeyEvent(keyboard.get(), kc, !release, 0);
XSync(disp.display, 0);
XFlush(disp.display);
XSync(keyboard.get(), 0);
XFlush(keyboard.get());
}
namespace gp {
@ -425,6 +411,11 @@ input_t input() {
input_t result { new input_raw_t() };
auto &gp = *(input_raw_t*)result.get();
gp.keyboard.reset(XOpenDisplay(nullptr));
if(!gp.keyboard) {
return nullptr;
}
if(gamepad(gp)) {
return nullptr;
}
@ -445,7 +436,6 @@ input_t input() {
std::filesystem::create_symlink(libevdev_uinput_get_devnode(gp.mouse_input.get()), mouse_path);
std::filesystem::create_symlink(libevdev_uinput_get_devnode(gp.gamepad_input.get()), gamepad_path);
gp.display = display();
return result;
}

View File

@ -34,7 +34,7 @@ void free_packet(AVPacket *packet) {
using ctx_t = util::safe_ptr<AVCodecContext, free_ctx>;
using frame_t = util::safe_ptr<AVFrame, free_frame>;
using sws_t = util::safe_ptr<SwsContext, sws_freeContext>;
using img_event_t = std::shared_ptr<safe::event_t<platf::img_t>>;
using img_event_t = std::shared_ptr<safe::event_t<std::unique_ptr<platf::img_t>>>;
auto open_codec(ctx_t &ctx, AVCodec *codec, AVDictionary **options) {
avcodec_open2(ctx.get(), codec, options);
@ -48,11 +48,11 @@ void encode(int64_t frame, ctx_t &ctx, sws_t &sws, frame_t &yuv_frame, platf::im
av_frame_make_writable(yuv_frame.get());
const int linesizes[2] {
(int)(platf::img_width(img) * sizeof(int)), 0
(int)(img.width * sizeof(int)), 0
};
auto data = platf::img_data(img);
int ret = sws_scale(sws.get(), (uint8_t*const*)&data, linesizes, 0, platf::img_height(img), yuv_frame->data, yuv_frame->linesize);
auto data = img.data;
int ret = sws_scale(sws.get(), (uint8_t*const*)&data, linesizes, 0, img.height, yuv_frame->data, yuv_frame->linesize);
if(ret <= 0) {
exit(1);
@ -149,8 +149,8 @@ void encodeThread(
// Initiate scaling context with correct height and width
sws_t sws;
while (auto img = images->pop()) {
auto new_width = platf::img_width(img);
auto new_height = platf::img_height(img);
auto new_width = img->width;
auto new_height = img->height;
if(img_width != new_width || img_height != new_height) {
img_width = new_width;
@ -177,7 +177,7 @@ void encodeThread(
yuv_frame->pict_type = AV_PICTURE_TYPE_I;
}
encode(frame++, ctx, sws, yuv_frame, img, packets);
encode(frame++, ctx, sws, yuv_frame, *img, packets);
yuv_frame->pict_type = AV_PICTURE_TYPE_NONE;
}
@ -197,7 +197,7 @@ void capture_display(packet_queue_t packets, idr_event_t idr_events, config_t co
auto time_span = std::chrono::floor<std::chrono::nanoseconds>(1s) / framerate;
while(packets->running()) {
auto next_snapshot = std::chrono::steady_clock::now() + time_span;
auto img = platf::snapshot(disp, display_cursor);
auto img = disp->snapshot(display_cursor);
images->raise(std::move(img));
img.reset();