diff --git a/input/common/wayland_common.c b/input/common/wayland_common.c
index f6e687fa83..30ba05dd64 100644
--- a/input/common/wayland_common.c
+++ b/input/common/wayland_common.c
@@ -162,12 +162,13 @@ static void pointer_handle_enter(void *data,
 {
    gfx_ctx_wayland_data_t *wl = (gfx_ctx_wayland_data_t*)data;
 
-   wl->input.mouse.last_x = wl_fixed_to_int(sx * (wl_fixed_t)wl->buffer_scale);
-   wl->input.mouse.last_y = wl_fixed_to_int(sy * (wl_fixed_t)wl->buffer_scale);
-   wl->input.mouse.x      = wl->input.mouse.last_x;
-   wl->input.mouse.y      = wl->input.mouse.last_y;
-   wl->input.mouse.focus  = true;
-   wl->cursor.serial      = serial;
+   wl->input.mouse.surface = surface;
+   wl->input.mouse.last_x  = wl_fixed_to_int(sx * (wl_fixed_t)wl->buffer_scale);
+   wl->input.mouse.last_y  = wl_fixed_to_int(sy * (wl_fixed_t)wl->buffer_scale);
+   wl->input.mouse.x       = wl->input.mouse.last_x;
+   wl->input.mouse.y       = wl->input.mouse.last_y;
+   wl->input.mouse.focus   = true;
+   wl->cursor.serial       = serial;
 
    gfx_ctx_wl_show_mouse(data, wl->cursor.visible);
 }
@@ -179,6 +180,9 @@ static void pointer_handle_leave(void *data,
 {
    gfx_ctx_wayland_data_t *wl = (gfx_ctx_wayland_data_t*)data;
    wl->input.mouse.focus      = false;
+
+   if (wl->input.mouse.surface == surface)
+      wl->input.mouse.surface = NULL;
 }
 
 static void pointer_handle_motion(void *data,
@@ -203,6 +207,9 @@ static void pointer_handle_button(void *data,
 {
    gfx_ctx_wayland_data_t *wl = (gfx_ctx_wayland_data_t*)data;
 
+   if (wl->input.mouse.surface != wl->surface)
+      return;
+
    if (state == WL_POINTER_BUTTON_STATE_PRESSED)
    {
       switch (button)
@@ -451,6 +458,8 @@ static void wl_surface_enter(void *data, struct wl_surface *wl_surface,
     output_info_t *oi;
     gfx_ctx_wayland_data_t *wl = (gfx_ctx_wayland_data_t*)data;
 
+    wl->input.mouse.surface = wl_surface;
+
     /* TODO: track all outputs the surface is on, pick highest scale */
 
     wl_list_for_each(oi, &wl->all_outputs, link)
diff --git a/input/common/wayland_common.h b/input/common/wayland_common.h
index fc44bf497d..6cf363357b 100644
--- a/input/common/wayland_common.h
+++ b/input/common/wayland_common.h
@@ -96,6 +96,7 @@ typedef struct input_ctx_wayland_data
 
    struct
    {
+      struct wl_pointer *surface;
       int last_x, last_y;
       int x, y;
       int delta_x, delta_y;