diff --git a/network/netplay/netplay_io.c b/network/netplay/netplay_io.c
index 68187931ba..0c610ff803 100644
--- a/network/netplay/netplay_io.c
+++ b/network/netplay/netplay_io.c
@@ -247,7 +247,7 @@ bool netplay_send_cur_input(netplay_t *netplay,
    {
       if (!send_input_frame(netplay, connection, NULL,
             netplay->self_frame_count,
-            (netplay->is_server ?  NETPLAY_CMD_INPUT_BIT_SERVER : 0) | netplay->self_player,
+            (netplay->is_server ? NETPLAY_CMD_INPUT_BIT_SERVER : 0) | netplay->self_player,
             dframe->self_state))
          return false;
    }
diff --git a/network/netplay/netplay_private.h b/network/netplay/netplay_private.h
index cbd7a999a5..fc4a7bd751 100644
--- a/network/netplay/netplay_private.h
+++ b/network/netplay/netplay_private.h
@@ -42,6 +42,7 @@
 
 #define MAX_SERVER_STALL_TIME_USEC  (5*1000*1000)
 #define MAX_CLIENT_STALL_TIME_USEC  (10*1000*1000)
+#define CATCH_UP_CHECK_TIME_USEC    (500*1000)
 #define MAX_RETRIES                 16
 #define RETRY_MS                    500
 
@@ -479,6 +480,12 @@ struct netplay
    /* Opposite of stalling, should we be catching up? */
    bool catch_up;
 
+   /* When did we start falling behind? */
+   retro_time_t catch_up_time;
+
+   /* How far behind did we fall? */
+   uint32_t catch_up_behind;
+
    /* Frequency with which to check CRCs */
    int check_frames;
 
diff --git a/network/netplay/netplay_sync.c b/network/netplay/netplay_sync.c
index ef9be28cd8..cf624c81de 100644
--- a/network/netplay/netplay_sync.c
+++ b/network/netplay/netplay_sync.c
@@ -555,7 +555,7 @@ void netplay_sync_post_frame(netplay_t *netplay, bool stalled)
    if (netplay->catch_up)
    {
       /* Are we caught up? */
-      if (netplay->self_frame_count >= lo_frame_count)
+      if (netplay->self_frame_count + 1 >= lo_frame_count)
       {
          netplay->catch_up = false;
          input_driver_unset_nonblock_state();
@@ -565,17 +565,44 @@ void netplay_sync_post_frame(netplay_t *netplay, bool stalled)
    }
    else if (!stalled)
    {
-      if (netplay->self_frame_count + 2 < lo_frame_count)
+      if (netplay->self_frame_count + 3 < lo_frame_count)
       {
-         /* Are we falling behind? */
-         netplay->catch_up = true;
-         input_driver_set_nonblock_state();
-         driver_set_nonblock_state();
+         retro_time_t cur_time = cpu_features_get_time_usec();
+         uint32_t cur_behind = lo_frame_count - netplay->self_frame_count;
+
+         /* We're behind, but we'll only try to catch up if we're actually
+          * falling behind, i.e. if we're more behind after some time */
+         if (netplay->catch_up_time == 0)
+         {
+            /* Record our current time to check for catch-up later */
+            netplay->catch_up_time = cur_time;
+            netplay->catch_up_behind = cur_behind;
+
+         }
+         else if (cur_time - netplay->catch_up_time > CATCH_UP_CHECK_TIME_USEC)
+         {
+            /* Time to check how far behind we are */
+            if (netplay->catch_up_behind <= cur_behind)
+            {
+               /* We're definitely falling behind! */
+               netplay->catch_up = true;
+               netplay->catch_up_time = 0;
+               input_driver_set_nonblock_state();
+               driver_set_nonblock_state();
+            }
+            else
+            {
+               /* Check again in another period */
+               netplay->catch_up_time = cur_time;
+               netplay->catch_up_behind = cur_behind;
+            }
+         }
 
       }
-      else if (netplay->self_frame_count + 2 < hi_frame_count)
+      else if (netplay->self_frame_count + 3 < hi_frame_count)
       {
          size_t i;
+         netplay->catch_up_time = 0;
 
          /* We're falling behind some clients but not others, so request that
           * clients ahead of us stall */
@@ -589,7 +616,7 @@ void netplay_sync_post_frame(netplay_t *netplay, bool stalled)
             player = connection->player;
 
             /* Are they ahead? */
-            if (netplay->self_frame_count + 2 < netplay->read_frame_count[player])
+            if (netplay->self_frame_count + 3 < netplay->read_frame_count[player])
             {
                /* Tell them to stall */
                if (connection->stall_frame + NETPLAY_MAX_REQ_STALL_FREQUENCY <
@@ -603,5 +630,9 @@ void netplay_sync_post_frame(netplay_t *netplay, bool stalled)
             }
          }
       }
+      else
+         netplay->catch_up_time = 0;
    }
+   else
+      netplay->catch_up_time =  0;
 }