From e0a2472706ee7af99cd00d35bc7e5829aea5caea Mon Sep 17 00:00:00 2001
From: goldsimon <goldsimon@gmx.de>
Date: Wed, 12 Apr 2017 12:37:16 +0200
Subject: [PATCH] netconn/sockets: remove fatal error handling, fix
 asynchronous error handling, ensure data before RST can be received

---
 CHANGELOG                       |   4 +
 src/api/api_lib.c               |  92 +++++---
 src/api/api_msg.c               | 373 ++++++++++++++++++--------------
 src/api/sockets.c               |  16 +-
 src/include/lwip/api.h          |  18 +-
 src/include/lwip/err.h          |   2 -
 src/include/lwip/priv/api_msg.h |   5 +-
 7 files changed, 285 insertions(+), 225 deletions(-)

diff --git a/CHANGELOG b/CHANGELOG
index 190b4f44..ed1d1427 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -6,6 +6,10 @@ HISTORY
 
   ++ New features:
 
+  2017-04-11: Simon Goldschmidt
+  * netconn/sockets: remove fatal error handling, fix asynchronous error handling,
+    ensure data before RST can be received
+
   2017-03-02: Simon Goldschmidt
   * netconn: added nonblocking accept/recv to netconn API (task #14396)
 
diff --git a/src/api/api_lib.c b/src/api/api_lib.c
index 521d6800..d3eaaf24 100644
--- a/src/api/api_lib.c
+++ b/src/api/api_lib.c
@@ -387,6 +387,7 @@ netconn_listen_with_backlog(struct netconn *conn, u8_t backlog)
 err_t
 netconn_accept(struct netconn *conn, struct netconn **new_conn)
 {
+  err_t err;
 #if LWIP_TCP
   void *accept_ptr;
   struct netconn *newconn;
@@ -398,10 +399,18 @@ netconn_accept(struct netconn *conn, struct netconn **new_conn)
   *new_conn = NULL;
   LWIP_ERROR("netconn_accept: invalid conn",       (conn != NULL),                      return ERR_ARG;);
 
-  if (ERR_IS_FATAL(conn->last_err)) {
-    /* don't recv on fatal errors: this might block the application task
+  /* NOTE: Although the opengroup spec says a pending error shall be returned to
+           send/recv/getsockopt(SO_ERROR) only, we return it for listening
+           connections also, to handle embedded-system errors */
+  err = netconn_err(conn);
+  if (err != ERR_OK) {
+    /* return pending error */
+    return err;
+  }
+  if (conn->flags & NETCONN_FLAG_MBOXCLOSED) {
+    /* don't accept if closed: this might block the application task
        waiting on acceptmbox forever! */
-    return conn->last_err;
+    return ERR_CLSD;
   }
   if (!sys_mbox_valid(&conn->acceptmbox)) {
     return ERR_CLSD;
@@ -432,29 +441,24 @@ netconn_accept(struct netconn *conn, struct netconn **new_conn)
     sys_arch_mbox_fetch(&conn->acceptmbox, &accept_ptr, 0);
 #endif /* LWIP_SO_RCVTIMEO*/
   }
-  newconn = (struct netconn *)accept_ptr;
   /* Register event with callback */
   API_EVENT(conn, NETCONN_EVT_RCVMINUS, 0);
 
-  if (accept_ptr == &netconn_aborted) {
-    /* a connection has been aborted: out of pcbs or out of netconns during accept */
-    /* @todo: set netconn error, but this would be fatal and thus block further accepts */
+  if (lwip_netconn_is_err_msg(accept_ptr, &err)) {
+    /* a connection has been aborted: e.g. out of pcbs or out of netconns during accept */
 #if TCP_LISTEN_BACKLOG
     API_MSG_VAR_FREE(msg);
 #endif /* TCP_LISTEN_BACKLOG */
-    return ERR_ABRT;
+    return err;
   }
-  if (newconn == NULL) {
+  if (accept_ptr == NULL) {
     /* connection has been aborted */
-    /* in this special case, we set the netconn error from application thread, as
-       on a ready-to-accept listening netconn, there should not be anything running
-       in tcpip_thread */
-    NETCONN_SET_SAFE_ERR(conn, ERR_CLSD);
 #if TCP_LISTEN_BACKLOG
     API_MSG_VAR_FREE(msg);
 #endif /* TCP_LISTEN_BACKLOG */
     return ERR_CLSD;
   }
+  newconn = (struct netconn *)accept_ptr;
 #if TCP_LISTEN_BACKLOG
   /* Let the stack know that we have accepted the connection. */
   API_MSG_VAR_REF(msg).conn = newconn;
@@ -494,23 +498,27 @@ netconn_recv_data(struct netconn *conn, void **new_buf, u8_t apiflags)
 {
   void *buf = NULL;
   u16_t len;
+  err_t err;
 
   LWIP_ERROR("netconn_recv: invalid pointer", (new_buf != NULL), return ERR_ARG;);
   *new_buf = NULL;
   LWIP_ERROR("netconn_recv: invalid conn",    (conn != NULL),    return ERR_ARG;);
-  LWIP_ERROR("netconn_recv: invalid recvmbox", sys_mbox_valid(&conn->recvmbox), return ERR_CONN;);
 
-  if (ERR_IS_FATAL(conn->last_err)) {
-    /* don't recv on fatal errors: this might block the application task
-       waiting on recvmbox forever! */
-    /* @todo: this does not allow us to fetch data that has been put into recvmbox
-       before the fatal error occurred - is that a problem? */
-    return conn->last_err;
+  err = netconn_err(conn);
+  if (err != ERR_OK) {
+    /* return pending error */
+    return err;
+  }
+  if (!sys_mbox_valid(&conn->recvmbox)) {
+    return ERR_CLSD;
   }
 
-  if (netconn_is_nonblocking(conn) || (apiflags & NETCONN_DONTBLOCK)) {
+  if (netconn_is_nonblocking(conn) || (apiflags & NETCONN_DONTBLOCK) || (conn->flags & NETCONN_FLAG_MBOXCLOSED)) {
     if (sys_arch_mbox_tryfetch(&conn->recvmbox, &buf) == SYS_ARCH_TIMEOUT) {
-     return ERR_WOULDBLOCK;
+      if (conn->flags & NETCONN_FLAG_MBOXCLOSED) {
+        return ERR_CLSD;
+      }
+      return ERR_WOULDBLOCK;
     }
   } else {
 #if LWIP_SO_RCVTIMEO
@@ -527,10 +535,14 @@ netconn_recv_data(struct netconn *conn, void **new_buf, u8_t apiflags)
   if (NETCONNTYPE_GROUP(conn->type) == NETCONN_TCP)
 #endif /* (LWIP_UDP || LWIP_RAW) */
   {
-    /* If we received a NULL pointer, we are closed */
-    if (buf == NULL) {
-      /* new_buf has been zeroed above alredy */
-      return ERR_OK;
+    /* Check if this is an error message or a pbuf */
+    if (lwip_netconn_is_err_msg(buf, &err)) {
+      /* new_buf has been zeroed above already */
+      if (err == ERR_CLSD) {
+        /* connection closed translates to ERR_OK with *new_buf == NULL */
+        return ERR_OK;
+      }
+      return err;
     }
     len = ((struct pbuf *)buf)->tot_len;
   }
@@ -630,7 +642,11 @@ netconn_recv_data_tcp(struct netconn *conn, struct pbuf **new_buf, u8_t apiflags
     API_EVENT(conn, NETCONN_EVT_RCVMINUS, 0);
     if (conn->pcb.ip == NULL) {
       /* race condition: RST during recv */
-      return conn->last_err == ERR_OK ? ERR_RST : conn->last_err;
+      err = netconn_err(conn);
+      if (err != ERR_OK) {
+        return err;
+      }
+      return ERR_RST;
     }
     /* RX side is closed, so deallocate the recvmbox */
     netconn_close_shutdown(conn, NETCONN_SHUT_RD);
@@ -994,6 +1010,28 @@ netconn_close(struct netconn *conn)
   return netconn_close_shutdown(conn, NETCONN_SHUT_RDWR);
 }
 
+/**
+ * @ingroup netconn_common
+ * Get and reset pending error on a netconn
+ *
+ * @param conn the netconn to get the error from
+ * @return and pending error or ERR_OK if no error was pending
+ */
+err_t
+netconn_err(struct netconn *conn)
+{
+  err_t err;
+  SYS_ARCH_DECL_PROTECT(lev);
+  if (conn == NULL) {
+    return ERR_OK;
+  }
+  SYS_ARCH_PROTECT(lev);
+  err = conn->pending_err;
+  conn->pending_err = ERR_OK;
+  SYS_ARCH_UNPROTECT(lev);
+  return err;
+}
+
 /**
  * @ingroup netconn_tcp
  * Shut down one or both sides of a TCP netconn (doesn't delete it).
diff --git a/src/api/api_msg.c b/src/api/api_msg.c
index db2228b6..063da344 100644
--- a/src/api/api_msg.c
+++ b/src/api/api_msg.c
@@ -79,15 +79,54 @@ static err_t lwip_netconn_do_close_internal(struct netconn *conn  WRITE_DELAYED_
 #endif
 
 #if LWIP_TCPIP_CORE_LOCKING
-#define TCPIP_APIMSG_ACK(m)   NETCONN_SET_SAFE_ERR((m)->conn, (m)->err)
+#define TCPIP_APIMSG_ACK(m)
 #else /* LWIP_TCPIP_CORE_LOCKING */
-#define TCPIP_APIMSG_ACK(m)   do { NETCONN_SET_SAFE_ERR((m)->conn, (m)->err); sys_sem_signal(LWIP_API_MSG_SEM(m)); } while(0)
+#define TCPIP_APIMSG_ACK(m)   do { sys_sem_signal(LWIP_API_MSG_SEM(m)); } while(0)
 #endif /* LWIP_TCPIP_CORE_LOCKING */
 
 #if LWIP_TCP
-u8_t netconn_aborted;
+const u8_t netconn_aborted = 0;
+const u8_t netconn_reset = 0;
+const u8_t netconn_closed = 0;
 #endif /* LWIP_TCP */
 
+/** Translate an error to a unique void* passed via an mbox */
+static void*
+lwip_netconn_err_to_msg(err_t err)
+{
+  switch(err)
+  {
+  case ERR_ABRT:
+    return LWIP_CONST_CAST(void*, &netconn_aborted);
+  case ERR_RST:
+    return LWIP_CONST_CAST(void*, &netconn_reset);
+  case ERR_CLSD:
+    return LWIP_CONST_CAST(void*, &netconn_closed);
+  default:
+    LWIP_ASSERT("unhandled error", err == ERR_OK);
+    return NULL;
+  }
+}
+
+int
+lwip_netconn_is_err_msg(void *msg, err_t *err)
+{
+  LWIP_ASSERT("err != NULL", err != NULL);
+
+  if (msg == &netconn_aborted) {
+    *err = ERR_ABRT;
+    return 1;
+  } else if (msg == &netconn_reset) {
+    *err = ERR_RST;
+    return 1;
+  } else if (msg == &netconn_closed) {
+    *err = ERR_CLSD;
+    return 1;
+  }
+  return 0;
+}
+
+
 #if LWIP_RAW
 /**
  * Receive callback function for RAW netconns.
@@ -244,10 +283,12 @@ recv_tcp(void *arg, struct tcp_pcb *pcb, struct pbuf *p, err_t err)
 {
   struct netconn *conn;
   u16_t len;
+  void *msg;
 
   LWIP_UNUSED_ARG(pcb);
   LWIP_ASSERT("recv_tcp must have a pcb argument", pcb != NULL);
   LWIP_ASSERT("recv_tcp must have an argument", arg != NULL);
+  LWIP_ASSERT("err != ERR_OK unhandled", err == ERR_OK);
   conn = (struct netconn *)arg;
 
   if (conn == NULL) {
@@ -267,18 +308,15 @@ recv_tcp(void *arg, struct tcp_pcb *pcb, struct pbuf *p, err_t err)
      using recv_avail since that could break the connection
      (data is already ACKed) */
 
-  /* don't overwrite fatal errors! */
-  if (err != ERR_OK) {
-    NETCONN_SET_SAFE_ERR(conn, err);
-  }
-
   if (p != NULL) {
+    msg = p;
     len = p->tot_len;
   } else {
+    msg = LWIP_CONST_CAST(void*, &netconn_closed);
     len = 0;
   }
 
-  if (sys_mbox_trypost(&conn->recvmbox, p) != ERR_OK) {
+  if (sys_mbox_trypost(&conn->recvmbox, msg) != ERR_OK) {
     /* don't deallocate p: it is presented to us later again from tcp_fasttmr! */
     return ERR_MEM;
   } else {
@@ -383,26 +421,26 @@ err_tcp(void *arg, err_t err)
 {
   struct netconn *conn;
   enum netconn_state old_state;
+  void *mbox_msg;
+  SYS_ARCH_DECL_PROTECT(lev);
 
   conn = (struct netconn *)arg;
   LWIP_ASSERT("conn != NULL", (conn != NULL));
 
+  SYS_ARCH_PROTECT(lev);
+
+  /* when err is called, the pcb is deallocated, so delete the reference */
   conn->pcb.tcp = NULL;
+  /* store pending error */
+  conn->pending_err = err;
+  /* prevent application threads from blocking on 'recvmbox'/'acceptmbox' */
+  conn->flags |= NETCONN_FLAG_MBOXCLOSED;
 
   /* reset conn->state now before waking up other threads */
   old_state = conn->state;
   conn->state = NETCONN_NONE;
 
-  if (old_state == NETCONN_CLOSE) {
-    /* RST during close: let close return success & dealloc the netconn */
-    err = ERR_OK;
-    NETCONN_SET_SAFE_ERR(conn, ERR_OK);
-  } else {
-    /* no check since this is always fatal! */
-    SYS_ARCH_SET(conn->last_err, err);
-  }
-
-  /* @todo: the type of NETCONN_EVT created should depend on 'old_state' */
+  SYS_ARCH_UNPROTECT(lev);
 
   /* Notify the user layer about a connection error. Used to signal select. */
   API_EVENT(conn, NETCONN_EVT_ERROR, 0);
@@ -411,15 +449,16 @@ err_tcp(void *arg, err_t err)
   API_EVENT(conn, NETCONN_EVT_RCVPLUS, 0);
   API_EVENT(conn, NETCONN_EVT_SENDPLUS, 0);
 
-  /* pass NULL-message to recvmbox to wake up pending recv */
+  mbox_msg = lwip_netconn_err_to_msg(err);
+  /* pass error message to recvmbox to wake up pending recv */
   if (sys_mbox_valid(&conn->recvmbox)) {
     /* use trypost to prevent deadlock */
-    sys_mbox_trypost(&conn->recvmbox, NULL);
+    sys_mbox_trypost(&conn->recvmbox, mbox_msg);
   }
-  /* pass NULL-message to acceptmbox to wake up pending accept */
+  /* pass error message to acceptmbox to wake up pending accept */
   if (sys_mbox_valid(&conn->acceptmbox)) {
     /* use trypost to preven deadlock */
-    sys_mbox_trypost(&conn->acceptmbox, NULL);
+    sys_mbox_trypost(&conn->acceptmbox, mbox_msg);
   }
 
   if ((old_state == NETCONN_WRITE) || (old_state == NETCONN_CLOSE) ||
@@ -433,13 +472,20 @@ err_tcp(void *arg, err_t err)
       sys_sem_t* op_completed_sem;
       /* set error return code */
       LWIP_ASSERT("conn->current_msg != NULL", conn->current_msg != NULL);
-      conn->current_msg->err = err;
+      if (old_state == NETCONN_CLOSE) {
+        /* let close succeed: the connection is closed after all... */
+        conn->current_msg->err = ERR_OK;
+      } else {
+        /* Write and connect fail */
+        conn->current_msg->err = err;
+      }
       op_completed_sem = LWIP_API_MSG_SEM(conn->current_msg);
       LWIP_ASSERT("inavlid op_completed_sem", sys_sem_valid(op_completed_sem));
       conn->current_msg = NULL;
       /* wake up the waiting task */
-      NETCONN_SET_SAFE_ERR(conn, err);
       sys_sem_signal(op_completed_sem);
+    } else {
+      /* @todo: test what happens for error on nonblocking connect */
     }
   } else {
     LWIP_ASSERT("conn->current_msg == NULL", conn->current_msg == NULL);
@@ -487,12 +533,14 @@ accept_function(void *arg, struct tcp_pcb *newpcb, err_t err)
 
   if (newpcb == NULL) {
     /* out-of-pcbs during connect: pass on this error to the application */
-    if (sys_mbox_trypost(&conn->acceptmbox, &netconn_aborted) == ERR_OK) {
+    if (sys_mbox_trypost(&conn->acceptmbox, lwip_netconn_err_to_msg(ERR_ABRT)) == ERR_OK) {
       /* Register event with callback */
       API_EVENT(conn, NETCONN_EVT_RCVPLUS, 0);
     }
     return ERR_VAL;
   }
+  LWIP_ASSERT("expect newpcb == NULL or err == ERR_OK", err == ERR_OK);
+  LWIP_UNUSED_ARG(err); /* for LWIP_NOASSERT */
 
   LWIP_DEBUGF(API_MSG_DEBUG, ("accept_function: newpcb->state: %s\n", tcp_debug_state_str(newpcb->state)));
 
@@ -501,7 +549,7 @@ accept_function(void *arg, struct tcp_pcb *newpcb, err_t err)
   newconn = netconn_alloc(conn->type, conn->callback);
   if (newconn == NULL) {
     /* outof netconns: pass on this error to the application */
-    if (sys_mbox_trypost(&conn->acceptmbox, &netconn_aborted) == ERR_OK) {
+    if (sys_mbox_trypost(&conn->acceptmbox, lwip_netconn_err_to_msg(ERR_ABRT)) == ERR_OK) {
       /* Register event with callback */
       API_EVENT(conn, NETCONN_EVT_RCVPLUS, 0);
     }
@@ -509,9 +557,6 @@ accept_function(void *arg, struct tcp_pcb *newpcb, err_t err)
   }
   newconn->pcb.tcp = newpcb;
   setup_tcp(newconn);
-  /* no protection: when creating the pcb, the netconn is not yet known
-     to the application thread */
-  newconn->last_err = err;
 
   /* handle backlog counter */
   tcp_backlog_delayed(newpcb);
@@ -655,7 +700,7 @@ netconn_alloc(enum netconn_type t, netconn_callback callback)
     return NULL;
   }
 
-  conn->last_err = ERR_OK;
+  conn->pending_err = ERR_OK;
   conn->type = t;
   conn->pcb.tcp = NULL;
 
@@ -978,7 +1023,6 @@ lwip_netconn_do_close_internal(struct netconn *conn  WRITE_DELAYED_PARAM)
         API_EVENT(conn, NETCONN_EVT_SENDPLUS, 0);
       }
     }
-    NETCONN_SET_SAFE_ERR(conn, err);
 #if LWIP_TCPIP_CORE_LOCKING
     if (delayed)
 #endif
@@ -1034,7 +1078,6 @@ lwip_netconn_do_delconn(void *m)
       msg->conn->current_msg->err = ERR_CLSD;
       msg->conn->current_msg = NULL;
       msg->conn->state = NETCONN_NONE;
-      NETCONN_SET_SAFE_ERR(msg->conn, ERR_CLSD);
       sys_sem_signal(op_completed_sem);
     }
   }
@@ -1118,33 +1161,33 @@ void
 lwip_netconn_do_bind(void *m)
 {
   struct api_msg *msg = (struct api_msg*)m;
+  err_t err;
 
-  if (ERR_IS_FATAL(msg->conn->last_err)) {
-    msg->err = msg->conn->last_err;
-  } else {
-    msg->err = ERR_VAL;
-    if (msg->conn->pcb.tcp != NULL) {
-      switch (NETCONNTYPE_GROUP(msg->conn->type)) {
+  if (msg->conn->pcb.tcp != NULL) {
+    switch (NETCONNTYPE_GROUP(msg->conn->type)) {
 #if LWIP_RAW
-      case NETCONN_RAW:
-        msg->err = raw_bind(msg->conn->pcb.raw, API_EXPR_REF(msg->msg.bc.ipaddr));
-        break;
+    case NETCONN_RAW:
+      err = raw_bind(msg->conn->pcb.raw, API_EXPR_REF(msg->msg.bc.ipaddr));
+      break;
 #endif /* LWIP_RAW */
 #if LWIP_UDP
-      case NETCONN_UDP:
-        msg->err = udp_bind(msg->conn->pcb.udp, API_EXPR_REF(msg->msg.bc.ipaddr), msg->msg.bc.port);
-        break;
+    case NETCONN_UDP:
+      err = udp_bind(msg->conn->pcb.udp, API_EXPR_REF(msg->msg.bc.ipaddr), msg->msg.bc.port);
+      break;
 #endif /* LWIP_UDP */
 #if LWIP_TCP
-      case NETCONN_TCP:
-        msg->err = tcp_bind(msg->conn->pcb.tcp, API_EXPR_REF(msg->msg.bc.ipaddr), msg->msg.bc.port);
-        break;
+    case NETCONN_TCP:
+      err = tcp_bind(msg->conn->pcb.tcp, API_EXPR_REF(msg->msg.bc.ipaddr), msg->msg.bc.port);
+      break;
 #endif /* LWIP_TCP */
-      default:
-        break;
-      }
+    default:
+      err = ERR_VAL;
+      break;
     }
+  } else {
+    err = ERR_VAL;
   }
+  msg->err = err;
   TCPIP_APIMSG_ACK(msg);
 }
 
@@ -1188,7 +1231,6 @@ lwip_netconn_do_connected(void *arg, struct tcp_pcb *pcb, err_t err)
     (!was_blocking && op_completed_sem == NULL));
   conn->current_msg = NULL;
   conn->state = NETCONN_NONE;
-  NETCONN_SET_SAFE_ERR(conn, ERR_OK);
   API_EVENT(conn, NETCONN_EVT_SENDPLUS, 0);
 
   if (was_blocking) {
@@ -1209,39 +1251,40 @@ void
 lwip_netconn_do_connect(void *m)
 {
   struct api_msg *msg = (struct api_msg*)m;
+  err_t err;
 
   if (msg->conn->pcb.tcp == NULL) {
     /* This may happen when calling netconn_connect() a second time */
-    msg->err = ERR_CLSD;
+    err = ERR_CLSD;
   } else {
     switch (NETCONNTYPE_GROUP(msg->conn->type)) {
 #if LWIP_RAW
     case NETCONN_RAW:
-      msg->err = raw_connect(msg->conn->pcb.raw, API_EXPR_REF(msg->msg.bc.ipaddr));
+      err = raw_connect(msg->conn->pcb.raw, API_EXPR_REF(msg->msg.bc.ipaddr));
       break;
 #endif /* LWIP_RAW */
 #if LWIP_UDP
     case NETCONN_UDP:
-      msg->err = udp_connect(msg->conn->pcb.udp, API_EXPR_REF(msg->msg.bc.ipaddr), msg->msg.bc.port);
+      err = udp_connect(msg->conn->pcb.udp, API_EXPR_REF(msg->msg.bc.ipaddr), msg->msg.bc.port);
       break;
 #endif /* LWIP_UDP */
 #if LWIP_TCP
     case NETCONN_TCP:
       /* Prevent connect while doing any other action. */
       if (msg->conn->state == NETCONN_CONNECT) {
-        msg->err = ERR_ALREADY;
+        err = ERR_ALREADY;
       } else if (msg->conn->state != NETCONN_NONE) {
-        msg->err = ERR_ISCONN;
+        err = ERR_ISCONN;
       } else {
         setup_tcp(msg->conn);
-        msg->err = tcp_connect(msg->conn->pcb.tcp, API_EXPR_REF(msg->msg.bc.ipaddr),
+        err = tcp_connect(msg->conn->pcb.tcp, API_EXPR_REF(msg->msg.bc.ipaddr),
           msg->msg.bc.port, lwip_netconn_do_connected);
-        if (msg->err == ERR_OK) {
+        if (err == ERR_OK) {
           u8_t non_blocking = netconn_is_nonblocking(msg->conn);
           msg->conn->state = NETCONN_CONNECT;
           SET_NONBLOCKING_CONNECT(msg->conn, non_blocking);
           if (non_blocking) {
-            msg->err = ERR_INPROGRESS;
+            err = ERR_INPROGRESS;
           } else {
             msg->conn->current_msg = msg;
             /* sys_sem_signal() is called from lwip_netconn_do_connected (or err_tcp()),
@@ -1260,10 +1303,11 @@ lwip_netconn_do_connect(void *m)
       break;
 #endif /* LWIP_TCP */
     default:
-      LWIP_ERROR("Invalid netconn type", 0, do{ msg->err = ERR_VAL; }while(0));
+      LWIP_ERROR("Invalid netconn type", 0, do{ err = ERR_VAL; }while(0));
       break;
     }
   }
+  msg->err = err;
   /* For all other protocols, netconn_connect() calls TCPIP_APIMSG(),
      so use TCPIP_APIMSG_ACK() here. */
   TCPIP_APIMSG_ACK(msg);
@@ -1304,76 +1348,76 @@ void
 lwip_netconn_do_listen(void *m)
 {
   struct api_msg *msg = (struct api_msg*)m;
+  err_t err;
 
-  if (ERR_IS_FATAL(msg->conn->last_err)) {
-    msg->err = msg->conn->last_err;
-  } else {
-    msg->err = ERR_CONN;
-    if (msg->conn->pcb.tcp != NULL) {
-      if (NETCONNTYPE_GROUP(msg->conn->type) == NETCONN_TCP) {
-        if (msg->conn->state == NETCONN_NONE) {
-          struct tcp_pcb* lpcb;
-          if (msg->conn->pcb.tcp->state != CLOSED) {
-            /* connection is not closed, cannot listen */
-            msg->err = ERR_VAL;
-          } else {
-            err_t err;
-            u8_t backlog;
+  if (msg->conn->pcb.tcp != NULL) {
+    if (NETCONNTYPE_GROUP(msg->conn->type) == NETCONN_TCP) {
+      if (msg->conn->state == NETCONN_NONE) {
+        struct tcp_pcb* lpcb;
+        if (msg->conn->pcb.tcp->state != CLOSED) {
+          /* connection is not closed, cannot listen */
+          err = ERR_VAL;
+        } else {
+          u8_t backlog;
 #if TCP_LISTEN_BACKLOG
-            backlog = msg->msg.lb.backlog;
+          backlog = msg->msg.lb.backlog;
 #else  /* TCP_LISTEN_BACKLOG */
-            backlog = TCP_DEFAULT_LISTEN_BACKLOG;
+          backlog = TCP_DEFAULT_LISTEN_BACKLOG;
 #endif /* TCP_LISTEN_BACKLOG */
 #if LWIP_IPV4 && LWIP_IPV6
-            /* "Socket API like" dual-stack support: If IP to listen to is IP6_ADDR_ANY,
-             * and NETCONN_FLAG_IPV6_V6ONLY is NOT set, use IP_ANY_TYPE to listen
-             */
-            if (ip_addr_cmp(&msg->conn->pcb.ip->local_ip, IP6_ADDR_ANY) &&
-                (netconn_get_ipv6only(msg->conn) == 0)) {
-              /* change PCB type to IPADDR_TYPE_ANY */
-              IP_SET_TYPE_VAL(msg->conn->pcb.tcp->local_ip,  IPADDR_TYPE_ANY);
-              IP_SET_TYPE_VAL(msg->conn->pcb.tcp->remote_ip, IPADDR_TYPE_ANY);
-            }
+          /* "Socket API like" dual-stack support: If IP to listen to is IP6_ADDR_ANY,
+            * and NETCONN_FLAG_IPV6_V6ONLY is NOT set, use IP_ANY_TYPE to listen
+            */
+          if (ip_addr_cmp(&msg->conn->pcb.ip->local_ip, IP6_ADDR_ANY) &&
+              (netconn_get_ipv6only(msg->conn) == 0)) {
+            /* change PCB type to IPADDR_TYPE_ANY */
+            IP_SET_TYPE_VAL(msg->conn->pcb.tcp->local_ip,  IPADDR_TYPE_ANY);
+            IP_SET_TYPE_VAL(msg->conn->pcb.tcp->remote_ip, IPADDR_TYPE_ANY);
+          }
 #endif /* LWIP_IPV4 && LWIP_IPV6 */
 
-            lpcb = tcp_listen_with_backlog_and_err(msg->conn->pcb.tcp, backlog, &err);
+          lpcb = tcp_listen_with_backlog_and_err(msg->conn->pcb.tcp, backlog, &err);
 
-            if (lpcb == NULL) {
-              /* in this case, the old pcb is still allocated */
-              msg->err = err;
+          if (lpcb == NULL) {
+            /* in this case, the old pcb is still allocated */
+          } else {
+            /* delete the recvmbox and allocate the acceptmbox */
+            if (sys_mbox_valid(&msg->conn->recvmbox)) {
+              /** @todo: should we drain the recvmbox here? */
+              sys_mbox_free(&msg->conn->recvmbox);
+              sys_mbox_set_invalid(&msg->conn->recvmbox);
+            }
+            err = ERR_OK;
+            if (!sys_mbox_valid(&msg->conn->acceptmbox)) {
+              err = sys_mbox_new(&msg->conn->acceptmbox, DEFAULT_ACCEPTMBOX_SIZE);
+            }
+            if (err == ERR_OK) {
+              msg->conn->state = NETCONN_LISTEN;
+              msg->conn->pcb.tcp = lpcb;
+              tcp_arg(msg->conn->pcb.tcp, msg->conn);
+              tcp_accept(msg->conn->pcb.tcp, accept_function);
             } else {
-              /* delete the recvmbox and allocate the acceptmbox */
-              if (sys_mbox_valid(&msg->conn->recvmbox)) {
-                /** @todo: should we drain the recvmbox here? */
-                sys_mbox_free(&msg->conn->recvmbox);
-                sys_mbox_set_invalid(&msg->conn->recvmbox);
-              }
-              msg->err = ERR_OK;
-              if (!sys_mbox_valid(&msg->conn->acceptmbox)) {
-                msg->err = sys_mbox_new(&msg->conn->acceptmbox, DEFAULT_ACCEPTMBOX_SIZE);
-              }
-              if (msg->err == ERR_OK) {
-                msg->conn->state = NETCONN_LISTEN;
-                msg->conn->pcb.tcp = lpcb;
-                tcp_arg(msg->conn->pcb.tcp, msg->conn);
-                tcp_accept(msg->conn->pcb.tcp, accept_function);
-              } else {
-                /* since the old pcb is already deallocated, free lpcb now */
-                tcp_close(lpcb);
-                msg->conn->pcb.tcp = NULL;
-              }
+              /* since the old pcb is already deallocated, free lpcb now */
+              tcp_close(lpcb);
+              msg->conn->pcb.tcp = NULL;
             }
           }
-        } else if (msg->conn->state == NETCONN_LISTEN) {
-          /* already listening, allow updating of the backlog */
-          msg->err = ERR_OK;
-          tcp_backlog_set(msg->conn->pcb.tcp, msg->msg.lb.backlog);
         }
-      } else {
-        msg->err = ERR_ARG;
+      } else if (msg->conn->state == NETCONN_LISTEN) {
+        /* already listening, allow updating of the backlog */
+        err = ERR_OK;
+        tcp_backlog_set(msg->conn->pcb.tcp, msg->msg.lb.backlog);
       }
+      else {
+        err = ERR_CONN;
+      }
+    } else {
+      err = ERR_ARG;
     }
+  } else {
+    err = ERR_CONN;
   }
+  msg->err = err;
   TCPIP_APIMSG_ACK(msg);
 }
 #endif /* LWIP_TCP */
@@ -1389,18 +1433,16 @@ lwip_netconn_do_send(void *m)
 {
   struct api_msg *msg = (struct api_msg*)m;
 
-  if (ERR_IS_FATAL(msg->conn->last_err)) {
-    msg->err = msg->conn->last_err;
-  } else {
-    msg->err = ERR_CONN;
+  err_t err = netconn_err(msg->conn);
+  if (err == ERR_OK) {
     if (msg->conn->pcb.tcp != NULL) {
       switch (NETCONNTYPE_GROUP(msg->conn->type)) {
 #if LWIP_RAW
       case NETCONN_RAW:
         if (ip_addr_isany(&msg->msg.b->addr) || IP_IS_ANY_TYPE_VAL(msg->msg.b->addr)) {
-          msg->err = raw_send(msg->conn->pcb.raw, msg->msg.b->p);
+          err = raw_send(msg->conn->pcb.raw, msg->msg.b->p);
         } else {
-          msg->err = raw_sendto(msg->conn->pcb.raw, msg->msg.b->p, &msg->msg.b->addr);
+          err = raw_sendto(msg->conn->pcb.raw, msg->msg.b->p, &msg->msg.b->addr);
         }
         break;
 #endif
@@ -1408,27 +1450,31 @@ lwip_netconn_do_send(void *m)
       case NETCONN_UDP:
 #if LWIP_CHECKSUM_ON_COPY
         if (ip_addr_isany(&msg->msg.b->addr) || IP_IS_ANY_TYPE_VAL(msg->msg.b->addr)) {
-          msg->err = udp_send_chksum(msg->conn->pcb.udp, msg->msg.b->p,
+          err = udp_send_chksum(msg->conn->pcb.udp, msg->msg.b->p,
             msg->msg.b->flags & NETBUF_FLAG_CHKSUM, msg->msg.b->toport_chksum);
         } else {
-          msg->err = udp_sendto_chksum(msg->conn->pcb.udp, msg->msg.b->p,
+          err = udp_sendto_chksum(msg->conn->pcb.udp, msg->msg.b->p,
             &msg->msg.b->addr, msg->msg.b->port,
             msg->msg.b->flags & NETBUF_FLAG_CHKSUM, msg->msg.b->toport_chksum);
         }
 #else /* LWIP_CHECKSUM_ON_COPY */
         if (ip_addr_isany_val(msg->msg.b->addr) || IP_IS_ANY_TYPE_VAL(msg->msg.b->addr)) {
-          msg->err = udp_send(msg->conn->pcb.udp, msg->msg.b->p);
+          err = udp_send(msg->conn->pcb.udp, msg->msg.b->p);
         } else {
-          msg->err = udp_sendto(msg->conn->pcb.udp, msg->msg.b->p, &msg->msg.b->addr, msg->msg.b->port);
+          err = udp_sendto(msg->conn->pcb.udp, msg->msg.b->p, &msg->msg.b->addr, msg->msg.b->port);
         }
 #endif /* LWIP_CHECKSUM_ON_COPY */
         break;
 #endif /* LWIP_UDP */
       default:
+        err = ERR_CONN;
         break;
       }
+    } else {
+      err = ERR_CONN;
     }
   }
+  msg->err = err;
   TCPIP_APIMSG_ACK(msg);
 }
 
@@ -1601,8 +1647,8 @@ err_mem:
         write_finished = 1;
       }
       out_err = tcp_output(conn->pcb.tcp);
-      if (ERR_IS_FATAL(out_err) || (out_err == ERR_RTE)) {
-        /* If tcp_output fails with fatal error or no route is found,
+      if (out_err == ERR_RTE) {
+        /* If tcp_output fails because no route is found,
            don't try writing any more but return the error
            to the application thread. */
         err = out_err;
@@ -1616,8 +1662,8 @@ err_mem:
 
       /* tcp_write returned ERR_MEM, try tcp_output anyway */
       err_t out_err = tcp_output(conn->pcb.tcp);
-      if (ERR_IS_FATAL(out_err) || (out_err == ERR_RTE)) {
-        /* If tcp_output fails with fatal error or no route is found,
+      if (out_err == ERR_RTE) {
+        /* If tcp_output fails because no route is found,
            don't try writing any more but return the error
            to the application thread. */
         err = out_err;
@@ -1641,7 +1687,6 @@ err_mem:
     conn->current_msg->err = err;
     conn->current_msg = NULL;
     conn->state = NETCONN_NONE;
-    NETCONN_SET_SAFE_ERR(conn, err);
 #if LWIP_TCPIP_CORE_LOCKING
     if (delayed)
 #endif
@@ -1669,14 +1714,13 @@ lwip_netconn_do_write(void *m)
 {
   struct api_msg *msg = (struct api_msg*)m;
 
-  if (ERR_IS_FATAL(msg->conn->last_err)) {
-    msg->err = msg->conn->last_err;
-  } else {
+  err_t err = netconn_err(msg->conn);
+  if (err == ERR_OK) {
     if (NETCONNTYPE_GROUP(msg->conn->type) == NETCONN_TCP) {
 #if LWIP_TCP
       if (msg->conn->state != NETCONN_NONE) {
         /* netconn is connecting, closing or in blocking write */
-        msg->err = ERR_INPROGRESS;
+        err = ERR_INPROGRESS;
       } else if (msg->conn->pcb.tcp != NULL) {
         msg->conn->state = NETCONN_WRITE;
         /* set all the variables used by lwip_netconn_do_writemore */
@@ -1698,17 +1742,18 @@ lwip_netconn_do_write(void *m)
            since lwip_netconn_do_writemore ACKs it! */
         return;
       } else {
-        msg->err = ERR_CONN;
+        err = ERR_CONN;
       }
 #else /* LWIP_TCP */
-      msg->err = ERR_VAL;
+      err = ERR_VAL;
 #endif /* LWIP_TCP */
 #if (LWIP_UDP || LWIP_RAW)
     } else {
-      msg->err = ERR_VAL;
+      err = ERR_VAL;
 #endif /* (LWIP_UDP || LWIP_RAW) */
     }
   }
+  msg->err = err;
   TCPIP_APIMSG_ACK(msg);
 }
 
@@ -1812,7 +1857,6 @@ lwip_netconn_do_close(void *m)
         msg->conn->current_msg = NULL;
         msg->conn->state = NETCONN_NONE;
         state = NETCONN_NONE;
-        NETCONN_SET_SAFE_ERR(msg->conn, ERR_CLSD);
         sys_sem_signal(write_completed_sem);
       } else {
         LWIP_ASSERT("msg->msg.sd.shut == NETCONN_SHUT_RD", msg->msg.sd.shut == NETCONN_SHUT_RD);
@@ -1867,43 +1911,38 @@ lwip_netconn_do_join_leave_group(void *m)
 {
   struct api_msg *msg = (struct api_msg*)m;
 
-  if (ERR_IS_FATAL(msg->conn->last_err)) {
-    msg->err = msg->conn->last_err;
-  } else {
-    if (msg->conn->pcb.tcp != NULL) {
-      if (NETCONNTYPE_GROUP(msg->conn->type) == NETCONN_UDP) {
+  msg->err = ERR_CONN;
+  if (msg->conn->pcb.tcp != NULL) {
+    if (NETCONNTYPE_GROUP(msg->conn->type) == NETCONN_UDP) {
 #if LWIP_UDP
 #if LWIP_IPV6 && LWIP_IPV6_MLD
-        if (NETCONNTYPE_ISIPV6(msg->conn->type)) {
-          if (msg->msg.jl.join_or_leave == NETCONN_JOIN) {
-            msg->err = mld6_joingroup(ip_2_ip6(API_EXPR_REF(msg->msg.jl.netif_addr)),
-              ip_2_ip6(API_EXPR_REF(msg->msg.jl.multiaddr)));
-          } else {
-            msg->err = mld6_leavegroup(ip_2_ip6(API_EXPR_REF(msg->msg.jl.netif_addr)),
-              ip_2_ip6(API_EXPR_REF(msg->msg.jl.multiaddr)));
-          }
+      if (NETCONNTYPE_ISIPV6(msg->conn->type)) {
+        if (msg->msg.jl.join_or_leave == NETCONN_JOIN) {
+          msg->err = mld6_joingroup(ip_2_ip6(API_EXPR_REF(msg->msg.jl.netif_addr)),
+            ip_2_ip6(API_EXPR_REF(msg->msg.jl.multiaddr)));
+        } else {
+          msg->err = mld6_leavegroup(ip_2_ip6(API_EXPR_REF(msg->msg.jl.netif_addr)),
+            ip_2_ip6(API_EXPR_REF(msg->msg.jl.multiaddr)));
         }
-        else
+      }
+      else
 #endif /* LWIP_IPV6 && LWIP_IPV6_MLD */
-        {
+      {
 #if LWIP_IGMP
-          if (msg->msg.jl.join_or_leave == NETCONN_JOIN) {
-            msg->err = igmp_joingroup(ip_2_ip4(API_EXPR_REF(msg->msg.jl.netif_addr)),
-              ip_2_ip4(API_EXPR_REF(msg->msg.jl.multiaddr)));
-          } else {
-            msg->err = igmp_leavegroup(ip_2_ip4(API_EXPR_REF(msg->msg.jl.netif_addr)),
-              ip_2_ip4(API_EXPR_REF(msg->msg.jl.multiaddr)));
-          }
-#endif /* LWIP_IGMP */
+        if (msg->msg.jl.join_or_leave == NETCONN_JOIN) {
+          msg->err = igmp_joingroup(ip_2_ip4(API_EXPR_REF(msg->msg.jl.netif_addr)),
+            ip_2_ip4(API_EXPR_REF(msg->msg.jl.multiaddr)));
+        } else {
+          msg->err = igmp_leavegroup(ip_2_ip4(API_EXPR_REF(msg->msg.jl.netif_addr)),
+            ip_2_ip4(API_EXPR_REF(msg->msg.jl.multiaddr)));
         }
+#endif /* LWIP_IGMP */
+      }
 #endif /* LWIP_UDP */
 #if (LWIP_TCP || LWIP_RAW)
-      } else {
-        msg->err = ERR_VAL;
-#endif /* (LWIP_TCP || LWIP_RAW) */
-      }
     } else {
-      msg->err = ERR_CONN;
+      msg->err = ERR_VAL;
+#endif /* (LWIP_TCP || LWIP_RAW) */
     }
   }
   TCPIP_APIMSG_ACK(msg);
diff --git a/src/api/sockets.c b/src/api/sockets.c
index 199a9898..320235fa 100644
--- a/src/api/sockets.c
+++ b/src/api/sockets.c
@@ -210,8 +210,6 @@ union lwip_sock_lastdata {
 struct lwip_sock {
   /** sockets currently are built on netconns, each socket has one netconn */
   struct netconn *conn;
-  /** last error that occurred on this socket */
-  int err;
   /** data that was left from the previous read */
   union lwip_sock_lastdata lastdata;
 #if LWIP_SOCKET_SELECT
@@ -325,7 +323,6 @@ static struct lwip_select_cb *select_cb_list;
 
 #define sock_set_errno(sk, e) do { \
   const int sockerr = (e); \
-  sk->err = sockerr; \
   set_errno(sockerr); \
 } while (0)
 
@@ -502,7 +499,6 @@ alloc_socket(struct netconn *newconn, int accepted)
       sockets[i].sendevent  = (NETCONNTYPE_GROUP(newconn->type) == NETCONN_TCP ? (accepted != 0) : 1);
       sockets[i].errevent   = 0;
 #endif /* LWIP_SOCKET_SELECT */
-      sockets[i].err        = 0;
       return i + LWIP_SOCKET_OFFSET;
     }
     SYS_ARCH_UNPROTECT(lev);
@@ -534,9 +530,8 @@ free_socket(struct lwip_sock *sock, int is_tcp)
   }
 #endif
 
-  lastdata         = sock->lastdata;
+  lastdata = sock->lastdata;
   sock->lastdata.pbuf = NULL;
-  sock->err        = 0;
   sock->conn = NULL;
   SYS_ARCH_UNPROTECT(lev);
   /* don't use 'sock' after this line, as another task might have allocated it */
@@ -2422,12 +2417,7 @@ lwip_getsockopt_impl(int s, int level, int optname, void *optval, socklen_t *opt
 
     case SO_ERROR:
       LWIP_SOCKOPT_CHECK_OPTLEN(sock, *optlen, int);
-      /* only overwrite ERR_OK or temporary errors */
-      if (((sock->err == 0) || (sock->err == EINPROGRESS)) && (sock->conn != NULL)) {
-        sock_set_errno(sock, err_to_errno(sock->conn->last_err));
-      }
-      *(int *)optval = sock->err;
-      sock->err = 0;
+      *(int *)optval = err_to_errno(netconn_err(sock->conn));
       LWIP_DEBUGF(SOCKETS_DEBUG, ("lwip_getsockopt(%d, SOL_SOCKET, SO_ERROR) = %d\n",
                   s, *(int *)optval));
       break;
@@ -3243,6 +3233,8 @@ lwip_fcntl(int s, int cmd, int val)
 
     break;
   case F_SETFL:
+    /* Bits corresponding to the file access mode and the file creation flags [..] that are set in arg shall be ignored */
+    val &= ~(O_RDONLY|O_WRONLY|O_RDWR);
     if ((val & ~O_NONBLOCK) == 0) {
       /* only O_NONBLOCK, all other bits are zero */
       netconn_set_nonblocking(sock->conn, val & O_NONBLOCK);
diff --git a/src/include/lwip/api.h b/src/include/lwip/api.h
index 4fb8038c..60711080 100644
--- a/src/include/lwip/api.h
+++ b/src/include/lwip/api.h
@@ -66,6 +66,8 @@ extern "C" {
 #define NETCONN_NOAUTORCVD  0x08 /* prevent netconn_recv_data_tcp() from updating the tcp window - must be done manually via netconn_tcp_recvd() */
 
 /* Flags for struct netconn.flags (u8_t) */
+/** This netconn had an error, don't block on recvmbox/acceptmbox any more */
+#define NETCONN_FLAG_MBOXCLOSED               0x01
 /** Should this netconn avoid blocking? */
 #define NETCONN_FLAG_NON_BLOCKING             0x02
 /** Was the last connect action a non-blocking one? */
@@ -215,8 +217,8 @@ struct netconn {
     struct udp_pcb *udp;
     struct raw_pcb *raw;
   } pcb;
-  /** the last error this netconn had */
-  err_t last_err;
+  /** the last asynchronous unreported error this netconn had */
+  err_t pending_err;
 #if !LWIP_NETCONN_SEM_PER_THREAD
   /** sem that is used to synchronously execute functions in the core context */
   sys_sem_t op_completed;
@@ -278,16 +280,6 @@ struct netvector {
                            (*c->callback)(c, e, l); \
                          }
 
-/** Set conn->last_err to err but don't overwrite fatal errors */
-#define NETCONN_SET_SAFE_ERR(conn, err) do { if ((conn) != NULL) { \
-  SYS_ARCH_DECL_PROTECT(netconn_set_safe_err_lev); \
-  SYS_ARCH_PROTECT(netconn_set_safe_err_lev); \
-  if (!ERR_IS_FATAL((conn)->last_err)) { \
-    (conn)->last_err = err; \
-  } \
-  SYS_ARCH_UNPROTECT(netconn_set_safe_err_lev); \
-}} while(0);
-
 /* Network connection functions: */
 
 /** @ingroup netconn_common
@@ -348,7 +340,7 @@ err_t   netconn_gethostbyname(const char *name, ip_addr_t *addr);
 #endif /* LWIP_IPV4 && LWIP_IPV6 */
 #endif /* LWIP_DNS */
 
-#define netconn_err(conn)               ((conn)->last_err)
+err_t   netconn_err(struct netconn *conn);
 #define netconn_recv_bufsize(conn)      ((conn)->recv_bufsize)
 
 /** Set the blocking status of netconn calls (@todo: write/send is missing) */
diff --git a/src/include/lwip/err.h b/src/include/lwip/err.h
index 84e528d1..77730987 100644
--- a/src/include/lwip/err.h
+++ b/src/include/lwip/err.h
@@ -96,8 +96,6 @@ typedef enum {
   ERR_ARG        = -16
 } err_enum_t;
 
-#define ERR_IS_FATAL(e) ((e) <= ERR_ABRT)
-
 /**
  * @}
  */
diff --git a/src/include/lwip/priv/api_msg.h b/src/include/lwip/priv/api_msg.h
index 225fbc3d..9005e24b 100644
--- a/src/include/lwip/priv/api_msg.h
+++ b/src/include/lwip/priv/api_msg.h
@@ -185,10 +185,7 @@ struct dns_api_msg {
 };
 #endif /* LWIP_DNS */
 
-#if LWIP_TCP
-extern u8_t netconn_aborted;
-#endif /* LWIP_TCP */
-
+int lwip_netconn_is_err_msg(void *msg, err_t *err);
 void lwip_netconn_do_newconn         (void *m);
 void lwip_netconn_do_delconn         (void *m);
 void lwip_netconn_do_bind            (void *m);