diff --git a/src/checks.c b/src/checks.c
index 7f195c9..8014a66 100644
--- a/src/checks.c
+++ b/src/checks.c
@@ -1443,10 +1443,9 @@
 		/* We're here because nobody wants to handle the error, so we
 		 * sure want to abort the hard way.
 		 */
-		if ((conn->flags & CO_FL_CTRL_READY) && !(conn->flags & CO_FL_SOCK_RD_SH)) {
-			if (conn->flags & CO_FL_WAIT_RD || !conn->ctrl->drain || !conn->ctrl->drain(conn->t.sock.fd))
-				setsockopt(conn->t.sock.fd, SOL_SOCKET, SO_LINGER,
-				           (struct linger *) &nolinger, sizeof(struct linger));
+		if (conn_ctrl_ready(conn) && !(conn->flags & CO_FL_SOCK_RD_SH)) {
+			if (!(conn->flags & CO_FL_WAIT_RD) && conn->ctrl->drain && conn->ctrl->drain(conn->t.sock.fd))
+				fdtab[conn->t.sock.fd].linger_risk = 0;
 		}
 		conn_force_close(conn);
 	}
@@ -1663,10 +1662,9 @@
 			 * as a failed response coupled with "observe layer7" caused the
 			 * server state to be suddenly changed.
 			 */
-			if ((conn->flags & CO_FL_CTRL_READY) && !(conn->flags & CO_FL_SOCK_RD_SH)) {
-				if (conn->flags & CO_FL_WAIT_RD || !conn->ctrl->drain || !conn->ctrl->drain(conn->t.sock.fd))
-					setsockopt(conn->t.sock.fd, SOL_SOCKET, SO_LINGER,
-					           (struct linger *) &nolinger, sizeof(struct linger));
+			if (conn_ctrl_ready(conn) && !(conn->flags & CO_FL_SOCK_RD_SH)) {
+				if (!(conn->flags & CO_FL_WAIT_RD) && conn->ctrl->drain && conn->ctrl->drain(conn->t.sock.fd))
+					fdtab[conn->t.sock.fd].linger_risk = 0;
 			}
 			conn_force_close(conn);
 		}
diff --git a/src/fd.c b/src/fd.c
index 5df0b45..7e69114 100644
--- a/src/fd.c
+++ b/src/fd.c
@@ -116,6 +116,11 @@
  */
 void fd_delete(int fd)
 {
+	if (fdtab[fd].linger_risk) {
+		/* this is generally set when connecting to servers */
+		setsockopt(fd, SOL_SOCKET, SO_LINGER,
+			   (struct linger *) &nolinger, sizeof(struct linger));
+	}
 	if (cur_poller.clo)
 		cur_poller.clo(fd);
 
diff --git a/src/frontend.c b/src/frontend.c
index 591c1f2..f94c6ab 100644
--- a/src/frontend.c
+++ b/src/frontend.c
@@ -85,8 +85,8 @@
 				   (char *) &one, sizeof(one));
 
 		if (s->fe->options & PR_O_TCP_NOLING)
-			setsockopt(cfd, SOL_SOCKET, SO_LINGER,
-				   (struct linger *) &nolinger, sizeof(struct linger));
+			fdtab[cfd].linger_risk = 1;
+
 #if defined(TCP_MAXSEG)
 		if (s->listener->maxseg < 0) {
 			/* we just want to reduce the current MSS by that value */
diff --git a/src/proto_tcp.c b/src/proto_tcp.c
index 2b6fbc7..c1b0d7b 100644
--- a/src/proto_tcp.c
+++ b/src/proto_tcp.c
@@ -475,6 +475,7 @@
 		conn->flags |= CO_FL_SEND_PROXY;
 
 	conn_ctrl_init(conn);       /* registers the FD */
+	fdtab[fd].linger_risk = 1;  /* close hard if needed */
 	conn_sock_want_send(conn);  /* for connect status */
 
 	if (conn_xprt_init(conn) < 0) {
diff --git a/src/stream_interface.c b/src/stream_interface.c
index f2650d3..d819ede 100644
--- a/src/stream_interface.c
+++ b/src/stream_interface.c
@@ -812,10 +812,6 @@
 			/* quick close, the socket is alredy shut anyway */
 		}
 		else if (si->flags & SI_FL_NOLINGER) {
-			if (conn_ctrl_ready(conn)) {
-				setsockopt(conn->t.sock.fd, SOL_SOCKET, SO_LINGER,
-					   (struct linger *) &nolinger, sizeof(struct linger));
-			}
 			/* unclean data-layer shutdown */
 			if (conn->xprt && conn->xprt->shutw)
 				conn->xprt->shutw(conn, 0);
@@ -1285,12 +1281,6 @@
 
 	if (si->flags & SI_FL_NOHALF) {
 		/* we want to immediately forward this close to the write side */
-		if (si->flags & SI_FL_NOLINGER) {
-			si->flags &= ~SI_FL_NOLINGER;
-			if (conn_ctrl_ready(conn))
-				setsockopt(conn->t.sock.fd, SOL_SOCKET, SO_LINGER,
-				           (struct linger *) &nolinger, sizeof(struct linger));
-		}
 		/* force flag on ssl to keep session in cache */
 		if (conn->xprt->shutw)
 			conn->xprt->shutw(conn, 0);
@@ -1298,6 +1288,8 @@
 	}
 
 	/* otherwise that's just a normal read shutdown */
+	if (conn_ctrl_ready(conn))
+		fdtab[conn->t.sock.fd].linger_risk = 0;
 	__conn_data_stop_recv(conn);
 	return;