diff --git a/src/listener.c b/src/listener.c
index aceb4cd..718f528 100644
--- a/src/listener.c
+++ b/src/listener.c
@@ -1058,6 +1058,9 @@
 		    (!p->fe_sps_lim || freq_ctr_remain(&p->fe_sess_per_sec, p->fe_sps_lim, 0) > 0))
 			dequeue_all_listeners(&p->listener_queue);
 	}
+
+	if (l->state != LI_READY)
+		fd_stop_recv(l->fd);
 }
 
 /* Notify the listener that a connection initiated from it was released. This