diff --git a/src/stream_interface.c b/src/stream_interface.c
index 8f89dba..950f55b 100644
--- a/src/stream_interface.c
+++ b/src/stream_interface.c
@@ -588,8 +588,14 @@
 
 	/* First step, report to the stream-int what was detected at the
 	 * connection layer : errors and connection establishment.
+	 * Only add SI_FL_ERR if we're connected, or we're attempting to
+	 * connect, we may get there because we got woken up, but only run
+	 * after process_stream() noticed there were an error, and decided
+	 * to retry to connect, the connection may still have CO_FL_ERROR,
+	 * and we don't want to add SI_FL_ERR back
 	 */
-	if (conn->flags & CO_FL_ERROR || cs->flags & CS_FL_ERROR)
+	if (si->state >= SI_ST_CON &&
+	    (conn->flags & CO_FL_ERROR || cs->flags & CS_FL_ERROR))
 		si->flags |= SI_FL_ERR;
 
 	/* If we had early data, and the handshake ended, then
@@ -645,6 +651,14 @@
 		return 0;
 
 	if (conn->flags & CO_FL_ERROR || cs->flags & (CS_FL_ERROR|CS_FL_ERR_PENDING)) {
+		/* We're probably there because the tasklet was woken up,
+		 * but process_stream() ran before, detected there were an
+		 * error and put the si back to SI_ST_TAR. There's still
+		 * CO_FL_ERROR on the connection but we don't want to add
+		 * SI_FL_ERR back, so give up
+		 */
+		if (si->state < SI_ST_CON)
+			return 0;
 		si->flags |= SI_FL_ERR;
 		return 1;
 	}
@@ -1126,7 +1140,8 @@
 
 	if (cs->flags & (CS_FL_ERROR|CS_FL_ERR_PENDING) || cs->conn->flags & CO_FL_ERROR) {
 		/* Write error on the file descriptor */
-		si->flags |= SI_FL_ERR;
+		if (si->state >= SI_ST_CON)
+			si->flags |= SI_FL_ERR;
 		goto out_wakeup;
 	}