REORG: fd: move the fd state management from ev_sepoll

ev_sepoll already provides everything needed to manage FD events
by only manipulating the speculative I/O list. Nothing there is
sepoll-specific so move all this to fd.
diff --git a/src/fd.c b/src/fd.c
index 5d63cc3..2718748 100644
--- a/src/fd.c
+++ b/src/fd.c
@@ -116,7 +116,12 @@
  */
 void fd_delete(int fd)
 {
-	cur_poller.clo(fd);
+	if (cur_poller.clo)
+		cur_poller.clo(fd);
+
+	release_spec_entry(fd);
+	fdtab[fd].spec_e &= ~(FD_EV_CURR_MASK | FD_EV_PREV_MASK);
+
 	port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port);
 	fdinfo[fd].port_range = NULL;
 	close(fd);