src/listener.c

/*
 * Listener management functions.
 *
 * Copyright 2000-2013 Willy Tarreau <w@1wt.eu>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 *
 */

#define _GNU_SOURCE
#include <ctype.h>
#include <errno.h>
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <fcntl.h>

#include <common/accept4.h>
#include <common/cfgparse.h>
#include <common/config.h>
#include <common/errors.h>
#include <common/initcall.h>
#include <common/mini-clist.h>
#include <common/standard.h>
#include <common/time.h>

#include <types/global.h>
#include <types/protocol.h>

#include <proto/acl.h>
#include <proto/connection.h>
#include <proto/fd.h>
#include <proto/freq_ctr.h>
#include <proto/log.h>
#include <proto/listener.h>
#include <proto/protocol.h>
#include <proto/proto_sockpair.h>
#include <proto/sample.h>
#include <proto/stream.h>
#include <proto/task.h>

/* List head of all known bind keywords */
static struct bind_kw_list bind_keywords = {
	.list = LIST_HEAD_INIT(bind_keywords.list)
};

struct xfer_sock_list *xfer_sock_list = NULL;

#if defined(USE_THREAD)

struct accept_queue_ring accept_queue_rings[MAX_THREADS] __attribute__((aligned(64))) = { };

/* dequeue and process a pending connection from the local accept queue (single
 * consumer). Returns the accepted fd or -1 if none was found. The listener is
 * placed into *li. The address is copied into *addr for no more than *addr_len
 * bytes, and the address length is returned into *addr_len.
 */
int accept_queue_pop_sc(struct accept_queue_ring *ring, struct listener **li, void *addr, int *addr_len)
{
	struct accept_queue_entry *e;
	unsigned int pos, next;
	struct listener *ptr;
	int len;
	int fd;

	pos = ring->head;

	if (pos == ring->tail)
		return -1;

	next = pos + 1;
	if (next >= ACCEPT_QUEUE_SIZE)
		next = 0;

	e = &ring->entry[pos];

	/* wait for the producer to update the listener's pointer */
	while (1) {
		ptr = e->listener;
		__ha_barrier_load();
		if (ptr)
			break;
		pl_cpu_relax();
	}

	fd = e->fd;
	len = e->addr_len;
	if (len > *addr_len)
		len = *addr_len;

	if (likely(len > 0))
		memcpy(addr, &e->addr, len);

	/* release the entry */
	e->listener = NULL;

	__ha_barrier_store();
	ring->head = next;

	*addr_len = len;
	*li = ptr;

	return fd;
}


/* tries to push a new accepted connection <fd> into ring <ring> for listener
 * <li>, from address <addr> whose length is <addr_len>. Returns non-zero if it
 * succeeds, or zero if the ring is full. Supports multiple producers.
 */
int accept_queue_push_mp(struct accept_queue_ring *ring, int fd,
                         struct listener *li, const void *addr, int addr_len)
{
	struct accept_queue_entry *e;
	unsigned int pos, next;

	pos = ring->tail;
	do {
		next = pos + 1;
		if (next >= ACCEPT_QUEUE_SIZE)
			next = 0;
		if (next == ring->head)
			return 0; // ring full
	} while (unlikely(!HA_ATOMIC_CAS(&ring->tail, &pos, next)));


	e = &ring->entry[pos];

	if (addr_len > sizeof(e->addr))
		addr_len = sizeof(e->addr);

	if (addr_len)
		memcpy(&e->addr, addr, addr_len);

	e->addr_len = addr_len;
	e->fd = fd;

	__ha_barrier_store();
	/* now commit the change */

	e->listener = li;
	return 1;
}

/* proceed with accepting new connections */
static struct task *accept_queue_process(struct task *t, void *context, unsigned short state)
{
	struct accept_queue_ring *ring = context;
	struct listener *li;
	struct sockaddr_storage addr;
	int max_accept = global.tune.maxaccept ? global.tune.maxaccept : 64;
	int addr_len;
	int ret;
	int fd;

	while (max_accept--) {
		addr_len = sizeof(addr);
		fd = accept_queue_pop_sc(ring, &li, &addr, &addr_len);
		if (fd < 0)
			break;

		HA_ATOMIC_ADD(&li->thr_conn[tid], 1);
		ret = li->accept(li, fd, &addr);
		if (ret <= 0) {
			/* connection was terminated by the application */
			continue;
		}

		/* increase the per-process number of cumulated sessions, this
		 * may only be done once l->accept() has accepted the connection.
		 */
		if (!(li->options & LI_O_UNLIMITED)) {
			HA_ATOMIC_UPDATE_MAX(&global.sps_max,
			                     update_freq_ctr(&global.sess_per_sec, 1));
			if (li->bind_conf && li->bind_conf->is_ssl) {
				HA_ATOMIC_UPDATE_MAX(&global.ssl_max,
				                     update_freq_ctr(&global.ssl_per_sec, 1));
			}
		}
	}

	/* ran out of budget ? Let's come here ASAP */
	if (max_accept < 0)
		task_wakeup(t, TASK_WOKEN_IO);

	return t;
}

/* Initializes the accept-queues. Returns 0 on success, otherwise ERR_* flags */
static int accept_queue_init()
{
	struct task *t;
	int i;

	for (i = 0; i < global.nbthread; i++) {
		t = task_new(1UL << i);
		if (!t) {
			ha_alert("Out of memory while initializing accept queue for thread %d\n", i);
			return ERR_FATAL|ERR_ABORT;
		}
		t->process = accept_queue_process;
		t->context = &accept_queue_rings[i];
		accept_queue_rings[i].task = t;
	}
	return 0;
}

REGISTER_CONFIG_POSTPARSER("multi-threaded accept queue", accept_queue_init);

#endif // USE_THREAD

/* This function adds the specified listener's file descriptor to the polling
 * lists if it is in the LI_LISTEN state. The listener enters LI_READY or
 * LI_FULL state depending on its number of connections. In deamon mode, we
 * also support binding only the relevant processes to their respective
 * listeners. We don't do that in debug mode however.
 */
static void enable_listener(struct listener *listener)
{
	HA_SPIN_LOCK(LISTENER_LOCK, &listener->lock);
	if (listener->state == LI_LISTEN) {
		if ((global.mode & (MODE_DAEMON | MODE_MWORKER)) &&
		    !(proc_mask(listener->bind_conf->bind_proc) & pid_bit)) {
			/* we don't want to enable this listener and don't
			 * want any fd event to reach it.
			 */
			if (!(global.tune.options & GTUNE_SOCKET_TRANSFER))
				do_unbind_listener(listener, 1);
			else {
				do_unbind_listener(listener, 0);
				listener->state = LI_LISTEN;
			}
		}
		else if (!listener->maxconn || listener->nbconn < listener->maxconn) {
			fd_want_recv(listener->fd);
			listener->state = LI_READY;
		}
		else {
			listener->state = LI_FULL;
		}
	}
	/* if this listener is supposed to be only in the master, close it in the workers */
	if ((global.mode & MODE_MWORKER) &&
	    (listener->options & LI_O_MWORKER) &&
	    master == 0) {
		do_unbind_listener(listener, 1);
	}
	HA_SPIN_UNLOCK(LISTENER_LOCK, &listener->lock);
}

/* This function removes the specified listener's file descriptor from the
 * polling lists if it is in the LI_READY or in the LI_FULL state. The listener
 * enters LI_LISTEN.
 */
static void disable_listener(struct listener *listener)
{
	HA_SPIN_LOCK(LISTENER_LOCK, &listener->lock);
	if (listener->state < LI_READY)
		goto end;
	if (listener->state == LI_READY)
		fd_stop_recv(listener->fd);
	LIST_DEL_LOCKED(&listener->wait_queue);
	listener->state = LI_LISTEN;
  end:
	HA_SPIN_UNLOCK(LISTENER_LOCK, &listener->lock);
}

/* This function tries to temporarily disable a listener, depending on the OS
 * capabilities. Linux unbinds the listen socket after a SHUT_RD, and ignores
 * SHUT_WR. Solaris refuses either shutdown(). OpenBSD ignores SHUT_RD but
 * closes upon SHUT_WR and refuses to rebind. So a common validation path
 * involves SHUT_WR && listen && SHUT_RD. In case of success, the FD's polling
 * is disabled. It normally returns non-zero, unless an error is reported.
 */
int pause_listener(struct listener *l)
{
	int ret = 1;

	HA_SPIN_LOCK(LISTENER_LOCK, &l->lock);

	if (l->state <= LI_ZOMBIE)
		goto end;

	if (l->proto->pause) {
		/* Returns < 0 in case of failure, 0 if the listener
		 * was totally stopped, or > 0 if correctly paused.
		 */
		int ret = l->proto->pause(l);

		if (ret < 0) {
			ret = 0;
			goto end;
		}
		else if (ret == 0)
			goto end;
	}

	LIST_DEL_LOCKED(&l->wait_queue);

	fd_stop_recv(l->fd);
	l->state = LI_PAUSED;
  end:
	HA_SPIN_UNLOCK(LISTENER_LOCK, &l->lock);
	return ret;
}

/* This function tries to resume a temporarily disabled listener. Paused, full,
 * limited and disabled listeners are handled, which means that this function
 * may replace enable_listener(). The resulting state will either be LI_READY
 * or LI_FULL. 0 is returned in case of failure to resume (eg: dead socket).
 * Listeners bound to a different process are not woken up unless we're in
 * foreground mode, and are ignored. If the listener was only in the assigned
 * state, it's totally rebound. This can happen if a pause() has completely
 * stopped it. If the resume fails, 0 is returned and an error might be
 * displayed.
 */
int resume_listener(struct listener *l)
{
	int ret = 1;

	HA_SPIN_LOCK(LISTENER_LOCK, &l->lock);

	if ((global.mode & (MODE_DAEMON | MODE_MWORKER)) &&
	    !(proc_mask(l->bind_conf->bind_proc) & pid_bit))
		goto end;

	if (l->state == LI_ASSIGNED) {
		char msg[100];
		int err;

		err = l->proto->bind(l, msg, sizeof(msg));
		if (err & ERR_ALERT)
			ha_alert("Resuming listener: %s\n", msg);
		else if (err & ERR_WARN)
			ha_warning("Resuming listener: %s\n", msg);

		if (err & (ERR_FATAL | ERR_ABORT)) {
			ret = 0;
			goto end;
		}
	}

	if (l->state < LI_PAUSED || l->state == LI_ZOMBIE) {
		ret = 0;
		goto end;
	}

	if (l->proto->sock_prot == IPPROTO_TCP &&
	    l->state == LI_PAUSED &&
	    listen(l->fd, listener_backlog(l)) != 0) {
		ret = 0;
		goto end;
	}

	if (l->state == LI_READY)
		goto end;

	LIST_DEL_LOCKED(&l->wait_queue);

	if (l->maxconn && l->nbconn >= l->maxconn) {
		l->state = LI_FULL;
		goto end;
	}

	fd_want_recv(l->fd);
	l->state = LI_READY;
  end:
	HA_SPIN_UNLOCK(LISTENER_LOCK, &l->lock);
	return ret;
}

/* Marks a ready listener as full so that the stream code tries to re-enable
 * it upon next close() using resume_listener().
 */
static void listener_full(struct listener *l)
{
	HA_SPIN_LOCK(LISTENER_LOCK, &l->lock);
	if (l->state >= LI_READY) {
		LIST_DEL_LOCKED(&l->wait_queue);
		if (l->state != LI_FULL) {
			fd_stop_recv(l->fd);
			l->state = LI_FULL;
		}
	}
	HA_SPIN_UNLOCK(LISTENER_LOCK, &l->lock);
}

/* Marks a ready listener as limited so that we only try to re-enable it when
 * resources are free again. It will be queued into the specified queue.
 */
static void limit_listener(struct listener *l, struct list *list)
{
	HA_SPIN_LOCK(LISTENER_LOCK, &l->lock);
	if (l->state == LI_READY) {
		LIST_ADDQ_LOCKED(list, &l->wait_queue);
		fd_stop_recv(l->fd);
		l->state = LI_LIMITED;
	}
	HA_SPIN_UNLOCK(LISTENER_LOCK, &l->lock);
}

/* This function adds all of the protocol's listener's file descriptors to the
 * polling lists when they are in the LI_LISTEN state. It is intended to be
 * used as a protocol's generic enable_all() primitive, for use after the
 * fork(). It puts the listeners into LI_READY or LI_FULL states depending on
 * their number of connections. It always returns ERR_NONE.
 */
int enable_all_listeners(struct protocol *proto)
{
	struct listener *listener;

	list_for_each_entry(listener, &proto->listeners, proto_list)
		enable_listener(listener);
	return ERR_NONE;
}

/* This function removes all of the protocol's listener's file descriptors from
 * the polling lists when they are in the LI_READY or LI_FULL states. It is
 * intended to be used as a protocol's generic disable_all() primitive. It puts
 * the listeners into LI_LISTEN, and always returns ERR_NONE.
 */
int disable_all_listeners(struct protocol *proto)
{
	struct listener *listener;

	list_for_each_entry(listener, &proto->listeners, proto_list)
		disable_listener(listener);
	return ERR_NONE;
}

/* Dequeues all of the listeners waiting for a resource in wait queue <queue>. */
void dequeue_all_listeners(struct list *list)
{
	struct listener *listener;

	while ((listener = LIST_POP_LOCKED(list, struct listener *, wait_queue))) {
		/* This cannot fail because the listeners are by definition in
		 * the LI_LIMITED state.
		 */
		resume_listener(listener);
	}
}

/* Must be called with the lock held. Depending on <do_close> value, it does
 * what unbind_listener or unbind_listener_no_close should do.
 */
void do_unbind_listener(struct listener *listener, int do_close)
{
	if (listener->state == LI_READY)
		fd_stop_recv(listener->fd);

	LIST_DEL_LOCKED(&listener->wait_queue);

	if (listener->state >= LI_PAUSED) {
		if (do_close) {
			fd_delete(listener->fd);
			listener->fd = -1;
		}
		else
			fd_remove(listener->fd);
		listener->state = LI_ASSIGNED;
	}
}

/* This function closes the listening socket for the specified listener,
 * provided that it's already in a listening state. The listener enters the
 * LI_ASSIGNED state. This function is intended to be used as a generic
 * function for standard protocols.
 */
void unbind_listener(struct listener *listener)
{
	HA_SPIN_LOCK(LISTENER_LOCK, &listener->lock);
	do_unbind_listener(listener, 1);
	HA_SPIN_UNLOCK(LISTENER_LOCK, &listener->lock);
}

/* This function pretends the listener is dead, but keeps the FD opened, so
 * that we can provide it, for conf reloading.
 */
void unbind_listener_no_close(struct listener *listener)
{
	HA_SPIN_LOCK(LISTENER_LOCK, &listener->lock);
	do_unbind_listener(listener, 0);
	HA_SPIN_UNLOCK(LISTENER_LOCK, &listener->lock);
}

/* This function closes all listening sockets bound to the protocol <proto>,
 * and the listeners end in LI_ASSIGNED state if they were higher. It does not
 * detach them from the protocol. It always returns ERR_NONE.
 */
int unbind_all_listeners(struct protocol *proto)
{
	struct listener *listener;

	list_for_each_entry(listener, &proto->listeners, proto_list)
		unbind_listener(listener);
	return ERR_NONE;
}

/* creates one or multiple listeners for bind_conf <bc> on sockaddr <ss> on port
 * range <portl> to <porth>, and possibly attached to fd <fd> (or -1 for auto
 * allocation). The address family is taken from ss->ss_family. The number of
 * jobs and listeners is automatically increased by the number of listeners
 * created. If the <inherited> argument is set to 1, it specifies that the FD
 * was obtained from a parent process.
 * It returns non-zero on success, zero on error with the error message
 * set in <err>.
 */
int create_listeners(struct bind_conf *bc, const struct sockaddr_storage *ss,
                     int portl, int porth, int fd, int inherited, char **err)
{
	struct protocol *proto = protocol_by_family(ss->ss_family);
	struct listener *l;
	int port;

	if (!proto) {
		memprintf(err, "unsupported protocol family %d", ss->ss_family);
		return 0;
	}

	for (port = portl; port <= porth; port++) {
		l = calloc(1, sizeof(*l));
		if (!l) {
			memprintf(err, "out of memory");
			return 0;
		}
		l->obj_type = OBJ_TYPE_LISTENER;
		LIST_ADDQ(&bc->frontend->conf.listeners, &l->by_fe);
		LIST_ADDQ(&bc->listeners, &l->by_bind);
		l->bind_conf = bc;

		l->fd = fd;
		memcpy(&l->addr, ss, sizeof(*ss));
		LIST_INIT(&l->wait_queue);
		l->state = LI_INIT;

		proto->add(l, port);

		if (inherited)
			l->options |= LI_O_INHERITED;

		HA_SPIN_INIT(&l->lock);
		HA_ATOMIC_ADD(&jobs, 1);
		HA_ATOMIC_ADD(&listeners, 1);
	}
	return 1;
}

/* Delete a listener from its protocol's list of listeners. The listener's
 * state is automatically updated from LI_ASSIGNED to LI_INIT. The protocol's
 * number of listeners is updated, as well as the global number of listeners
 * and jobs. Note that the listener must have previously been unbound. This
 * is the generic function to use to remove a listener.
 */
void delete_listener(struct listener *listener)
{
	HA_SPIN_LOCK(LISTENER_LOCK, &listener->lock);
	if (listener->state == LI_ASSIGNED) {
		listener->state = LI_INIT;
		LIST_DEL(&listener->proto_list);
		listener->proto->nb_listeners--;
		HA_ATOMIC_SUB(&jobs, 1);
		HA_ATOMIC_SUB(&listeners, 1);
	}
	HA_SPIN_UNLOCK(LISTENER_LOCK, &listener->lock);
}

/* Returns a suitable value for a listener's backlog. It uses the listener's,
 * otherwise the frontend's backlog, otherwise the listener's maxconn,
 * otherwise the frontend's maxconn, otherwise 1024.
 */
int listener_backlog(const struct listener *l)
{
	if (l->backlog)
		return l->backlog;

	if (l->bind_conf->frontend->backlog)
		return l->bind_conf->frontend->backlog;

	if (l->maxconn)
		return l->maxconn;

	if (l->bind_conf->frontend->maxconn)
		return l->bind_conf->frontend->maxconn;

	return 1024;
}

/* This function is called on a read event from a listening socket, corresponding
 * to an accept. It tries to accept as many connections as possible, and for each
 * calls the listener's accept handler (generally the frontend's accept handler).
 */
void listener_accept(int fd)
{
	struct listener *l = fdtab[fd].owner;
	struct proxy *p;
	int max_accept;
	int next_conn = 0;
	int next_feconn = 0;
	int next_actconn = 0;
	int expire;
	int cfd;
	int ret;
#ifdef USE_ACCEPT4
	static int accept4_broken;
#endif

	if (!l)
		return;
	p = l->bind_conf->frontend;
	max_accept = l->maxaccept ? l->maxaccept : 1;

	if (!(l->options & LI_O_UNLIMITED) && global.sps_lim) {
		int max = freq_ctr_remain(&global.sess_per_sec, global.sps_lim, 0);

		if (unlikely(!max)) {
			/* frontend accept rate limit was reached */
			expire = tick_add(now_ms, next_event_delay(&global.sess_per_sec, global.sps_lim, 0));
			goto wait_expire;
		}

		if (max_accept > max)
			max_accept = max;
	}

	if (!(l->options & LI_O_UNLIMITED) && global.cps_lim) {
		int max = freq_ctr_remain(&global.conn_per_sec, global.cps_lim, 0);

		if (unlikely(!max)) {
			/* frontend accept rate limit was reached */
			expire = tick_add(now_ms, next_event_delay(&global.conn_per_sec, global.cps_lim, 0));
			goto wait_expire;
		}

		if (max_accept > max)
			max_accept = max;
	}
#ifdef USE_OPENSSL
	if (!(l->options & LI_O_UNLIMITED) && global.ssl_lim && l->bind_conf && l->bind_conf->is_ssl) {
		int max = freq_ctr_remain(&global.ssl_per_sec, global.ssl_lim, 0);

		if (unlikely(!max)) {
			/* frontend accept rate limit was reached */
			expire = tick_add(now_ms, next_event_delay(&global.ssl_per_sec, global.ssl_lim, 0));
			goto wait_expire;
		}

		if (max_accept > max)
			max_accept = max;
	}
#endif
	if (p && p->fe_sps_lim) {
		int max = freq_ctr_remain(&p->fe_sess_per_sec, p->fe_sps_lim, 0);

		if (unlikely(!max)) {
			/* frontend accept rate limit was reached */
			limit_listener(l, &p->listener_queue);
			task_schedule(p->task, tick_add(now_ms, next_event_delay(&p->fe_sess_per_sec, p->fe_sps_lim, 0)));
			goto end;
		}

		if (max_accept > max)
			max_accept = max;
	}

	/* Note: if we fail to allocate a connection because of configured
	 * limits, we'll schedule a new attempt worst 1 second later in the
	 * worst case. If we fail due to system limits or temporary resource
	 * shortage, we try again 100ms later in the worst case.
	 */
	for (; max_accept-- > 0; next_conn = next_feconn = next_actconn = 0) {
		struct sockaddr_storage addr;
		socklen_t laddr = sizeof(addr);
		unsigned int count;

		/* pre-increase the number of connections without going too far.
		 * We process the listener, then the proxy, then the process.
		 * We know which ones to unroll based on the next_xxx value.
		 */
		do {
			count = l->nbconn;
			if (l->maxconn && count >= l->maxconn) {
				/* the listener was marked full or another
				 * thread is going to do it.
				 */
				next_conn = 0;
				goto end;
			}
			next_conn = count + 1;
		} while (!HA_ATOMIC_CAS(&l->nbconn, &count, next_conn));

		if (l->maxconn && next_conn == l->maxconn) {
			/* we filled it, mark it full */
			listener_full(l);
		}

		if (p) {
			do {
				count = p->feconn;
				if (count >= p->maxconn) {
					/* the frontend was marked full or another
					 * thread is going to do it.
					 */
					next_feconn = 0;
					goto end;
				}
				next_feconn = count + 1;
			} while (!HA_ATOMIC_CAS(&p->feconn, &count, next_feconn));

			if (unlikely(next_feconn == p->maxconn)) {
				/* we just filled it */
				limit_listener(l, &p->listener_queue);
			}
		}

		if (!(l->options & LI_O_UNLIMITED)) {
			do {
				count = actconn;
				if (count >= global.maxconn) {
					/* the process was marked full or another
					 * thread is going to do it.
					 */
					next_actconn = 0;
					goto end;
				}
				next_actconn = count + 1;
			} while (!HA_ATOMIC_CAS(&actconn, &count, next_actconn));

			if (unlikely(next_actconn == global.maxconn)) {
				limit_listener(l, &global_listener_queue);
				task_schedule(global_listener_queue_task, tick_add(now_ms, 1000)); /* try again in 1 second */
			}
		}

		/* with sockpair@ we don't want to do an accept */
		if (unlikely(l->addr.ss_family == AF_CUST_SOCKPAIR)) {
			if ((cfd = recv_fd_uxst(fd)) != -1)
				fcntl(cfd, F_SETFL, O_NONBLOCK);
			/* just like with UNIX sockets, only the family is filled */
			addr.ss_family = AF_UNIX;
			laddr = sizeof(addr.ss_family);
		} else

#ifdef USE_ACCEPT4
		/* only call accept4() if it's known to be safe, otherwise
		 * fallback to the legacy accept() + fcntl().
		 */
		if (unlikely(accept4_broken ||
			((cfd = accept4(fd, (struct sockaddr *)&addr, &laddr, SOCK_NONBLOCK)) == -1 &&
			(errno == ENOSYS || errno == EINVAL || errno == EBADF) &&
			(accept4_broken = 1))))
#endif
			if ((cfd = accept(fd, (struct sockaddr *)&addr, &laddr)) != -1)
				fcntl(cfd, F_SETFL, O_NONBLOCK);

		if (unlikely(cfd == -1)) {
			switch (errno) {
			case EAGAIN:
				if (fdtab[fd].ev & FD_POLL_HUP) {
					/* the listening socket might have been disabled in a shared
					 * process and we're a collateral victim. We'll just pause for
					 * a while in case it comes back. In the mean time, we need to
					 * clear this sticky flag.
					 */
					fdtab[fd].ev &= ~FD_POLL_HUP;
					goto transient_error;
				}
				fd_cant_recv(fd);
				goto end; /* nothing more to accept */
			case EINVAL:
				/* might be trying to accept on a shut fd (eg: soft stop) */
				goto transient_error;
			case EINTR:
			case ECONNABORTED:
				HA_ATOMIC_SUB(&l->nbconn, 1);
				if (p)
					HA_ATOMIC_SUB(&p->feconn, 1);
				if (!(l->options & LI_O_UNLIMITED))
					HA_ATOMIC_SUB(&actconn, 1);
				continue;
			case ENFILE:
				if (p)
					send_log(p, LOG_EMERG,
						 "Proxy %s reached system FD limit (maxsock=%d). Please check system tunables.\n",
						 p->id, global.maxsock);
				goto transient_error;
			case EMFILE:
				if (p)
					send_log(p, LOG_EMERG,
						 "Proxy %s reached process FD limit (maxsock=%d). Please check 'ulimit-n' and restart.\n",
						 p->id, global.maxsock);
				goto transient_error;
			case ENOBUFS:
			case ENOMEM:
				if (p)
					send_log(p, LOG_EMERG,
						 "Proxy %s reached system memory limit (maxsock=%d). Please check system tunables.\n",
						 p->id, global.maxsock);
				goto transient_error;
			default:
				/* unexpected result, let's give up and let other tasks run */
				goto stop;
			}
		}

		/* we don't want to leak the FD upon reload if it's in the master */
		if (unlikely(master == 1))
			fcntl(cfd, F_SETFD, FD_CLOEXEC);

		/* The connection was accepted, it must be counted as such */
		if (l->counters)
			HA_ATOMIC_UPDATE_MAX(&l->counters->conn_max, next_conn);

		if (p)
			HA_ATOMIC_UPDATE_MAX(&p->fe_counters.conn_max, next_feconn);

		proxy_inc_fe_conn_ctr(l, p);

		if (!(l->options & LI_O_UNLIMITED)) {
			count = update_freq_ctr(&global.conn_per_sec, 1);
			HA_ATOMIC_UPDATE_MAX(&global.cps_max, count);
		}

		if (unlikely(cfd >= global.maxsock)) {
			send_log(p, LOG_EMERG,
				 "Proxy %s reached the configured maximum connection limit. Please check the global 'maxconn' value.\n",
				 p->id);
			close(cfd);
			limit_listener(l, &global_listener_queue);
			task_schedule(global_listener_queue_task, tick_add(now_ms, 1000)); /* try again in 1 second */
			goto end;
		}

		/* past this point, l->accept() will automatically decrement
		 * l->nbconn, feconn and actconn once done. Setting next_*conn=0
		 * allows the error path not to rollback on nbconn. It's more
		 * convenient than duplicating all exit labels.
		 */
		next_conn = 0;
		next_feconn = 0;
		next_actconn = 0;

#if defined(USE_THREAD)
		count = l->bind_conf->thr_count;
		if (count > 1 && (global.tune.options & GTUNE_LISTENER_MQ)) {
			struct accept_queue_ring *ring;
			int t1, t2, q1, q2;

			/* pick a first thread ID using a round robin index,
			 * and a second thread ID using a random. The
			 * connection will be assigned to the one with the
			 * least connections. This provides fairness on short
			 * connections (round robin) and on long ones (conn
			 * count).
			 */
			t1 = l->bind_conf->thr_idx;
			do {
				t2 = t1 + 1;
				if (t2 >= count)
					t2 = 0;
			} while (!HA_ATOMIC_CAS(&l->bind_conf->thr_idx, &t1, t2));

			t2 = (random() >> 8) % (count - 1);  // 0..thr_count-2
			t2 += t1 + 1;   // necessarily different from t1

			if (t2 >= count)
				t2 -= count;

			t1 = bind_map_thread_id(l->bind_conf, t1);
			t2 = bind_map_thread_id(l->bind_conf, t2);

			q1 = accept_queue_rings[t1].tail - accept_queue_rings[t1].head + ACCEPT_QUEUE_SIZE;
			if (q1 >= ACCEPT_QUEUE_SIZE)
				q1 -= ACCEPT_QUEUE_SIZE;

			q2 = accept_queue_rings[t2].tail - accept_queue_rings[t2].head + ACCEPT_QUEUE_SIZE;
			if (q2 >= ACCEPT_QUEUE_SIZE)
				q2 -= ACCEPT_QUEUE_SIZE;

			/* make t1 the lowest loaded thread */
			if (q1 >= ACCEPT_QUEUE_SIZE || l->thr_conn[t1] + q1 > l->thr_conn[t2] + q2)
				t1 = t2;

			/* We use deferred accepts even if it's the local thread because
			 * tests show that it's the best performing model, likely due to
			 * better cache locality when processing this loop.
			 */
			ring = &accept_queue_rings[t1];
			if (accept_queue_push_mp(ring, cfd, l, &addr, laddr)) {
				HA_ATOMIC_ADD(&activity[t1].accq_pushed, 1);
				task_wakeup(ring->task, TASK_WOKEN_IO);
				continue;
			}
			/* If the ring is full we do a synchronous accept on
			 * the local thread here.
			 */
			HA_ATOMIC_ADD(&activity[t1].accq_full, 1);
		}
#endif // USE_THREAD

		HA_ATOMIC_ADD(&l->thr_conn[tid], 1);
		ret = l->accept(l, cfd, &addr);
		if (unlikely(ret <= 0)) {
			/* The connection was closed by stream_accept(). Either
			 * we just have to ignore it (ret == 0) or it's a critical
			 * error due to a resource shortage, and we must stop the
			 * listener (ret < 0).
			 */
			if (ret == 0) /* successful termination */
				continue;

			goto transient_error;
		}

		/* increase the per-process number of cumulated sessions, this
		 * may only be done once l->accept() has accepted the connection.
		 */
		if (!(l->options & LI_O_UNLIMITED)) {
			count = update_freq_ctr(&global.sess_per_sec, 1);
			HA_ATOMIC_UPDATE_MAX(&global.sps_max, count);
		}
#ifdef USE_OPENSSL
		if (!(l->options & LI_O_UNLIMITED) && l->bind_conf && l->bind_conf->is_ssl) {
			count = update_freq_ctr(&global.ssl_per_sec, 1);
			HA_ATOMIC_UPDATE_MAX(&global.ssl_max, count);
		}
#endif

	} /* end of for (max_accept--) */

	/* we've exhausted max_accept, so there is no need to poll again */
 stop:
	fd_done_recv(fd);
	goto end;

 transient_error:
	/* pause the listener and try again in 100 ms */
	expire = tick_add(now_ms, 100);

 wait_expire:
	limit_listener(l, &global_listener_queue);
	task_schedule(global_listener_queue_task, tick_first(expire, global_listener_queue_task->expire));
 end:
	if (next_conn)
		HA_ATOMIC_SUB(&l->nbconn, 1);

	if (p && next_feconn)
		HA_ATOMIC_SUB(&p->feconn, 1);

	if (next_actconn)
		HA_ATOMIC_SUB(&actconn, 1);

	if ((l->state == LI_FULL && (!l->maxconn || l->nbconn < l->maxconn)) ||
	    (l->state == LI_LIMITED && ((!p || p->feconn < p->maxconn) && (actconn < global.maxconn)))) {
		/* at least one thread has to this when quitting */
		resume_listener(l);

		/* Dequeues all of the listeners waiting for a resource */
		if (!LIST_ISEMPTY(&global_listener_queue))
			dequeue_all_listeners(&global_listener_queue);

		if (!LIST_ISEMPTY(&p->listener_queue) &&
		    (!p->fe_sps_lim || freq_ctr_remain(&p->fe_sess_per_sec, p->fe_sps_lim, 0) > 0))
			dequeue_all_listeners(&p->listener_queue);
	}
}

/* Notify the listener that a connection initiated from it was released. This
 * is used to keep the connection count consistent and to possibly re-open
 * listening when it was limited.
 */
void listener_release(struct listener *l)
{
	struct proxy *fe = l->bind_conf->frontend;

	if (!(l->options & LI_O_UNLIMITED))
		HA_ATOMIC_SUB(&actconn, 1);
	if (fe)
		HA_ATOMIC_SUB(&fe->feconn, 1);
	HA_ATOMIC_SUB(&l->nbconn, 1);
	HA_ATOMIC_SUB(&l->thr_conn[tid], 1);

	if (l->state == LI_FULL || l->state == LI_LIMITED)
		resume_listener(l);

	/* Dequeues all of the listeners waiting for a resource */
	if (!LIST_ISEMPTY(&global_listener_queue))
		dequeue_all_listeners(&global_listener_queue);

	if (!LIST_ISEMPTY(&fe->listener_queue) &&
	    (!fe->fe_sps_lim || freq_ctr_remain(&fe->fe_sess_per_sec, fe->fe_sps_lim, 0) > 0))
		dequeue_all_listeners(&fe->listener_queue);
}

/*
 * Registers the bind keyword list <kwl> as a list of valid keywords for next
 * parsing sessions.
 */
void bind_register_keywords(struct bind_kw_list *kwl)
{
	LIST_ADDQ(&bind_keywords.list, &kwl->list);
}

/* Return a pointer to the bind keyword <kw>, or NULL if not found. If the
 * keyword is found with a NULL ->parse() function, then an attempt is made to
 * find one with a valid ->parse() function. This way it is possible to declare
 * platform-dependant, known keywords as NULL, then only declare them as valid
 * if some options are met. Note that if the requested keyword contains an
 * opening parenthesis, everything from this point is ignored.
 */
struct bind_kw *bind_find_kw(const char *kw)
{
	int index;
	const char *kwend;
	struct bind_kw_list *kwl;
	struct bind_kw *ret = NULL;

	kwend = strchr(kw, '(');
	if (!kwend)
		kwend = kw + strlen(kw);

	list_for_each_entry(kwl, &bind_keywords.list, list) {
		for (index = 0; kwl->kw[index].kw != NULL; index++) {
			if ((strncmp(kwl->kw[index].kw, kw, kwend - kw) == 0) &&
			    kwl->kw[index].kw[kwend-kw] == 0) {
				if (kwl->kw[index].parse)
					return &kwl->kw[index]; /* found it !*/
				else
					ret = &kwl->kw[index];  /* may be OK */
			}
		}
	}
	return ret;
}

/* Dumps all registered "bind" keywords to the <out> string pointer. The
 * unsupported keywords are only dumped if their supported form was not
 * found.
 */
void bind_dump_kws(char **out)
{
	struct bind_kw_list *kwl;
	int index;

	*out = NULL;
	list_for_each_entry(kwl, &bind_keywords.list, list) {
		for (index = 0; kwl->kw[index].kw != NULL; index++) {
			if (kwl->kw[index].parse ||
			    bind_find_kw(kwl->kw[index].kw) == &kwl->kw[index]) {
				memprintf(out, "%s[%4s] %s%s%s\n", *out ? *out : "",
				          kwl->scope,
				          kwl->kw[index].kw,
				          kwl->kw[index].skip ? " <arg>" : "",
				          kwl->kw[index].parse ? "" : " (not supported)");
			}
		}
	}
}

/* recompute the bit counts per parity for the bind_thread value. This will be
 * used to quickly map a thread number from 1 to #thread to a thread ID among
 * the ones bound. This is the preparation phase of the bit rank counting algo
 * described here: https://graphics.stanford.edu/~seander/bithacks.html
 */
void bind_recount_thread_bits(struct bind_conf *conf)
{
	unsigned long m;

	m = thread_mask(conf->bind_thread);
	conf->thr_count = my_popcountl(m);
	mask_prep_rank_map(m, &conf->thr_2, &conf->thr_4, &conf->thr_8, &conf->thr_16);
}

/* Report the ID of thread <r> in bind_conf <conf> according to its thread_mask.
 * <r> must be between 0 and LONGBITS-1. This makes use of the pre-computed
 * bits resulting from bind_recount_thread_bits. See this function for more
 * info.
 */
unsigned int bind_map_thread_id(const struct bind_conf *conf, unsigned int r)
{
	unsigned long m;

	m = thread_mask(conf->bind_thread);
	return mask_find_rank_bit_fast(r, m, conf->thr_2, conf->thr_4, conf->thr_8, conf->thr_16);
}

/************************************************************************/
/*      All supported sample and ACL keywords must be declared here.    */
/************************************************************************/

/* set temp integer to the number of connexions to the same listening socket */
static int
smp_fetch_dconn(const struct arg *args, struct sample *smp, const char *kw, void *private)
{
	smp->data.type = SMP_T_SINT;
	smp->data.u.sint = smp->sess->listener->nbconn;
	return 1;
}

/* set temp integer to the id of the socket (listener) */
static int
smp_fetch_so_id(const struct arg *args, struct sample *smp, const char *kw, void *private)
{
	smp->data.type = SMP_T_SINT;
	smp->data.u.sint = smp->sess->listener->luid;
	return 1;
}

/* parse the "accept-proxy" bind keyword */
static int bind_parse_accept_proxy(char **args, int cur_arg, struct proxy *px, struct bind_conf *conf, char **err)
{
	struct listener *l;

	list_for_each_entry(l, &conf->listeners, by_bind)
		l->options |= LI_O_ACC_PROXY;

	return 0;
}

/* parse the "accept-netscaler-cip" bind keyword */
static int bind_parse_accept_netscaler_cip(char **args, int cur_arg, struct proxy *px, struct bind_conf *conf, char **err)
{
	struct listener *l;
	uint32_t val;

	if (!*args[cur_arg + 1]) {
		memprintf(err, "'%s' : missing value", args[cur_arg]);
		return ERR_ALERT | ERR_FATAL;
	}

	val = atol(args[cur_arg + 1]);
	if (val <= 0) {
		memprintf(err, "'%s' : invalid value %d, must be >= 0", args[cur_arg], val);
		return ERR_ALERT | ERR_FATAL;
	}

	list_for_each_entry(l, &conf->listeners, by_bind) {
		l->options |= LI_O_ACC_CIP;
		conf->ns_cip_magic = val;
	}

	return 0;
}

/* parse the "backlog" bind keyword */
static int bind_parse_backlog(char **args, int cur_arg, struct proxy *px, struct bind_conf *conf, char **err)
{
	struct listener *l;
	int val;

	if (!*args[cur_arg + 1]) {
		memprintf(err, "'%s' : missing value", args[cur_arg]);
		return ERR_ALERT | ERR_FATAL;
	}

	val = atol(args[cur_arg + 1]);
	if (val < 0) {
		memprintf(err, "'%s' : invalid value %d, must be > 0", args[cur_arg], val);
		return ERR_ALERT | ERR_FATAL;
	}

	list_for_each_entry(l, &conf->listeners, by_bind)
		l->backlog = val;

	return 0;
}

/* parse the "id" bind keyword */
static int bind_parse_id(char **args, int cur_arg, struct proxy *px, struct bind_conf *conf, char **err)
{
	struct eb32_node *node;
	struct listener *l, *new;
	char *error;

	if (conf->listeners.n != conf->listeners.p) {
		memprintf(err, "'%s' can only be used with a single socket", args[cur_arg]);
		return ERR_ALERT | ERR_FATAL;
	}

	if (!*args[cur_arg + 1]) {
		memprintf(err, "'%s' : expects an integer argument", args[cur_arg]);
		return ERR_ALERT | ERR_FATAL;
	}

	new = LIST_NEXT(&conf->listeners, struct listener *, by_bind);
	new->luid = strtol(args[cur_arg + 1], &error, 10);
	if (*error != '\0') {
		memprintf(err, "'%s' : expects an integer argument, found '%s'", args[cur_arg], args[cur_arg + 1]);
		return ERR_ALERT | ERR_FATAL;
	}
	new->conf.id.key = new->luid;

	if (new->luid <= 0) {
		memprintf(err, "'%s' : custom id has to be > 0", args[cur_arg]);
		return ERR_ALERT | ERR_FATAL;
	}

	node = eb32_lookup(&px->conf.used_listener_id, new->luid);
	if (node) {
		l = container_of(node, struct listener, conf.id);
		memprintf(err, "'%s' : custom id %d already used at %s:%d ('bind %s')",
		          args[cur_arg], l->luid, l->bind_conf->file, l->bind_conf->line,
		          l->bind_conf->arg);
		return ERR_ALERT | ERR_FATAL;
	}

	eb32_insert(&px->conf.used_listener_id, &new->conf.id);
	return 0;
}

/* parse the "maxconn" bind keyword */
static int bind_parse_maxconn(char **args, int cur_arg, struct proxy *px, struct bind_conf *conf, char **err)
{
	struct listener *l;
	int val;

	if (!*args[cur_arg + 1]) {
		memprintf(err, "'%s' : missing value", args[cur_arg]);
		return ERR_ALERT | ERR_FATAL;
	}

	val = atol(args[cur_arg + 1]);
	if (val < 0) {
		memprintf(err, "'%s' : invalid value %d, must be >= 0", args[cur_arg], val);
		return ERR_ALERT | ERR_FATAL;
	}

	list_for_each_entry(l, &conf->listeners, by_bind)
		l->maxconn = val;

	return 0;
}

/* parse the "name" bind keyword */
static int bind_parse_name(char **args, int cur_arg, struct proxy *px, struct bind_conf *conf, char **err)
{
	struct listener *l;

	if (!*args[cur_arg + 1]) {
		memprintf(err, "'%s' : missing name", args[cur_arg]);
		return ERR_ALERT | ERR_FATAL;
	}

	list_for_each_entry(l, &conf->listeners, by_bind)
		l->name = strdup(args[cur_arg + 1]);

	return 0;
}

/* parse the "nice" bind keyword */
static int bind_parse_nice(char **args, int cur_arg, struct proxy *px, struct bind_conf *conf, char **err)
{
	struct listener *l;
	int val;

	if (!*args[cur_arg + 1]) {
		memprintf(err, "'%s' : missing value", args[cur_arg]);
		return ERR_ALERT | ERR_FATAL;
	}

	val = atol(args[cur_arg + 1]);
	if (val < -1024 || val > 1024) {
		memprintf(err, "'%s' : invalid value %d, allowed range is -1024..1024", args[cur_arg], val);
		return ERR_ALERT | ERR_FATAL;
	}

	list_for_each_entry(l, &conf->listeners, by_bind)
		l->nice = val;

	return 0;
}

/* parse the "process" bind keyword */
static int bind_parse_process(char **args, int cur_arg, struct proxy *px, struct bind_conf *conf, char **err)
{
	char *slash;
	unsigned long proc = 0, thread = 0;

	if ((slash = strchr(args[cur_arg + 1], '/')) != NULL)
		*slash = 0;

	if (parse_process_number(args[cur_arg + 1], &proc, MAX_PROCS, NULL, err)) {
		memprintf(err, "'%s' : %s", args[cur_arg], *err);
		return ERR_ALERT | ERR_FATAL;
	}

	if (slash) {
		if (parse_process_number(slash+1, &thread, MAX_THREADS, NULL, err)) {
			memprintf(err, "'%s' : %s", args[cur_arg], *err);
			return ERR_ALERT | ERR_FATAL;
		}
		*slash = '/';
	}

	conf->bind_proc |= proc;
	conf->bind_thread |= thread;
	bind_recount_thread_bits(conf);
	return 0;
}

/* parse the "proto" bind keyword */
static int bind_parse_proto(char **args, int cur_arg, struct proxy *px, struct bind_conf *conf, char **err)
{
	struct ist proto;

	if (!*args[cur_arg + 1]) {
		memprintf(err, "'%s' : missing value", args[cur_arg]);
		return ERR_ALERT | ERR_FATAL;
	}

	proto = ist2(args[cur_arg + 1], strlen(args[cur_arg + 1]));
	conf->mux_proto = get_mux_proto(proto);
	if (!conf->mux_proto) {
		memprintf(err, "'%s' :  unknown MUX protocol '%s'", args[cur_arg], args[cur_arg+1]);
		return ERR_ALERT | ERR_FATAL;
	}
	return 0;
}

/* config parser for global "tune.listener.multi-queue", accepts "on" or "off" */
static int cfg_parse_tune_listener_mq(char **args, int section_type, struct proxy *curpx,
                                      struct proxy *defpx, const char *file, int line,
                                      char **err)
{
	if (too_many_args(1, args, err, NULL))
		return -1;

	if (strcmp(args[1], "on") == 0)
		global.tune.options |= GTUNE_LISTENER_MQ;
	else if (strcmp(args[1], "off") == 0)
		global.tune.options &= ~GTUNE_LISTENER_MQ;
	else {
		memprintf(err, "'%s' expects either 'on' or 'off' but got '%s'.", args[0], args[1]);
		return -1;
	}
	return 0;
}

/* Note: must not be declared <const> as its list will be overwritten.
 * Please take care of keeping this list alphabetically sorted.
 */
static struct sample_fetch_kw_list smp_kws = {ILH, {
	{ "dst_conn", smp_fetch_dconn, 0, NULL, SMP_T_SINT, SMP_USE_FTEND, },
	{ "so_id",    smp_fetch_so_id, 0, NULL, SMP_T_SINT, SMP_USE_FTEND, },
	{ /* END */ },
}};

INITCALL1(STG_REGISTER, sample_register_fetches, &smp_kws);

/* Note: must not be declared <const> as its list will be overwritten.
 * Please take care of keeping this list alphabetically sorted.
 */
static struct acl_kw_list acl_kws = {ILH, {
	{ /* END */ },
}};

INITCALL1(STG_REGISTER, acl_register_keywords, &acl_kws);

/* Note: must not be declared <const> as its list will be overwritten.
 * Please take care of keeping this list alphabetically sorted, doing so helps
 * all code contributors.
 * Optional keywords are also declared with a NULL ->parse() function so that
 * the config parser can report an appropriate error when a known keyword was
 * not enabled.
 */
static struct bind_kw_list bind_kws = { "ALL", { }, {
	{ "accept-netscaler-cip", bind_parse_accept_netscaler_cip, 1 }, /* enable NetScaler Client IP insertion protocol */
	{ "accept-proxy", bind_parse_accept_proxy, 0 }, /* enable PROXY protocol */
	{ "backlog",      bind_parse_backlog,      1 }, /* set backlog of listening socket */
	{ "id",           bind_parse_id,           1 }, /* set id of listening socket */
	{ "maxconn",      bind_parse_maxconn,      1 }, /* set maxconn of listening socket */
	{ "name",         bind_parse_name,         1 }, /* set name of listening socket */
	{ "nice",         bind_parse_nice,         1 }, /* set nice of listening socket */
	{ "process",      bind_parse_process,      1 }, /* set list of allowed process for this socket */
	{ "proto",        bind_parse_proto,        1 }, /* set the proto to use for all incoming connections */
	{ /* END */ },
}};

INITCALL1(STG_REGISTER, bind_register_keywords, &bind_kws);

/* config keyword parsers */
static struct cfg_kw_list cfg_kws = {ILH, {
	{ CFG_GLOBAL, "tune.listener.multi-queue",      cfg_parse_tune_listener_mq      },
	{ 0, NULL, NULL }
}};

INITCALL1(STG_REGISTER, cfg_register_keywords, &cfg_kws);

/*
 * Local variables:
 *  c-indent-level: 8
 *  c-basic-offset: 8
 * End:
 */