blob: 3d361e89f3b59d0f02067b0a00cfd374f48e3527 [file] [log] [blame]
Willy Tarreaubaaee002006-06-26 02:48:02 +02001/*
2 * File descriptors management functions.
3 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +01004 * Copyright 2000-2014 Willy Tarreau <w@1wt.eu>
Willy Tarreaubaaee002006-06-26 02:48:02 +02005 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010011 * This code implements an events cache for file descriptors. It remembers the
12 * readiness of a file descriptor after a return from poll() and the fact that
13 * an I/O attempt failed on EAGAIN. Events in the cache which are still marked
14 * ready and active are processed just as if they were reported by poll().
Willy Tarreau7be79a42012-11-11 15:02:54 +010015 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010016 * This serves multiple purposes. First, it significantly improves performance
17 * by avoiding to subscribe to polling unless absolutely necessary, so most
18 * events are processed without polling at all, especially send() which
19 * benefits from the socket buffers. Second, it is the only way to support
20 * edge-triggered pollers (eg: EPOLL_ET). And third, it enables I/O operations
21 * that are backed by invisible buffers. For example, SSL is able to read a
22 * whole socket buffer and not deliver it to the application buffer because
23 * it's full. Unfortunately, it won't be reported by a poller anymore until
24 * some new activity happens. The only way to call it again thus is to keep
25 * this readiness information in the cache and to access it without polling
26 * once the FD is enabled again.
Willy Tarreau7be79a42012-11-11 15:02:54 +010027 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010028 * One interesting feature of the cache is that it maintains the principle
29 * of speculative I/O introduced in haproxy 1.3 : the first time an event is
30 * enabled, the FD is considered as ready so that the I/O attempt is performed
31 * via the cache without polling. And the polling happens only when EAGAIN is
32 * first met. This avoids polling for HTTP requests, especially when the
33 * defer-accept mode is used. It also avoids polling for sending short data
34 * such as requests to servers or short responses to clients.
35 *
36 * The cache consists in a list of active events and a list of updates.
37 * Active events are events that are expected to come and that we must report
38 * to the application until it asks to stop or asks to poll. Updates are new
39 * requests for changing an FD state. Updates are the only way to create new
40 * events. This is important because it means that the number of cached events
41 * cannot increase between updates and will only grow one at a time while
42 * processing updates. All updates must always be processed, though events
43 * might be processed by small batches if required.
Willy Tarreau7be79a42012-11-11 15:02:54 +010044 *
45 * There is no direct link between the FD and the updates list. There is only a
46 * bit in the fdtab[] to indicate than a file descriptor is already present in
47 * the updates list. Once an fd is present in the updates list, it will have to
48 * be considered even if its changes are reverted in the middle or if the fd is
49 * replaced.
50 *
51 * It is important to understand that as long as all expected events are
52 * processed, they might starve the polled events, especially because polled
Willy Tarreauf817e9f2014-01-10 16:58:45 +010053 * I/O starvation quickly induces more cached I/O. One solution to this
Willy Tarreau7be79a42012-11-11 15:02:54 +010054 * consists in only processing a part of the events at once, but one drawback
Willy Tarreauf817e9f2014-01-10 16:58:45 +010055 * is that unhandled events will still wake the poller up. Using an edge-
56 * triggered poller such as EPOLL_ET will solve this issue though.
Willy Tarreau7be79a42012-11-11 15:02:54 +010057 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010058 * Since we do not want to scan all the FD list to find cached I/O events,
59 * we store them in a list consisting in a linear array holding only the FD
60 * indexes right now. Note that a closed FD cannot exist in the cache, because
61 * it is closed by fd_delete() which in turn calls fd_release_cache_entry()
62 * which always removes it from the list.
Willy Tarreau7be79a42012-11-11 15:02:54 +010063 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010064 * The FD array has to hold a back reference to the cache. This reference is
65 * always valid unless the FD is not in the cache and is not updated, in which
66 * case the reference points to index 0.
Willy Tarreau7be79a42012-11-11 15:02:54 +010067 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010068 * The event state for an FD, as found in fdtab[].state, is maintained for each
69 * direction. The state field is built this way, with R bits in the low nibble
70 * and W bits in the high nibble for ease of access and debugging :
Willy Tarreau7be79a42012-11-11 15:02:54 +010071 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010072 * 7 6 5 4 3 2 1 0
73 * [ 0 | PW | RW | AW | 0 | PR | RR | AR ]
74 *
75 * A* = active *R = read
76 * P* = polled *W = write
77 * R* = ready
78 *
79 * An FD is marked "active" when there is a desire to use it.
80 * An FD is marked "polled" when it is registered in the polling.
81 * An FD is marked "ready" when it has not faced a new EAGAIN since last wake-up
82 * (it is a cache of the last EAGAIN regardless of polling changes).
Willy Tarreau7be79a42012-11-11 15:02:54 +010083 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010084 * We have 8 possible states for each direction based on these 3 flags :
Willy Tarreau7be79a42012-11-11 15:02:54 +010085 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010086 * +---+---+---+----------+---------------------------------------------+
87 * | P | R | A | State | Description |
88 * +---+---+---+----------+---------------------------------------------+
89 * | 0 | 0 | 0 | DISABLED | No activity desired, not ready. |
90 * | 0 | 0 | 1 | MUSTPOLL | Activity desired via polling. |
91 * | 0 | 1 | 0 | STOPPED | End of activity without polling. |
92 * | 0 | 1 | 1 | ACTIVE | Activity desired without polling. |
93 * | 1 | 0 | 0 | ABORT | Aborted poll(). Not frequently seen. |
94 * | 1 | 0 | 1 | POLLED | FD is being polled. |
95 * | 1 | 1 | 0 | PAUSED | FD was paused while ready (eg: buffer full) |
96 * | 1 | 1 | 1 | READY | FD was marked ready by poll() |
97 * +---+---+---+----------+---------------------------------------------+
Willy Tarreau7be79a42012-11-11 15:02:54 +010098 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010099 * The transitions are pretty simple :
100 * - fd_want_*() : set flag A
101 * - fd_stop_*() : clear flag A
102 * - fd_cant_*() : clear flag R (when facing EAGAIN)
103 * - fd_may_*() : set flag R (upon return from poll())
104 * - sync() : if (A) { if (!R) P := 1 } else { P := 0 }
105 *
106 * The PAUSED, ABORT and MUSTPOLL states are transient for level-trigerred
107 * pollers and are fixed by the sync() which happens at the beginning of the
108 * poller. For event-triggered pollers, only the MUSTPOLL state will be
109 * transient and ABORT will lead to PAUSED. The ACTIVE state is the only stable
110 * one which has P != A.
111 *
112 * The READY state is a bit special as activity on the FD might be notified
113 * both by the poller or by the cache. But it is needed for some multi-layer
114 * protocols (eg: SSL) where connection activity is not 100% linked to FD
115 * activity. Also some pollers might prefer to implement it as ACTIVE if
116 * enabling/disabling the FD is cheap. The READY and ACTIVE states are the
117 * two states for which a cache entry is allocated.
118 *
119 * The state transitions look like the diagram below. Only the 4 right states
120 * have polling enabled :
121 *
122 * (POLLED=0) (POLLED=1)
123 *
124 * +----------+ sync +-------+
125 * | DISABLED | <----- | ABORT | (READY=0, ACTIVE=0)
126 * +----------+ +-------+
127 * clr | ^ set | ^
128 * | | | |
129 * v | set v | clr
130 * +----------+ sync +--------+
131 * | MUSTPOLL | -----> | POLLED | (READY=0, ACTIVE=1)
132 * +----------+ +--------+
133 * ^ poll | ^
134 * | | |
135 * | EAGAIN v | EAGAIN
136 * +--------+ +-------+
137 * | ACTIVE | | READY | (READY=1, ACTIVE=1)
138 * +--------+ +-------+
139 * clr | ^ set | ^
140 * | | | |
141 * v | set v | clr
142 * +---------+ sync +--------+
143 * | STOPPED | <------ | PAUSED | (READY=1, ACTIVE=0)
144 * +---------+ +--------+
Willy Tarreaubaaee002006-06-26 02:48:02 +0200145 */
146
Willy Tarreau2ff76222007-04-09 19:29:56 +0200147#include <stdio.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +0200148#include <string.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +0200149#include <unistd.h>
Olivier Houchard79321b92018-07-26 17:55:11 +0200150#include <fcntl.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +0200151#include <sys/types.h>
Willy Tarreau2d7f81b2019-02-21 22:19:17 +0100152#include <sys/resource.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +0200153
Willy Tarreaue5733232019-05-22 19:24:06 +0200154#if defined(USE_POLL)
Willy Tarreau9188ac62019-02-21 22:12:47 +0100155#include <poll.h>
156#include <errno.h>
157#endif
158
Willy Tarreau2dd0d472006-06-29 17:53:05 +0200159#include <common/compat.h>
160#include <common/config.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +0200161
Willy Tarreau7be79a42012-11-11 15:02:54 +0100162#include <types/global.h>
163
Willy Tarreau2a429502006-10-15 14:52:29 +0200164#include <proto/fd.h>
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200165#include <proto/log.h>
Willy Tarreauc6f4ce82009-06-10 11:09:37 +0200166#include <proto/port_range.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +0200167
168struct fdtab *fdtab = NULL; /* array of all the file descriptors */
Olivier Houchardcb92f5c2018-04-26 14:23:07 +0200169unsigned long *polled_mask = NULL; /* Array for the polled_mask of each fd */
Willy Tarreau8d5d77e2009-10-18 07:25:52 +0200170struct fdinfo *fdinfo = NULL; /* less-often used infos for file descriptors */
Willy Tarreaubaaee002006-06-26 02:48:02 +0200171int totalconn; /* total # of terminated sessions */
172int actconn; /* # of active sessions */
173
Willy Tarreau4f60f162007-04-08 16:39:58 +0200174struct poller pollers[MAX_POLLERS];
175struct poller cur_poller;
176int nbpollers = 0;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200177
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100178volatile struct fdlist fd_cache ; // FD events cache
179volatile struct fdlist fd_cache_local[MAX_THREADS]; // FD events local for each thread
Olivier Houchard6b96f722018-04-25 16:58:25 +0200180volatile struct fdlist update_list; // Global update list
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100181
Christopher Faulet69553fe2018-01-15 11:57:03 +0100182unsigned long fd_cache_mask = 0; // Mask of threads with events in the cache
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200183
184THREAD_LOCAL int *fd_updt = NULL; // FD updates list
185THREAD_LOCAL int fd_nbupdt = 0; // number of updates in the list
Olivier Houchard79321b92018-07-26 17:55:11 +0200186THREAD_LOCAL int poller_rd_pipe = -1; // Pipe to wake the thread
187int poller_wr_pipe[MAX_THREADS]; // Pipe to wake the threads
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200188
Olivier Houchard7c49d2e2019-04-16 18:37:05 +0200189volatile int ha_used_fds = 0; // Number of FD we're currently using
190
Willy Tarreau1c90f322019-12-20 07:20:00 +0100191#define _GET_NEXT(fd, off) ((volatile struct fdlist_entry *)(void *)((char *)(&fdtab[fd]) + off))->next
192#define _GET_PREV(fd, off) ((volatile struct fdlist_entry *)(void *)((char *)(&fdtab[fd]) + off))->prev
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100193/* adds fd <fd> to fd list <list> if it was not yet in it */
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200194void fd_add_to_fd_list(volatile struct fdlist *list, int fd, int off)
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100195{
196 int next;
197 int new;
198 int old;
199 int last;
200
201redo_next:
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200202 next = _GET_NEXT(fd, off);
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100203 /* Check that we're not already in the cache, and if not, lock us. */
Olivier Houcharda29cdde2019-12-19 18:33:08 +0100204 if (next > -2)
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100205 goto done;
Olivier Houcharda29cdde2019-12-19 18:33:08 +0100206 if (next == -2)
207 goto redo_next;
Olivier Houchardd3608792019-03-08 18:47:42 +0100208 if (!_HA_ATOMIC_CAS(&_GET_NEXT(fd, off), &next, -2))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100209 goto redo_next;
Olivier Houchardd2b5d162019-03-08 13:47:21 +0100210 __ha_barrier_atomic_store();
Willy Tarreau11559a72018-02-05 17:52:24 +0100211
212 new = fd;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100213redo_last:
214 /* First, insert in the linked list */
215 last = list->last;
216 old = -1;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100217
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200218 _GET_PREV(fd, off) = -2;
Willy Tarreau11559a72018-02-05 17:52:24 +0100219 /* Make sure the "prev" store is visible before we update the last entry */
220 __ha_barrier_store();
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100221
Willy Tarreau11559a72018-02-05 17:52:24 +0100222 if (unlikely(last == -1)) {
223 /* list is empty, try to add ourselves alone so that list->last=fd */
Olivier Houchardd3608792019-03-08 18:47:42 +0100224 if (unlikely(!_HA_ATOMIC_CAS(&list->last, &old, new)))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100225 goto redo_last;
226
227 /* list->first was necessary -1, we're guaranteed to be alone here */
228 list->first = fd;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100229 } else {
Willy Tarreau11559a72018-02-05 17:52:24 +0100230 /* adding ourselves past the last element
231 * The CAS will only succeed if its next is -1,
232 * which means it's in the cache, and the last element.
233 */
Olivier Houchardd3608792019-03-08 18:47:42 +0100234 if (unlikely(!_HA_ATOMIC_CAS(&_GET_NEXT(last, off), &old, new)))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100235 goto redo_last;
Willy Tarreau11559a72018-02-05 17:52:24 +0100236
237 /* Then, update the last entry */
238 list->last = fd;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100239 }
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100240 __ha_barrier_store();
Willy Tarreau11559a72018-02-05 17:52:24 +0100241 /* since we're alone at the end of the list and still locked(-2),
242 * we know noone tried to add past us. Mark the end of list.
243 */
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200244 _GET_PREV(fd, off) = last;
245 _GET_NEXT(fd, off) = -1;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100246 __ha_barrier_store();
247done:
248 return;
249}
250
251/* removes fd <fd> from fd list <list> */
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200252void fd_rm_from_fd_list(volatile struct fdlist *list, int fd, int off)
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100253{
254#if defined(HA_HAVE_CAS_DW) || defined(HA_CAS_IS_8B)
255 volatile struct fdlist_entry cur_list, next_list;
256#endif
257 int old;
258 int new = -2;
259 int prev;
260 int next;
261 int last;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100262lock_self:
263#if (defined(HA_CAS_IS_8B) || defined(HA_HAVE_CAS_DW))
264 next_list.next = next_list.prev = -2;
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200265 cur_list = *(volatile struct fdlist_entry *)(((char *)&fdtab[fd]) + off);
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100266 /* First, attempt to lock our own entries */
267 do {
268 /* The FD is not in the FD cache, give up */
269 if (unlikely(cur_list.next <= -3))
270 return;
271 if (unlikely(cur_list.prev == -2 || cur_list.next == -2))
272 goto lock_self;
273 } while (
274#ifdef HA_CAS_IS_8B
Olivier Houchardd3608792019-03-08 18:47:42 +0100275 unlikely(!_HA_ATOMIC_CAS(((void **)(void *)&_GET_NEXT(fd, off)), ((void **)(void *)&cur_list), (*(void **)(void *)&next_list))))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100276#else
Willy Tarreauc3b59582019-05-27 17:37:20 +0200277 unlikely(!_HA_ATOMIC_DWCAS(((void *)&_GET_NEXT(fd, off)), ((void *)&cur_list), ((void *)&next_list))))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100278#endif
279 ;
280 next = cur_list.next;
281 prev = cur_list.prev;
282
283#else
284lock_self_next:
Willy Tarreau1c90f322019-12-20 07:20:00 +0100285 next = _GET_NEXT(fd, off);
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100286 if (next == -2)
287 goto lock_self_next;
288 if (next <= -3)
289 goto done;
Olivier Houchardd3608792019-03-08 18:47:42 +0100290 if (unlikely(!_HA_ATOMIC_CAS(&_GET_NEXT(fd, off), &next, -2)))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100291 goto lock_self_next;
292lock_self_prev:
Willy Tarreau1c90f322019-12-20 07:20:00 +0100293 prev = _GET_PREV(fd, off);
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100294 if (prev == -2)
295 goto lock_self_prev;
Olivier Houchardd3608792019-03-08 18:47:42 +0100296 if (unlikely(!_HA_ATOMIC_CAS(&_GET_PREV(fd, off), &prev, -2)))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100297 goto lock_self_prev;
298#endif
Olivier Houchardd2b5d162019-03-08 13:47:21 +0100299 __ha_barrier_atomic_store();
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100300
301 /* Now, lock the entries of our neighbours */
302 if (likely(prev != -1)) {
303redo_prev:
304 old = fd;
305
Olivier Houchardd3608792019-03-08 18:47:42 +0100306 if (unlikely(!_HA_ATOMIC_CAS(&_GET_NEXT(prev, off), &old, new))) {
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100307 if (unlikely(old == -2)) {
308 /* Neighbour already locked, give up and
309 * retry again once he's done
310 */
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200311 _GET_PREV(fd, off) = prev;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100312 __ha_barrier_store();
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200313 _GET_NEXT(fd, off) = next;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100314 __ha_barrier_store();
315 goto lock_self;
316 }
317 goto redo_prev;
318 }
319 }
320 if (likely(next != -1)) {
321redo_next:
322 old = fd;
Olivier Houchardd3608792019-03-08 18:47:42 +0100323 if (unlikely(!_HA_ATOMIC_CAS(&_GET_PREV(next, off), &old, new))) {
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100324 if (unlikely(old == -2)) {
325 /* Neighbour already locked, give up and
326 * retry again once he's done
327 */
328 if (prev != -1) {
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200329 _GET_NEXT(prev, off) = fd;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100330 __ha_barrier_store();
331 }
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200332 _GET_PREV(fd, off) = prev;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100333 __ha_barrier_store();
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200334 _GET_NEXT(fd, off) = next;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100335 __ha_barrier_store();
336 goto lock_self;
337 }
338 goto redo_next;
339 }
340 }
341 if (list->first == fd)
342 list->first = next;
343 __ha_barrier_store();
344 last = list->last;
Olivier Houchardd3608792019-03-08 18:47:42 +0100345 while (unlikely(last == fd && (!_HA_ATOMIC_CAS(&list->last, &last, prev))))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100346 __ha_compiler_barrier();
347 /* Make sure we let other threads know we're no longer in cache,
348 * before releasing our neighbours.
349 */
350 __ha_barrier_store();
351 if (likely(prev != -1))
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200352 _GET_NEXT(prev, off) = next;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100353 __ha_barrier_store();
354 if (likely(next != -1))
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200355 _GET_PREV(next, off) = prev;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100356 __ha_barrier_store();
357 /* Ok, now we're out of the fd cache */
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200358 _GET_NEXT(fd, off) = -(next + 4);
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100359 __ha_barrier_store();
360done:
361 return;
362}
363
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200364#undef _GET_NEXT
365#undef _GET_PREV
366
Willy Tarreau173d9952018-01-26 21:48:23 +0100367/* Deletes an FD from the fdsets.
Willy Tarreau4f60f162007-04-08 16:39:58 +0200368 * The file descriptor is also closed.
369 */
Olivier Houchard1fc05162017-04-06 01:05:05 +0200370static void fd_dodelete(int fd, int do_close)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200371{
Willy Tarreau87d54a92018-10-15 09:44:46 +0200372 unsigned long locked = atleast2(fdtab[fd].thread_mask);
373
374 if (locked)
375 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreauad38ace2013-12-15 14:19:38 +0100376 if (fdtab[fd].linger_risk) {
377 /* this is generally set when connecting to servers */
378 setsockopt(fd, SOL_SOCKET, SO_LINGER,
379 (struct linger *) &nolinger, sizeof(struct linger));
380 }
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100381 if (cur_poller.clo)
382 cur_poller.clo(fd);
Olivier Houchard15a2e682019-08-05 18:51:52 +0200383 polled_mask[fd] = 0;
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100384
Willy Tarreau899d9572014-01-25 19:20:35 +0100385 fd_release_cache_entry(fd);
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100386 fdtab[fd].state = 0;
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100387
Willy Tarreau8d5d77e2009-10-18 07:25:52 +0200388 port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port);
389 fdinfo[fd].port_range = NULL;
Willy Tarreaudb3b3262012-07-05 23:19:22 +0200390 fdtab[fd].owner = NULL;
Willy Tarreauf65610a2017-10-31 16:06:06 +0100391 fdtab[fd].thread_mask = 0;
Willy Tarreauc9c83782018-01-17 18:44:46 +0100392 if (do_close) {
Christopher Fauletd531f882017-06-01 16:55:03 +0200393 close(fd);
Olivier Houchard7c49d2e2019-04-16 18:37:05 +0200394 _HA_ATOMIC_SUB(&ha_used_fds, 1);
Willy Tarreauc9c83782018-01-17 18:44:46 +0100395 }
Willy Tarreau87d54a92018-10-15 09:44:46 +0200396 if (locked)
397 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200398}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200399
Willy Tarreau173d9952018-01-26 21:48:23 +0100400/* Deletes an FD from the fdsets.
Olivier Houchard1fc05162017-04-06 01:05:05 +0200401 * The file descriptor is also closed.
402 */
403void fd_delete(int fd)
404{
405 fd_dodelete(fd, 1);
406}
407
Willy Tarreau173d9952018-01-26 21:48:23 +0100408/* Deletes an FD from the fdsets.
Olivier Houchard1fc05162017-04-06 01:05:05 +0200409 * The file descriptor is kept open.
410 */
411void fd_remove(int fd)
412{
413 fd_dodelete(fd, 0);
414}
415
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100416static inline void fdlist_process_cached_events(volatile struct fdlist *fdlist)
Willy Tarreau09f24562012-11-11 16:43:45 +0100417{
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100418 int fd, old_fd, e;
Richard Russobc9d9842019-02-20 12:43:45 -0800419 unsigned long locked;
Willy Tarreau09f24562012-11-11 16:43:45 +0100420
Olivier Houchard12568362018-01-31 18:07:29 +0100421 for (old_fd = fd = fdlist->first; fd != -1; fd = fdtab[fd].cache.next) {
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100422 if (fd == -2) {
423 fd = old_fd;
424 continue;
425 } else if (fd <= -3)
426 fd = -fd - 4;
427 if (fd == -1)
428 break;
429 old_fd = fd;
430 if (!(fdtab[fd].thread_mask & tid_bit))
431 continue;
Olivier Houchard12568362018-01-31 18:07:29 +0100432 if (fdtab[fd].cache.next < -3)
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100433 continue;
Christopher Faulet69553fe2018-01-15 11:57:03 +0100434
Olivier Houchardd3608792019-03-08 18:47:42 +0100435 _HA_ATOMIC_OR(&fd_cache_mask, tid_bit);
Richard Russobc9d9842019-02-20 12:43:45 -0800436 locked = atleast2(fdtab[fd].thread_mask);
437 if (locked && HA_SPIN_TRYLOCK(FD_LOCK, &fdtab[fd].lock)) {
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100438 activity[tid].fd_lock++;
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100439 continue;
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100440 }
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200441
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200442 e = fdtab[fd].state;
Willy Tarreau09f24562012-11-11 16:43:45 +0100443 fdtab[fd].ev &= FD_POLL_STICKY;
444
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100445 if ((e & (FD_EV_READY_R | FD_EV_ACTIVE_R)) == (FD_EV_READY_R | FD_EV_ACTIVE_R))
Willy Tarreau09f24562012-11-11 16:43:45 +0100446 fdtab[fd].ev |= FD_POLL_IN;
447
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100448 if ((e & (FD_EV_READY_W | FD_EV_ACTIVE_W)) == (FD_EV_READY_W | FD_EV_ACTIVE_W))
Willy Tarreau09f24562012-11-11 16:43:45 +0100449 fdtab[fd].ev |= FD_POLL_OUT;
450
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200451 if (fdtab[fd].iocb && fdtab[fd].owner && fdtab[fd].ev) {
Richard Russobc9d9842019-02-20 12:43:45 -0800452 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200453 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau09f24562012-11-11 16:43:45 +0100454 fdtab[fd].iocb(fd);
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200455 }
456 else {
Willy Tarreau5be2f352014-11-19 19:43:05 +0100457 fd_release_cache_entry(fd);
Richard Russobc9d9842019-02-20 12:43:45 -0800458 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200459 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200460 }
Willy Tarreaue1aed242019-08-16 16:06:14 +0200461 ti->flags &= ~TI_FL_STUCK; // this thread is still running
Willy Tarreau09f24562012-11-11 16:43:45 +0100462 }
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100463}
464
465/* Scan and process the cached events. This should be called right after
466 * the poller. The loop may cause new entries to be created, for example
467 * if a listener causes an accept() to initiate a new incoming connection
468 * wanting to attempt an recv().
469 */
470void fd_process_cached_events()
471{
Olivier Houchardd3608792019-03-08 18:47:42 +0100472 _HA_ATOMIC_AND(&fd_cache_mask, ~tid_bit);
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100473 fdlist_process_cached_events(&fd_cache_local[tid]);
474 fdlist_process_cached_events(&fd_cache);
Willy Tarreau09f24562012-11-11 16:43:45 +0100475}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200476
Olivier Houchard2292edf2019-02-25 14:26:54 +0100477#if defined(USE_CLOSEFROM)
478void my_closefrom(int start)
479{
480 closefrom(start);
481}
482
Willy Tarreaue5733232019-05-22 19:24:06 +0200483#elif defined(USE_POLL)
Willy Tarreau9188ac62019-02-21 22:12:47 +0100484/* This is a portable implementation of closefrom(). It closes all open file
485 * descriptors starting at <start> and above. It relies on the fact that poll()
486 * will return POLLNVAL for each invalid (hence close) file descriptor passed
487 * in argument in order to skip them. It acts with batches of FDs and will
488 * typically perform one poll() call per 1024 FDs so the overhead is low in
489 * case all FDs have to be closed.
490 */
491void my_closefrom(int start)
492{
493 struct pollfd poll_events[1024];
494 struct rlimit limit;
495 int nbfds, fd, ret, idx;
496 int step, next;
497
498 if (getrlimit(RLIMIT_NOFILE, &limit) == 0)
499 step = nbfds = limit.rlim_cur;
500 else
501 step = nbfds = 0;
502
503 if (nbfds <= 0) {
504 /* set safe limit */
505 nbfds = 1024;
506 step = 256;
507 }
508
509 if (step > sizeof(poll_events) / sizeof(poll_events[0]))
510 step = sizeof(poll_events) / sizeof(poll_events[0]);
511
512 while (start < nbfds) {
513 next = (start / step + 1) * step;
514
515 for (fd = start; fd < next && fd < nbfds; fd++) {
516 poll_events[fd - start].fd = fd;
517 poll_events[fd - start].events = 0;
518 }
519
520 do {
521 ret = poll(poll_events, fd - start, 0);
522 if (ret >= 0)
523 break;
524 } while (errno == EAGAIN || errno == EINTR || errno == ENOMEM);
525
Willy Tarreaub8e602c2019-02-22 09:07:42 +0100526 if (ret)
527 ret = fd - start;
528
Willy Tarreau9188ac62019-02-21 22:12:47 +0100529 for (idx = 0; idx < ret; idx++) {
530 if (poll_events[idx].revents & POLLNVAL)
531 continue; /* already closed */
532
533 fd = poll_events[idx].fd;
534 close(fd);
535 }
536 start = next;
537 }
538}
539
Willy Tarreaue5733232019-05-22 19:24:06 +0200540#else // defined(USE_POLL)
Willy Tarreau9188ac62019-02-21 22:12:47 +0100541
Willy Tarreau2d7f81b2019-02-21 22:19:17 +0100542/* This is a portable implementation of closefrom(). It closes all open file
543 * descriptors starting at <start> and above. This is a naive version for use
544 * when the operating system provides no alternative.
545 */
546void my_closefrom(int start)
547{
548 struct rlimit limit;
549 int nbfds;
550
551 if (getrlimit(RLIMIT_NOFILE, &limit) == 0)
552 nbfds = limit.rlim_cur;
553 else
554 nbfds = 0;
555
556 if (nbfds <= 0)
557 nbfds = 1024; /* safe limit */
558
559 while (start < nbfds)
560 close(start++);
561}
Willy Tarreaue5733232019-05-22 19:24:06 +0200562#endif // defined(USE_POLL)
Willy Tarreau2d7f81b2019-02-21 22:19:17 +0100563
Willy Tarreau4f60f162007-04-08 16:39:58 +0200564/* disable the specified poller */
565void disable_poller(const char *poller_name)
566{
567 int p;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200568
Willy Tarreau4f60f162007-04-08 16:39:58 +0200569 for (p = 0; p < nbpollers; p++)
570 if (strcmp(pollers[p].name, poller_name) == 0)
571 pollers[p].pref = 0;
572}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200573
Olivier Houchard79321b92018-07-26 17:55:11 +0200574void poller_pipe_io_handler(int fd)
575{
576 char buf[1024];
577 /* Flush the pipe */
578 while (read(fd, buf, sizeof(buf)) > 0);
579 fd_cant_recv(fd);
580}
581
Willy Tarreau082b6282019-05-22 14:42:12 +0200582/* allocate the per-thread fd_updt thus needs to be called early after
583 * thread creation.
584 */
585static int alloc_pollers_per_thread()
586{
587 fd_updt = calloc(global.maxsock, sizeof(*fd_updt));
588 return fd_updt != NULL;
589}
590
591/* Initialize the pollers per thread.*/
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200592static int init_pollers_per_thread()
593{
Olivier Houchard79321b92018-07-26 17:55:11 +0200594 int mypipe[2];
Willy Tarreau082b6282019-05-22 14:42:12 +0200595
596 if (pipe(mypipe) < 0)
Olivier Houchard79321b92018-07-26 17:55:11 +0200597 return 0;
Willy Tarreau082b6282019-05-22 14:42:12 +0200598
Olivier Houchard79321b92018-07-26 17:55:11 +0200599 poller_rd_pipe = mypipe[0];
600 poller_wr_pipe[tid] = mypipe[1];
601 fcntl(poller_rd_pipe, F_SETFL, O_NONBLOCK);
602 fd_insert(poller_rd_pipe, poller_pipe_io_handler, poller_pipe_io_handler,
603 tid_bit);
604 fd_want_recv(poller_rd_pipe);
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200605 return 1;
606}
607
608/* Deinitialize the pollers per thread */
609static void deinit_pollers_per_thread()
610{
William Lallemand808e1b72018-12-15 22:34:31 +0100611 /* rd and wr are init at the same place, but only rd is init to -1, so
612 we rely to rd to close. */
613 if (poller_rd_pipe > -1) {
614 close(poller_rd_pipe);
615 poller_rd_pipe = -1;
616 close(poller_wr_pipe[tid]);
617 poller_wr_pipe[tid] = -1;
618 }
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200619}
620
Willy Tarreau082b6282019-05-22 14:42:12 +0200621/* Release the pollers per thread, to be called late */
622static void free_pollers_per_thread()
623{
624 free(fd_updt);
625 fd_updt = NULL;
626}
627
Willy Tarreaubaaee002006-06-26 02:48:02 +0200628/*
Willy Tarreau4f60f162007-04-08 16:39:58 +0200629 * Initialize the pollers till the best one is found.
630 * If none works, returns 0, otherwise 1.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200631 */
Willy Tarreau4f60f162007-04-08 16:39:58 +0200632int init_pollers()
Willy Tarreaubaaee002006-06-26 02:48:02 +0200633{
Willy Tarreau4f60f162007-04-08 16:39:58 +0200634 int p;
635 struct poller *bp;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200636
Christopher Faulet63fe6522017-08-31 17:52:09 +0200637 if ((fdtab = calloc(global.maxsock, sizeof(struct fdtab))) == NULL)
638 goto fail_tab;
639
Olivier Houchardcb92f5c2018-04-26 14:23:07 +0200640 if ((polled_mask = calloc(global.maxsock, sizeof(unsigned long))) == NULL)
641 goto fail_polledmask;
Uman Shahzadda7eeed2019-01-17 08:21:39 +0000642
Christopher Faulet63fe6522017-08-31 17:52:09 +0200643 if ((fdinfo = calloc(global.maxsock, sizeof(struct fdinfo))) == NULL)
644 goto fail_info;
645
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100646 fd_cache.first = fd_cache.last = -1;
Olivier Houchard6b96f722018-04-25 16:58:25 +0200647 update_list.first = update_list.last = -1;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200648
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100649 for (p = 0; p < global.maxsock; p++) {
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100650 HA_SPIN_INIT(&fdtab[p].lock);
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100651 /* Mark the fd as out of the fd cache */
Olivier Houchard12568362018-01-31 18:07:29 +0100652 fdtab[p].cache.next = -3;
Olivier Houchard6b96f722018-04-25 16:58:25 +0200653 fdtab[p].update.next = -3;
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100654 }
655 for (p = 0; p < global.nbthread; p++)
656 fd_cache_local[p].first = fd_cache_local[p].last = -1;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200657
Willy Tarreau4f60f162007-04-08 16:39:58 +0200658 do {
659 bp = NULL;
660 for (p = 0; p < nbpollers; p++)
661 if (!bp || (pollers[p].pref > bp->pref))
662 bp = &pollers[p];
Willy Tarreaubaaee002006-06-26 02:48:02 +0200663
Willy Tarreau4f60f162007-04-08 16:39:58 +0200664 if (!bp || bp->pref == 0)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200665 break;
666
Willy Tarreau4f60f162007-04-08 16:39:58 +0200667 if (bp->init(bp)) {
668 memcpy(&cur_poller, bp, sizeof(*bp));
669 return 1;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200670 }
Willy Tarreau4f60f162007-04-08 16:39:58 +0200671 } while (!bp || bp->pref == 0);
Willy Tarreau7be79a42012-11-11 15:02:54 +0100672
Christopher Faulet63fe6522017-08-31 17:52:09 +0200673 free(fdinfo);
674 fail_info:
Olivier Houchardcb92f5c2018-04-26 14:23:07 +0200675 free(polled_mask);
676 fail_polledmask:
Uman Shahzadda7eeed2019-01-17 08:21:39 +0000677 free(fdtab);
678 fail_tab:
Willy Tarreau7be79a42012-11-11 15:02:54 +0100679 return 0;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200680}
681
Willy Tarreaubaaee002006-06-26 02:48:02 +0200682/*
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +0200683 * Deinitialize the pollers.
684 */
685void deinit_pollers() {
686
687 struct poller *bp;
688 int p;
689
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200690 for (p = 0; p < global.maxsock; p++)
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100691 HA_SPIN_DESTROY(&fdtab[p].lock);
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200692
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +0200693 for (p = 0; p < nbpollers; p++) {
694 bp = &pollers[p];
695
696 if (bp && bp->pref)
697 bp->term(bp);
698 }
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200699
Christopher Faulet63fe6522017-08-31 17:52:09 +0200700 free(fdinfo); fdinfo = NULL;
701 free(fdtab); fdtab = NULL;
Olivier Houchardcb92f5c2018-04-26 14:23:07 +0200702 free(polled_mask); polled_mask = NULL;
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +0200703}
704
705/*
Willy Tarreau2ff76222007-04-09 19:29:56 +0200706 * Lists the known pollers on <out>.
707 * Should be performed only before initialization.
708 */
709int list_pollers(FILE *out)
710{
711 int p;
712 int last, next;
713 int usable;
714 struct poller *bp;
715
716 fprintf(out, "Available polling systems :\n");
717
718 usable = 0;
719 bp = NULL;
720 last = next = -1;
721 while (1) {
722 for (p = 0; p < nbpollers; p++) {
Willy Tarreau2ff76222007-04-09 19:29:56 +0200723 if ((next < 0 || pollers[p].pref > next)
Willy Tarreaue79c3b22010-11-19 10:20:36 +0100724 && (last < 0 || pollers[p].pref < last)) {
Willy Tarreau2ff76222007-04-09 19:29:56 +0200725 next = pollers[p].pref;
Willy Tarreaue79c3b22010-11-19 10:20:36 +0100726 if (!bp || (pollers[p].pref > bp->pref))
727 bp = &pollers[p];
728 }
Willy Tarreau2ff76222007-04-09 19:29:56 +0200729 }
730
731 if (next == -1)
732 break;
733
734 for (p = 0; p < nbpollers; p++) {
735 if (pollers[p].pref == next) {
736 fprintf(out, " %10s : ", pollers[p].name);
737 if (pollers[p].pref == 0)
738 fprintf(out, "disabled, ");
739 else
740 fprintf(out, "pref=%3d, ", pollers[p].pref);
741 if (pollers[p].test(&pollers[p])) {
742 fprintf(out, " test result OK");
743 if (next > 0)
744 usable++;
Willy Tarreaue79c3b22010-11-19 10:20:36 +0100745 } else {
Willy Tarreau2ff76222007-04-09 19:29:56 +0200746 fprintf(out, " test result FAILED");
Willy Tarreaue79c3b22010-11-19 10:20:36 +0100747 if (bp == &pollers[p])
748 bp = NULL;
749 }
Willy Tarreau2ff76222007-04-09 19:29:56 +0200750 fprintf(out, "\n");
751 }
752 }
753 last = next;
754 next = -1;
755 };
756 fprintf(out, "Total: %d (%d usable), will use %s.\n", nbpollers, usable, bp ? bp->name : "none");
757 return 0;
758}
759
760/*
761 * Some pollers may lose their connection after a fork(). It may be necessary
762 * to create initialize part of them again. Returns 0 in case of failure,
763 * otherwise 1. The fork() function may be NULL if unused. In case of error,
764 * the the current poller is destroyed and the caller is responsible for trying
765 * another one by calling init_pollers() again.
766 */
767int fork_poller()
768{
Conrad Hoffmann041751c2014-05-20 14:28:24 +0200769 int fd;
Willy Tarreauce036bc2018-01-29 14:58:02 +0100770 for (fd = 0; fd < global.maxsock; fd++) {
Conrad Hoffmann041751c2014-05-20 14:28:24 +0200771 if (fdtab[fd].owner) {
772 fdtab[fd].cloned = 1;
773 }
774 }
775
Willy Tarreau2ff76222007-04-09 19:29:56 +0200776 if (cur_poller.fork) {
777 if (cur_poller.fork(&cur_poller))
778 return 1;
779 cur_poller.term(&cur_poller);
780 return 0;
781 }
782 return 1;
783}
784
Willy Tarreau082b6282019-05-22 14:42:12 +0200785REGISTER_PER_THREAD_ALLOC(alloc_pollers_per_thread);
Willy Tarreau172f5ce2018-11-26 11:21:50 +0100786REGISTER_PER_THREAD_INIT(init_pollers_per_thread);
787REGISTER_PER_THREAD_DEINIT(deinit_pollers_per_thread);
Willy Tarreau082b6282019-05-22 14:42:12 +0200788REGISTER_PER_THREAD_FREE(free_pollers_per_thread);
Willy Tarreau172f5ce2018-11-26 11:21:50 +0100789
Willy Tarreau2ff76222007-04-09 19:29:56 +0200790/*
Willy Tarreaubaaee002006-06-26 02:48:02 +0200791 * Local variables:
792 * c-indent-level: 8
793 * c-basic-offset: 8
794 * End:
795 */