blob: 8dab4977a968b346858279417cd33b1b02b318a8 [file] [log] [blame]
Willy Tarreaubaaee002006-06-26 02:48:02 +02001/*
2 * File descriptors management functions.
3 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +01004 * Copyright 2000-2014 Willy Tarreau <w@1wt.eu>
Willy Tarreaubaaee002006-06-26 02:48:02 +02005 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010011 * This code implements an events cache for file descriptors. It remembers the
12 * readiness of a file descriptor after a return from poll() and the fact that
13 * an I/O attempt failed on EAGAIN. Events in the cache which are still marked
14 * ready and active are processed just as if they were reported by poll().
Willy Tarreau7be79a42012-11-11 15:02:54 +010015 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010016 * This serves multiple purposes. First, it significantly improves performance
17 * by avoiding to subscribe to polling unless absolutely necessary, so most
18 * events are processed without polling at all, especially send() which
19 * benefits from the socket buffers. Second, it is the only way to support
20 * edge-triggered pollers (eg: EPOLL_ET). And third, it enables I/O operations
21 * that are backed by invisible buffers. For example, SSL is able to read a
22 * whole socket buffer and not deliver it to the application buffer because
23 * it's full. Unfortunately, it won't be reported by a poller anymore until
24 * some new activity happens. The only way to call it again thus is to keep
25 * this readiness information in the cache and to access it without polling
26 * once the FD is enabled again.
Willy Tarreau7be79a42012-11-11 15:02:54 +010027 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010028 * One interesting feature of the cache is that it maintains the principle
29 * of speculative I/O introduced in haproxy 1.3 : the first time an event is
30 * enabled, the FD is considered as ready so that the I/O attempt is performed
31 * via the cache without polling. And the polling happens only when EAGAIN is
32 * first met. This avoids polling for HTTP requests, especially when the
33 * defer-accept mode is used. It also avoids polling for sending short data
34 * such as requests to servers or short responses to clients.
35 *
36 * The cache consists in a list of active events and a list of updates.
37 * Active events are events that are expected to come and that we must report
38 * to the application until it asks to stop or asks to poll. Updates are new
39 * requests for changing an FD state. Updates are the only way to create new
40 * events. This is important because it means that the number of cached events
41 * cannot increase between updates and will only grow one at a time while
42 * processing updates. All updates must always be processed, though events
43 * might be processed by small batches if required.
Willy Tarreau7be79a42012-11-11 15:02:54 +010044 *
45 * There is no direct link between the FD and the updates list. There is only a
46 * bit in the fdtab[] to indicate than a file descriptor is already present in
47 * the updates list. Once an fd is present in the updates list, it will have to
48 * be considered even if its changes are reverted in the middle or if the fd is
49 * replaced.
50 *
51 * It is important to understand that as long as all expected events are
52 * processed, they might starve the polled events, especially because polled
Willy Tarreauf817e9f2014-01-10 16:58:45 +010053 * I/O starvation quickly induces more cached I/O. One solution to this
Willy Tarreau7be79a42012-11-11 15:02:54 +010054 * consists in only processing a part of the events at once, but one drawback
Willy Tarreauf817e9f2014-01-10 16:58:45 +010055 * is that unhandled events will still wake the poller up. Using an edge-
56 * triggered poller such as EPOLL_ET will solve this issue though.
Willy Tarreau7be79a42012-11-11 15:02:54 +010057 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010058 * Since we do not want to scan all the FD list to find cached I/O events,
59 * we store them in a list consisting in a linear array holding only the FD
60 * indexes right now. Note that a closed FD cannot exist in the cache, because
61 * it is closed by fd_delete() which in turn calls fd_release_cache_entry()
62 * which always removes it from the list.
Willy Tarreau7be79a42012-11-11 15:02:54 +010063 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010064 * The FD array has to hold a back reference to the cache. This reference is
65 * always valid unless the FD is not in the cache and is not updated, in which
66 * case the reference points to index 0.
Willy Tarreau7be79a42012-11-11 15:02:54 +010067 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010068 * The event state for an FD, as found in fdtab[].state, is maintained for each
69 * direction. The state field is built this way, with R bits in the low nibble
70 * and W bits in the high nibble for ease of access and debugging :
Willy Tarreau7be79a42012-11-11 15:02:54 +010071 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010072 * 7 6 5 4 3 2 1 0
73 * [ 0 | PW | RW | AW | 0 | PR | RR | AR ]
74 *
75 * A* = active *R = read
76 * P* = polled *W = write
77 * R* = ready
78 *
79 * An FD is marked "active" when there is a desire to use it.
80 * An FD is marked "polled" when it is registered in the polling.
81 * An FD is marked "ready" when it has not faced a new EAGAIN since last wake-up
82 * (it is a cache of the last EAGAIN regardless of polling changes).
Willy Tarreau7be79a42012-11-11 15:02:54 +010083 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010084 * We have 8 possible states for each direction based on these 3 flags :
Willy Tarreau7be79a42012-11-11 15:02:54 +010085 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010086 * +---+---+---+----------+---------------------------------------------+
87 * | P | R | A | State | Description |
88 * +---+---+---+----------+---------------------------------------------+
89 * | 0 | 0 | 0 | DISABLED | No activity desired, not ready. |
90 * | 0 | 0 | 1 | MUSTPOLL | Activity desired via polling. |
91 * | 0 | 1 | 0 | STOPPED | End of activity without polling. |
92 * | 0 | 1 | 1 | ACTIVE | Activity desired without polling. |
93 * | 1 | 0 | 0 | ABORT | Aborted poll(). Not frequently seen. |
94 * | 1 | 0 | 1 | POLLED | FD is being polled. |
95 * | 1 | 1 | 0 | PAUSED | FD was paused while ready (eg: buffer full) |
96 * | 1 | 1 | 1 | READY | FD was marked ready by poll() |
97 * +---+---+---+----------+---------------------------------------------+
Willy Tarreau7be79a42012-11-11 15:02:54 +010098 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010099 * The transitions are pretty simple :
100 * - fd_want_*() : set flag A
101 * - fd_stop_*() : clear flag A
102 * - fd_cant_*() : clear flag R (when facing EAGAIN)
103 * - fd_may_*() : set flag R (upon return from poll())
104 * - sync() : if (A) { if (!R) P := 1 } else { P := 0 }
105 *
106 * The PAUSED, ABORT and MUSTPOLL states are transient for level-trigerred
107 * pollers and are fixed by the sync() which happens at the beginning of the
108 * poller. For event-triggered pollers, only the MUSTPOLL state will be
109 * transient and ABORT will lead to PAUSED. The ACTIVE state is the only stable
110 * one which has P != A.
111 *
112 * The READY state is a bit special as activity on the FD might be notified
113 * both by the poller or by the cache. But it is needed for some multi-layer
114 * protocols (eg: SSL) where connection activity is not 100% linked to FD
115 * activity. Also some pollers might prefer to implement it as ACTIVE if
116 * enabling/disabling the FD is cheap. The READY and ACTIVE states are the
117 * two states for which a cache entry is allocated.
118 *
119 * The state transitions look like the diagram below. Only the 4 right states
120 * have polling enabled :
121 *
122 * (POLLED=0) (POLLED=1)
123 *
124 * +----------+ sync +-------+
125 * | DISABLED | <----- | ABORT | (READY=0, ACTIVE=0)
126 * +----------+ +-------+
127 * clr | ^ set | ^
128 * | | | |
129 * v | set v | clr
130 * +----------+ sync +--------+
131 * | MUSTPOLL | -----> | POLLED | (READY=0, ACTIVE=1)
132 * +----------+ +--------+
133 * ^ poll | ^
134 * | | |
135 * | EAGAIN v | EAGAIN
136 * +--------+ +-------+
137 * | ACTIVE | | READY | (READY=1, ACTIVE=1)
138 * +--------+ +-------+
139 * clr | ^ set | ^
140 * | | | |
141 * v | set v | clr
142 * +---------+ sync +--------+
143 * | STOPPED | <------ | PAUSED | (READY=1, ACTIVE=0)
144 * +---------+ +--------+
Willy Tarreaubaaee002006-06-26 02:48:02 +0200145 */
146
Willy Tarreau2ff76222007-04-09 19:29:56 +0200147#include <stdio.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +0200148#include <string.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +0200149#include <unistd.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +0200150#include <sys/types.h>
151
Willy Tarreau2dd0d472006-06-29 17:53:05 +0200152#include <common/compat.h>
153#include <common/config.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +0200154
Willy Tarreau7be79a42012-11-11 15:02:54 +0100155#include <types/global.h>
156
Willy Tarreau2a429502006-10-15 14:52:29 +0200157#include <proto/fd.h>
Willy Tarreauc6f4ce82009-06-10 11:09:37 +0200158#include <proto/port_range.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +0200159
160struct fdtab *fdtab = NULL; /* array of all the file descriptors */
Willy Tarreau8d5d77e2009-10-18 07:25:52 +0200161struct fdinfo *fdinfo = NULL; /* less-often used infos for file descriptors */
Willy Tarreaubaaee002006-06-26 02:48:02 +0200162int maxfd; /* # of the highest fd + 1 */
163int totalconn; /* total # of terminated sessions */
164int actconn; /* # of active sessions */
165
Willy Tarreau4f60f162007-04-08 16:39:58 +0200166struct poller pollers[MAX_POLLERS];
167struct poller cur_poller;
168int nbpollers = 0;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200169
Willy Tarreau16f649c2014-01-25 19:10:48 +0100170unsigned int *fd_cache = NULL; // FD events cache
Willy Tarreau7be79a42012-11-11 15:02:54 +0100171unsigned int *fd_updt = NULL; // FD updates list
Willy Tarreau16f649c2014-01-25 19:10:48 +0100172int fd_cache_num = 0; // number of events in the cache
173int fd_nbupdt = 0; // number of updates in the list
Willy Tarreaubaaee002006-06-26 02:48:02 +0200174
Willy Tarreau4f60f162007-04-08 16:39:58 +0200175/* Deletes an FD from the fdsets, and recomputes the maxfd limit.
176 * The file descriptor is also closed.
177 */
Olivier Houchard1fc05162017-04-06 01:05:05 +0200178static void fd_dodelete(int fd, int do_close)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200179{
Willy Tarreauad38ace2013-12-15 14:19:38 +0100180 if (fdtab[fd].linger_risk) {
181 /* this is generally set when connecting to servers */
182 setsockopt(fd, SOL_SOCKET, SO_LINGER,
183 (struct linger *) &nolinger, sizeof(struct linger));
184 }
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100185 if (cur_poller.clo)
186 cur_poller.clo(fd);
187
Willy Tarreau899d9572014-01-25 19:20:35 +0100188 fd_release_cache_entry(fd);
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100189 fdtab[fd].state = 0;
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100190
Willy Tarreau8d5d77e2009-10-18 07:25:52 +0200191 port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port);
192 fdinfo[fd].port_range = NULL;
Willy Tarreaudb3b3262012-07-05 23:19:22 +0200193 fdtab[fd].owner = NULL;
Christopher Fauletd531f882017-06-01 16:55:03 +0200194 fdtab[fd].updated = 0;
Willy Tarreau1720abd2012-11-11 17:08:32 +0100195 fdtab[fd].new = 0;
Christopher Fauletd531f882017-06-01 16:55:03 +0200196 if (do_close)
197 close(fd);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200198
Willy Tarreaudb3b3262012-07-05 23:19:22 +0200199 while ((maxfd-1 >= 0) && !fdtab[maxfd-1].owner)
Willy Tarreau4f60f162007-04-08 16:39:58 +0200200 maxfd--;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200201}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200202
Olivier Houchard1fc05162017-04-06 01:05:05 +0200203/* Deletes an FD from the fdsets, and recomputes the maxfd limit.
204 * The file descriptor is also closed.
205 */
206void fd_delete(int fd)
207{
208 fd_dodelete(fd, 1);
209}
210
211/* Deletes an FD from the fdsets, and recomputes the maxfd limit.
212 * The file descriptor is kept open.
213 */
214void fd_remove(int fd)
215{
216 fd_dodelete(fd, 0);
217}
218
Willy Tarreau033cd9d2014-01-25 19:24:15 +0100219/* Scan and process the cached events. This should be called right after
Willy Tarreau5be2f352014-11-19 19:43:05 +0100220 * the poller. The loop may cause new entries to be created, for example
221 * if a listener causes an accept() to initiate a new incoming connection
222 * wanting to attempt an recv().
Willy Tarreau09f24562012-11-11 16:43:45 +0100223 */
Willy Tarreau033cd9d2014-01-25 19:24:15 +0100224void fd_process_cached_events()
Willy Tarreau09f24562012-11-11 16:43:45 +0100225{
Willy Tarreau033cd9d2014-01-25 19:24:15 +0100226 int fd, entry, e;
Willy Tarreau09f24562012-11-11 16:43:45 +0100227
Willy Tarreau033cd9d2014-01-25 19:24:15 +0100228 for (entry = 0; entry < fd_cache_num; ) {
229 fd = fd_cache[entry];
Willy Tarreau69a41fa2014-01-20 11:02:59 +0100230 e = fdtab[fd].state;
Willy Tarreau09f24562012-11-11 16:43:45 +0100231
Willy Tarreau09f24562012-11-11 16:43:45 +0100232 fdtab[fd].ev &= FD_POLL_STICKY;
233
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100234 if ((e & (FD_EV_READY_R | FD_EV_ACTIVE_R)) == (FD_EV_READY_R | FD_EV_ACTIVE_R))
Willy Tarreau09f24562012-11-11 16:43:45 +0100235 fdtab[fd].ev |= FD_POLL_IN;
236
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100237 if ((e & (FD_EV_READY_W | FD_EV_ACTIVE_W)) == (FD_EV_READY_W | FD_EV_ACTIVE_W))
Willy Tarreau09f24562012-11-11 16:43:45 +0100238 fdtab[fd].ev |= FD_POLL_OUT;
239
240 if (fdtab[fd].iocb && fdtab[fd].owner && fdtab[fd].ev)
241 fdtab[fd].iocb(fd);
Willy Tarreaufa7fc952014-01-20 20:18:59 +0100242 else
Willy Tarreau5be2f352014-11-19 19:43:05 +0100243 fd_release_cache_entry(fd);
Willy Tarreau09f24562012-11-11 16:43:45 +0100244
Willy Tarreau033cd9d2014-01-25 19:24:15 +0100245 /* If the fd was removed from the cache, it has been
Willy Tarreau09f24562012-11-11 16:43:45 +0100246 * replaced by the next one that we don't want to skip !
247 */
Willy Tarreau033cd9d2014-01-25 19:24:15 +0100248 if (entry < fd_cache_num && fd_cache[entry] != fd)
Willy Tarreau09f24562012-11-11 16:43:45 +0100249 continue;
Willy Tarreau033cd9d2014-01-25 19:24:15 +0100250 entry++;
Willy Tarreau09f24562012-11-11 16:43:45 +0100251 }
252}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200253
Willy Tarreau4f60f162007-04-08 16:39:58 +0200254/* disable the specified poller */
255void disable_poller(const char *poller_name)
256{
257 int p;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200258
Willy Tarreau4f60f162007-04-08 16:39:58 +0200259 for (p = 0; p < nbpollers; p++)
260 if (strcmp(pollers[p].name, poller_name) == 0)
261 pollers[p].pref = 0;
262}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200263
264/*
Willy Tarreau4f60f162007-04-08 16:39:58 +0200265 * Initialize the pollers till the best one is found.
266 * If none works, returns 0, otherwise 1.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200267 */
Willy Tarreau4f60f162007-04-08 16:39:58 +0200268int init_pollers()
Willy Tarreaubaaee002006-06-26 02:48:02 +0200269{
Willy Tarreau4f60f162007-04-08 16:39:58 +0200270 int p;
271 struct poller *bp;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200272
Christopher Faulet63fe6522017-08-31 17:52:09 +0200273 if ((fdtab = calloc(global.maxsock, sizeof(struct fdtab))) == NULL)
274 goto fail_tab;
275
276 if ((fdinfo = calloc(global.maxsock, sizeof(struct fdinfo))) == NULL)
277 goto fail_info;
278
279 if ((fd_cache = calloc(global.maxsock, sizeof(*fd_cache))) == NULL)
Willy Tarreau16f649c2014-01-25 19:10:48 +0100280 goto fail_cache;
Willy Tarreau7be79a42012-11-11 15:02:54 +0100281
Christopher Faulet63fe6522017-08-31 17:52:09 +0200282 if ((fd_updt = calloc(global.maxsock, sizeof(*fd_updt))) == NULL)
Willy Tarreau7be79a42012-11-11 15:02:54 +0100283 goto fail_updt;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200284
Willy Tarreau4f60f162007-04-08 16:39:58 +0200285 do {
286 bp = NULL;
287 for (p = 0; p < nbpollers; p++)
288 if (!bp || (pollers[p].pref > bp->pref))
289 bp = &pollers[p];
Willy Tarreaubaaee002006-06-26 02:48:02 +0200290
Willy Tarreau4f60f162007-04-08 16:39:58 +0200291 if (!bp || bp->pref == 0)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200292 break;
293
Willy Tarreau4f60f162007-04-08 16:39:58 +0200294 if (bp->init(bp)) {
295 memcpy(&cur_poller, bp, sizeof(*bp));
296 return 1;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200297 }
Willy Tarreau4f60f162007-04-08 16:39:58 +0200298 } while (!bp || bp->pref == 0);
299 return 0;
Willy Tarreau7be79a42012-11-11 15:02:54 +0100300
301 fail_updt:
Willy Tarreau16f649c2014-01-25 19:10:48 +0100302 free(fd_cache);
303 fail_cache:
Christopher Faulet63fe6522017-08-31 17:52:09 +0200304 free(fdinfo);
305 fail_info:
306 free(fdtab);
307 fail_tab:
Willy Tarreau7be79a42012-11-11 15:02:54 +0100308 return 0;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200309}
310
Willy Tarreaubaaee002006-06-26 02:48:02 +0200311/*
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +0200312 * Deinitialize the pollers.
313 */
314void deinit_pollers() {
315
316 struct poller *bp;
317 int p;
318
319 for (p = 0; p < nbpollers; p++) {
320 bp = &pollers[p];
321
322 if (bp && bp->pref)
323 bp->term(bp);
324 }
Christopher Faulet63fe6522017-08-31 17:52:09 +0200325 free(fd_updt); fd_updt = NULL;
326 free(fd_cache); fd_cache = NULL;
327 free(fdinfo); fdinfo = NULL;
328 free(fdtab); fdtab = NULL;
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +0200329}
330
331/*
Willy Tarreau2ff76222007-04-09 19:29:56 +0200332 * Lists the known pollers on <out>.
333 * Should be performed only before initialization.
334 */
335int list_pollers(FILE *out)
336{
337 int p;
338 int last, next;
339 int usable;
340 struct poller *bp;
341
342 fprintf(out, "Available polling systems :\n");
343
344 usable = 0;
345 bp = NULL;
346 last = next = -1;
347 while (1) {
348 for (p = 0; p < nbpollers; p++) {
Willy Tarreau2ff76222007-04-09 19:29:56 +0200349 if ((next < 0 || pollers[p].pref > next)
Willy Tarreaue79c3b22010-11-19 10:20:36 +0100350 && (last < 0 || pollers[p].pref < last)) {
Willy Tarreau2ff76222007-04-09 19:29:56 +0200351 next = pollers[p].pref;
Willy Tarreaue79c3b22010-11-19 10:20:36 +0100352 if (!bp || (pollers[p].pref > bp->pref))
353 bp = &pollers[p];
354 }
Willy Tarreau2ff76222007-04-09 19:29:56 +0200355 }
356
357 if (next == -1)
358 break;
359
360 for (p = 0; p < nbpollers; p++) {
361 if (pollers[p].pref == next) {
362 fprintf(out, " %10s : ", pollers[p].name);
363 if (pollers[p].pref == 0)
364 fprintf(out, "disabled, ");
365 else
366 fprintf(out, "pref=%3d, ", pollers[p].pref);
367 if (pollers[p].test(&pollers[p])) {
368 fprintf(out, " test result OK");
369 if (next > 0)
370 usable++;
Willy Tarreaue79c3b22010-11-19 10:20:36 +0100371 } else {
Willy Tarreau2ff76222007-04-09 19:29:56 +0200372 fprintf(out, " test result FAILED");
Willy Tarreaue79c3b22010-11-19 10:20:36 +0100373 if (bp == &pollers[p])
374 bp = NULL;
375 }
Willy Tarreau2ff76222007-04-09 19:29:56 +0200376 fprintf(out, "\n");
377 }
378 }
379 last = next;
380 next = -1;
381 };
382 fprintf(out, "Total: %d (%d usable), will use %s.\n", nbpollers, usable, bp ? bp->name : "none");
383 return 0;
384}
385
386/*
387 * Some pollers may lose their connection after a fork(). It may be necessary
388 * to create initialize part of them again. Returns 0 in case of failure,
389 * otherwise 1. The fork() function may be NULL if unused. In case of error,
390 * the the current poller is destroyed and the caller is responsible for trying
391 * another one by calling init_pollers() again.
392 */
393int fork_poller()
394{
Conrad Hoffmann041751c2014-05-20 14:28:24 +0200395 int fd;
396 for (fd = 0; fd <= maxfd; fd++) {
397 if (fdtab[fd].owner) {
398 fdtab[fd].cloned = 1;
399 }
400 }
401
Willy Tarreau2ff76222007-04-09 19:29:56 +0200402 if (cur_poller.fork) {
403 if (cur_poller.fork(&cur_poller))
404 return 1;
405 cur_poller.term(&cur_poller);
406 return 0;
407 }
408 return 1;
409}
410
411/*
Willy Tarreaubaaee002006-06-26 02:48:02 +0200412 * Local variables:
413 * c-indent-level: 8
414 * c-basic-offset: 8
415 * End:
416 */