blob: b5048157bd8dc62ca31937dee44d95fc7b20f838 [file] [log] [blame]
Willy Tarreaude99e992007-04-16 00:53:59 +02001/*
2 * FD polling functions for Speculative I/O combined with Linux epoll()
3 *
Willy Tarreau43d8fb22011-08-22 17:12:02 +02004 * Copyright 2000-2011 Willy Tarreau <w@1wt.eu>
Willy Tarreaude99e992007-04-16 00:53:59 +02005 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
Willy Tarreauf2e8ee22008-05-25 10:39:02 +020011 *
12 * This code implements "speculative I/O" under Linux. The principle is to
13 * try to perform expected I/O before registering the events in the poller.
14 * Each time this succeeds, it saves an expensive epoll_ctl(). It generally
15 * succeeds for all reads after an accept(), and for writes after a connect().
16 * It also improves performance for streaming connections because even if only
17 * one side is polled, the other one may react accordingly depending on the
18 * level of the buffer.
19 *
20 * It has a presents drawbacks though. If too many events are set for spec I/O,
21 * those ones can starve the polled events. Experiments show that when polled
22 * events starve, they quickly turn into spec I/O, making the situation even
23 * worse. While we can reduce the number of polled events processed at once,
24 * we cannot do this on speculative events because most of them are new ones
25 * (avg 2/3 new - 1/3 old from experiments).
26 *
27 * The solution against this problem relies on those two factors :
28 * 1) one FD registered as a spec event cannot be polled at the same time
29 * 2) even during very high loads, we will almost never be interested in
30 * simultaneous read and write streaming on the same FD.
31 *
32 * The first point implies that during starvation, we will not have more than
33 * half of our FDs in the poll list, otherwise it means there is less than that
34 * in the spec list, implying there is no starvation.
35 *
36 * The second point implies that we're statically only interested in half of
37 * the maximum number of file descriptors at once, because we will unlikely
38 * have simultaneous read and writes for a same buffer during long periods.
39 *
40 * So, if we make it possible to drain maxsock/2/2 during peak loads, then we
41 * can ensure that there will be no starvation effect. This means that we must
42 * always allocate maxsock/4 events for the poller.
43 *
44 *
Willy Tarreaude99e992007-04-16 00:53:59 +020045 */
46
47#include <unistd.h>
48#include <sys/time.h>
49#include <sys/types.h>
50
51#include <common/compat.h>
52#include <common/config.h>
Willy Tarreaud6f087e2008-01-18 17:20:13 +010053#include <common/debug.h>
Willy Tarreau43d8fb22011-08-22 17:12:02 +020054#include <common/epoll.h>
Willy Tarreaude99e992007-04-16 00:53:59 +020055#include <common/standard.h>
Willy Tarreau0c303ee2008-07-07 00:09:58 +020056#include <common/ticks.h>
Willy Tarreaude99e992007-04-16 00:53:59 +020057#include <common/time.h>
Willy Tarreau1db37712007-06-03 17:16:49 +020058#include <common/tools.h>
Willy Tarreaude99e992007-04-16 00:53:59 +020059
Willy Tarreaude99e992007-04-16 00:53:59 +020060#include <types/global.h>
61
62#include <proto/fd.h>
Willy Tarreau332740d2009-05-10 09:57:21 +020063#include <proto/signal.h>
Willy Tarreaude99e992007-04-16 00:53:59 +020064#include <proto/task.h>
65
Willy Tarreaude99e992007-04-16 00:53:59 +020066/*
67 * We define 4 states for each direction of a file descriptor, which we store
68 * as 2 bits :
69 *
70 * 00 = IDLE : we're not interested in this event
71 * 01 = SPEC : perform speculative I/O on this FD
72 * 10 = WAIT : really wait for an availability event on this FD (poll)
73 * 11 = STOP : was marked WAIT, but disabled. It can switch back to WAIT if
74 * the application changes its mind, otherwise disable FD polling
75 * and switch back to IDLE.
76 *
77 * Since we do not want to scan all the FD list to find speculative I/O events,
78 * we store them in a list consisting in a linear array holding only the FD
Willy Tarreau7a52a5c2008-08-16 16:06:02 +020079 * indexes right now. Note that a closed FD cannot exist in the spec list,
80 * because it is closed by fd_delete() which in turn calls __fd_clo() which
81 * always removes it from the list.
Willy Tarreaude99e992007-04-16 00:53:59 +020082 *
83 * The STOP state requires the event to be present in the spec list so that
84 * it can be detected and flushed upon next scan without having to scan the
85 * whole FD list.
86 *
87 * This translates like this :
88 *
89 * EVENT_IN_SPEC_LIST = 01
90 * EVENT_IN_POLL_LIST = 10
91 *
92 * IDLE = 0
93 * SPEC = (EVENT_IN_SPEC_LIST)
94 * WAIT = (EVENT_IN_POLL_LIST)
95 * STOP = (EVENT_IN_SPEC_LIST|EVENT_IN_POLL_LIST)
96 *
97 * fd_is_set() just consists in checking that the status is 01 or 10.
98 *
99 * For efficiency reasons, we will store the Read and Write bits interlaced to
100 * form a 4-bit field, so that we can simply shift the value right by 0/1 and
101 * get what we want :
102 * 3 2 1 0
103 * Wp Rp Ws Rs
104 *
105 * The FD array has to hold a back reference to the speculative list. This
106 * reference is only valid if at least one of the directions is marked SPEC.
107 *
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200108 * We store the FD state in the 4 lower bits of fdtab[fd].spec.e, and save the
109 * previous state upon changes in the 4 higher bits, so that changes are easy
110 * to spot.
Willy Tarreaude99e992007-04-16 00:53:59 +0200111 */
112
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200113#define FD_EV_IN_SL 1U
114#define FD_EV_IN_PL 4U
Willy Tarreaude99e992007-04-16 00:53:59 +0200115
116#define FD_EV_IDLE 0
117#define FD_EV_SPEC (FD_EV_IN_SL)
118#define FD_EV_WAIT (FD_EV_IN_PL)
119#define FD_EV_STOP (FD_EV_IN_SL|FD_EV_IN_PL)
120
121/* Those match any of R or W for Spec list or Poll list */
122#define FD_EV_RW_SL (FD_EV_IN_SL | (FD_EV_IN_SL << 1))
123#define FD_EV_RW_PL (FD_EV_IN_PL | (FD_EV_IN_PL << 1))
124#define FD_EV_MASK_DIR (FD_EV_IN_SL|FD_EV_IN_PL)
125
126#define FD_EV_IDLE_R 0
127#define FD_EV_SPEC_R (FD_EV_IN_SL)
128#define FD_EV_WAIT_R (FD_EV_IN_PL)
129#define FD_EV_STOP_R (FD_EV_IN_SL|FD_EV_IN_PL)
130#define FD_EV_MASK_R (FD_EV_IN_SL|FD_EV_IN_PL)
131
132#define FD_EV_IDLE_W (FD_EV_IDLE_R << 1)
133#define FD_EV_SPEC_W (FD_EV_SPEC_R << 1)
134#define FD_EV_WAIT_W (FD_EV_WAIT_R << 1)
135#define FD_EV_STOP_W (FD_EV_STOP_R << 1)
136#define FD_EV_MASK_W (FD_EV_MASK_R << 1)
137
138#define FD_EV_MASK (FD_EV_MASK_W | FD_EV_MASK_R)
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200139#define FD_EV_MASK_OLD (FD_EV_MASK << 4)
Willy Tarreaude99e992007-04-16 00:53:59 +0200140
Willy Tarreau6653d172007-05-13 01:52:05 +0200141/* This is the minimum number of events successfully processed in speculative
142 * mode above which we agree to return without checking epoll() (1/2 times).
143 */
144#define MIN_RETURN_EVENTS 25
Willy Tarreaude99e992007-04-16 00:53:59 +0200145
Willy Tarreaude99e992007-04-16 00:53:59 +0200146static int nbspec = 0; // current size of the spec list
Willy Tarreauf2e8ee22008-05-25 10:39:02 +0200147static int absmaxevents = 0; // absolute maximum amounts of polled events
Willy Tarreaude99e992007-04-16 00:53:59 +0200148
Willy Tarreaude99e992007-04-16 00:53:59 +0200149static unsigned int *spec_list = NULL; // speculative I/O list
150
151/* private data */
152static struct epoll_event *epoll_events;
153static int epoll_fd;
154
155/* This structure may be used for any purpose. Warning! do not use it in
156 * recursive functions !
157 */
158static struct epoll_event ev;
159
160
Willy Tarreauff9d5ba2009-10-17 21:43:03 +0200161REGPRM1 static inline void alloc_spec_entry(const int fd)
Willy Tarreaude99e992007-04-16 00:53:59 +0200162{
Willy Tarreaub48b3232009-10-17 22:54:17 +0200163 if (fdtab[fd].spec.s1)
Willy Tarreauff9d5ba2009-10-17 21:43:03 +0200164 /* sometimes the entry already exists for the other direction */
Willy Tarreaude99e992007-04-16 00:53:59 +0200165 return;
Willy Tarreaub48b3232009-10-17 22:54:17 +0200166 fdtab[fd].spec.s1 = nbspec + 1;
Willy Tarreau4eac2092007-08-31 17:01:18 +0200167 spec_list[nbspec] = fd;
168 nbspec++;
Willy Tarreaude99e992007-04-16 00:53:59 +0200169}
170
Willy Tarreau4eac2092007-08-31 17:01:18 +0200171/* Removes entry used by fd <fd> from the spec list and replaces it with the
Willy Tarreaub48b3232009-10-17 22:54:17 +0200172 * last one. The fdtab.spec is adjusted to match the back reference if needed.
Willy Tarreau4eac2092007-08-31 17:01:18 +0200173 * If the fd has no entry assigned, return immediately.
Willy Tarreaude99e992007-04-16 00:53:59 +0200174 */
Willy Tarreau4eac2092007-08-31 17:01:18 +0200175REGPRM1 static void release_spec_entry(int fd)
Willy Tarreaude99e992007-04-16 00:53:59 +0200176{
Willy Tarreau4eac2092007-08-31 17:01:18 +0200177 unsigned int pos;
178
Willy Tarreaub48b3232009-10-17 22:54:17 +0200179 pos = fdtab[fd].spec.s1;
Willy Tarreau4eac2092007-08-31 17:01:18 +0200180 if (!pos)
181 return;
182
Willy Tarreaub48b3232009-10-17 22:54:17 +0200183 fdtab[fd].spec.s1 = 0;
Willy Tarreau4eac2092007-08-31 17:01:18 +0200184 pos--;
185 /* we have spec_list[pos]==fd */
Willy Tarreaude99e992007-04-16 00:53:59 +0200186
187 nbspec--;
188 if (pos == nbspec)
189 return;
190
Willy Tarreau4eac2092007-08-31 17:01:18 +0200191 /* we replace current FD by the highest one, which may sometimes be the same */
Willy Tarreaude99e992007-04-16 00:53:59 +0200192 fd = spec_list[nbspec];
193 spec_list[pos] = fd;
Willy Tarreaub48b3232009-10-17 22:54:17 +0200194 fdtab[fd].spec.s1 = pos + 1;
Willy Tarreaude99e992007-04-16 00:53:59 +0200195}
196
197/*
198 * Returns non-zero if <fd> is already monitored for events in direction <dir>.
199 */
200REGPRM2 static int __fd_is_set(const int fd, int dir)
201{
202 int ret;
203
Willy Tarreau7a52a5c2008-08-16 16:06:02 +0200204#if DEBUG_DEV
205 if (fdtab[fd].state == FD_STCLOSE) {
206 fprintf(stderr, "sepoll.fd_isset called on closed fd #%d.\n", fd);
207 ABORT_NOW();
208 }
209#endif
Willy Tarreaub48b3232009-10-17 22:54:17 +0200210 ret = ((unsigned)fdtab[fd].spec.e >> dir) & FD_EV_MASK_DIR;
Willy Tarreaude99e992007-04-16 00:53:59 +0200211 return (ret == FD_EV_SPEC || ret == FD_EV_WAIT);
212}
213
214/*
215 * Don't worry about the strange constructs in __fd_set/__fd_clr, they are
216 * designed like this in order to reduce the number of jumps (verified).
217 */
218REGPRM2 static int __fd_set(const int fd, int dir)
219{
Willy Tarreaude99e992007-04-16 00:53:59 +0200220 unsigned int i;
221
Willy Tarreau7a52a5c2008-08-16 16:06:02 +0200222#if DEBUG_DEV
223 if (fdtab[fd].state == FD_STCLOSE) {
224 fprintf(stderr, "sepoll.fd_set called on closed fd #%d.\n", fd);
225 ABORT_NOW();
226 }
227#endif
Willy Tarreaub48b3232009-10-17 22:54:17 +0200228 i = ((unsigned)fdtab[fd].spec.e >> dir) & FD_EV_MASK_DIR;
Willy Tarreaude99e992007-04-16 00:53:59 +0200229
Willy Tarreauff9d5ba2009-10-17 21:43:03 +0200230 if (i != FD_EV_STOP) {
231 if (unlikely(i != FD_EV_IDLE))
232 return 0;
Willy Tarreaude99e992007-04-16 00:53:59 +0200233 // switch to SPEC state and allocate a SPEC entry.
234 alloc_spec_entry(fd);
Willy Tarreaude99e992007-04-16 00:53:59 +0200235 }
Willy Tarreaub48b3232009-10-17 22:54:17 +0200236 fdtab[fd].spec.e ^= (unsigned int)(FD_EV_IN_SL << dir);
Willy Tarreauff9d5ba2009-10-17 21:43:03 +0200237 return 1;
Willy Tarreaude99e992007-04-16 00:53:59 +0200238}
239
240REGPRM2 static int __fd_clr(const int fd, int dir)
241{
Willy Tarreaude99e992007-04-16 00:53:59 +0200242 unsigned int i;
243
Willy Tarreau7a52a5c2008-08-16 16:06:02 +0200244#if DEBUG_DEV
245 if (fdtab[fd].state == FD_STCLOSE) {
246 fprintf(stderr, "sepoll.fd_clr called on closed fd #%d.\n", fd);
247 ABORT_NOW();
248 }
249#endif
Willy Tarreaub48b3232009-10-17 22:54:17 +0200250 i = ((unsigned)fdtab[fd].spec.e >> dir) & FD_EV_MASK_DIR;
Willy Tarreaude99e992007-04-16 00:53:59 +0200251
Willy Tarreauff9d5ba2009-10-17 21:43:03 +0200252 if (i != FD_EV_SPEC) {
253 if (unlikely(i != FD_EV_WAIT))
254 return 0;
Willy Tarreaude99e992007-04-16 00:53:59 +0200255 // switch to STOP state
256 /* We will create a queue entry for this one because we want to
257 * process it later in order to merge it with other events on
258 * the same FD.
259 */
260 alloc_spec_entry(fd);
Willy Tarreaude99e992007-04-16 00:53:59 +0200261 }
Willy Tarreaub48b3232009-10-17 22:54:17 +0200262 fdtab[fd].spec.e ^= (unsigned int)(FD_EV_IN_SL << dir);
Willy Tarreauff9d5ba2009-10-17 21:43:03 +0200263 return 1;
Willy Tarreaude99e992007-04-16 00:53:59 +0200264}
265
Willy Tarreau6653d172007-05-13 01:52:05 +0200266/* normally unused */
Willy Tarreaude99e992007-04-16 00:53:59 +0200267REGPRM1 static void __fd_rem(int fd)
268{
269 __fd_clr(fd, DIR_RD);
270 __fd_clr(fd, DIR_WR);
271}
272
273/*
274 * On valid epoll() implementations, a call to close() automatically removes
275 * the fds. This means that the FD will appear as previously unset.
276 */
277REGPRM1 static void __fd_clo(int fd)
278{
Willy Tarreau7a52a5c2008-08-16 16:06:02 +0200279 release_spec_entry(fd);
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200280 fdtab[fd].spec.e &= ~(FD_EV_MASK | FD_EV_MASK_OLD);
Willy Tarreaude99e992007-04-16 00:53:59 +0200281}
282
Willy Tarreaudc246a72007-05-09 21:57:51 +0200283/*
Willy Tarreaude99e992007-04-16 00:53:59 +0200284 * speculative epoll() poller
285 */
Willy Tarreau0c303ee2008-07-07 00:09:58 +0200286REGPRM2 static void _do_poll(struct poller *p, int exp)
Willy Tarreaude99e992007-04-16 00:53:59 +0200287{
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200288 int status, eo, en;
Willy Tarreaude99e992007-04-16 00:53:59 +0200289 int fd, opcode;
290 int count;
291 int spec_idx;
Willy Tarreaud825eef2007-05-12 22:35:00 +0200292 int wait_time;
Willy Tarreaude99e992007-04-16 00:53:59 +0200293
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200294 /* first, update the poll list according to what changed in the spec list */
Willy Tarreaude99e992007-04-16 00:53:59 +0200295 spec_idx = nbspec;
296 while (likely(spec_idx > 0)) {
297 spec_idx--;
298 fd = spec_list[spec_idx];
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200299 en = fdtab[fd].spec.e & 15; /* new events */
300 eo = fdtab[fd].spec.e >> 4; /* previous events */
Willy Tarreaude99e992007-04-16 00:53:59 +0200301
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200302 /* If an fd with a poll bit is present here, it means that it
303 * has last requested a poll, or is leaving from a poll. Given
304 * that an FD is fully in the poll list or in the spec list but
305 * not in both at once, we'll first ensure that if it is
306 * already being polled in one direction and requests a spec
307 * access, then we'll turn that into a polled access in order
308 * to save a syscall which will likely return EAGAIN.
Willy Tarreaude99e992007-04-16 00:53:59 +0200309 */
310
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200311 if ((en & FD_EV_RW_PL) && (en & FD_EV_RW_SL)) {
312 /* convert SPEC to WAIT if fd already being polled */
313 if ((en & FD_EV_MASK_R) == FD_EV_SPEC_R)
314 en = (en & ~FD_EV_MASK_R) | FD_EV_WAIT_R;
315
316 if ((en & FD_EV_MASK_W) == FD_EV_SPEC_W)
317 en = (en & ~FD_EV_MASK_W) | FD_EV_WAIT_W;
Willy Tarreaude99e992007-04-16 00:53:59 +0200318 }
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200319
320 if ((en & FD_EV_MASK_R) == FD_EV_STOP_R) {
Willy Tarreau6653d172007-05-13 01:52:05 +0200321 /* This FD was being polled and is now being removed. */
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200322 en &= ~FD_EV_MASK_R;
Willy Tarreaude99e992007-04-16 00:53:59 +0200323 }
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200324
325 if ((en & FD_EV_MASK_W) == FD_EV_STOP_W) {
Willy Tarreau6653d172007-05-13 01:52:05 +0200326 /* This FD was being polled and is now being removed. */
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200327 en &= ~FD_EV_MASK_W;
Willy Tarreau6653d172007-05-13 01:52:05 +0200328 }
Willy Tarreaude99e992007-04-16 00:53:59 +0200329
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200330 if ((eo ^ en) & FD_EV_RW_PL) {
331 /* poll status changed */
332 if ((en & FD_EV_RW_PL) == 0) {
Willy Tarreau6653d172007-05-13 01:52:05 +0200333 /* fd removed from poll list */
334 opcode = EPOLL_CTL_DEL;
335 }
336 else if ((eo & FD_EV_RW_PL) == 0) {
337 /* new fd in the poll list */
Willy Tarreaude99e992007-04-16 00:53:59 +0200338 opcode = EPOLL_CTL_ADD;
339 }
Willy Tarreau6653d172007-05-13 01:52:05 +0200340 else {
341 /* fd status changed */
342 opcode = EPOLL_CTL_MOD;
343 }
344
345 /* construct the epoll events based on new state */
346 ev.events = 0;
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200347 if (en & FD_EV_WAIT_R)
Willy Tarreau6653d172007-05-13 01:52:05 +0200348 ev.events |= EPOLLIN;
349
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200350 if (en & FD_EV_WAIT_W)
Willy Tarreau6653d172007-05-13 01:52:05 +0200351 ev.events |= EPOLLOUT;
352
353 ev.data.fd = fd;
Willy Tarreaude99e992007-04-16 00:53:59 +0200354 epoll_ctl(epoll_fd, opcode, fd, &ev);
Willy Tarreau6653d172007-05-13 01:52:05 +0200355 }
Willy Tarreaude99e992007-04-16 00:53:59 +0200356
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200357 fdtab[fd].spec.e = (en << 4) + en; /* save new events */
Willy Tarreaude99e992007-04-16 00:53:59 +0200358
Willy Tarreaub48b3232009-10-17 22:54:17 +0200359 if (!(fdtab[fd].spec.e & FD_EV_RW_SL)) {
Willy Tarreau6653d172007-05-13 01:52:05 +0200360 /* This fd switched to combinations of either WAIT or
361 * IDLE. It must be removed from the spec list.
362 */
Willy Tarreau4eac2092007-08-31 17:01:18 +0200363 release_spec_entry(fd);
Willy Tarreaude99e992007-04-16 00:53:59 +0200364 }
365 }
366
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200367 /* compute the epoll_wait() timeout */
Willy Tarreaucb651252008-08-29 13:57:30 +0200368
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200369 if (nbspec || run_queue || signal_queue_len) {
370 /* Maybe we still have events in the spec list, or there are
Willy Tarreau3a628112008-06-13 21:06:56 +0200371 * some tasks left pending in the run_queue, so we must not
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200372 * wait in epoll() otherwise we would delay their delivery by
Willy Tarreau6653d172007-05-13 01:52:05 +0200373 * the next timeout.
374 */
Willy Tarreaude99e992007-04-16 00:53:59 +0200375 wait_time = 0;
376 }
Willy Tarreaud825eef2007-05-12 22:35:00 +0200377 else {
Willy Tarreau0c303ee2008-07-07 00:09:58 +0200378 if (!exp)
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200379 wait_time = MAX_DELAY_MS;
Willy Tarreau0c303ee2008-07-07 00:09:58 +0200380 else if (tick_is_expired(exp, now_ms))
Willy Tarreaubdefc512007-05-14 02:02:04 +0200381 wait_time = 0;
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200382 else {
Willy Tarreau0c303ee2008-07-07 00:09:58 +0200383 wait_time = TICKS_TO_MS(tick_remain(now_ms, exp)) + 1;
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200384 if (wait_time > MAX_DELAY_MS)
385 wait_time = MAX_DELAY_MS;
386 }
Willy Tarreaud825eef2007-05-12 22:35:00 +0200387 }
Willy Tarreaude99e992007-04-16 00:53:59 +0200388
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200389 /* now let's wait for real events */
390 fd = MIN(maxfd, global.tune.maxpollevents);
Willy Tarreau45a12512011-09-10 16:56:42 +0200391 gettimeofday(&before_poll, NULL);
Willy Tarreau1db37712007-06-03 17:16:49 +0200392 status = epoll_wait(epoll_fd, epoll_events, fd, wait_time);
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200393 tv_update_date(wait_time, status);
Willy Tarreau45a12512011-09-10 16:56:42 +0200394 measure_idle();
Willy Tarreaude99e992007-04-16 00:53:59 +0200395
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200396 /* process events */
Willy Tarreaude99e992007-04-16 00:53:59 +0200397 for (count = 0; count < status; count++) {
398 int e = epoll_events[count].events;
399 fd = epoll_events[count].data.fd;
400
401 /* it looks complicated but gcc can optimize it away when constants
402 * have same values.
403 */
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100404 fdtab[fd].ev &= FD_POLL_STICKY;
Willy Tarreau491c4982012-07-06 11:16:01 +0200405 fdtab[fd].ev |=
Willy Tarreaude99e992007-04-16 00:53:59 +0200406 ((e & EPOLLIN ) ? FD_POLL_IN : 0) |
407 ((e & EPOLLPRI) ? FD_POLL_PRI : 0) |
408 ((e & EPOLLOUT) ? FD_POLL_OUT : 0) |
409 ((e & EPOLLERR) ? FD_POLL_ERR : 0) |
410 ((e & EPOLLHUP) ? FD_POLL_HUP : 0);
Willy Tarreau491c4982012-07-06 11:16:01 +0200411
Willy Tarreaub48b3232009-10-17 22:54:17 +0200412 if ((fdtab[fd].spec.e & FD_EV_MASK_R) == FD_EV_WAIT_R) {
Willy Tarreau8bb46f42007-04-30 12:56:21 +0200413 if (fdtab[fd].state == FD_STCLOSE || fdtab[fd].state == FD_STERROR)
Willy Tarreaude99e992007-04-16 00:53:59 +0200414 continue;
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100415 if (fdtab[fd].ev & (FD_POLL_IN|FD_POLL_HUP|FD_POLL_ERR))
Willy Tarreaude99e992007-04-16 00:53:59 +0200416 fdtab[fd].cb[DIR_RD].f(fd);
417 }
418
Willy Tarreaub48b3232009-10-17 22:54:17 +0200419 if ((fdtab[fd].spec.e & FD_EV_MASK_W) == FD_EV_WAIT_W) {
Willy Tarreau8bb46f42007-04-30 12:56:21 +0200420 if (fdtab[fd].state == FD_STCLOSE || fdtab[fd].state == FD_STERROR)
Willy Tarreaude99e992007-04-16 00:53:59 +0200421 continue;
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100422 if (fdtab[fd].ev & (FD_POLL_OUT|FD_POLL_ERR))
Willy Tarreaude99e992007-04-16 00:53:59 +0200423 fdtab[fd].cb[DIR_WR].f(fd);
424 }
425 }
Willy Tarreaucb651252008-08-29 13:57:30 +0200426
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200427 /* now process speculative events if any */
428
429 /* Here we have two options :
430 * - either walk the list forwards and hope to match more events
431 * - or walk it backwards to minimize the number of changes and
432 * to make better use of the cache.
433 * Tests have shown that walking backwards improves perf by 0.2%.
434 */
435
436 spec_idx = nbspec;
437 while (likely(spec_idx > 0)) {
438 spec_idx--;
439 fd = spec_list[spec_idx];
440 eo = fdtab[fd].spec.e; /* save old events */
441
442 /*
443 * Process the speculative events.
444 *
445 * Principle: events which are marked FD_EV_SPEC are processed
446 * with their assigned function. If the function returns 0, it
447 * means there is nothing doable without polling first. We will
448 * then convert the event to a pollable one by assigning them
449 * the WAIT status.
Willy Tarreaucb651252008-08-29 13:57:30 +0200450 */
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200451
452 fdtab[fd].ev &= FD_POLL_STICKY;
453 if ((eo & FD_EV_MASK_R) == FD_EV_SPEC_R) {
454 /* The owner is interested in reading from this FD */
455 if (fdtab[fd].state != FD_STERROR) {
456 /* Pretend there is something to read */
457 fdtab[fd].ev |= FD_POLL_IN;
458 if (!fdtab[fd].cb[DIR_RD].f(fd))
459 fdtab[fd].spec.e ^= (FD_EV_WAIT_R ^ FD_EV_SPEC_R);
460 }
461 }
462
463 if ((eo & FD_EV_MASK_W) == FD_EV_SPEC_W) {
464 /* The owner is interested in writing to this FD */
465 if (fdtab[fd].state != FD_STERROR) {
466 /* Pretend there is something to write */
467 fdtab[fd].ev |= FD_POLL_OUT;
468 if (!fdtab[fd].cb[DIR_WR].f(fd))
469 fdtab[fd].spec.e ^= (FD_EV_WAIT_W ^ FD_EV_SPEC_W);
470 }
471 }
472
473 /* one callback might already have closed the fd by itself */
474 if (fdtab[fd].state == FD_STCLOSE)
475 continue;
476
477 if (!(fdtab[fd].spec.e & (FD_EV_RW_SL|FD_EV_RW_PL))) {
478 /* This fd switched to IDLE, it can be removed from the spec list. */
479 release_spec_entry(fd);
480 continue;
481 }
Willy Tarreaucb651252008-08-29 13:57:30 +0200482 }
Willy Tarreaudbcd47e2012-05-13 09:42:26 +0200483
484 /* in the end, we have processed status + spec_processed FDs */
Willy Tarreaude99e992007-04-16 00:53:59 +0200485}
486
487/*
488 * Initialization of the speculative epoll() poller.
489 * Returns 0 in case of failure, non-zero in case of success. If it fails, it
490 * disables the poller by setting its pref to 0.
491 */
492REGPRM1 static int _do_init(struct poller *p)
493{
Willy Tarreaub48b3232009-10-17 22:54:17 +0200494 __label__ fail_spec, fail_ee, fail_fd;
Willy Tarreaude99e992007-04-16 00:53:59 +0200495
496 p->private = NULL;
497
498 epoll_fd = epoll_create(global.maxsock + 1);
499 if (epoll_fd < 0)
500 goto fail_fd;
501
Willy Tarreauf2e8ee22008-05-25 10:39:02 +0200502 /* See comments at the top of the file about this formula. */
503 absmaxevents = MAX(global.tune.maxpollevents, global.maxsock/4);
Willy Tarreaude99e992007-04-16 00:53:59 +0200504 epoll_events = (struct epoll_event*)
Willy Tarreauf2e8ee22008-05-25 10:39:02 +0200505 calloc(1, sizeof(struct epoll_event) * absmaxevents);
Willy Tarreaude99e992007-04-16 00:53:59 +0200506
507 if (epoll_events == NULL)
508 goto fail_ee;
509
510 if ((spec_list = (uint32_t *)calloc(1, sizeof(uint32_t) * global.maxsock)) == NULL)
511 goto fail_spec;
512
Willy Tarreaude99e992007-04-16 00:53:59 +0200513 return 1;
514
Willy Tarreaude99e992007-04-16 00:53:59 +0200515 fail_spec:
516 free(epoll_events);
517 fail_ee:
518 close(epoll_fd);
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200519 epoll_fd = -1;
Willy Tarreaude99e992007-04-16 00:53:59 +0200520 fail_fd:
521 p->pref = 0;
522 return 0;
523}
524
525/*
526 * Termination of the speculative epoll() poller.
527 * Memory is released and the poller is marked as unselectable.
528 */
529REGPRM1 static void _do_term(struct poller *p)
530{
Willy Tarreaua534fea2008-08-03 12:19:50 +0200531 free(spec_list);
532 free(epoll_events);
Willy Tarreaude99e992007-04-16 00:53:59 +0200533
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200534 if (epoll_fd >= 0) {
535 close(epoll_fd);
536 epoll_fd = -1;
537 }
Willy Tarreaude99e992007-04-16 00:53:59 +0200538
Willy Tarreaude99e992007-04-16 00:53:59 +0200539 spec_list = NULL;
540 epoll_events = NULL;
541
542 p->private = NULL;
543 p->pref = 0;
544}
545
546/*
547 * Check that the poller works.
548 * Returns 1 if OK, otherwise 0.
549 */
550REGPRM1 static int _do_test(struct poller *p)
551{
552 int fd;
553
554 fd = epoll_create(global.maxsock + 1);
555 if (fd < 0)
556 return 0;
557 close(fd);
558 return 1;
559}
560
561/*
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200562 * Recreate the epoll file descriptor after a fork(). Returns 1 if OK,
563 * otherwise 0. It will ensure that all processes will not share their
564 * epoll_fd. Some side effects were encountered because of this, such
565 * as epoll_wait() returning an FD which was previously deleted.
566 */
567REGPRM1 static int _do_fork(struct poller *p)
568{
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200569 if (epoll_fd >= 0)
570 close(epoll_fd);
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200571 epoll_fd = epoll_create(global.maxsock + 1);
572 if (epoll_fd < 0)
573 return 0;
574 return 1;
575}
576
577/*
Willy Tarreaude99e992007-04-16 00:53:59 +0200578 * It is a constructor, which means that it will automatically be called before
579 * main(). This is GCC-specific but it works at least since 2.95.
580 * Special care must be taken so that it does not need any uninitialized data.
581 */
582__attribute__((constructor))
583static void _do_register(void)
584{
585 struct poller *p;
586
587 if (nbpollers >= MAX_POLLERS)
588 return;
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200589
590 epoll_fd = -1;
Willy Tarreaude99e992007-04-16 00:53:59 +0200591 p = &pollers[nbpollers++];
592
593 p->name = "sepoll";
594 p->pref = 400;
595 p->private = NULL;
596
597 p->test = _do_test;
598 p->init = _do_init;
599 p->term = _do_term;
600 p->poll = _do_poll;
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200601 p->fork = _do_fork;
Willy Tarreaude99e992007-04-16 00:53:59 +0200602
603 p->is_set = __fd_is_set;
604 p->cond_s = p->set = __fd_set;
605 p->cond_c = p->clr = __fd_clr;
606 p->rem = __fd_rem;
607 p->clo = __fd_clo;
608}
609
610
611/*
612 * Local variables:
613 * c-indent-level: 8
614 * c-basic-offset: 8
615 * End:
616 */