blob: 70c934db2a344ab5655b7337109b6c36dbf024ff [file] [log] [blame]
Willy Tarreaude99e992007-04-16 00:53:59 +02001/*
2 * FD polling functions for Speculative I/O combined with Linux epoll()
3 *
Willy Tarreauf2e8ee22008-05-25 10:39:02 +02004 * Copyright 2000-2008 Willy Tarreau <w@1wt.eu>
Willy Tarreaude99e992007-04-16 00:53:59 +02005 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
Willy Tarreauf2e8ee22008-05-25 10:39:02 +020011 *
12 * This code implements "speculative I/O" under Linux. The principle is to
13 * try to perform expected I/O before registering the events in the poller.
14 * Each time this succeeds, it saves an expensive epoll_ctl(). It generally
15 * succeeds for all reads after an accept(), and for writes after a connect().
16 * It also improves performance for streaming connections because even if only
17 * one side is polled, the other one may react accordingly depending on the
18 * level of the buffer.
19 *
20 * It has a presents drawbacks though. If too many events are set for spec I/O,
21 * those ones can starve the polled events. Experiments show that when polled
22 * events starve, they quickly turn into spec I/O, making the situation even
23 * worse. While we can reduce the number of polled events processed at once,
24 * we cannot do this on speculative events because most of them are new ones
25 * (avg 2/3 new - 1/3 old from experiments).
26 *
27 * The solution against this problem relies on those two factors :
28 * 1) one FD registered as a spec event cannot be polled at the same time
29 * 2) even during very high loads, we will almost never be interested in
30 * simultaneous read and write streaming on the same FD.
31 *
32 * The first point implies that during starvation, we will not have more than
33 * half of our FDs in the poll list, otherwise it means there is less than that
34 * in the spec list, implying there is no starvation.
35 *
36 * The second point implies that we're statically only interested in half of
37 * the maximum number of file descriptors at once, because we will unlikely
38 * have simultaneous read and writes for a same buffer during long periods.
39 *
40 * So, if we make it possible to drain maxsock/2/2 during peak loads, then we
41 * can ensure that there will be no starvation effect. This means that we must
42 * always allocate maxsock/4 events for the poller.
43 *
44 *
Willy Tarreaude99e992007-04-16 00:53:59 +020045 */
46
47#include <unistd.h>
48#include <sys/time.h>
49#include <sys/types.h>
50
51#include <common/compat.h>
52#include <common/config.h>
Willy Tarreaud6f087e2008-01-18 17:20:13 +010053#include <common/debug.h>
Willy Tarreaude99e992007-04-16 00:53:59 +020054#include <common/standard.h>
Willy Tarreau0c303ee2008-07-07 00:09:58 +020055#include <common/ticks.h>
Willy Tarreaude99e992007-04-16 00:53:59 +020056#include <common/time.h>
Willy Tarreau1db37712007-06-03 17:16:49 +020057#include <common/tools.h>
Willy Tarreaude99e992007-04-16 00:53:59 +020058
Willy Tarreaude99e992007-04-16 00:53:59 +020059#include <types/global.h>
60
61#include <proto/fd.h>
62#include <proto/task.h>
63
64#if defined(USE_MY_EPOLL)
65#include <common/epoll.h>
66#include <errno.h>
67#include <sys/syscall.h>
68static _syscall1 (int, epoll_create, int, size);
69static _syscall4 (int, epoll_ctl, int, epfd, int, op, int, fd, struct epoll_event *, event);
70static _syscall4 (int, epoll_wait, int, epfd, struct epoll_event *, events, int, maxevents, int, timeout);
71#else
72#include <sys/epoll.h>
73#endif
74
75/*
76 * We define 4 states for each direction of a file descriptor, which we store
77 * as 2 bits :
78 *
79 * 00 = IDLE : we're not interested in this event
80 * 01 = SPEC : perform speculative I/O on this FD
81 * 10 = WAIT : really wait for an availability event on this FD (poll)
82 * 11 = STOP : was marked WAIT, but disabled. It can switch back to WAIT if
83 * the application changes its mind, otherwise disable FD polling
84 * and switch back to IDLE.
85 *
86 * Since we do not want to scan all the FD list to find speculative I/O events,
87 * we store them in a list consisting in a linear array holding only the FD
Willy Tarreau7a52a5c2008-08-16 16:06:02 +020088 * indexes right now. Note that a closed FD cannot exist in the spec list,
89 * because it is closed by fd_delete() which in turn calls __fd_clo() which
90 * always removes it from the list.
Willy Tarreaude99e992007-04-16 00:53:59 +020091 *
92 * The STOP state requires the event to be present in the spec list so that
93 * it can be detected and flushed upon next scan without having to scan the
94 * whole FD list.
95 *
96 * This translates like this :
97 *
98 * EVENT_IN_SPEC_LIST = 01
99 * EVENT_IN_POLL_LIST = 10
100 *
101 * IDLE = 0
102 * SPEC = (EVENT_IN_SPEC_LIST)
103 * WAIT = (EVENT_IN_POLL_LIST)
104 * STOP = (EVENT_IN_SPEC_LIST|EVENT_IN_POLL_LIST)
105 *
106 * fd_is_set() just consists in checking that the status is 01 or 10.
107 *
108 * For efficiency reasons, we will store the Read and Write bits interlaced to
109 * form a 4-bit field, so that we can simply shift the value right by 0/1 and
110 * get what we want :
111 * 3 2 1 0
112 * Wp Rp Ws Rs
113 *
114 * The FD array has to hold a back reference to the speculative list. This
115 * reference is only valid if at least one of the directions is marked SPEC.
116 *
117 */
118
119#define FD_EV_IN_SL 1
120#define FD_EV_IN_PL 4
121
122#define FD_EV_IDLE 0
123#define FD_EV_SPEC (FD_EV_IN_SL)
124#define FD_EV_WAIT (FD_EV_IN_PL)
125#define FD_EV_STOP (FD_EV_IN_SL|FD_EV_IN_PL)
126
127/* Those match any of R or W for Spec list or Poll list */
128#define FD_EV_RW_SL (FD_EV_IN_SL | (FD_EV_IN_SL << 1))
129#define FD_EV_RW_PL (FD_EV_IN_PL | (FD_EV_IN_PL << 1))
130#define FD_EV_MASK_DIR (FD_EV_IN_SL|FD_EV_IN_PL)
131
132#define FD_EV_IDLE_R 0
133#define FD_EV_SPEC_R (FD_EV_IN_SL)
134#define FD_EV_WAIT_R (FD_EV_IN_PL)
135#define FD_EV_STOP_R (FD_EV_IN_SL|FD_EV_IN_PL)
136#define FD_EV_MASK_R (FD_EV_IN_SL|FD_EV_IN_PL)
137
138#define FD_EV_IDLE_W (FD_EV_IDLE_R << 1)
139#define FD_EV_SPEC_W (FD_EV_SPEC_R << 1)
140#define FD_EV_WAIT_W (FD_EV_WAIT_R << 1)
141#define FD_EV_STOP_W (FD_EV_STOP_R << 1)
142#define FD_EV_MASK_W (FD_EV_MASK_R << 1)
143
144#define FD_EV_MASK (FD_EV_MASK_W | FD_EV_MASK_R)
145
Willy Tarreau6653d172007-05-13 01:52:05 +0200146/* This is the minimum number of events successfully processed in speculative
147 * mode above which we agree to return without checking epoll() (1/2 times).
148 */
149#define MIN_RETURN_EVENTS 25
Willy Tarreaude99e992007-04-16 00:53:59 +0200150
151/* descriptor of one FD.
152 * FIXME: should be a bit field */
153struct fd_status {
154 unsigned int e:4; // read and write events status.
Willy Tarreau4eac2092007-08-31 17:01:18 +0200155 unsigned int s1:28; // Position in spec list+1. 0=not in list. Should be last.
Willy Tarreaude99e992007-04-16 00:53:59 +0200156};
157
158static int nbspec = 0; // current size of the spec list
Willy Tarreauf2e8ee22008-05-25 10:39:02 +0200159static int absmaxevents = 0; // absolute maximum amounts of polled events
Willy Tarreaucb651252008-08-29 13:57:30 +0200160static int fd_created = 0; // fd creation detector, reset upon poll() entry.
Willy Tarreaude99e992007-04-16 00:53:59 +0200161
162static struct fd_status *fd_list = NULL; // list of FDs
163static unsigned int *spec_list = NULL; // speculative I/O list
164
165/* private data */
166static struct epoll_event *epoll_events;
167static int epoll_fd;
168
169/* This structure may be used for any purpose. Warning! do not use it in
170 * recursive functions !
171 */
172static struct epoll_event ev;
173
174
175REGPRM1 static void alloc_spec_entry(const int fd)
176{
Willy Tarreau4eac2092007-08-31 17:01:18 +0200177 if (fd_list[fd].s1)
Willy Tarreaude99e992007-04-16 00:53:59 +0200178 return;
Willy Tarreau4eac2092007-08-31 17:01:18 +0200179 fd_list[fd].s1 = nbspec + 1;
180 spec_list[nbspec] = fd;
181 nbspec++;
Willy Tarreaude99e992007-04-16 00:53:59 +0200182}
183
Willy Tarreau4eac2092007-08-31 17:01:18 +0200184/* Removes entry used by fd <fd> from the spec list and replaces it with the
185 * last one. The fd_list is adjusted to match the back reference if needed.
186 * If the fd has no entry assigned, return immediately.
Willy Tarreaude99e992007-04-16 00:53:59 +0200187 */
Willy Tarreau4eac2092007-08-31 17:01:18 +0200188REGPRM1 static void release_spec_entry(int fd)
Willy Tarreaude99e992007-04-16 00:53:59 +0200189{
Willy Tarreau4eac2092007-08-31 17:01:18 +0200190 unsigned int pos;
191
192 pos = fd_list[fd].s1;
193 if (!pos)
194 return;
195
196 fd_list[fd].s1 = 0;
197 pos--;
198 /* we have spec_list[pos]==fd */
Willy Tarreaude99e992007-04-16 00:53:59 +0200199
200 nbspec--;
201 if (pos == nbspec)
202 return;
203
Willy Tarreau4eac2092007-08-31 17:01:18 +0200204 /* we replace current FD by the highest one, which may sometimes be the same */
Willy Tarreaude99e992007-04-16 00:53:59 +0200205 fd = spec_list[nbspec];
Willy Tarreau4eac2092007-08-31 17:01:18 +0200206 fd_list[fd].s1 = pos + 1;
Willy Tarreaude99e992007-04-16 00:53:59 +0200207 spec_list[pos] = fd;
Willy Tarreaude99e992007-04-16 00:53:59 +0200208}
209
210/*
211 * Returns non-zero if <fd> is already monitored for events in direction <dir>.
212 */
213REGPRM2 static int __fd_is_set(const int fd, int dir)
214{
215 int ret;
216
Willy Tarreau7a52a5c2008-08-16 16:06:02 +0200217#if DEBUG_DEV
218 if (fdtab[fd].state == FD_STCLOSE) {
219 fprintf(stderr, "sepoll.fd_isset called on closed fd #%d.\n", fd);
220 ABORT_NOW();
221 }
222#endif
Willy Tarreaude99e992007-04-16 00:53:59 +0200223 ret = ((unsigned)fd_list[fd].e >> dir) & FD_EV_MASK_DIR;
224 return (ret == FD_EV_SPEC || ret == FD_EV_WAIT);
225}
226
227/*
228 * Don't worry about the strange constructs in __fd_set/__fd_clr, they are
229 * designed like this in order to reduce the number of jumps (verified).
230 */
231REGPRM2 static int __fd_set(const int fd, int dir)
232{
233 __label__ switch_state;
234 unsigned int i;
235
Willy Tarreau7a52a5c2008-08-16 16:06:02 +0200236#if DEBUG_DEV
237 if (fdtab[fd].state == FD_STCLOSE) {
238 fprintf(stderr, "sepoll.fd_set called on closed fd #%d.\n", fd);
239 ABORT_NOW();
240 }
241#endif
Willy Tarreaude99e992007-04-16 00:53:59 +0200242 i = ((unsigned)fd_list[fd].e >> dir) & FD_EV_MASK_DIR;
243
244 if (i == FD_EV_IDLE) {
245 // switch to SPEC state and allocate a SPEC entry.
Willy Tarreaucb651252008-08-29 13:57:30 +0200246 fd_created = 1;
Willy Tarreaude99e992007-04-16 00:53:59 +0200247 alloc_spec_entry(fd);
248 switch_state:
249 fd_list[fd].e ^= (unsigned int)(FD_EV_IN_SL << dir);
250 return 1;
251 }
252 else if (i == FD_EV_STOP) {
253 // switch to WAIT state
254 goto switch_state;
255 }
256 else
257 return 0;
258}
259
260REGPRM2 static int __fd_clr(const int fd, int dir)
261{
262 __label__ switch_state;
263 unsigned int i;
264
Willy Tarreau7a52a5c2008-08-16 16:06:02 +0200265#if DEBUG_DEV
266 if (fdtab[fd].state == FD_STCLOSE) {
267 fprintf(stderr, "sepoll.fd_clr called on closed fd #%d.\n", fd);
268 ABORT_NOW();
269 }
270#endif
Willy Tarreaude99e992007-04-16 00:53:59 +0200271 i = ((unsigned)fd_list[fd].e >> dir) & FD_EV_MASK_DIR;
272
273 if (i == FD_EV_SPEC) {
274 // switch to IDLE state
275 goto switch_state;
276 }
277 else if (likely(i == FD_EV_WAIT)) {
278 // switch to STOP state
279 /* We will create a queue entry for this one because we want to
280 * process it later in order to merge it with other events on
281 * the same FD.
282 */
283 alloc_spec_entry(fd);
284 switch_state:
285 fd_list[fd].e ^= (unsigned int)(FD_EV_IN_SL << dir);
286 return 1;
287 }
288 return 0;
289}
290
Willy Tarreau6653d172007-05-13 01:52:05 +0200291/* normally unused */
Willy Tarreaude99e992007-04-16 00:53:59 +0200292REGPRM1 static void __fd_rem(int fd)
293{
294 __fd_clr(fd, DIR_RD);
295 __fd_clr(fd, DIR_WR);
296}
297
298/*
299 * On valid epoll() implementations, a call to close() automatically removes
300 * the fds. This means that the FD will appear as previously unset.
301 */
302REGPRM1 static void __fd_clo(int fd)
303{
Willy Tarreau7a52a5c2008-08-16 16:06:02 +0200304 release_spec_entry(fd);
Willy Tarreaude99e992007-04-16 00:53:59 +0200305 fd_list[fd].e &= ~(FD_EV_MASK);
306}
307
Willy Tarreaudc246a72007-05-09 21:57:51 +0200308/*
Willy Tarreaude99e992007-04-16 00:53:59 +0200309 * speculative epoll() poller
310 */
Willy Tarreau0c303ee2008-07-07 00:09:58 +0200311REGPRM2 static void _do_poll(struct poller *p, int exp)
Willy Tarreaude99e992007-04-16 00:53:59 +0200312{
313 static unsigned int last_skipped;
Willy Tarreauf2e8ee22008-05-25 10:39:02 +0200314 static unsigned int spec_processed;
Willy Tarreau6653d172007-05-13 01:52:05 +0200315 int status, eo;
Willy Tarreaude99e992007-04-16 00:53:59 +0200316 int fd, opcode;
317 int count;
318 int spec_idx;
Willy Tarreaud825eef2007-05-12 22:35:00 +0200319 int wait_time;
Willy Tarreaucb651252008-08-29 13:57:30 +0200320 int looping = 0;
Willy Tarreaude99e992007-04-16 00:53:59 +0200321
322
Willy Tarreaucb651252008-08-29 13:57:30 +0200323 re_poll_once:
Willy Tarreaude99e992007-04-16 00:53:59 +0200324 /* Here we have two options :
Willy Tarreau6653d172007-05-13 01:52:05 +0200325 * - either walk the list forwards and hope to match more events
Willy Tarreaude99e992007-04-16 00:53:59 +0200326 * - or walk it backwards to minimize the number of changes and
327 * to make better use of the cache.
328 * Tests have shown that walking backwards improves perf by 0.2%.
329 */
330
Willy Tarreau6653d172007-05-13 01:52:05 +0200331 status = 0;
Willy Tarreaude99e992007-04-16 00:53:59 +0200332 spec_idx = nbspec;
333 while (likely(spec_idx > 0)) {
334 spec_idx--;
335 fd = spec_list[spec_idx];
Willy Tarreau6653d172007-05-13 01:52:05 +0200336 eo = fd_list[fd].e; /* save old events */
Willy Tarreaude99e992007-04-16 00:53:59 +0200337
Willy Tarreau6653d172007-05-13 01:52:05 +0200338 /*
339 * Process the speculative events.
340 *
341 * Principle: events which are marked FD_EV_SPEC are processed
342 * with their assigned function. If the function returns 0, it
343 * means there is nothing doable without polling first. We will
344 * then convert the event to a pollable one by assigning them
345 * the WAIT status.
Willy Tarreaude99e992007-04-16 00:53:59 +0200346 */
347
Willy Tarreau7a52a5c2008-08-16 16:06:02 +0200348#ifdef DEBUG_DEV
349 if (fdtab[fd].state == FD_STCLOSE) {
350 fprintf(stderr,"fd=%d, fdtab[].ev=%x, fd_list[].e=%x, .s=%d, idx=%d\n",
351 fd, fdtab[fd].ev, fd_list[fd].e, fd_list[fd].s1, spec_idx);
352 }
353#endif
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100354 fdtab[fd].ev &= FD_POLL_STICKY;
Willy Tarreau6653d172007-05-13 01:52:05 +0200355 if ((eo & FD_EV_MASK_R) == FD_EV_SPEC_R) {
356 /* The owner is interested in reading from this FD */
Willy Tarreau7a52a5c2008-08-16 16:06:02 +0200357 if (fdtab[fd].state != FD_STERROR) {
Willy Tarreau6653d172007-05-13 01:52:05 +0200358 /* Pretend there is something to read */
Willy Tarreaude99e992007-04-16 00:53:59 +0200359 fdtab[fd].ev |= FD_POLL_IN;
Willy Tarreau6653d172007-05-13 01:52:05 +0200360 if (!fdtab[fd].cb[DIR_RD].f(fd))
361 fd_list[fd].e ^= (FD_EV_WAIT_R ^ FD_EV_SPEC_R);
362 else
363 status++;
Willy Tarreaude99e992007-04-16 00:53:59 +0200364 }
365 }
Willy Tarreau6653d172007-05-13 01:52:05 +0200366 else if ((eo & FD_EV_MASK_R) == FD_EV_STOP_R) {
367 /* This FD was being polled and is now being removed. */
368 fd_list[fd].e &= ~FD_EV_MASK_R;
369 }
Willy Tarreaude99e992007-04-16 00:53:59 +0200370
Willy Tarreau6653d172007-05-13 01:52:05 +0200371 if ((eo & FD_EV_MASK_W) == FD_EV_SPEC_W) {
372 /* The owner is interested in writing to this FD */
Willy Tarreau7a52a5c2008-08-16 16:06:02 +0200373 if (fdtab[fd].state != FD_STERROR) {
Willy Tarreau6653d172007-05-13 01:52:05 +0200374 /* Pretend there is something to write */
Willy Tarreaude99e992007-04-16 00:53:59 +0200375 fdtab[fd].ev |= FD_POLL_OUT;
Willy Tarreau6653d172007-05-13 01:52:05 +0200376 if (!fdtab[fd].cb[DIR_WR].f(fd))
377 fd_list[fd].e ^= (FD_EV_WAIT_W ^ FD_EV_SPEC_W);
378 else
379 status++;
Willy Tarreaude99e992007-04-16 00:53:59 +0200380 }
381 }
Willy Tarreau6653d172007-05-13 01:52:05 +0200382 else if ((eo & FD_EV_MASK_W) == FD_EV_STOP_W) {
383 /* This FD was being polled and is now being removed. */
384 fd_list[fd].e &= ~FD_EV_MASK_W;
385 }
Willy Tarreaude99e992007-04-16 00:53:59 +0200386
Willy Tarreau6653d172007-05-13 01:52:05 +0200387 /* Now, we will adjust the event in the poll list. Indeed, it
388 * is possible that an event which was previously in the poll
389 * list now goes out, and the opposite is possible too. We can
390 * have opposite changes for READ and WRITE too.
391 */
392
393 if ((eo ^ fd_list[fd].e) & FD_EV_RW_PL) {
394 /* poll status changed*/
395 if ((fd_list[fd].e & FD_EV_RW_PL) == 0) {
396 /* fd removed from poll list */
397 opcode = EPOLL_CTL_DEL;
398 }
399 else if ((eo & FD_EV_RW_PL) == 0) {
400 /* new fd in the poll list */
Willy Tarreaude99e992007-04-16 00:53:59 +0200401 opcode = EPOLL_CTL_ADD;
402 }
Willy Tarreau6653d172007-05-13 01:52:05 +0200403 else {
404 /* fd status changed */
405 opcode = EPOLL_CTL_MOD;
406 }
407
408 /* construct the epoll events based on new state */
409 ev.events = 0;
410 if (fd_list[fd].e & FD_EV_WAIT_R)
411 ev.events |= EPOLLIN;
412
413 if (fd_list[fd].e & FD_EV_WAIT_W)
414 ev.events |= EPOLLOUT;
415
416 ev.data.fd = fd;
Willy Tarreaude99e992007-04-16 00:53:59 +0200417 epoll_ctl(epoll_fd, opcode, fd, &ev);
Willy Tarreau6653d172007-05-13 01:52:05 +0200418 }
Willy Tarreaude99e992007-04-16 00:53:59 +0200419
Willy Tarreaude99e992007-04-16 00:53:59 +0200420
Willy Tarreau6653d172007-05-13 01:52:05 +0200421 if (!(fd_list[fd].e & FD_EV_RW_SL)) {
422 /* This fd switched to combinations of either WAIT or
423 * IDLE. It must be removed from the spec list.
424 */
Willy Tarreau4eac2092007-08-31 17:01:18 +0200425 release_spec_entry(fd);
Willy Tarreau6653d172007-05-13 01:52:05 +0200426 continue;
Willy Tarreaude99e992007-04-16 00:53:59 +0200427 }
428 }
429
Willy Tarreau6653d172007-05-13 01:52:05 +0200430 /* It may make sense to immediately return here if there are enough
431 * processed events, without passing through epoll_wait() because we
432 * have exactly done a poll.
433 * Measures have shown a great performance increase if we call the
434 * epoll_wait() only the second time after speculative accesses have
435 * succeeded. This reduces the number of unsucessful calls to
436 * epoll_wait() by a factor of about 3, and the total number of calls
437 * by about 2.
Willy Tarreauf2e8ee22008-05-25 10:39:02 +0200438 * However, when we do that after having processed too many events,
439 * events waiting in epoll() starve for too long a time and tend to
440 * become themselves eligible for speculative polling. So we try to
441 * limit this practise to reasonable situations.
Willy Tarreaude99e992007-04-16 00:53:59 +0200442 */
Willy Tarreau6653d172007-05-13 01:52:05 +0200443
Willy Tarreauf2e8ee22008-05-25 10:39:02 +0200444 spec_processed += status;
Willy Tarreaucb651252008-08-29 13:57:30 +0200445
446 if (looping) {
447 last_skipped++;
448 return;
449 }
450
Willy Tarreauf2e8ee22008-05-25 10:39:02 +0200451 if (status >= MIN_RETURN_EVENTS && spec_processed < absmaxevents) {
Willy Tarreau6653d172007-05-13 01:52:05 +0200452 /* We have processed at least MIN_RETURN_EVENTS, it's worth
453 * returning now without checking epoll_wait().
454 */
455 if (++last_skipped <= 1) {
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200456 tv_update_date(0, 1);
Willy Tarreaude99e992007-04-16 00:53:59 +0200457 return;
458 }
Willy Tarreaude99e992007-04-16 00:53:59 +0200459 }
460 last_skipped = 0;
461
Willy Tarreau3a628112008-06-13 21:06:56 +0200462 if (nbspec || status || run_queue) {
Willy Tarreau6653d172007-05-13 01:52:05 +0200463 /* Maybe we have processed some events that we must report, or
Willy Tarreau3a628112008-06-13 21:06:56 +0200464 * maybe we still have events in the spec list, or there are
465 * some tasks left pending in the run_queue, so we must not
Willy Tarreau6653d172007-05-13 01:52:05 +0200466 * wait in epoll() otherwise we will delay their delivery by
467 * the next timeout.
468 */
Willy Tarreaude99e992007-04-16 00:53:59 +0200469 wait_time = 0;
470 }
Willy Tarreaud825eef2007-05-12 22:35:00 +0200471 else {
Willy Tarreau0c303ee2008-07-07 00:09:58 +0200472 if (!exp)
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200473 wait_time = MAX_DELAY_MS;
Willy Tarreau0c303ee2008-07-07 00:09:58 +0200474 else if (tick_is_expired(exp, now_ms))
Willy Tarreaubdefc512007-05-14 02:02:04 +0200475 wait_time = 0;
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200476 else {
Willy Tarreau0c303ee2008-07-07 00:09:58 +0200477 wait_time = TICKS_TO_MS(tick_remain(now_ms, exp)) + 1;
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200478 if (wait_time > MAX_DELAY_MS)
479 wait_time = MAX_DELAY_MS;
480 }
Willy Tarreaud825eef2007-05-12 22:35:00 +0200481 }
Willy Tarreaude99e992007-04-16 00:53:59 +0200482
Willy Tarreauf2e8ee22008-05-25 10:39:02 +0200483 /* now let's wait for real events. We normally use maxpollevents as a
484 * high limit, unless <nbspec> is already big, in which case we need
485 * to compensate for the high number of events processed there.
486 */
487 fd = MIN(absmaxevents, spec_processed);
488 fd = MAX(global.tune.maxpollevents, fd);
489 fd = MIN(maxfd, fd);
Willy Tarreaucb651252008-08-29 13:57:30 +0200490 /* we want to detect if an accept() will create new speculative FDs here */
491 fd_created = 0;
Willy Tarreauf2e8ee22008-05-25 10:39:02 +0200492 spec_processed = 0;
Willy Tarreau1db37712007-06-03 17:16:49 +0200493 status = epoll_wait(epoll_fd, epoll_events, fd, wait_time);
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200494 tv_update_date(wait_time, status);
Willy Tarreaude99e992007-04-16 00:53:59 +0200495
496 for (count = 0; count < status; count++) {
497 int e = epoll_events[count].events;
498 fd = epoll_events[count].data.fd;
499
500 /* it looks complicated but gcc can optimize it away when constants
501 * have same values.
502 */
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100503 DPRINTF(stderr, "%s:%d: fd=%d, ev=0x%08x, e=0x%08x\n",
504 __FUNCTION__, __LINE__,
505 fd, fdtab[fd].ev, e);
506
507 fdtab[fd].ev &= FD_POLL_STICKY;
508 fdtab[fd].ev |=
Willy Tarreaude99e992007-04-16 00:53:59 +0200509 ((e & EPOLLIN ) ? FD_POLL_IN : 0) |
510 ((e & EPOLLPRI) ? FD_POLL_PRI : 0) |
511 ((e & EPOLLOUT) ? FD_POLL_OUT : 0) |
512 ((e & EPOLLERR) ? FD_POLL_ERR : 0) |
513 ((e & EPOLLHUP) ? FD_POLL_HUP : 0);
514
515 if ((fd_list[fd].e & FD_EV_MASK_R) == FD_EV_WAIT_R) {
Willy Tarreau8bb46f42007-04-30 12:56:21 +0200516 if (fdtab[fd].state == FD_STCLOSE || fdtab[fd].state == FD_STERROR)
Willy Tarreaude99e992007-04-16 00:53:59 +0200517 continue;
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100518 if (fdtab[fd].ev & (FD_POLL_IN|FD_POLL_HUP|FD_POLL_ERR))
Willy Tarreaude99e992007-04-16 00:53:59 +0200519 fdtab[fd].cb[DIR_RD].f(fd);
520 }
521
522 if ((fd_list[fd].e & FD_EV_MASK_W) == FD_EV_WAIT_W) {
Willy Tarreau8bb46f42007-04-30 12:56:21 +0200523 if (fdtab[fd].state == FD_STCLOSE || fdtab[fd].state == FD_STERROR)
Willy Tarreaude99e992007-04-16 00:53:59 +0200524 continue;
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100525 if (fdtab[fd].ev & (FD_POLL_OUT|FD_POLL_ERR))
Willy Tarreaude99e992007-04-16 00:53:59 +0200526 fdtab[fd].cb[DIR_WR].f(fd);
527 }
528 }
Willy Tarreaucb651252008-08-29 13:57:30 +0200529
530 if (fd_created) {
531 /* we have created some fds, certainly in return of an accept(),
532 * and they're marked as speculative. If we can manage to perform
533 * a read(), we're almost sure to collect all the request at once
534 * and avoid several expensive wakeups. So let's try now. Anyway,
535 * if we fail, the tasks are still woken up, and the FD gets marked
536 * for poll mode.
537 */
538 fd_created = 0;
539 looping = 1;
540 goto re_poll_once;
541 }
Willy Tarreaude99e992007-04-16 00:53:59 +0200542}
543
544/*
545 * Initialization of the speculative epoll() poller.
546 * Returns 0 in case of failure, non-zero in case of success. If it fails, it
547 * disables the poller by setting its pref to 0.
548 */
549REGPRM1 static int _do_init(struct poller *p)
550{
551 __label__ fail_fd_list, fail_spec, fail_ee, fail_fd;
552
553 p->private = NULL;
554
555 epoll_fd = epoll_create(global.maxsock + 1);
556 if (epoll_fd < 0)
557 goto fail_fd;
558
Willy Tarreauf2e8ee22008-05-25 10:39:02 +0200559 /* See comments at the top of the file about this formula. */
560 absmaxevents = MAX(global.tune.maxpollevents, global.maxsock/4);
Willy Tarreaude99e992007-04-16 00:53:59 +0200561 epoll_events = (struct epoll_event*)
Willy Tarreauf2e8ee22008-05-25 10:39:02 +0200562 calloc(1, sizeof(struct epoll_event) * absmaxevents);
Willy Tarreaude99e992007-04-16 00:53:59 +0200563
564 if (epoll_events == NULL)
565 goto fail_ee;
566
567 if ((spec_list = (uint32_t *)calloc(1, sizeof(uint32_t) * global.maxsock)) == NULL)
568 goto fail_spec;
569
570 fd_list = (struct fd_status *)calloc(1, sizeof(struct fd_status) * global.maxsock);
571 if (fd_list == NULL)
572 goto fail_fd_list;
573
574 return 1;
575
576 fail_fd_list:
577 free(spec_list);
578 fail_spec:
579 free(epoll_events);
580 fail_ee:
581 close(epoll_fd);
582 epoll_fd = 0;
583 fail_fd:
584 p->pref = 0;
585 return 0;
586}
587
588/*
589 * Termination of the speculative epoll() poller.
590 * Memory is released and the poller is marked as unselectable.
591 */
592REGPRM1 static void _do_term(struct poller *p)
593{
Willy Tarreaua534fea2008-08-03 12:19:50 +0200594 free(fd_list);
595 free(spec_list);
596 free(epoll_events);
Willy Tarreaude99e992007-04-16 00:53:59 +0200597
598 close(epoll_fd);
599 epoll_fd = 0;
600
601 fd_list = NULL;
602 spec_list = NULL;
603 epoll_events = NULL;
604
605 p->private = NULL;
606 p->pref = 0;
607}
608
609/*
610 * Check that the poller works.
611 * Returns 1 if OK, otherwise 0.
612 */
613REGPRM1 static int _do_test(struct poller *p)
614{
615 int fd;
616
617 fd = epoll_create(global.maxsock + 1);
618 if (fd < 0)
619 return 0;
620 close(fd);
621 return 1;
622}
623
624/*
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200625 * Recreate the epoll file descriptor after a fork(). Returns 1 if OK,
626 * otherwise 0. It will ensure that all processes will not share their
627 * epoll_fd. Some side effects were encountered because of this, such
628 * as epoll_wait() returning an FD which was previously deleted.
629 */
630REGPRM1 static int _do_fork(struct poller *p)
631{
632 close(epoll_fd);
633 epoll_fd = epoll_create(global.maxsock + 1);
634 if (epoll_fd < 0)
635 return 0;
636 return 1;
637}
638
639/*
Willy Tarreaude99e992007-04-16 00:53:59 +0200640 * It is a constructor, which means that it will automatically be called before
641 * main(). This is GCC-specific but it works at least since 2.95.
642 * Special care must be taken so that it does not need any uninitialized data.
643 */
644__attribute__((constructor))
645static void _do_register(void)
646{
647 struct poller *p;
648
649 if (nbpollers >= MAX_POLLERS)
650 return;
651 p = &pollers[nbpollers++];
652
653 p->name = "sepoll";
654 p->pref = 400;
655 p->private = NULL;
656
657 p->test = _do_test;
658 p->init = _do_init;
659 p->term = _do_term;
660 p->poll = _do_poll;
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200661 p->fork = _do_fork;
Willy Tarreaude99e992007-04-16 00:53:59 +0200662
663 p->is_set = __fd_is_set;
664 p->cond_s = p->set = __fd_set;
665 p->cond_c = p->clr = __fd_clr;
666 p->rem = __fd_rem;
667 p->clo = __fd_clo;
668}
669
670
671/*
672 * Local variables:
673 * c-indent-level: 8
674 * c-basic-offset: 8
675 * End:
676 */