blob: c9c59787703f91993b8fdfa319a5ccb6630d3608 [file] [log] [blame]
Willy Tarreaude99e992007-04-16 00:53:59 +02001/*
2 * FD polling functions for Speculative I/O combined with Linux epoll()
3 *
Willy Tarreau43d8fb22011-08-22 17:12:02 +02004 * Copyright 2000-2011 Willy Tarreau <w@1wt.eu>
Willy Tarreaude99e992007-04-16 00:53:59 +02005 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
Willy Tarreauf2e8ee22008-05-25 10:39:02 +020011 *
12 * This code implements "speculative I/O" under Linux. The principle is to
13 * try to perform expected I/O before registering the events in the poller.
14 * Each time this succeeds, it saves an expensive epoll_ctl(). It generally
15 * succeeds for all reads after an accept(), and for writes after a connect().
16 * It also improves performance for streaming connections because even if only
17 * one side is polled, the other one may react accordingly depending on the
18 * level of the buffer.
19 *
20 * It has a presents drawbacks though. If too many events are set for spec I/O,
21 * those ones can starve the polled events. Experiments show that when polled
22 * events starve, they quickly turn into spec I/O, making the situation even
23 * worse. While we can reduce the number of polled events processed at once,
24 * we cannot do this on speculative events because most of them are new ones
25 * (avg 2/3 new - 1/3 old from experiments).
26 *
27 * The solution against this problem relies on those two factors :
28 * 1) one FD registered as a spec event cannot be polled at the same time
29 * 2) even during very high loads, we will almost never be interested in
30 * simultaneous read and write streaming on the same FD.
31 *
32 * The first point implies that during starvation, we will not have more than
33 * half of our FDs in the poll list, otherwise it means there is less than that
34 * in the spec list, implying there is no starvation.
35 *
36 * The second point implies that we're statically only interested in half of
37 * the maximum number of file descriptors at once, because we will unlikely
38 * have simultaneous read and writes for a same buffer during long periods.
39 *
40 * So, if we make it possible to drain maxsock/2/2 during peak loads, then we
41 * can ensure that there will be no starvation effect. This means that we must
42 * always allocate maxsock/4 events for the poller.
43 *
44 *
Willy Tarreaude99e992007-04-16 00:53:59 +020045 */
46
47#include <unistd.h>
48#include <sys/time.h>
49#include <sys/types.h>
50
51#include <common/compat.h>
52#include <common/config.h>
Willy Tarreaud6f087e2008-01-18 17:20:13 +010053#include <common/debug.h>
Willy Tarreau43d8fb22011-08-22 17:12:02 +020054#include <common/epoll.h>
Willy Tarreaude99e992007-04-16 00:53:59 +020055#include <common/standard.h>
Willy Tarreau0c303ee2008-07-07 00:09:58 +020056#include <common/ticks.h>
Willy Tarreaude99e992007-04-16 00:53:59 +020057#include <common/time.h>
Willy Tarreau1db37712007-06-03 17:16:49 +020058#include <common/tools.h>
Willy Tarreaude99e992007-04-16 00:53:59 +020059
Willy Tarreaude99e992007-04-16 00:53:59 +020060#include <types/global.h>
61
62#include <proto/fd.h>
Willy Tarreau332740d2009-05-10 09:57:21 +020063#include <proto/signal.h>
Willy Tarreaude99e992007-04-16 00:53:59 +020064#include <proto/task.h>
65
Willy Tarreaude99e992007-04-16 00:53:59 +020066/*
67 * We define 4 states for each direction of a file descriptor, which we store
68 * as 2 bits :
69 *
70 * 00 = IDLE : we're not interested in this event
71 * 01 = SPEC : perform speculative I/O on this FD
72 * 10 = WAIT : really wait for an availability event on this FD (poll)
73 * 11 = STOP : was marked WAIT, but disabled. It can switch back to WAIT if
74 * the application changes its mind, otherwise disable FD polling
75 * and switch back to IDLE.
76 *
77 * Since we do not want to scan all the FD list to find speculative I/O events,
78 * we store them in a list consisting in a linear array holding only the FD
Willy Tarreau7a52a5c2008-08-16 16:06:02 +020079 * indexes right now. Note that a closed FD cannot exist in the spec list,
80 * because it is closed by fd_delete() which in turn calls __fd_clo() which
81 * always removes it from the list.
Willy Tarreaude99e992007-04-16 00:53:59 +020082 *
83 * The STOP state requires the event to be present in the spec list so that
84 * it can be detected and flushed upon next scan without having to scan the
85 * whole FD list.
86 *
87 * This translates like this :
88 *
89 * EVENT_IN_SPEC_LIST = 01
90 * EVENT_IN_POLL_LIST = 10
91 *
92 * IDLE = 0
93 * SPEC = (EVENT_IN_SPEC_LIST)
94 * WAIT = (EVENT_IN_POLL_LIST)
95 * STOP = (EVENT_IN_SPEC_LIST|EVENT_IN_POLL_LIST)
96 *
97 * fd_is_set() just consists in checking that the status is 01 or 10.
98 *
99 * For efficiency reasons, we will store the Read and Write bits interlaced to
100 * form a 4-bit field, so that we can simply shift the value right by 0/1 and
101 * get what we want :
102 * 3 2 1 0
103 * Wp Rp Ws Rs
104 *
105 * The FD array has to hold a back reference to the speculative list. This
106 * reference is only valid if at least one of the directions is marked SPEC.
107 *
108 */
109
110#define FD_EV_IN_SL 1
111#define FD_EV_IN_PL 4
112
113#define FD_EV_IDLE 0
114#define FD_EV_SPEC (FD_EV_IN_SL)
115#define FD_EV_WAIT (FD_EV_IN_PL)
116#define FD_EV_STOP (FD_EV_IN_SL|FD_EV_IN_PL)
117
118/* Those match any of R or W for Spec list or Poll list */
119#define FD_EV_RW_SL (FD_EV_IN_SL | (FD_EV_IN_SL << 1))
120#define FD_EV_RW_PL (FD_EV_IN_PL | (FD_EV_IN_PL << 1))
121#define FD_EV_MASK_DIR (FD_EV_IN_SL|FD_EV_IN_PL)
122
123#define FD_EV_IDLE_R 0
124#define FD_EV_SPEC_R (FD_EV_IN_SL)
125#define FD_EV_WAIT_R (FD_EV_IN_PL)
126#define FD_EV_STOP_R (FD_EV_IN_SL|FD_EV_IN_PL)
127#define FD_EV_MASK_R (FD_EV_IN_SL|FD_EV_IN_PL)
128
129#define FD_EV_IDLE_W (FD_EV_IDLE_R << 1)
130#define FD_EV_SPEC_W (FD_EV_SPEC_R << 1)
131#define FD_EV_WAIT_W (FD_EV_WAIT_R << 1)
132#define FD_EV_STOP_W (FD_EV_STOP_R << 1)
133#define FD_EV_MASK_W (FD_EV_MASK_R << 1)
134
135#define FD_EV_MASK (FD_EV_MASK_W | FD_EV_MASK_R)
136
Willy Tarreau6653d172007-05-13 01:52:05 +0200137/* This is the minimum number of events successfully processed in speculative
138 * mode above which we agree to return without checking epoll() (1/2 times).
139 */
140#define MIN_RETURN_EVENTS 25
Willy Tarreaude99e992007-04-16 00:53:59 +0200141
Willy Tarreaude99e992007-04-16 00:53:59 +0200142static int nbspec = 0; // current size of the spec list
Willy Tarreauf2e8ee22008-05-25 10:39:02 +0200143static int absmaxevents = 0; // absolute maximum amounts of polled events
Willy Tarreaucb651252008-08-29 13:57:30 +0200144static int fd_created = 0; // fd creation detector, reset upon poll() entry.
Willy Tarreaude99e992007-04-16 00:53:59 +0200145
Willy Tarreaude99e992007-04-16 00:53:59 +0200146static unsigned int *spec_list = NULL; // speculative I/O list
147
148/* private data */
149static struct epoll_event *epoll_events;
150static int epoll_fd;
151
152/* This structure may be used for any purpose. Warning! do not use it in
153 * recursive functions !
154 */
155static struct epoll_event ev;
156
157
Willy Tarreauff9d5ba2009-10-17 21:43:03 +0200158REGPRM1 static inline void alloc_spec_entry(const int fd)
Willy Tarreaude99e992007-04-16 00:53:59 +0200159{
Willy Tarreaub48b3232009-10-17 22:54:17 +0200160 if (fdtab[fd].spec.s1)
Willy Tarreauff9d5ba2009-10-17 21:43:03 +0200161 /* sometimes the entry already exists for the other direction */
Willy Tarreaude99e992007-04-16 00:53:59 +0200162 return;
Willy Tarreaub48b3232009-10-17 22:54:17 +0200163 fdtab[fd].spec.s1 = nbspec + 1;
Willy Tarreau4eac2092007-08-31 17:01:18 +0200164 spec_list[nbspec] = fd;
165 nbspec++;
Willy Tarreaude99e992007-04-16 00:53:59 +0200166}
167
Willy Tarreau4eac2092007-08-31 17:01:18 +0200168/* Removes entry used by fd <fd> from the spec list and replaces it with the
Willy Tarreaub48b3232009-10-17 22:54:17 +0200169 * last one. The fdtab.spec is adjusted to match the back reference if needed.
Willy Tarreau4eac2092007-08-31 17:01:18 +0200170 * If the fd has no entry assigned, return immediately.
Willy Tarreaude99e992007-04-16 00:53:59 +0200171 */
Willy Tarreau4eac2092007-08-31 17:01:18 +0200172REGPRM1 static void release_spec_entry(int fd)
Willy Tarreaude99e992007-04-16 00:53:59 +0200173{
Willy Tarreau4eac2092007-08-31 17:01:18 +0200174 unsigned int pos;
175
Willy Tarreaub48b3232009-10-17 22:54:17 +0200176 pos = fdtab[fd].spec.s1;
Willy Tarreau4eac2092007-08-31 17:01:18 +0200177 if (!pos)
178 return;
179
Willy Tarreaub48b3232009-10-17 22:54:17 +0200180 fdtab[fd].spec.s1 = 0;
Willy Tarreau4eac2092007-08-31 17:01:18 +0200181 pos--;
182 /* we have spec_list[pos]==fd */
Willy Tarreaude99e992007-04-16 00:53:59 +0200183
184 nbspec--;
185 if (pos == nbspec)
186 return;
187
Willy Tarreau4eac2092007-08-31 17:01:18 +0200188 /* we replace current FD by the highest one, which may sometimes be the same */
Willy Tarreaude99e992007-04-16 00:53:59 +0200189 fd = spec_list[nbspec];
190 spec_list[pos] = fd;
Willy Tarreaub48b3232009-10-17 22:54:17 +0200191 fdtab[fd].spec.s1 = pos + 1;
Willy Tarreaude99e992007-04-16 00:53:59 +0200192}
193
194/*
195 * Returns non-zero if <fd> is already monitored for events in direction <dir>.
196 */
197REGPRM2 static int __fd_is_set(const int fd, int dir)
198{
199 int ret;
200
Willy Tarreau7a52a5c2008-08-16 16:06:02 +0200201#if DEBUG_DEV
202 if (fdtab[fd].state == FD_STCLOSE) {
203 fprintf(stderr, "sepoll.fd_isset called on closed fd #%d.\n", fd);
204 ABORT_NOW();
205 }
206#endif
Willy Tarreaub48b3232009-10-17 22:54:17 +0200207 ret = ((unsigned)fdtab[fd].spec.e >> dir) & FD_EV_MASK_DIR;
Willy Tarreaude99e992007-04-16 00:53:59 +0200208 return (ret == FD_EV_SPEC || ret == FD_EV_WAIT);
209}
210
211/*
212 * Don't worry about the strange constructs in __fd_set/__fd_clr, they are
213 * designed like this in order to reduce the number of jumps (verified).
214 */
215REGPRM2 static int __fd_set(const int fd, int dir)
216{
Willy Tarreaude99e992007-04-16 00:53:59 +0200217 unsigned int i;
218
Willy Tarreau7a52a5c2008-08-16 16:06:02 +0200219#if DEBUG_DEV
220 if (fdtab[fd].state == FD_STCLOSE) {
221 fprintf(stderr, "sepoll.fd_set called on closed fd #%d.\n", fd);
222 ABORT_NOW();
223 }
224#endif
Willy Tarreaub48b3232009-10-17 22:54:17 +0200225 i = ((unsigned)fdtab[fd].spec.e >> dir) & FD_EV_MASK_DIR;
Willy Tarreaude99e992007-04-16 00:53:59 +0200226
Willy Tarreauff9d5ba2009-10-17 21:43:03 +0200227 if (i != FD_EV_STOP) {
228 if (unlikely(i != FD_EV_IDLE))
229 return 0;
Willy Tarreaude99e992007-04-16 00:53:59 +0200230 // switch to SPEC state and allocate a SPEC entry.
Willy Tarreau573fd802009-03-22 19:25:46 +0100231 fd_created++;
Willy Tarreaude99e992007-04-16 00:53:59 +0200232 alloc_spec_entry(fd);
Willy Tarreaude99e992007-04-16 00:53:59 +0200233 }
Willy Tarreaub48b3232009-10-17 22:54:17 +0200234 fdtab[fd].spec.e ^= (unsigned int)(FD_EV_IN_SL << dir);
Willy Tarreauff9d5ba2009-10-17 21:43:03 +0200235 return 1;
Willy Tarreaude99e992007-04-16 00:53:59 +0200236}
237
238REGPRM2 static int __fd_clr(const int fd, int dir)
239{
Willy Tarreaude99e992007-04-16 00:53:59 +0200240 unsigned int i;
241
Willy Tarreau7a52a5c2008-08-16 16:06:02 +0200242#if DEBUG_DEV
243 if (fdtab[fd].state == FD_STCLOSE) {
244 fprintf(stderr, "sepoll.fd_clr called on closed fd #%d.\n", fd);
245 ABORT_NOW();
246 }
247#endif
Willy Tarreaub48b3232009-10-17 22:54:17 +0200248 i = ((unsigned)fdtab[fd].spec.e >> dir) & FD_EV_MASK_DIR;
Willy Tarreaude99e992007-04-16 00:53:59 +0200249
Willy Tarreauff9d5ba2009-10-17 21:43:03 +0200250 if (i != FD_EV_SPEC) {
251 if (unlikely(i != FD_EV_WAIT))
252 return 0;
Willy Tarreaude99e992007-04-16 00:53:59 +0200253 // switch to STOP state
254 /* We will create a queue entry for this one because we want to
255 * process it later in order to merge it with other events on
256 * the same FD.
257 */
258 alloc_spec_entry(fd);
Willy Tarreaude99e992007-04-16 00:53:59 +0200259 }
Willy Tarreaub48b3232009-10-17 22:54:17 +0200260 fdtab[fd].spec.e ^= (unsigned int)(FD_EV_IN_SL << dir);
Willy Tarreauff9d5ba2009-10-17 21:43:03 +0200261 return 1;
Willy Tarreaude99e992007-04-16 00:53:59 +0200262}
263
Willy Tarreau6653d172007-05-13 01:52:05 +0200264/* normally unused */
Willy Tarreaude99e992007-04-16 00:53:59 +0200265REGPRM1 static void __fd_rem(int fd)
266{
267 __fd_clr(fd, DIR_RD);
268 __fd_clr(fd, DIR_WR);
269}
270
271/*
272 * On valid epoll() implementations, a call to close() automatically removes
273 * the fds. This means that the FD will appear as previously unset.
274 */
275REGPRM1 static void __fd_clo(int fd)
276{
Willy Tarreau7a52a5c2008-08-16 16:06:02 +0200277 release_spec_entry(fd);
Willy Tarreaub48b3232009-10-17 22:54:17 +0200278 fdtab[fd].spec.e &= ~(FD_EV_MASK);
Willy Tarreaude99e992007-04-16 00:53:59 +0200279}
280
Willy Tarreaudc246a72007-05-09 21:57:51 +0200281/*
Willy Tarreaude99e992007-04-16 00:53:59 +0200282 * speculative epoll() poller
283 */
Willy Tarreau0c303ee2008-07-07 00:09:58 +0200284REGPRM2 static void _do_poll(struct poller *p, int exp)
Willy Tarreaude99e992007-04-16 00:53:59 +0200285{
286 static unsigned int last_skipped;
Willy Tarreauf2e8ee22008-05-25 10:39:02 +0200287 static unsigned int spec_processed;
Willy Tarreau6653d172007-05-13 01:52:05 +0200288 int status, eo;
Willy Tarreaude99e992007-04-16 00:53:59 +0200289 int fd, opcode;
290 int count;
291 int spec_idx;
Willy Tarreaud825eef2007-05-12 22:35:00 +0200292 int wait_time;
Willy Tarreaucb651252008-08-29 13:57:30 +0200293 int looping = 0;
Willy Tarreaude99e992007-04-16 00:53:59 +0200294
295
Willy Tarreaucb651252008-08-29 13:57:30 +0200296 re_poll_once:
Willy Tarreaude99e992007-04-16 00:53:59 +0200297 /* Here we have two options :
Willy Tarreau6653d172007-05-13 01:52:05 +0200298 * - either walk the list forwards and hope to match more events
Willy Tarreaude99e992007-04-16 00:53:59 +0200299 * - or walk it backwards to minimize the number of changes and
300 * to make better use of the cache.
301 * Tests have shown that walking backwards improves perf by 0.2%.
302 */
303
Willy Tarreau6653d172007-05-13 01:52:05 +0200304 status = 0;
Willy Tarreaude99e992007-04-16 00:53:59 +0200305 spec_idx = nbspec;
306 while (likely(spec_idx > 0)) {
Willy Tarreau65745192009-03-28 21:10:48 +0100307 int done;
308
Willy Tarreaude99e992007-04-16 00:53:59 +0200309 spec_idx--;
310 fd = spec_list[spec_idx];
Willy Tarreaub48b3232009-10-17 22:54:17 +0200311 eo = fdtab[fd].spec.e; /* save old events */
Willy Tarreaude99e992007-04-16 00:53:59 +0200312
Willy Tarreau573fd802009-03-22 19:25:46 +0100313 if (looping && --fd_created < 0) {
314 /* we were just checking the newly created FDs */
315 break;
316 }
Willy Tarreau6653d172007-05-13 01:52:05 +0200317 /*
318 * Process the speculative events.
319 *
320 * Principle: events which are marked FD_EV_SPEC are processed
321 * with their assigned function. If the function returns 0, it
322 * means there is nothing doable without polling first. We will
323 * then convert the event to a pollable one by assigning them
324 * the WAIT status.
Willy Tarreaude99e992007-04-16 00:53:59 +0200325 */
326
Willy Tarreau7a52a5c2008-08-16 16:06:02 +0200327#ifdef DEBUG_DEV
328 if (fdtab[fd].state == FD_STCLOSE) {
Willy Tarreaub48b3232009-10-17 22:54:17 +0200329 fprintf(stderr,"fd=%d, fdtab[].ev=%x, fdtab[].spec.e=%x, .s=%d, idx=%d\n",
330 fd, fdtab[fd].ev, fdtab[fd].spec.e, fdtab[fd].spec.s1, spec_idx);
Willy Tarreau7a52a5c2008-08-16 16:06:02 +0200331 }
332#endif
Willy Tarreau65745192009-03-28 21:10:48 +0100333 done = 0;
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100334 fdtab[fd].ev &= FD_POLL_STICKY;
Willy Tarreau6653d172007-05-13 01:52:05 +0200335 if ((eo & FD_EV_MASK_R) == FD_EV_SPEC_R) {
336 /* The owner is interested in reading from this FD */
Willy Tarreau7a52a5c2008-08-16 16:06:02 +0200337 if (fdtab[fd].state != FD_STERROR) {
Willy Tarreau6653d172007-05-13 01:52:05 +0200338 /* Pretend there is something to read */
Willy Tarreaude99e992007-04-16 00:53:59 +0200339 fdtab[fd].ev |= FD_POLL_IN;
Willy Tarreau6653d172007-05-13 01:52:05 +0200340 if (!fdtab[fd].cb[DIR_RD].f(fd))
Willy Tarreaub48b3232009-10-17 22:54:17 +0200341 fdtab[fd].spec.e ^= (FD_EV_WAIT_R ^ FD_EV_SPEC_R);
Willy Tarreau6653d172007-05-13 01:52:05 +0200342 else
Willy Tarreau65745192009-03-28 21:10:48 +0100343 done = 1;
Willy Tarreaude99e992007-04-16 00:53:59 +0200344 }
345 }
Willy Tarreau6653d172007-05-13 01:52:05 +0200346 else if ((eo & FD_EV_MASK_R) == FD_EV_STOP_R) {
347 /* This FD was being polled and is now being removed. */
Willy Tarreaub48b3232009-10-17 22:54:17 +0200348 fdtab[fd].spec.e &= ~FD_EV_MASK_R;
Willy Tarreau6653d172007-05-13 01:52:05 +0200349 }
Willy Tarreaude99e992007-04-16 00:53:59 +0200350
Willy Tarreau6653d172007-05-13 01:52:05 +0200351 if ((eo & FD_EV_MASK_W) == FD_EV_SPEC_W) {
352 /* The owner is interested in writing to this FD */
Willy Tarreau7a52a5c2008-08-16 16:06:02 +0200353 if (fdtab[fd].state != FD_STERROR) {
Willy Tarreau6653d172007-05-13 01:52:05 +0200354 /* Pretend there is something to write */
Willy Tarreaude99e992007-04-16 00:53:59 +0200355 fdtab[fd].ev |= FD_POLL_OUT;
Willy Tarreau6653d172007-05-13 01:52:05 +0200356 if (!fdtab[fd].cb[DIR_WR].f(fd))
Willy Tarreaub48b3232009-10-17 22:54:17 +0200357 fdtab[fd].spec.e ^= (FD_EV_WAIT_W ^ FD_EV_SPEC_W);
Willy Tarreau6653d172007-05-13 01:52:05 +0200358 else
Willy Tarreau65745192009-03-28 21:10:48 +0100359 done = 1;
Willy Tarreaude99e992007-04-16 00:53:59 +0200360 }
361 }
Willy Tarreau6653d172007-05-13 01:52:05 +0200362 else if ((eo & FD_EV_MASK_W) == FD_EV_STOP_W) {
363 /* This FD was being polled and is now being removed. */
Willy Tarreaub48b3232009-10-17 22:54:17 +0200364 fdtab[fd].spec.e &= ~FD_EV_MASK_W;
Willy Tarreau6653d172007-05-13 01:52:05 +0200365 }
Willy Tarreaude99e992007-04-16 00:53:59 +0200366
Willy Tarreau65745192009-03-28 21:10:48 +0100367 status += done;
Willy Tarreau1eead502009-03-28 19:43:06 +0100368 /* one callback might already have closed the fd by itself */
369 if (fdtab[fd].state == FD_STCLOSE)
370 continue;
371
Willy Tarreau6653d172007-05-13 01:52:05 +0200372 /* Now, we will adjust the event in the poll list. Indeed, it
373 * is possible that an event which was previously in the poll
374 * list now goes out, and the opposite is possible too. We can
375 * have opposite changes for READ and WRITE too.
376 */
377
Willy Tarreaub48b3232009-10-17 22:54:17 +0200378 if ((eo ^ fdtab[fd].spec.e) & FD_EV_RW_PL) {
Willy Tarreau6653d172007-05-13 01:52:05 +0200379 /* poll status changed*/
Willy Tarreaub48b3232009-10-17 22:54:17 +0200380 if ((fdtab[fd].spec.e & FD_EV_RW_PL) == 0) {
Willy Tarreau6653d172007-05-13 01:52:05 +0200381 /* fd removed from poll list */
382 opcode = EPOLL_CTL_DEL;
383 }
384 else if ((eo & FD_EV_RW_PL) == 0) {
385 /* new fd in the poll list */
Willy Tarreaude99e992007-04-16 00:53:59 +0200386 opcode = EPOLL_CTL_ADD;
387 }
Willy Tarreau6653d172007-05-13 01:52:05 +0200388 else {
389 /* fd status changed */
390 opcode = EPOLL_CTL_MOD;
391 }
392
393 /* construct the epoll events based on new state */
394 ev.events = 0;
Willy Tarreaub48b3232009-10-17 22:54:17 +0200395 if (fdtab[fd].spec.e & FD_EV_WAIT_R)
Willy Tarreau6653d172007-05-13 01:52:05 +0200396 ev.events |= EPOLLIN;
397
Willy Tarreaub48b3232009-10-17 22:54:17 +0200398 if (fdtab[fd].spec.e & FD_EV_WAIT_W)
Willy Tarreau6653d172007-05-13 01:52:05 +0200399 ev.events |= EPOLLOUT;
400
401 ev.data.fd = fd;
Willy Tarreaude99e992007-04-16 00:53:59 +0200402 epoll_ctl(epoll_fd, opcode, fd, &ev);
Willy Tarreau6653d172007-05-13 01:52:05 +0200403 }
Willy Tarreaude99e992007-04-16 00:53:59 +0200404
Willy Tarreaude99e992007-04-16 00:53:59 +0200405
Willy Tarreaub48b3232009-10-17 22:54:17 +0200406 if (!(fdtab[fd].spec.e & FD_EV_RW_SL)) {
Willy Tarreau6653d172007-05-13 01:52:05 +0200407 /* This fd switched to combinations of either WAIT or
408 * IDLE. It must be removed from the spec list.
409 */
Willy Tarreau4eac2092007-08-31 17:01:18 +0200410 release_spec_entry(fd);
Willy Tarreau6653d172007-05-13 01:52:05 +0200411 continue;
Willy Tarreaude99e992007-04-16 00:53:59 +0200412 }
413 }
414
Willy Tarreau6653d172007-05-13 01:52:05 +0200415 /* It may make sense to immediately return here if there are enough
416 * processed events, without passing through epoll_wait() because we
417 * have exactly done a poll.
418 * Measures have shown a great performance increase if we call the
419 * epoll_wait() only the second time after speculative accesses have
420 * succeeded. This reduces the number of unsucessful calls to
421 * epoll_wait() by a factor of about 3, and the total number of calls
422 * by about 2.
Willy Tarreauf2e8ee22008-05-25 10:39:02 +0200423 * However, when we do that after having processed too many events,
424 * events waiting in epoll() starve for too long a time and tend to
425 * become themselves eligible for speculative polling. So we try to
426 * limit this practise to reasonable situations.
Willy Tarreaude99e992007-04-16 00:53:59 +0200427 */
Willy Tarreau6653d172007-05-13 01:52:05 +0200428
Willy Tarreauf2e8ee22008-05-25 10:39:02 +0200429 spec_processed += status;
Willy Tarreaucb651252008-08-29 13:57:30 +0200430
431 if (looping) {
432 last_skipped++;
433 return;
434 }
435
Willy Tarreauf2e8ee22008-05-25 10:39:02 +0200436 if (status >= MIN_RETURN_EVENTS && spec_processed < absmaxevents) {
Willy Tarreau6653d172007-05-13 01:52:05 +0200437 /* We have processed at least MIN_RETURN_EVENTS, it's worth
438 * returning now without checking epoll_wait().
439 */
440 if (++last_skipped <= 1) {
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200441 tv_update_date(0, 1);
Willy Tarreaude99e992007-04-16 00:53:59 +0200442 return;
443 }
Willy Tarreaude99e992007-04-16 00:53:59 +0200444 }
445 last_skipped = 0;
446
Willy Tarreau332740d2009-05-10 09:57:21 +0200447 if (nbspec || status || run_queue || signal_queue_len) {
Willy Tarreau6653d172007-05-13 01:52:05 +0200448 /* Maybe we have processed some events that we must report, or
Willy Tarreau3a628112008-06-13 21:06:56 +0200449 * maybe we still have events in the spec list, or there are
450 * some tasks left pending in the run_queue, so we must not
Willy Tarreau6653d172007-05-13 01:52:05 +0200451 * wait in epoll() otherwise we will delay their delivery by
452 * the next timeout.
453 */
Willy Tarreaude99e992007-04-16 00:53:59 +0200454 wait_time = 0;
455 }
Willy Tarreaud825eef2007-05-12 22:35:00 +0200456 else {
Willy Tarreau0c303ee2008-07-07 00:09:58 +0200457 if (!exp)
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200458 wait_time = MAX_DELAY_MS;
Willy Tarreau0c303ee2008-07-07 00:09:58 +0200459 else if (tick_is_expired(exp, now_ms))
Willy Tarreaubdefc512007-05-14 02:02:04 +0200460 wait_time = 0;
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200461 else {
Willy Tarreau0c303ee2008-07-07 00:09:58 +0200462 wait_time = TICKS_TO_MS(tick_remain(now_ms, exp)) + 1;
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200463 if (wait_time > MAX_DELAY_MS)
464 wait_time = MAX_DELAY_MS;
465 }
Willy Tarreaud825eef2007-05-12 22:35:00 +0200466 }
Willy Tarreaude99e992007-04-16 00:53:59 +0200467
Willy Tarreauf2e8ee22008-05-25 10:39:02 +0200468 /* now let's wait for real events. We normally use maxpollevents as a
469 * high limit, unless <nbspec> is already big, in which case we need
470 * to compensate for the high number of events processed there.
471 */
472 fd = MIN(absmaxevents, spec_processed);
473 fd = MAX(global.tune.maxpollevents, fd);
474 fd = MIN(maxfd, fd);
Willy Tarreaucb651252008-08-29 13:57:30 +0200475 /* we want to detect if an accept() will create new speculative FDs here */
476 fd_created = 0;
Willy Tarreauf2e8ee22008-05-25 10:39:02 +0200477 spec_processed = 0;
Willy Tarreau1db37712007-06-03 17:16:49 +0200478 status = epoll_wait(epoll_fd, epoll_events, fd, wait_time);
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200479 tv_update_date(wait_time, status);
Willy Tarreaude99e992007-04-16 00:53:59 +0200480
481 for (count = 0; count < status; count++) {
482 int e = epoll_events[count].events;
483 fd = epoll_events[count].data.fd;
484
485 /* it looks complicated but gcc can optimize it away when constants
486 * have same values.
487 */
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100488 DPRINTF(stderr, "%s:%d: fd=%d, ev=0x%08x, e=0x%08x\n",
489 __FUNCTION__, __LINE__,
490 fd, fdtab[fd].ev, e);
491
492 fdtab[fd].ev &= FD_POLL_STICKY;
493 fdtab[fd].ev |=
Willy Tarreaude99e992007-04-16 00:53:59 +0200494 ((e & EPOLLIN ) ? FD_POLL_IN : 0) |
495 ((e & EPOLLPRI) ? FD_POLL_PRI : 0) |
496 ((e & EPOLLOUT) ? FD_POLL_OUT : 0) |
497 ((e & EPOLLERR) ? FD_POLL_ERR : 0) |
498 ((e & EPOLLHUP) ? FD_POLL_HUP : 0);
499
Willy Tarreaub48b3232009-10-17 22:54:17 +0200500 if ((fdtab[fd].spec.e & FD_EV_MASK_R) == FD_EV_WAIT_R) {
Willy Tarreau8bb46f42007-04-30 12:56:21 +0200501 if (fdtab[fd].state == FD_STCLOSE || fdtab[fd].state == FD_STERROR)
Willy Tarreaude99e992007-04-16 00:53:59 +0200502 continue;
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100503 if (fdtab[fd].ev & (FD_POLL_IN|FD_POLL_HUP|FD_POLL_ERR))
Willy Tarreaude99e992007-04-16 00:53:59 +0200504 fdtab[fd].cb[DIR_RD].f(fd);
505 }
506
Willy Tarreaub48b3232009-10-17 22:54:17 +0200507 if ((fdtab[fd].spec.e & FD_EV_MASK_W) == FD_EV_WAIT_W) {
Willy Tarreau8bb46f42007-04-30 12:56:21 +0200508 if (fdtab[fd].state == FD_STCLOSE || fdtab[fd].state == FD_STERROR)
Willy Tarreaude99e992007-04-16 00:53:59 +0200509 continue;
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100510 if (fdtab[fd].ev & (FD_POLL_OUT|FD_POLL_ERR))
Willy Tarreaude99e992007-04-16 00:53:59 +0200511 fdtab[fd].cb[DIR_WR].f(fd);
512 }
513 }
Willy Tarreaucb651252008-08-29 13:57:30 +0200514
515 if (fd_created) {
516 /* we have created some fds, certainly in return of an accept(),
517 * and they're marked as speculative. If we can manage to perform
518 * a read(), we're almost sure to collect all the request at once
519 * and avoid several expensive wakeups. So let's try now. Anyway,
520 * if we fail, the tasks are still woken up, and the FD gets marked
521 * for poll mode.
522 */
Willy Tarreaucb651252008-08-29 13:57:30 +0200523 looping = 1;
524 goto re_poll_once;
525 }
Willy Tarreaude99e992007-04-16 00:53:59 +0200526}
527
528/*
529 * Initialization of the speculative epoll() poller.
530 * Returns 0 in case of failure, non-zero in case of success. If it fails, it
531 * disables the poller by setting its pref to 0.
532 */
533REGPRM1 static int _do_init(struct poller *p)
534{
Willy Tarreaub48b3232009-10-17 22:54:17 +0200535 __label__ fail_spec, fail_ee, fail_fd;
Willy Tarreaude99e992007-04-16 00:53:59 +0200536
537 p->private = NULL;
538
539 epoll_fd = epoll_create(global.maxsock + 1);
540 if (epoll_fd < 0)
541 goto fail_fd;
542
Willy Tarreauf2e8ee22008-05-25 10:39:02 +0200543 /* See comments at the top of the file about this formula. */
544 absmaxevents = MAX(global.tune.maxpollevents, global.maxsock/4);
Willy Tarreaude99e992007-04-16 00:53:59 +0200545 epoll_events = (struct epoll_event*)
Willy Tarreauf2e8ee22008-05-25 10:39:02 +0200546 calloc(1, sizeof(struct epoll_event) * absmaxevents);
Willy Tarreaude99e992007-04-16 00:53:59 +0200547
548 if (epoll_events == NULL)
549 goto fail_ee;
550
551 if ((spec_list = (uint32_t *)calloc(1, sizeof(uint32_t) * global.maxsock)) == NULL)
552 goto fail_spec;
553
Willy Tarreaude99e992007-04-16 00:53:59 +0200554 return 1;
555
Willy Tarreaude99e992007-04-16 00:53:59 +0200556 fail_spec:
557 free(epoll_events);
558 fail_ee:
559 close(epoll_fd);
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200560 epoll_fd = -1;
Willy Tarreaude99e992007-04-16 00:53:59 +0200561 fail_fd:
562 p->pref = 0;
563 return 0;
564}
565
566/*
567 * Termination of the speculative epoll() poller.
568 * Memory is released and the poller is marked as unselectable.
569 */
570REGPRM1 static void _do_term(struct poller *p)
571{
Willy Tarreaua534fea2008-08-03 12:19:50 +0200572 free(spec_list);
573 free(epoll_events);
Willy Tarreaude99e992007-04-16 00:53:59 +0200574
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200575 if (epoll_fd >= 0) {
576 close(epoll_fd);
577 epoll_fd = -1;
578 }
Willy Tarreaude99e992007-04-16 00:53:59 +0200579
Willy Tarreaude99e992007-04-16 00:53:59 +0200580 spec_list = NULL;
581 epoll_events = NULL;
582
583 p->private = NULL;
584 p->pref = 0;
585}
586
587/*
588 * Check that the poller works.
589 * Returns 1 if OK, otherwise 0.
590 */
591REGPRM1 static int _do_test(struct poller *p)
592{
593 int fd;
594
595 fd = epoll_create(global.maxsock + 1);
596 if (fd < 0)
597 return 0;
598 close(fd);
599 return 1;
600}
601
602/*
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200603 * Recreate the epoll file descriptor after a fork(). Returns 1 if OK,
604 * otherwise 0. It will ensure that all processes will not share their
605 * epoll_fd. Some side effects were encountered because of this, such
606 * as epoll_wait() returning an FD which was previously deleted.
607 */
608REGPRM1 static int _do_fork(struct poller *p)
609{
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200610 if (epoll_fd >= 0)
611 close(epoll_fd);
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200612 epoll_fd = epoll_create(global.maxsock + 1);
613 if (epoll_fd < 0)
614 return 0;
615 return 1;
616}
617
618/*
Willy Tarreaude99e992007-04-16 00:53:59 +0200619 * It is a constructor, which means that it will automatically be called before
620 * main(). This is GCC-specific but it works at least since 2.95.
621 * Special care must be taken so that it does not need any uninitialized data.
622 */
623__attribute__((constructor))
624static void _do_register(void)
625{
626 struct poller *p;
627
628 if (nbpollers >= MAX_POLLERS)
629 return;
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200630
631 epoll_fd = -1;
Willy Tarreaude99e992007-04-16 00:53:59 +0200632 p = &pollers[nbpollers++];
633
634 p->name = "sepoll";
635 p->pref = 400;
636 p->private = NULL;
637
638 p->test = _do_test;
639 p->init = _do_init;
640 p->term = _do_term;
641 p->poll = _do_poll;
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200642 p->fork = _do_fork;
Willy Tarreaude99e992007-04-16 00:53:59 +0200643
644 p->is_set = __fd_is_set;
645 p->cond_s = p->set = __fd_set;
646 p->cond_c = p->clr = __fd_clr;
647 p->rem = __fd_rem;
648 p->clo = __fd_clo;
649}
650
651
652/*
653 * Local variables:
654 * c-indent-level: 8
655 * c-basic-offset: 8
656 * End:
657 */