Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 1 | /* |
| 2 | * FD polling functions for Speculative I/O combined with Linux epoll() |
| 3 | * |
| 4 | * Copyright 2000-2007 Willy Tarreau <w@1wt.eu> |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or |
| 7 | * modify it under the terms of the GNU General Public License |
| 8 | * as published by the Free Software Foundation; either version |
| 9 | * 2 of the License, or (at your option) any later version. |
| 10 | * |
| 11 | */ |
| 12 | |
| 13 | #include <unistd.h> |
| 14 | #include <sys/time.h> |
| 15 | #include <sys/types.h> |
| 16 | |
| 17 | #include <common/compat.h> |
| 18 | #include <common/config.h> |
| 19 | #include <common/standard.h> |
| 20 | #include <common/time.h> |
| 21 | |
| 22 | #include <types/fd.h> |
| 23 | #include <types/global.h> |
| 24 | |
| 25 | #include <proto/fd.h> |
| 26 | #include <proto/task.h> |
| 27 | |
| 28 | #if defined(USE_MY_EPOLL) |
| 29 | #include <common/epoll.h> |
| 30 | #include <errno.h> |
| 31 | #include <sys/syscall.h> |
| 32 | static _syscall1 (int, epoll_create, int, size); |
| 33 | static _syscall4 (int, epoll_ctl, int, epfd, int, op, int, fd, struct epoll_event *, event); |
| 34 | static _syscall4 (int, epoll_wait, int, epfd, struct epoll_event *, events, int, maxevents, int, timeout); |
| 35 | #else |
| 36 | #include <sys/epoll.h> |
| 37 | #endif |
| 38 | |
| 39 | /* |
| 40 | * We define 4 states for each direction of a file descriptor, which we store |
| 41 | * as 2 bits : |
| 42 | * |
| 43 | * 00 = IDLE : we're not interested in this event |
| 44 | * 01 = SPEC : perform speculative I/O on this FD |
| 45 | * 10 = WAIT : really wait for an availability event on this FD (poll) |
| 46 | * 11 = STOP : was marked WAIT, but disabled. It can switch back to WAIT if |
| 47 | * the application changes its mind, otherwise disable FD polling |
| 48 | * and switch back to IDLE. |
| 49 | * |
| 50 | * Since we do not want to scan all the FD list to find speculative I/O events, |
| 51 | * we store them in a list consisting in a linear array holding only the FD |
| 52 | * indexes right now. |
| 53 | * |
| 54 | * The STOP state requires the event to be present in the spec list so that |
| 55 | * it can be detected and flushed upon next scan without having to scan the |
| 56 | * whole FD list. |
| 57 | * |
| 58 | * This translates like this : |
| 59 | * |
| 60 | * EVENT_IN_SPEC_LIST = 01 |
| 61 | * EVENT_IN_POLL_LIST = 10 |
| 62 | * |
| 63 | * IDLE = 0 |
| 64 | * SPEC = (EVENT_IN_SPEC_LIST) |
| 65 | * WAIT = (EVENT_IN_POLL_LIST) |
| 66 | * STOP = (EVENT_IN_SPEC_LIST|EVENT_IN_POLL_LIST) |
| 67 | * |
| 68 | * fd_is_set() just consists in checking that the status is 01 or 10. |
| 69 | * |
| 70 | * For efficiency reasons, we will store the Read and Write bits interlaced to |
| 71 | * form a 4-bit field, so that we can simply shift the value right by 0/1 and |
| 72 | * get what we want : |
| 73 | * 3 2 1 0 |
| 74 | * Wp Rp Ws Rs |
| 75 | * |
| 76 | * The FD array has to hold a back reference to the speculative list. This |
| 77 | * reference is only valid if at least one of the directions is marked SPEC. |
| 78 | * |
| 79 | */ |
| 80 | |
| 81 | #define FD_EV_IN_SL 1 |
| 82 | #define FD_EV_IN_PL 4 |
| 83 | |
| 84 | #define FD_EV_IDLE 0 |
| 85 | #define FD_EV_SPEC (FD_EV_IN_SL) |
| 86 | #define FD_EV_WAIT (FD_EV_IN_PL) |
| 87 | #define FD_EV_STOP (FD_EV_IN_SL|FD_EV_IN_PL) |
| 88 | |
| 89 | /* Those match any of R or W for Spec list or Poll list */ |
| 90 | #define FD_EV_RW_SL (FD_EV_IN_SL | (FD_EV_IN_SL << 1)) |
| 91 | #define FD_EV_RW_PL (FD_EV_IN_PL | (FD_EV_IN_PL << 1)) |
| 92 | #define FD_EV_MASK_DIR (FD_EV_IN_SL|FD_EV_IN_PL) |
| 93 | |
| 94 | #define FD_EV_IDLE_R 0 |
| 95 | #define FD_EV_SPEC_R (FD_EV_IN_SL) |
| 96 | #define FD_EV_WAIT_R (FD_EV_IN_PL) |
| 97 | #define FD_EV_STOP_R (FD_EV_IN_SL|FD_EV_IN_PL) |
| 98 | #define FD_EV_MASK_R (FD_EV_IN_SL|FD_EV_IN_PL) |
| 99 | |
| 100 | #define FD_EV_IDLE_W (FD_EV_IDLE_R << 1) |
| 101 | #define FD_EV_SPEC_W (FD_EV_SPEC_R << 1) |
| 102 | #define FD_EV_WAIT_W (FD_EV_WAIT_R << 1) |
| 103 | #define FD_EV_STOP_W (FD_EV_STOP_R << 1) |
| 104 | #define FD_EV_MASK_W (FD_EV_MASK_R << 1) |
| 105 | |
| 106 | #define FD_EV_MASK (FD_EV_MASK_W | FD_EV_MASK_R) |
| 107 | |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 108 | /* This is the minimum number of events successfully processed in speculative |
| 109 | * mode above which we agree to return without checking epoll() (1/2 times). |
| 110 | */ |
| 111 | #define MIN_RETURN_EVENTS 25 |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 112 | |
| 113 | /* descriptor of one FD. |
| 114 | * FIXME: should be a bit field */ |
| 115 | struct fd_status { |
| 116 | unsigned int e:4; // read and write events status. |
| 117 | unsigned int s:28; // Position in spec list. Should be last. |
| 118 | }; |
| 119 | |
| 120 | static int nbspec = 0; // current size of the spec list |
| 121 | |
| 122 | static struct fd_status *fd_list = NULL; // list of FDs |
| 123 | static unsigned int *spec_list = NULL; // speculative I/O list |
| 124 | |
| 125 | /* private data */ |
| 126 | static struct epoll_event *epoll_events; |
| 127 | static int epoll_fd; |
| 128 | |
| 129 | /* This structure may be used for any purpose. Warning! do not use it in |
| 130 | * recursive functions ! |
| 131 | */ |
| 132 | static struct epoll_event ev; |
| 133 | |
| 134 | |
| 135 | REGPRM1 static void alloc_spec_entry(const int fd) |
| 136 | { |
| 137 | if (fd_list[fd].e & FD_EV_RW_SL) |
| 138 | return; |
| 139 | fd_list[fd].s = nbspec; |
| 140 | spec_list[nbspec++] = fd; |
| 141 | } |
| 142 | |
| 143 | /* removes entry <pos> from the spec list and replaces it with the last one. |
| 144 | * The fd_list is adjusted to match the back reference if needed. |
| 145 | */ |
| 146 | REGPRM1 static void delete_spec_entry(const int pos) |
| 147 | { |
| 148 | int fd; |
| 149 | |
| 150 | nbspec--; |
| 151 | if (pos == nbspec) |
| 152 | return; |
| 153 | |
| 154 | /* we replace current FD by the highest one */ |
| 155 | fd = spec_list[nbspec]; |
| 156 | spec_list[pos] = fd; |
| 157 | fd_list[fd].s = pos; |
| 158 | } |
| 159 | |
| 160 | /* |
| 161 | * Returns non-zero if <fd> is already monitored for events in direction <dir>. |
| 162 | */ |
| 163 | REGPRM2 static int __fd_is_set(const int fd, int dir) |
| 164 | { |
| 165 | int ret; |
| 166 | |
| 167 | ret = ((unsigned)fd_list[fd].e >> dir) & FD_EV_MASK_DIR; |
| 168 | return (ret == FD_EV_SPEC || ret == FD_EV_WAIT); |
| 169 | } |
| 170 | |
| 171 | /* |
| 172 | * Don't worry about the strange constructs in __fd_set/__fd_clr, they are |
| 173 | * designed like this in order to reduce the number of jumps (verified). |
| 174 | */ |
| 175 | REGPRM2 static int __fd_set(const int fd, int dir) |
| 176 | { |
| 177 | __label__ switch_state; |
| 178 | unsigned int i; |
| 179 | |
| 180 | i = ((unsigned)fd_list[fd].e >> dir) & FD_EV_MASK_DIR; |
| 181 | |
| 182 | if (i == FD_EV_IDLE) { |
| 183 | // switch to SPEC state and allocate a SPEC entry. |
| 184 | alloc_spec_entry(fd); |
| 185 | switch_state: |
| 186 | fd_list[fd].e ^= (unsigned int)(FD_EV_IN_SL << dir); |
| 187 | return 1; |
| 188 | } |
| 189 | else if (i == FD_EV_STOP) { |
| 190 | // switch to WAIT state |
| 191 | goto switch_state; |
| 192 | } |
| 193 | else |
| 194 | return 0; |
| 195 | } |
| 196 | |
| 197 | REGPRM2 static int __fd_clr(const int fd, int dir) |
| 198 | { |
| 199 | __label__ switch_state; |
| 200 | unsigned int i; |
| 201 | |
| 202 | i = ((unsigned)fd_list[fd].e >> dir) & FD_EV_MASK_DIR; |
| 203 | |
| 204 | if (i == FD_EV_SPEC) { |
| 205 | // switch to IDLE state |
| 206 | goto switch_state; |
| 207 | } |
| 208 | else if (likely(i == FD_EV_WAIT)) { |
| 209 | // switch to STOP state |
| 210 | /* We will create a queue entry for this one because we want to |
| 211 | * process it later in order to merge it with other events on |
| 212 | * the same FD. |
| 213 | */ |
| 214 | alloc_spec_entry(fd); |
| 215 | switch_state: |
| 216 | fd_list[fd].e ^= (unsigned int)(FD_EV_IN_SL << dir); |
| 217 | return 1; |
| 218 | } |
| 219 | return 0; |
| 220 | } |
| 221 | |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 222 | /* normally unused */ |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 223 | REGPRM1 static void __fd_rem(int fd) |
| 224 | { |
| 225 | __fd_clr(fd, DIR_RD); |
| 226 | __fd_clr(fd, DIR_WR); |
| 227 | } |
| 228 | |
| 229 | /* |
| 230 | * On valid epoll() implementations, a call to close() automatically removes |
| 231 | * the fds. This means that the FD will appear as previously unset. |
| 232 | */ |
| 233 | REGPRM1 static void __fd_clo(int fd) |
| 234 | { |
| 235 | if (fd_list[fd].e & FD_EV_RW_SL) |
| 236 | delete_spec_entry(fd_list[fd].s); |
| 237 | fd_list[fd].e &= ~(FD_EV_MASK); |
| 238 | } |
| 239 | |
Willy Tarreau | dc246a7 | 2007-05-09 21:57:51 +0200 | [diff] [blame] | 240 | /* |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 241 | * speculative epoll() poller |
| 242 | */ |
Willy Tarreau | d825eef | 2007-05-12 22:35:00 +0200 | [diff] [blame] | 243 | REGPRM2 static void _do_poll(struct poller *p, struct timeval *exp) |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 244 | { |
| 245 | static unsigned int last_skipped; |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 246 | int status, eo; |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 247 | int fd, opcode; |
| 248 | int count; |
| 249 | int spec_idx; |
Willy Tarreau | d825eef | 2007-05-12 22:35:00 +0200 | [diff] [blame] | 250 | int wait_time; |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 251 | |
| 252 | |
| 253 | /* Here we have two options : |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 254 | * - either walk the list forwards and hope to match more events |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 255 | * - or walk it backwards to minimize the number of changes and |
| 256 | * to make better use of the cache. |
| 257 | * Tests have shown that walking backwards improves perf by 0.2%. |
| 258 | */ |
| 259 | |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 260 | status = 0; |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 261 | spec_idx = nbspec; |
| 262 | while (likely(spec_idx > 0)) { |
| 263 | spec_idx--; |
| 264 | fd = spec_list[spec_idx]; |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 265 | eo = fd_list[fd].e; /* save old events */ |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 266 | |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 267 | /* |
| 268 | * Process the speculative events. |
| 269 | * |
| 270 | * Principle: events which are marked FD_EV_SPEC are processed |
| 271 | * with their assigned function. If the function returns 0, it |
| 272 | * means there is nothing doable without polling first. We will |
| 273 | * then convert the event to a pollable one by assigning them |
| 274 | * the WAIT status. |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 275 | */ |
| 276 | |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 277 | fdtab[fd].ev = 0; |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 278 | if ((eo & FD_EV_MASK_R) == FD_EV_SPEC_R) { |
| 279 | /* The owner is interested in reading from this FD */ |
Willy Tarreau | 8bb46f4 | 2007-04-30 12:56:21 +0200 | [diff] [blame] | 280 | if (fdtab[fd].state != FD_STCLOSE && fdtab[fd].state != FD_STERROR) { |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 281 | /* Pretend there is something to read */ |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 282 | fdtab[fd].ev |= FD_POLL_IN; |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 283 | if (!fdtab[fd].cb[DIR_RD].f(fd)) |
| 284 | fd_list[fd].e ^= (FD_EV_WAIT_R ^ FD_EV_SPEC_R); |
| 285 | else |
| 286 | status++; |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 287 | } |
| 288 | } |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 289 | else if ((eo & FD_EV_MASK_R) == FD_EV_STOP_R) { |
| 290 | /* This FD was being polled and is now being removed. */ |
| 291 | fd_list[fd].e &= ~FD_EV_MASK_R; |
| 292 | } |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 293 | |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 294 | if ((eo & FD_EV_MASK_W) == FD_EV_SPEC_W) { |
| 295 | /* The owner is interested in writing to this FD */ |
Willy Tarreau | 8bb46f4 | 2007-04-30 12:56:21 +0200 | [diff] [blame] | 296 | if (fdtab[fd].state != FD_STCLOSE && fdtab[fd].state != FD_STERROR) { |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 297 | /* Pretend there is something to write */ |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 298 | fdtab[fd].ev |= FD_POLL_OUT; |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 299 | if (!fdtab[fd].cb[DIR_WR].f(fd)) |
| 300 | fd_list[fd].e ^= (FD_EV_WAIT_W ^ FD_EV_SPEC_W); |
| 301 | else |
| 302 | status++; |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 303 | } |
| 304 | } |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 305 | else if ((eo & FD_EV_MASK_W) == FD_EV_STOP_W) { |
| 306 | /* This FD was being polled and is now being removed. */ |
| 307 | fd_list[fd].e &= ~FD_EV_MASK_W; |
| 308 | } |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 309 | |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 310 | /* Now, we will adjust the event in the poll list. Indeed, it |
| 311 | * is possible that an event which was previously in the poll |
| 312 | * list now goes out, and the opposite is possible too. We can |
| 313 | * have opposite changes for READ and WRITE too. |
| 314 | */ |
| 315 | |
| 316 | if ((eo ^ fd_list[fd].e) & FD_EV_RW_PL) { |
| 317 | /* poll status changed*/ |
| 318 | if ((fd_list[fd].e & FD_EV_RW_PL) == 0) { |
| 319 | /* fd removed from poll list */ |
| 320 | opcode = EPOLL_CTL_DEL; |
| 321 | } |
| 322 | else if ((eo & FD_EV_RW_PL) == 0) { |
| 323 | /* new fd in the poll list */ |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 324 | opcode = EPOLL_CTL_ADD; |
| 325 | } |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 326 | else { |
| 327 | /* fd status changed */ |
| 328 | opcode = EPOLL_CTL_MOD; |
| 329 | } |
| 330 | |
| 331 | /* construct the epoll events based on new state */ |
| 332 | ev.events = 0; |
| 333 | if (fd_list[fd].e & FD_EV_WAIT_R) |
| 334 | ev.events |= EPOLLIN; |
| 335 | |
| 336 | if (fd_list[fd].e & FD_EV_WAIT_W) |
| 337 | ev.events |= EPOLLOUT; |
| 338 | |
| 339 | ev.data.fd = fd; |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 340 | epoll_ctl(epoll_fd, opcode, fd, &ev); |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 341 | } |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 342 | |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 343 | |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 344 | if (!(fd_list[fd].e & FD_EV_RW_SL)) { |
| 345 | /* This fd switched to combinations of either WAIT or |
| 346 | * IDLE. It must be removed from the spec list. |
| 347 | */ |
| 348 | delete_spec_entry(spec_idx); |
| 349 | continue; |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 350 | } |
| 351 | } |
| 352 | |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 353 | /* It may make sense to immediately return here if there are enough |
| 354 | * processed events, without passing through epoll_wait() because we |
| 355 | * have exactly done a poll. |
| 356 | * Measures have shown a great performance increase if we call the |
| 357 | * epoll_wait() only the second time after speculative accesses have |
| 358 | * succeeded. This reduces the number of unsucessful calls to |
| 359 | * epoll_wait() by a factor of about 3, and the total number of calls |
| 360 | * by about 2. |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 361 | */ |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 362 | |
| 363 | if (status >= MIN_RETURN_EVENTS) { |
| 364 | /* We have processed at least MIN_RETURN_EVENTS, it's worth |
| 365 | * returning now without checking epoll_wait(). |
| 366 | */ |
| 367 | if (++last_skipped <= 1) { |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 368 | tv_now(&now); |
| 369 | return; |
| 370 | } |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 371 | } |
| 372 | last_skipped = 0; |
| 373 | |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 374 | if (nbspec || status) { |
| 375 | /* Maybe we have processed some events that we must report, or |
| 376 | * maybe we still have events in the spec list, so we must not |
| 377 | * wait in epoll() otherwise we will delay their delivery by |
| 378 | * the next timeout. |
| 379 | */ |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 380 | wait_time = 0; |
| 381 | } |
Willy Tarreau | d825eef | 2007-05-12 22:35:00 +0200 | [diff] [blame] | 382 | else { |
Willy Tarreau | bdefc51 | 2007-05-14 02:02:04 +0200 | [diff] [blame] | 383 | if (tv_iseternity(exp)) |
Willy Tarreau | d825eef | 2007-05-12 22:35:00 +0200 | [diff] [blame] | 384 | wait_time = -1; |
Willy Tarreau | bdefc51 | 2007-05-14 02:02:04 +0200 | [diff] [blame] | 385 | else if (tv_isge(&now, exp)) |
| 386 | wait_time = 0; |
| 387 | else |
| 388 | wait_time = __tv_ms_elapsed(&now, exp) + 1; |
Willy Tarreau | d825eef | 2007-05-12 22:35:00 +0200 | [diff] [blame] | 389 | } |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 390 | |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 391 | /* now let's wait for real events */ |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 392 | status = epoll_wait(epoll_fd, epoll_events, maxfd, wait_time); |
Willy Tarreau | 6653d17 | 2007-05-13 01:52:05 +0200 | [diff] [blame] | 393 | |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 394 | tv_now(&now); |
| 395 | |
| 396 | for (count = 0; count < status; count++) { |
| 397 | int e = epoll_events[count].events; |
| 398 | fd = epoll_events[count].data.fd; |
| 399 | |
| 400 | /* it looks complicated but gcc can optimize it away when constants |
| 401 | * have same values. |
| 402 | */ |
| 403 | fdtab[fd].ev = |
| 404 | ((e & EPOLLIN ) ? FD_POLL_IN : 0) | |
| 405 | ((e & EPOLLPRI) ? FD_POLL_PRI : 0) | |
| 406 | ((e & EPOLLOUT) ? FD_POLL_OUT : 0) | |
| 407 | ((e & EPOLLERR) ? FD_POLL_ERR : 0) | |
| 408 | ((e & EPOLLHUP) ? FD_POLL_HUP : 0); |
| 409 | |
| 410 | if ((fd_list[fd].e & FD_EV_MASK_R) == FD_EV_WAIT_R) { |
Willy Tarreau | 8bb46f4 | 2007-04-30 12:56:21 +0200 | [diff] [blame] | 411 | if (fdtab[fd].state == FD_STCLOSE || fdtab[fd].state == FD_STERROR) |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 412 | continue; |
Willy Tarreau | 8bb46f4 | 2007-04-30 12:56:21 +0200 | [diff] [blame] | 413 | if (fdtab[fd].ev & (FD_POLL_RD|FD_POLL_HUP|FD_POLL_ERR)) |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 414 | fdtab[fd].cb[DIR_RD].f(fd); |
| 415 | } |
| 416 | |
| 417 | if ((fd_list[fd].e & FD_EV_MASK_W) == FD_EV_WAIT_W) { |
Willy Tarreau | 8bb46f4 | 2007-04-30 12:56:21 +0200 | [diff] [blame] | 418 | if (fdtab[fd].state == FD_STCLOSE || fdtab[fd].state == FD_STERROR) |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 419 | continue; |
Willy Tarreau | 8bb46f4 | 2007-04-30 12:56:21 +0200 | [diff] [blame] | 420 | if (fdtab[fd].ev & (FD_POLL_WR|FD_POLL_ERR)) |
Willy Tarreau | de99e99 | 2007-04-16 00:53:59 +0200 | [diff] [blame] | 421 | fdtab[fd].cb[DIR_WR].f(fd); |
| 422 | } |
| 423 | } |
| 424 | } |
| 425 | |
| 426 | /* |
| 427 | * Initialization of the speculative epoll() poller. |
| 428 | * Returns 0 in case of failure, non-zero in case of success. If it fails, it |
| 429 | * disables the poller by setting its pref to 0. |
| 430 | */ |
| 431 | REGPRM1 static int _do_init(struct poller *p) |
| 432 | { |
| 433 | __label__ fail_fd_list, fail_spec, fail_ee, fail_fd; |
| 434 | |
| 435 | p->private = NULL; |
| 436 | |
| 437 | epoll_fd = epoll_create(global.maxsock + 1); |
| 438 | if (epoll_fd < 0) |
| 439 | goto fail_fd; |
| 440 | |
| 441 | epoll_events = (struct epoll_event*) |
| 442 | calloc(1, sizeof(struct epoll_event) * global.maxsock); |
| 443 | |
| 444 | if (epoll_events == NULL) |
| 445 | goto fail_ee; |
| 446 | |
| 447 | if ((spec_list = (uint32_t *)calloc(1, sizeof(uint32_t) * global.maxsock)) == NULL) |
| 448 | goto fail_spec; |
| 449 | |
| 450 | fd_list = (struct fd_status *)calloc(1, sizeof(struct fd_status) * global.maxsock); |
| 451 | if (fd_list == NULL) |
| 452 | goto fail_fd_list; |
| 453 | |
| 454 | return 1; |
| 455 | |
| 456 | fail_fd_list: |
| 457 | free(spec_list); |
| 458 | fail_spec: |
| 459 | free(epoll_events); |
| 460 | fail_ee: |
| 461 | close(epoll_fd); |
| 462 | epoll_fd = 0; |
| 463 | fail_fd: |
| 464 | p->pref = 0; |
| 465 | return 0; |
| 466 | } |
| 467 | |
| 468 | /* |
| 469 | * Termination of the speculative epoll() poller. |
| 470 | * Memory is released and the poller is marked as unselectable. |
| 471 | */ |
| 472 | REGPRM1 static void _do_term(struct poller *p) |
| 473 | { |
| 474 | if (fd_list) |
| 475 | free(fd_list); |
| 476 | if (spec_list) |
| 477 | free(spec_list); |
| 478 | if (epoll_events) |
| 479 | free(epoll_events); |
| 480 | |
| 481 | close(epoll_fd); |
| 482 | epoll_fd = 0; |
| 483 | |
| 484 | fd_list = NULL; |
| 485 | spec_list = NULL; |
| 486 | epoll_events = NULL; |
| 487 | |
| 488 | p->private = NULL; |
| 489 | p->pref = 0; |
| 490 | } |
| 491 | |
| 492 | /* |
| 493 | * Check that the poller works. |
| 494 | * Returns 1 if OK, otherwise 0. |
| 495 | */ |
| 496 | REGPRM1 static int _do_test(struct poller *p) |
| 497 | { |
| 498 | int fd; |
| 499 | |
| 500 | fd = epoll_create(global.maxsock + 1); |
| 501 | if (fd < 0) |
| 502 | return 0; |
| 503 | close(fd); |
| 504 | return 1; |
| 505 | } |
| 506 | |
| 507 | /* |
| 508 | * It is a constructor, which means that it will automatically be called before |
| 509 | * main(). This is GCC-specific but it works at least since 2.95. |
| 510 | * Special care must be taken so that it does not need any uninitialized data. |
| 511 | */ |
| 512 | __attribute__((constructor)) |
| 513 | static void _do_register(void) |
| 514 | { |
| 515 | struct poller *p; |
| 516 | |
| 517 | if (nbpollers >= MAX_POLLERS) |
| 518 | return; |
| 519 | p = &pollers[nbpollers++]; |
| 520 | |
| 521 | p->name = "sepoll"; |
| 522 | p->pref = 400; |
| 523 | p->private = NULL; |
| 524 | |
| 525 | p->test = _do_test; |
| 526 | p->init = _do_init; |
| 527 | p->term = _do_term; |
| 528 | p->poll = _do_poll; |
| 529 | |
| 530 | p->is_set = __fd_is_set; |
| 531 | p->cond_s = p->set = __fd_set; |
| 532 | p->cond_c = p->clr = __fd_clr; |
| 533 | p->rem = __fd_rem; |
| 534 | p->clo = __fd_clo; |
| 535 | } |
| 536 | |
| 537 | |
| 538 | /* |
| 539 | * Local variables: |
| 540 | * c-indent-level: 8 |
| 541 | * c-basic-offset: 8 |
| 542 | * End: |
| 543 | */ |