blob: e826af3cb0726998c58906f2da9f8039a8cb8e61 [file] [log] [blame]
Willy Tarreau4f60f162007-04-08 16:39:58 +02001/*
Willy Tarreaue9f49e72012-11-11 17:42:00 +01002 * FD polling functions for Linux epoll
Willy Tarreau4f60f162007-04-08 16:39:58 +02003 *
Willy Tarreaue9f49e72012-11-11 17:42:00 +01004 * Copyright 2000-2012 Willy Tarreau <w@1wt.eu>
Willy Tarreau4f60f162007-04-08 16:39:58 +02005 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
Willy Tarreau4f60f162007-04-08 16:39:58 +020010 */
11
12#include <unistd.h>
13#include <sys/time.h>
14#include <sys/types.h>
15
16#include <common/compat.h>
17#include <common/config.h>
Willy Tarreaue9f49e72012-11-11 17:42:00 +010018#include <common/debug.h>
Willy Tarreau43d8fb22011-08-22 17:12:02 +020019#include <common/epoll.h>
Willy Tarreau58094f22007-04-10 01:33:20 +020020#include <common/standard.h>
Willy Tarreau0c303ee2008-07-07 00:09:58 +020021#include <common/ticks.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020022#include <common/time.h>
Willy Tarreau1db37712007-06-03 17:16:49 +020023#include <common/tools.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020024
Willy Tarreau4f60f162007-04-08 16:39:58 +020025#include <types/global.h>
26
Willy Tarreaue9f49e72012-11-11 17:42:00 +010027#include <proto/fd.h>
Willy Tarreau332740d2009-05-10 09:57:21 +020028#include <proto/signal.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020029#include <proto/task.h>
30
Willy Tarreau58094f22007-04-10 01:33:20 +020031
Willy Tarreaue9f49e72012-11-11 17:42:00 +010032static int absmaxevents = 0; // absolute maximum amounts of polled events
Willy Tarreau4f60f162007-04-08 16:39:58 +020033
34/* private data */
35static struct epoll_event *epoll_events;
36static int epoll_fd;
37
Willy Tarreau58094f22007-04-10 01:33:20 +020038/* This structure may be used for any purpose. Warning! do not use it in
39 * recursive functions !
40 */
41static struct epoll_event ev;
42
Willy Tarreau1c07b072013-01-07 16:19:18 +010043#ifndef EPOLLRDHUP
44/* EPOLLRDHUP was defined late in libc, and it appeared in kernel 2.6.17 */
45#define EPOLLRDHUP 0x2000
46#endif
47
Willy Tarreau4f60f162007-04-08 16:39:58 +020048/*
Willy Tarreaue9f49e72012-11-11 17:42:00 +010049 * speculative epoll() poller
Willy Tarreau4f60f162007-04-08 16:39:58 +020050 */
Willy Tarreaue9f49e72012-11-11 17:42:00 +010051REGPRM2 static void _do_poll(struct poller *p, int exp)
Willy Tarreau4f60f162007-04-08 16:39:58 +020052{
Willy Tarreaue9f49e72012-11-11 17:42:00 +010053 int status, eo, en;
54 int fd, opcode;
55 int count;
56 int updt_idx;
57 int wait_time;
Willy Tarreau58094f22007-04-10 01:33:20 +020058
Willy Tarreaue9f49e72012-11-11 17:42:00 +010059 /* first, scan the update list to find changes */
60 for (updt_idx = 0; updt_idx < fd_nbupdt; updt_idx++) {
61 fd = fd_updt[updt_idx];
Willy Tarreau69a41fa2014-01-20 11:02:59 +010062 en = fdtab[fd].state & 15; /* new events */
63 eo = fdtab[fd].state >> 4; /* previous events */
Willy Tarreau58094f22007-04-10 01:33:20 +020064
Willy Tarreaue9f49e72012-11-11 17:42:00 +010065 if (fdtab[fd].owner && (eo ^ en)) {
66 if ((eo ^ en) & FD_EV_POLLED_RW) {
Willy Tarreau3ef5af32013-12-20 16:03:41 +010067 /* poll status changed */
Willy Tarreaue9f49e72012-11-11 17:42:00 +010068 if ((en & FD_EV_POLLED_RW) == 0) {
69 /* fd removed from poll list */
70 opcode = EPOLL_CTL_DEL;
71 }
72 else if ((eo & FD_EV_POLLED_RW) == 0) {
73 /* new fd in the poll list */
74 opcode = EPOLL_CTL_ADD;
75 }
76 else {
77 /* fd status changed */
78 opcode = EPOLL_CTL_MOD;
79 }
Willy Tarreau58094f22007-04-10 01:33:20 +020080
Willy Tarreaue9f49e72012-11-11 17:42:00 +010081 /* construct the epoll events based on new state */
82 ev.events = 0;
83 if (en & FD_EV_POLLED_R)
Willy Tarreau1c07b072013-01-07 16:19:18 +010084 ev.events |= EPOLLIN | EPOLLRDHUP;
Willy Tarreau58094f22007-04-10 01:33:20 +020085
Willy Tarreaue9f49e72012-11-11 17:42:00 +010086 if (en & FD_EV_POLLED_W)
87 ev.events |= EPOLLOUT;
Willy Tarreau58094f22007-04-10 01:33:20 +020088
Willy Tarreaue9f49e72012-11-11 17:42:00 +010089 ev.data.fd = fd;
90 epoll_ctl(epoll_fd, opcode, fd, &ev);
Willy Tarreau58094f22007-04-10 01:33:20 +020091 }
Willy Tarreau58094f22007-04-10 01:33:20 +020092
Willy Tarreau69a41fa2014-01-20 11:02:59 +010093 fdtab[fd].state = (en << 4) + en; /* save new events */
Willy Tarreauf8cfa442012-10-04 21:54:41 +020094
Willy Tarreaue9f49e72012-11-11 17:42:00 +010095 if (!(en & FD_EV_ACTIVE_RW)) {
96 /* This fd doesn't use any active entry anymore, we can
97 * kill its entry.
98 */
99 release_spec_entry(fd);
100 }
101 else if ((en & ~eo) & FD_EV_ACTIVE_RW) {
102 /* we need a new spec entry now */
103 alloc_spec_entry(fd);
104 }
Willy Tarreau4f60f162007-04-08 16:39:58 +0200105
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100106 }
107 fdtab[fd].updated = 0;
108 fdtab[fd].new = 0;
Willy Tarreau58094f22007-04-10 01:33:20 +0200109 }
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100110 fd_nbupdt = 0;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200111
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100112 /* compute the epoll_wait() timeout */
Willy Tarreau4f60f162007-04-08 16:39:58 +0200113
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100114 if (fd_nbspec || run_queue || signal_queue_len) {
115 /* Maybe we still have events in the spec list, or there are
116 * some tasks left pending in the run_queue, so we must not
117 * wait in epoll() otherwise we would delay their delivery by
118 * the next timeout.
119 */
Willy Tarreaubdefc512007-05-14 02:02:04 +0200120 wait_time = 0;
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100121 }
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200122 else {
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100123 if (!exp)
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200124 wait_time = MAX_DELAY_MS;
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100125 else if (tick_is_expired(exp, now_ms))
126 wait_time = 0;
127 else {
128 wait_time = TICKS_TO_MS(tick_remain(now_ms, exp)) + 1;
129 if (wait_time > MAX_DELAY_MS)
130 wait_time = MAX_DELAY_MS;
131 }
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200132 }
Willy Tarreaud825eef2007-05-12 22:35:00 +0200133
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100134 /* now let's wait for polled events */
135
Willy Tarreau45a12512011-09-10 16:56:42 +0200136 gettimeofday(&before_poll, NULL);
Willy Tarreaucf181c92013-01-18 15:22:41 +0100137 status = epoll_wait(epoll_fd, epoll_events, global.tune.maxpollevents, wait_time);
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200138 tv_update_date(wait_time, status);
Willy Tarreau45a12512011-09-10 16:56:42 +0200139 measure_idle();
Willy Tarreau4f60f162007-04-08 16:39:58 +0200140
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100141 /* process polled events */
142
Willy Tarreau4f60f162007-04-08 16:39:58 +0200143 for (count = 0; count < status; count++) {
Willy Tarreau1c07b072013-01-07 16:19:18 +0100144 unsigned int n;
145 unsigned int e = epoll_events[count].events;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200146 fd = epoll_events[count].data.fd;
147
Willy Tarreau076be252012-07-06 16:02:29 +0200148 if (!fdtab[fd].owner)
149 continue;
150
Willy Tarreau491c4982012-07-06 11:16:01 +0200151 /* it looks complicated but gcc can optimize it away when constants
Willy Tarreau462c7202012-12-13 22:26:37 +0100152 * have same values... In fact it depends on gcc :-(
Willy Tarreau491c4982012-07-06 11:16:01 +0200153 */
154 fdtab[fd].ev &= FD_POLL_STICKY;
Willy Tarreau462c7202012-12-13 22:26:37 +0100155 if (EPOLLIN == FD_POLL_IN && EPOLLOUT == FD_POLL_OUT &&
156 EPOLLPRI == FD_POLL_PRI && EPOLLERR == FD_POLL_ERR &&
157 EPOLLHUP == FD_POLL_HUP) {
Willy Tarreau6320c3c2012-12-13 23:52:58 +0100158 n = e & (EPOLLIN|EPOLLOUT|EPOLLPRI|EPOLLERR|EPOLLHUP);
Willy Tarreau462c7202012-12-13 22:26:37 +0100159 }
160 else {
Willy Tarreau6320c3c2012-12-13 23:52:58 +0100161 n = ((e & EPOLLIN ) ? FD_POLL_IN : 0) |
Willy Tarreau462c7202012-12-13 22:26:37 +0100162 ((e & EPOLLPRI) ? FD_POLL_PRI : 0) |
163 ((e & EPOLLOUT) ? FD_POLL_OUT : 0) |
164 ((e & EPOLLERR) ? FD_POLL_ERR : 0) |
165 ((e & EPOLLHUP) ? FD_POLL_HUP : 0);
166 }
Willy Tarreau491c4982012-07-06 11:16:01 +0200167
Willy Tarreau1c07b072013-01-07 16:19:18 +0100168 /* always remap RDHUP to HUP as they're used similarly */
169 if (e & EPOLLRDHUP)
170 n |= FD_POLL_HUP;
171
Willy Tarreau6320c3c2012-12-13 23:52:58 +0100172 if (!n)
173 continue;
174
175 fdtab[fd].ev |= n;
176
177 if (fdtab[fd].iocb) {
Willy Tarreaufb5470d2012-12-14 00:02:33 +0100178 int new_updt, old_updt;
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100179
180 /* Mark the events as speculative before processing
181 * them so that if nothing can be done we don't need
182 * to poll again.
183 */
Willy Tarreau26d7cfc2012-12-07 00:09:43 +0100184 if (fdtab[fd].ev & FD_POLL_IN)
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100185 fd_ev_set(fd, DIR_RD);
186
Willy Tarreau26d7cfc2012-12-07 00:09:43 +0100187 if (fdtab[fd].ev & FD_POLL_OUT)
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100188 fd_ev_set(fd, DIR_WR);
189
Willy Tarreau15a4dec2014-01-20 11:09:39 +0100190 if (fdtab[fd].cache) {
Willy Tarreau39ebef82012-12-14 00:17:03 +0100191 /* This fd was already scheduled for being called as a speculative I/O */
192 continue;
193 }
194
Willy Tarreaufb5470d2012-12-14 00:02:33 +0100195 /* Save number of updates to detect creation of new FDs. */
196 old_updt = fd_nbupdt;
Willy Tarreau9845e752012-07-06 11:44:28 +0200197 fdtab[fd].iocb(fd);
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100198
199 /* One or more fd might have been created during the iocb().
200 * This mainly happens with new incoming connections that have
201 * just been accepted, so we'd like to process them immediately
202 * for better efficiency. Second benefit, if at the end the fds
203 * are disabled again, we can safely destroy their update entry
204 * to reduce the scope of later scans. This is the reason we
205 * scan the new entries backwards.
206 */
207
208 for (new_updt = fd_nbupdt; new_updt > old_updt; new_updt--) {
209 fd = fd_updt[new_updt - 1];
210 if (!fdtab[fd].new)
211 continue;
212
213 fdtab[fd].new = 0;
214 fdtab[fd].ev &= FD_POLL_STICKY;
215
Willy Tarreau69a41fa2014-01-20 11:02:59 +0100216 if ((fdtab[fd].state & FD_EV_STATUS_R) == FD_EV_ACTIVE_R)
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100217 fdtab[fd].ev |= FD_POLL_IN;
218
Willy Tarreau69a41fa2014-01-20 11:02:59 +0100219 if ((fdtab[fd].state & FD_EV_STATUS_W) == FD_EV_ACTIVE_W)
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100220 fdtab[fd].ev |= FD_POLL_OUT;
221
222 if (fdtab[fd].ev && fdtab[fd].iocb && fdtab[fd].owner)
223 fdtab[fd].iocb(fd);
224
225 /* we can remove this update entry if it's the last one and is
226 * unused, otherwise we don't touch anything.
227 */
Willy Tarreau69a41fa2014-01-20 11:02:59 +0100228 if (new_updt == fd_nbupdt && fdtab[fd].state == 0) {
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100229 fdtab[fd].updated = 0;
230 fd_nbupdt--;
231 }
232 }
233 }
Willy Tarreau4f60f162007-04-08 16:39:58 +0200234 }
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100235
236 /* the caller will take care of speculative events */
Willy Tarreau4f60f162007-04-08 16:39:58 +0200237}
238
239/*
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100240 * Initialization of the speculative epoll() poller.
Willy Tarreaue54e9172007-04-09 09:23:31 +0200241 * Returns 0 in case of failure, non-zero in case of success. If it fails, it
242 * disables the poller by setting its pref to 0.
243 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200244REGPRM1 static int _do_init(struct poller *p)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200245{
Willy Tarreaue54e9172007-04-09 09:23:31 +0200246 p->private = NULL;
Willy Tarreaue54e9172007-04-09 09:23:31 +0200247
248 epoll_fd = epoll_create(global.maxsock + 1);
249 if (epoll_fd < 0)
250 goto fail_fd;
251
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100252 /* See comments at the top of the file about this formula. */
253 absmaxevents = MAX(global.tune.maxpollevents, global.maxsock);
Willy Tarreaue54e9172007-04-09 09:23:31 +0200254 epoll_events = (struct epoll_event*)
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100255 calloc(1, sizeof(struct epoll_event) * absmaxevents);
Willy Tarreaue54e9172007-04-09 09:23:31 +0200256
257 if (epoll_events == NULL)
258 goto fail_ee;
259
Willy Tarreaue54e9172007-04-09 09:23:31 +0200260 return 1;
261
Willy Tarreaue54e9172007-04-09 09:23:31 +0200262 fail_ee:
263 close(epoll_fd);
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200264 epoll_fd = -1;
Willy Tarreaue54e9172007-04-09 09:23:31 +0200265 fail_fd:
266 p->pref = 0;
267 return 0;
268}
269
270/*
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100271 * Termination of the speculative epoll() poller.
Willy Tarreaue54e9172007-04-09 09:23:31 +0200272 * Memory is released and the poller is marked as unselectable.
273 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200274REGPRM1 static void _do_term(struct poller *p)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200275{
Willy Tarreaua534fea2008-08-03 12:19:50 +0200276 free(epoll_events);
Willy Tarreaue54e9172007-04-09 09:23:31 +0200277
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200278 if (epoll_fd >= 0) {
279 close(epoll_fd);
280 epoll_fd = -1;
281 }
Willy Tarreaue54e9172007-04-09 09:23:31 +0200282
Willy Tarreau58094f22007-04-10 01:33:20 +0200283 epoll_events = NULL;
Willy Tarreaue54e9172007-04-09 09:23:31 +0200284 p->private = NULL;
285 p->pref = 0;
286}
287
288/*
Willy Tarreau2ff76222007-04-09 19:29:56 +0200289 * Check that the poller works.
290 * Returns 1 if OK, otherwise 0.
291 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200292REGPRM1 static int _do_test(struct poller *p)
Willy Tarreau2ff76222007-04-09 19:29:56 +0200293{
294 int fd;
295
296 fd = epoll_create(global.maxsock + 1);
297 if (fd < 0)
298 return 0;
299 close(fd);
300 return 1;
301}
302
303/*
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200304 * Recreate the epoll file descriptor after a fork(). Returns 1 if OK,
305 * otherwise 0. It will ensure that all processes will not share their
306 * epoll_fd. Some side effects were encountered because of this, such
307 * as epoll_wait() returning an FD which was previously deleted.
308 */
309REGPRM1 static int _do_fork(struct poller *p)
310{
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200311 if (epoll_fd >= 0)
312 close(epoll_fd);
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200313 epoll_fd = epoll_create(global.maxsock + 1);
314 if (epoll_fd < 0)
315 return 0;
316 return 1;
317}
318
319/*
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200320 * It is a constructor, which means that it will automatically be called before
321 * main(). This is GCC-specific but it works at least since 2.95.
322 * Special care must be taken so that it does not need any uninitialized data.
Willy Tarreau4f60f162007-04-08 16:39:58 +0200323 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200324__attribute__((constructor))
325static void _do_register(void)
Willy Tarreau4f60f162007-04-08 16:39:58 +0200326{
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200327 struct poller *p;
328
329 if (nbpollers >= MAX_POLLERS)
330 return;
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200331
332 epoll_fd = -1;
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200333 p = &pollers[nbpollers++];
334
Willy Tarreau4f60f162007-04-08 16:39:58 +0200335 p->name = "epoll";
336 p->pref = 300;
337 p->private = NULL;
338
Willy Tarreau70c6fd82012-11-11 21:02:34 +0100339 p->clo = NULL;
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200340 p->test = _do_test;
341 p->init = _do_init;
342 p->term = _do_term;
343 p->poll = _do_poll;
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200344 p->fork = _do_fork;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200345}
346
347
348/*
349 * Local variables:
350 * c-indent-level: 8
351 * c-basic-offset: 8
352 * End:
353 */