blob: a8e57973fbe0047b2e73df0950ad72247550b548 [file] [log] [blame]
Willy Tarreau4f60f162007-04-08 16:39:58 +02001/*
Willy Tarreaue9f49e72012-11-11 17:42:00 +01002 * FD polling functions for Linux epoll
Willy Tarreau4f60f162007-04-08 16:39:58 +02003 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +01004 * Copyright 2000-2014 Willy Tarreau <w@1wt.eu>
Willy Tarreau4f60f162007-04-08 16:39:58 +02005 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
Willy Tarreau4f60f162007-04-08 16:39:58 +020010 */
11
12#include <unistd.h>
13#include <sys/time.h>
14#include <sys/types.h>
15
16#include <common/compat.h>
17#include <common/config.h>
Willy Tarreaue9f49e72012-11-11 17:42:00 +010018#include <common/debug.h>
Willy Tarreau43d8fb22011-08-22 17:12:02 +020019#include <common/epoll.h>
Willy Tarreau58094f22007-04-10 01:33:20 +020020#include <common/standard.h>
Willy Tarreau0c303ee2008-07-07 00:09:58 +020021#include <common/ticks.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020022#include <common/time.h>
Willy Tarreau1db37712007-06-03 17:16:49 +020023#include <common/tools.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020024
Willy Tarreau4f60f162007-04-08 16:39:58 +020025#include <types/global.h>
26
Willy Tarreaue9f49e72012-11-11 17:42:00 +010027#include <proto/fd.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020028
Willy Tarreau58094f22007-04-10 01:33:20 +020029
Willy Tarreau4f60f162007-04-08 16:39:58 +020030/* private data */
Christopher Fauletd4604ad2017-05-29 10:40:41 +020031static THREAD_LOCAL struct epoll_event *epoll_events = NULL;
Willy Tarreaud9e7e362018-01-18 19:16:02 +010032static int epoll_fd[MAX_THREADS]; // per-thread epoll_fd
Willy Tarreau4f60f162007-04-08 16:39:58 +020033
Willy Tarreau58094f22007-04-10 01:33:20 +020034/* This structure may be used for any purpose. Warning! do not use it in
35 * recursive functions !
36 */
Christopher Fauletd4604ad2017-05-29 10:40:41 +020037static THREAD_LOCAL struct epoll_event ev;
Willy Tarreau58094f22007-04-10 01:33:20 +020038
Willy Tarreau1c07b072013-01-07 16:19:18 +010039#ifndef EPOLLRDHUP
40/* EPOLLRDHUP was defined late in libc, and it appeared in kernel 2.6.17 */
41#define EPOLLRDHUP 0x2000
42#endif
43
Willy Tarreau4f60f162007-04-08 16:39:58 +020044/*
Conrad Hoffmann041751c2014-05-20 14:28:24 +020045 * Immediately remove file descriptor from epoll set upon close.
46 * Since we forked, some fds share inodes with the other process, and epoll may
47 * send us events even though this process closed the fd (see man 7 epoll,
48 * "Questions and answers", Q 6).
49 */
50REGPRM1 static void __fd_clo(int fd)
51{
Willy Tarreaud9e7e362018-01-18 19:16:02 +010052 if (unlikely(fdtab[fd].cloned)) {
Willy Tarreau49795922018-01-31 09:49:29 +010053 unsigned long m = fdtab[fd].polled_mask;
Willy Tarreaud9e7e362018-01-18 19:16:02 +010054 int i;
55
56 for (i = global.nbthread - 1; i >= 0; i--)
57 if (m & (1UL << i))
58 epoll_ctl(epoll_fd[i], EPOLL_CTL_DEL, fd, &ev);
59 }
Conrad Hoffmann041751c2014-05-20 14:28:24 +020060}
61
62/*
Willy Tarreauf817e9f2014-01-10 16:58:45 +010063 * Linux epoll() poller
Willy Tarreau4f60f162007-04-08 16:39:58 +020064 */
Willy Tarreaue9f49e72012-11-11 17:42:00 +010065REGPRM2 static void _do_poll(struct poller *p, int exp)
Willy Tarreau4f60f162007-04-08 16:39:58 +020066{
Willy Tarreau038e54c2018-01-17 21:51:21 +010067 int status, en;
Willy Tarreaue9f49e72012-11-11 17:42:00 +010068 int fd, opcode;
69 int count;
70 int updt_idx;
71 int wait_time;
Willy Tarreau58094f22007-04-10 01:33:20 +020072
Willy Tarreau5be2f352014-11-19 19:43:05 +010073 /* first, scan the update list to find polling changes */
Willy Tarreaue9f49e72012-11-11 17:42:00 +010074 for (updt_idx = 0; updt_idx < fd_nbupdt; updt_idx++) {
75 fd = fd_updt[updt_idx];
Willy Tarreau58094f22007-04-10 01:33:20 +020076
Olivier Houchard8ef1a6b2018-04-03 19:06:18 +020077 HA_ATOMIC_AND(&fdtab[fd].update_mask, ~tid_bit);
Willy Tarreaud80cb4e2018-01-20 19:30:13 +010078 if (!fdtab[fd].owner) {
79 activity[tid].poll_drop++;
Willy Tarreauf817e9f2014-01-10 16:58:45 +010080 continue;
Willy Tarreaud80cb4e2018-01-20 19:30:13 +010081 }
Willy Tarreau58094f22007-04-10 01:33:20 +020082
Willy Tarreau038e54c2018-01-17 21:51:21 +010083 en = fdtab[fd].state;
Willy Tarreau58094f22007-04-10 01:33:20 +020084
Willy Tarreaud9e7e362018-01-18 19:16:02 +010085 if (fdtab[fd].polled_mask & tid_bit) {
86 if (!(fdtab[fd].thread_mask & tid_bit) || !(en & FD_EV_POLLED_RW)) {
Willy Tarreauf817e9f2014-01-10 16:58:45 +010087 /* fd removed from poll list */
88 opcode = EPOLL_CTL_DEL;
Willy Tarreaud9e7e362018-01-18 19:16:02 +010089 HA_ATOMIC_AND(&fdtab[fd].polled_mask, ~tid_bit);
Willy Tarreaue9f49e72012-11-11 17:42:00 +010090 }
Willy Tarreauf817e9f2014-01-10 16:58:45 +010091 else {
92 /* fd status changed */
93 opcode = EPOLL_CTL_MOD;
94 }
Willy Tarreaud9e7e362018-01-18 19:16:02 +010095 }
96 else if ((fdtab[fd].thread_mask & tid_bit) && (en & FD_EV_POLLED_RW)) {
97 /* new fd in the poll list */
98 opcode = EPOLL_CTL_ADD;
99 HA_ATOMIC_OR(&fdtab[fd].polled_mask, tid_bit);
100 }
101 else {
102 continue;
103 }
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100104
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100105 /* construct the epoll events based on new state */
106 ev.events = 0;
107 if (en & FD_EV_POLLED_R)
108 ev.events |= EPOLLIN | EPOLLRDHUP;
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100109
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100110 if (en & FD_EV_POLLED_W)
111 ev.events |= EPOLLOUT;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200112
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100113 ev.data.fd = fd;
114 epoll_ctl(epoll_fd[tid], opcode, fd, &ev);
Willy Tarreau58094f22007-04-10 01:33:20 +0200115 }
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100116 fd_nbupdt = 0;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200117
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100118 /* compute the epoll_wait() timeout */
Willy Tarreau10146c92015-04-13 20:44:19 +0200119 if (!exp)
120 wait_time = MAX_DELAY_MS;
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100121 else if (tick_is_expired(exp, now_ms)) {
122 activity[tid].poll_exp++;
Willy Tarreaubdefc512007-05-14 02:02:04 +0200123 wait_time = 0;
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100124 }
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200125 else {
Willy Tarreau10146c92015-04-13 20:44:19 +0200126 wait_time = TICKS_TO_MS(tick_remain(now_ms, exp)) + 1;
127 if (wait_time > MAX_DELAY_MS)
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200128 wait_time = MAX_DELAY_MS;
129 }
Willy Tarreaud825eef2007-05-12 22:35:00 +0200130
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100131 /* now let's wait for polled events */
132
Willy Tarreau45a12512011-09-10 16:56:42 +0200133 gettimeofday(&before_poll, NULL);
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100134 status = epoll_wait(epoll_fd[tid], epoll_events, global.tune.maxpollevents, wait_time);
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200135 tv_update_date(wait_time, status);
Willy Tarreau45a12512011-09-10 16:56:42 +0200136 measure_idle();
Willy Tarreau4f60f162007-04-08 16:39:58 +0200137
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100138 /* process polled events */
139
Willy Tarreau4f60f162007-04-08 16:39:58 +0200140 for (count = 0; count < status; count++) {
Willy Tarreau1c07b072013-01-07 16:19:18 +0100141 unsigned int n;
142 unsigned int e = epoll_events[count].events;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200143 fd = epoll_events[count].data.fd;
144
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100145 if (!fdtab[fd].owner) {
146 activity[tid].poll_dead++;
147 continue;
148 }
149
150 if (!(fdtab[fd].thread_mask & tid_bit)) {
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100151 /* FD has been migrated */
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100152 activity[tid].poll_skip++;
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100153 epoll_ctl(epoll_fd[tid], EPOLL_CTL_DEL, fd, &ev);
154 HA_ATOMIC_AND(&fdtab[fd].polled_mask, ~tid_bit);
Willy Tarreau076be252012-07-06 16:02:29 +0200155 continue;
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100156 }
Willy Tarreau076be252012-07-06 16:02:29 +0200157
Willy Tarreau491c4982012-07-06 11:16:01 +0200158 /* it looks complicated but gcc can optimize it away when constants
Willy Tarreau462c7202012-12-13 22:26:37 +0100159 * have same values... In fact it depends on gcc :-(
Willy Tarreau491c4982012-07-06 11:16:01 +0200160 */
Willy Tarreau462c7202012-12-13 22:26:37 +0100161 if (EPOLLIN == FD_POLL_IN && EPOLLOUT == FD_POLL_OUT &&
162 EPOLLPRI == FD_POLL_PRI && EPOLLERR == FD_POLL_ERR &&
163 EPOLLHUP == FD_POLL_HUP) {
Willy Tarreau6320c3c2012-12-13 23:52:58 +0100164 n = e & (EPOLLIN|EPOLLOUT|EPOLLPRI|EPOLLERR|EPOLLHUP);
Willy Tarreau462c7202012-12-13 22:26:37 +0100165 }
166 else {
Willy Tarreau6320c3c2012-12-13 23:52:58 +0100167 n = ((e & EPOLLIN ) ? FD_POLL_IN : 0) |
Willy Tarreau462c7202012-12-13 22:26:37 +0100168 ((e & EPOLLPRI) ? FD_POLL_PRI : 0) |
169 ((e & EPOLLOUT) ? FD_POLL_OUT : 0) |
170 ((e & EPOLLERR) ? FD_POLL_ERR : 0) |
171 ((e & EPOLLHUP) ? FD_POLL_HUP : 0);
172 }
Willy Tarreau491c4982012-07-06 11:16:01 +0200173
Willy Tarreau1c07b072013-01-07 16:19:18 +0100174 /* always remap RDHUP to HUP as they're used similarly */
Willy Tarreau5a767692017-03-13 11:38:28 +0100175 if (e & EPOLLRDHUP) {
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200176 HA_ATOMIC_OR(&cur_poller.flags, HAP_POLL_F_RDHUP);
Willy Tarreau1c07b072013-01-07 16:19:18 +0100177 n |= FD_POLL_HUP;
Willy Tarreau5a767692017-03-13 11:38:28 +0100178 }
Christopher Fauletab62f512017-08-30 10:34:36 +0200179 fd_update_events(fd, n);
Willy Tarreau4f60f162007-04-08 16:39:58 +0200180 }
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100181 /* the caller will take care of cached events */
Willy Tarreau4f60f162007-04-08 16:39:58 +0200182}
183
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200184static int init_epoll_per_thread()
185{
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100186 int fd;
187
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200188 epoll_events = calloc(1, sizeof(struct epoll_event) * global.tune.maxpollevents);
189 if (epoll_events == NULL)
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100190 goto fail_alloc;
191
Christopher Faulet3e805ed2018-01-25 16:18:09 +0100192 if (MAX_THREADS > 1 && tid) {
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100193 epoll_fd[tid] = epoll_create(global.maxsock + 1);
194 if (epoll_fd[tid] < 0)
195 goto fail_fd;
196 }
197
198 /* we may have to unregister some events initially registered on the
199 * original fd when it was alone, and/or to register events on the new
200 * fd for this thread. Let's just mark them as updated, the poller will
201 * do the rest.
202 */
Willy Tarreauce036bc2018-01-29 14:58:02 +0100203 for (fd = 0; fd < global.maxsock; fd++)
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100204 updt_fd_polling(fd);
205
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200206 return 1;
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100207 fail_fd:
208 free(epoll_events);
209 fail_alloc:
210 return 0;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200211}
212
213static void deinit_epoll_per_thread()
214{
Christopher Faulet3e805ed2018-01-25 16:18:09 +0100215 if (MAX_THREADS > 1 && tid)
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100216 close(epoll_fd[tid]);
217
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200218 free(epoll_events);
Christopher Fauletcd7879a2017-10-27 13:53:47 +0200219 epoll_events = NULL;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200220}
221
Willy Tarreau4f60f162007-04-08 16:39:58 +0200222/*
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100223 * Initialization of the epoll() poller.
Willy Tarreaue54e9172007-04-09 09:23:31 +0200224 * Returns 0 in case of failure, non-zero in case of success. If it fails, it
225 * disables the poller by setting its pref to 0.
226 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200227REGPRM1 static int _do_init(struct poller *p)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200228{
Willy Tarreaue54e9172007-04-09 09:23:31 +0200229 p->private = NULL;
Willy Tarreaue54e9172007-04-09 09:23:31 +0200230
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100231 epoll_fd[tid] = epoll_create(global.maxsock + 1);
232 if (epoll_fd[tid] < 0)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200233 goto fail_fd;
234
Christopher Fauletcd7879a2017-10-27 13:53:47 +0200235 hap_register_per_thread_init(init_epoll_per_thread);
236 hap_register_per_thread_deinit(deinit_epoll_per_thread);
Willy Tarreaue54e9172007-04-09 09:23:31 +0200237
Willy Tarreaue54e9172007-04-09 09:23:31 +0200238 return 1;
239
Willy Tarreaue54e9172007-04-09 09:23:31 +0200240 fail_fd:
241 p->pref = 0;
242 return 0;
243}
244
245/*
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100246 * Termination of the epoll() poller.
Willy Tarreaue54e9172007-04-09 09:23:31 +0200247 * Memory is released and the poller is marked as unselectable.
248 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200249REGPRM1 static void _do_term(struct poller *p)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200250{
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100251 if (epoll_fd[tid] >= 0) {
252 close(epoll_fd[tid]);
253 epoll_fd[tid] = -1;
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200254 }
Willy Tarreaue54e9172007-04-09 09:23:31 +0200255
256 p->private = NULL;
257 p->pref = 0;
258}
259
260/*
Willy Tarreau2ff76222007-04-09 19:29:56 +0200261 * Check that the poller works.
262 * Returns 1 if OK, otherwise 0.
263 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200264REGPRM1 static int _do_test(struct poller *p)
Willy Tarreau2ff76222007-04-09 19:29:56 +0200265{
266 int fd;
267
268 fd = epoll_create(global.maxsock + 1);
269 if (fd < 0)
270 return 0;
271 close(fd);
272 return 1;
273}
274
275/*
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200276 * Recreate the epoll file descriptor after a fork(). Returns 1 if OK,
277 * otherwise 0. It will ensure that all processes will not share their
278 * epoll_fd. Some side effects were encountered because of this, such
279 * as epoll_wait() returning an FD which was previously deleted.
280 */
281REGPRM1 static int _do_fork(struct poller *p)
282{
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100283 if (epoll_fd[tid] >= 0)
284 close(epoll_fd[tid]);
285 epoll_fd[tid] = epoll_create(global.maxsock + 1);
286 if (epoll_fd[tid] < 0)
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200287 return 0;
288 return 1;
289}
290
291/*
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200292 * It is a constructor, which means that it will automatically be called before
293 * main(). This is GCC-specific but it works at least since 2.95.
294 * Special care must be taken so that it does not need any uninitialized data.
Willy Tarreau4f60f162007-04-08 16:39:58 +0200295 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200296__attribute__((constructor))
297static void _do_register(void)
Willy Tarreau4f60f162007-04-08 16:39:58 +0200298{
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200299 struct poller *p;
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100300 int i;
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200301
302 if (nbpollers >= MAX_POLLERS)
303 return;
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200304
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100305 for (i = 0; i < MAX_THREADS; i++)
306 epoll_fd[i] = -1;
307
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200308 p = &pollers[nbpollers++];
309
Willy Tarreau4f60f162007-04-08 16:39:58 +0200310 p->name = "epoll";
311 p->pref = 300;
Willy Tarreau5a767692017-03-13 11:38:28 +0100312 p->flags = 0;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200313 p->private = NULL;
314
Conrad Hoffmann041751c2014-05-20 14:28:24 +0200315 p->clo = __fd_clo;
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200316 p->test = _do_test;
317 p->init = _do_init;
318 p->term = _do_term;
319 p->poll = _do_poll;
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200320 p->fork = _do_fork;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200321}
322
323
324/*
325 * Local variables:
326 * c-indent-level: 8
327 * c-basic-offset: 8
328 * End:
329 */