blob: 924c083b251a5091a71b8b7f1eed1f0b974e60aa [file] [log] [blame]
Willy Tarreau4f60f162007-04-08 16:39:58 +02001/*
Willy Tarreaue9f49e72012-11-11 17:42:00 +01002 * FD polling functions for Linux epoll
Willy Tarreau4f60f162007-04-08 16:39:58 +02003 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +01004 * Copyright 2000-2014 Willy Tarreau <w@1wt.eu>
Willy Tarreau4f60f162007-04-08 16:39:58 +02005 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
Willy Tarreau4f60f162007-04-08 16:39:58 +020010 */
11
12#include <unistd.h>
13#include <sys/time.h>
14#include <sys/types.h>
15
16#include <common/compat.h>
17#include <common/config.h>
Willy Tarreaue9f49e72012-11-11 17:42:00 +010018#include <common/debug.h>
Willy Tarreau43d8fb22011-08-22 17:12:02 +020019#include <common/epoll.h>
Willy Tarreau58094f22007-04-10 01:33:20 +020020#include <common/standard.h>
Willy Tarreau0c303ee2008-07-07 00:09:58 +020021#include <common/ticks.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020022#include <common/time.h>
Willy Tarreau1db37712007-06-03 17:16:49 +020023#include <common/tools.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020024
Willy Tarreau4f60f162007-04-08 16:39:58 +020025#include <types/global.h>
26
Willy Tarreaue9f49e72012-11-11 17:42:00 +010027#include <proto/fd.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020028
Willy Tarreau58094f22007-04-10 01:33:20 +020029
Willy Tarreau4f60f162007-04-08 16:39:58 +020030/* private data */
Christopher Fauletd4604ad2017-05-29 10:40:41 +020031static THREAD_LOCAL struct epoll_event *epoll_events = NULL;
Willy Tarreaud9e7e362018-01-18 19:16:02 +010032static int epoll_fd[MAX_THREADS]; // per-thread epoll_fd
Willy Tarreau4f60f162007-04-08 16:39:58 +020033
Willy Tarreau58094f22007-04-10 01:33:20 +020034/* This structure may be used for any purpose. Warning! do not use it in
35 * recursive functions !
36 */
Christopher Fauletd4604ad2017-05-29 10:40:41 +020037static THREAD_LOCAL struct epoll_event ev;
Willy Tarreau58094f22007-04-10 01:33:20 +020038
Willy Tarreau1c07b072013-01-07 16:19:18 +010039#ifndef EPOLLRDHUP
40/* EPOLLRDHUP was defined late in libc, and it appeared in kernel 2.6.17 */
41#define EPOLLRDHUP 0x2000
42#endif
43
Willy Tarreau4f60f162007-04-08 16:39:58 +020044/*
Conrad Hoffmann041751c2014-05-20 14:28:24 +020045 * Immediately remove file descriptor from epoll set upon close.
46 * Since we forked, some fds share inodes with the other process, and epoll may
47 * send us events even though this process closed the fd (see man 7 epoll,
48 * "Questions and answers", Q 6).
49 */
50REGPRM1 static void __fd_clo(int fd)
51{
Willy Tarreaud9e7e362018-01-18 19:16:02 +010052 if (unlikely(fdtab[fd].cloned)) {
Willy Tarreau49795922018-01-31 09:49:29 +010053 unsigned long m = fdtab[fd].polled_mask;
Willy Tarreaud9e7e362018-01-18 19:16:02 +010054 int i;
55
56 for (i = global.nbthread - 1; i >= 0; i--)
57 if (m & (1UL << i))
58 epoll_ctl(epoll_fd[i], EPOLL_CTL_DEL, fd, &ev);
59 }
Conrad Hoffmann041751c2014-05-20 14:28:24 +020060}
61
62/*
Willy Tarreauf817e9f2014-01-10 16:58:45 +010063 * Linux epoll() poller
Willy Tarreau4f60f162007-04-08 16:39:58 +020064 */
Willy Tarreaue9f49e72012-11-11 17:42:00 +010065REGPRM2 static void _do_poll(struct poller *p, int exp)
Willy Tarreau4f60f162007-04-08 16:39:58 +020066{
Willy Tarreaue9f49e72012-11-11 17:42:00 +010067 int status, eo, en;
68 int fd, opcode;
69 int count;
70 int updt_idx;
71 int wait_time;
Willy Tarreau58094f22007-04-10 01:33:20 +020072
Willy Tarreau5be2f352014-11-19 19:43:05 +010073 /* first, scan the update list to find polling changes */
Willy Tarreaue9f49e72012-11-11 17:42:00 +010074 for (updt_idx = 0; updt_idx < fd_nbupdt; updt_idx++) {
75 fd = fd_updt[updt_idx];
Willy Tarreau58094f22007-04-10 01:33:20 +020076
Willy Tarreaud80cb4e2018-01-20 19:30:13 +010077 if (!fdtab[fd].owner) {
78 activity[tid].poll_drop++;
Willy Tarreauf817e9f2014-01-10 16:58:45 +010079 continue;
Willy Tarreaud80cb4e2018-01-20 19:30:13 +010080 }
Willy Tarreau58094f22007-04-10 01:33:20 +020081
Christopher Faulet2a944ee2017-11-07 10:42:54 +010082 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreauebc78d72018-01-20 23:53:50 +010083 fdtab[fd].update_mask &= ~tid_bit;
Willy Tarreau25002d22014-01-25 10:32:56 +010084 eo = fdtab[fd].state;
85 en = fd_compute_new_polled_status(eo);
Christopher Fauletd4604ad2017-05-29 10:40:41 +020086 fdtab[fd].state = en;
Christopher Faulet2a944ee2017-11-07 10:42:54 +010087 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau58094f22007-04-10 01:33:20 +020088
Willy Tarreaud9e7e362018-01-18 19:16:02 +010089 if (fdtab[fd].polled_mask & tid_bit) {
90 if (!(fdtab[fd].thread_mask & tid_bit) || !(en & FD_EV_POLLED_RW)) {
Willy Tarreauf817e9f2014-01-10 16:58:45 +010091 /* fd removed from poll list */
92 opcode = EPOLL_CTL_DEL;
Willy Tarreaud9e7e362018-01-18 19:16:02 +010093 HA_ATOMIC_AND(&fdtab[fd].polled_mask, ~tid_bit);
Willy Tarreaue9f49e72012-11-11 17:42:00 +010094 }
Willy Tarreauf817e9f2014-01-10 16:58:45 +010095 else {
96 /* fd status changed */
97 opcode = EPOLL_CTL_MOD;
98 }
Willy Tarreaud9e7e362018-01-18 19:16:02 +010099 }
100 else if ((fdtab[fd].thread_mask & tid_bit) && (en & FD_EV_POLLED_RW)) {
101 /* new fd in the poll list */
102 opcode = EPOLL_CTL_ADD;
103 HA_ATOMIC_OR(&fdtab[fd].polled_mask, tid_bit);
104 }
105 else {
106 continue;
107 }
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100108
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100109 /* construct the epoll events based on new state */
110 ev.events = 0;
111 if (en & FD_EV_POLLED_R)
112 ev.events |= EPOLLIN | EPOLLRDHUP;
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100113
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100114 if (en & FD_EV_POLLED_W)
115 ev.events |= EPOLLOUT;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200116
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100117 ev.data.fd = fd;
118 epoll_ctl(epoll_fd[tid], opcode, fd, &ev);
Willy Tarreau58094f22007-04-10 01:33:20 +0200119 }
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100120 fd_nbupdt = 0;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200121
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100122 /* compute the epoll_wait() timeout */
Willy Tarreau10146c92015-04-13 20:44:19 +0200123 if (!exp)
124 wait_time = MAX_DELAY_MS;
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100125 else if (tick_is_expired(exp, now_ms)) {
126 activity[tid].poll_exp++;
Willy Tarreaubdefc512007-05-14 02:02:04 +0200127 wait_time = 0;
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100128 }
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200129 else {
Willy Tarreau10146c92015-04-13 20:44:19 +0200130 wait_time = TICKS_TO_MS(tick_remain(now_ms, exp)) + 1;
131 if (wait_time > MAX_DELAY_MS)
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200132 wait_time = MAX_DELAY_MS;
133 }
Willy Tarreaud825eef2007-05-12 22:35:00 +0200134
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100135 /* now let's wait for polled events */
136
Willy Tarreau45a12512011-09-10 16:56:42 +0200137 gettimeofday(&before_poll, NULL);
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100138 status = epoll_wait(epoll_fd[tid], epoll_events, global.tune.maxpollevents, wait_time);
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200139 tv_update_date(wait_time, status);
Willy Tarreau45a12512011-09-10 16:56:42 +0200140 measure_idle();
Willy Tarreau4f60f162007-04-08 16:39:58 +0200141
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100142 /* process polled events */
143
Willy Tarreau4f60f162007-04-08 16:39:58 +0200144 for (count = 0; count < status; count++) {
Willy Tarreau1c07b072013-01-07 16:19:18 +0100145 unsigned int n;
146 unsigned int e = epoll_events[count].events;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200147 fd = epoll_events[count].data.fd;
148
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100149 if (!fdtab[fd].owner) {
150 activity[tid].poll_dead++;
151 continue;
152 }
153
154 if (!(fdtab[fd].thread_mask & tid_bit)) {
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100155 /* FD has been migrated */
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100156 activity[tid].poll_skip++;
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100157 epoll_ctl(epoll_fd[tid], EPOLL_CTL_DEL, fd, &ev);
158 HA_ATOMIC_AND(&fdtab[fd].polled_mask, ~tid_bit);
Willy Tarreau076be252012-07-06 16:02:29 +0200159 continue;
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100160 }
Willy Tarreau076be252012-07-06 16:02:29 +0200161
Willy Tarreau491c4982012-07-06 11:16:01 +0200162 /* it looks complicated but gcc can optimize it away when constants
Willy Tarreau462c7202012-12-13 22:26:37 +0100163 * have same values... In fact it depends on gcc :-(
Willy Tarreau491c4982012-07-06 11:16:01 +0200164 */
Willy Tarreau462c7202012-12-13 22:26:37 +0100165 if (EPOLLIN == FD_POLL_IN && EPOLLOUT == FD_POLL_OUT &&
166 EPOLLPRI == FD_POLL_PRI && EPOLLERR == FD_POLL_ERR &&
167 EPOLLHUP == FD_POLL_HUP) {
Willy Tarreau6320c3c2012-12-13 23:52:58 +0100168 n = e & (EPOLLIN|EPOLLOUT|EPOLLPRI|EPOLLERR|EPOLLHUP);
Willy Tarreau462c7202012-12-13 22:26:37 +0100169 }
170 else {
Willy Tarreau6320c3c2012-12-13 23:52:58 +0100171 n = ((e & EPOLLIN ) ? FD_POLL_IN : 0) |
Willy Tarreau462c7202012-12-13 22:26:37 +0100172 ((e & EPOLLPRI) ? FD_POLL_PRI : 0) |
173 ((e & EPOLLOUT) ? FD_POLL_OUT : 0) |
174 ((e & EPOLLERR) ? FD_POLL_ERR : 0) |
175 ((e & EPOLLHUP) ? FD_POLL_HUP : 0);
176 }
Willy Tarreau491c4982012-07-06 11:16:01 +0200177
Willy Tarreau1c07b072013-01-07 16:19:18 +0100178 /* always remap RDHUP to HUP as they're used similarly */
Willy Tarreau5a767692017-03-13 11:38:28 +0100179 if (e & EPOLLRDHUP) {
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200180 HA_ATOMIC_OR(&cur_poller.flags, HAP_POLL_F_RDHUP);
Willy Tarreau1c07b072013-01-07 16:19:18 +0100181 n |= FD_POLL_HUP;
Willy Tarreau5a767692017-03-13 11:38:28 +0100182 }
Christopher Fauletab62f512017-08-30 10:34:36 +0200183 fd_update_events(fd, n);
Willy Tarreau4f60f162007-04-08 16:39:58 +0200184 }
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100185 /* the caller will take care of cached events */
Willy Tarreau4f60f162007-04-08 16:39:58 +0200186}
187
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200188static int init_epoll_per_thread()
189{
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100190 int fd;
191
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200192 epoll_events = calloc(1, sizeof(struct epoll_event) * global.tune.maxpollevents);
193 if (epoll_events == NULL)
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100194 goto fail_alloc;
195
Christopher Faulet3e805ed2018-01-25 16:18:09 +0100196 if (MAX_THREADS > 1 && tid) {
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100197 epoll_fd[tid] = epoll_create(global.maxsock + 1);
198 if (epoll_fd[tid] < 0)
199 goto fail_fd;
200 }
201
202 /* we may have to unregister some events initially registered on the
203 * original fd when it was alone, and/or to register events on the new
204 * fd for this thread. Let's just mark them as updated, the poller will
205 * do the rest.
206 */
Willy Tarreauce036bc2018-01-29 14:58:02 +0100207 for (fd = 0; fd < global.maxsock; fd++)
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100208 updt_fd_polling(fd);
209
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200210 return 1;
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100211 fail_fd:
212 free(epoll_events);
213 fail_alloc:
214 return 0;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200215}
216
217static void deinit_epoll_per_thread()
218{
Christopher Faulet3e805ed2018-01-25 16:18:09 +0100219 if (MAX_THREADS > 1 && tid)
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100220 close(epoll_fd[tid]);
221
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200222 free(epoll_events);
Christopher Fauletcd7879a2017-10-27 13:53:47 +0200223 epoll_events = NULL;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200224}
225
Willy Tarreau4f60f162007-04-08 16:39:58 +0200226/*
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100227 * Initialization of the epoll() poller.
Willy Tarreaue54e9172007-04-09 09:23:31 +0200228 * Returns 0 in case of failure, non-zero in case of success. If it fails, it
229 * disables the poller by setting its pref to 0.
230 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200231REGPRM1 static int _do_init(struct poller *p)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200232{
Willy Tarreaue54e9172007-04-09 09:23:31 +0200233 p->private = NULL;
Willy Tarreaue54e9172007-04-09 09:23:31 +0200234
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100235 epoll_fd[tid] = epoll_create(global.maxsock + 1);
236 if (epoll_fd[tid] < 0)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200237 goto fail_fd;
238
Christopher Fauletcd7879a2017-10-27 13:53:47 +0200239 hap_register_per_thread_init(init_epoll_per_thread);
240 hap_register_per_thread_deinit(deinit_epoll_per_thread);
Willy Tarreaue54e9172007-04-09 09:23:31 +0200241
Willy Tarreaue54e9172007-04-09 09:23:31 +0200242 return 1;
243
Willy Tarreaue54e9172007-04-09 09:23:31 +0200244 fail_fd:
245 p->pref = 0;
246 return 0;
247}
248
249/*
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100250 * Termination of the epoll() poller.
Willy Tarreaue54e9172007-04-09 09:23:31 +0200251 * Memory is released and the poller is marked as unselectable.
252 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200253REGPRM1 static void _do_term(struct poller *p)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200254{
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100255 if (epoll_fd[tid] >= 0) {
256 close(epoll_fd[tid]);
257 epoll_fd[tid] = -1;
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200258 }
Willy Tarreaue54e9172007-04-09 09:23:31 +0200259
260 p->private = NULL;
261 p->pref = 0;
262}
263
264/*
Willy Tarreau2ff76222007-04-09 19:29:56 +0200265 * Check that the poller works.
266 * Returns 1 if OK, otherwise 0.
267 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200268REGPRM1 static int _do_test(struct poller *p)
Willy Tarreau2ff76222007-04-09 19:29:56 +0200269{
270 int fd;
271
272 fd = epoll_create(global.maxsock + 1);
273 if (fd < 0)
274 return 0;
275 close(fd);
276 return 1;
277}
278
279/*
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200280 * Recreate the epoll file descriptor after a fork(). Returns 1 if OK,
281 * otherwise 0. It will ensure that all processes will not share their
282 * epoll_fd. Some side effects were encountered because of this, such
283 * as epoll_wait() returning an FD which was previously deleted.
284 */
285REGPRM1 static int _do_fork(struct poller *p)
286{
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100287 if (epoll_fd[tid] >= 0)
288 close(epoll_fd[tid]);
289 epoll_fd[tid] = epoll_create(global.maxsock + 1);
290 if (epoll_fd[tid] < 0)
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200291 return 0;
292 return 1;
293}
294
295/*
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200296 * It is a constructor, which means that it will automatically be called before
297 * main(). This is GCC-specific but it works at least since 2.95.
298 * Special care must be taken so that it does not need any uninitialized data.
Willy Tarreau4f60f162007-04-08 16:39:58 +0200299 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200300__attribute__((constructor))
301static void _do_register(void)
Willy Tarreau4f60f162007-04-08 16:39:58 +0200302{
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200303 struct poller *p;
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100304 int i;
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200305
306 if (nbpollers >= MAX_POLLERS)
307 return;
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200308
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100309 for (i = 0; i < MAX_THREADS; i++)
310 epoll_fd[i] = -1;
311
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200312 p = &pollers[nbpollers++];
313
Willy Tarreau4f60f162007-04-08 16:39:58 +0200314 p->name = "epoll";
315 p->pref = 300;
Willy Tarreau5a767692017-03-13 11:38:28 +0100316 p->flags = 0;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200317 p->private = NULL;
318
Conrad Hoffmann041751c2014-05-20 14:28:24 +0200319 p->clo = __fd_clo;
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200320 p->test = _do_test;
321 p->init = _do_init;
322 p->term = _do_term;
323 p->poll = _do_poll;
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200324 p->fork = _do_fork;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200325}
326
327
328/*
329 * Local variables:
330 * c-indent-level: 8
331 * c-basic-offset: 8
332 * End:
333 */