blob: 635b8a5fe51e98b165048b33b2d1789910e50fd4 [file] [log] [blame]
Willy Tarreau4f60f162007-04-08 16:39:58 +02001/*
Willy Tarreaue9f49e72012-11-11 17:42:00 +01002 * FD polling functions for Linux epoll
Willy Tarreau4f60f162007-04-08 16:39:58 +02003 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +01004 * Copyright 2000-2014 Willy Tarreau <w@1wt.eu>
Willy Tarreau4f60f162007-04-08 16:39:58 +02005 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
Willy Tarreau4f60f162007-04-08 16:39:58 +020010 */
11
12#include <unistd.h>
13#include <sys/time.h>
14#include <sys/types.h>
15
16#include <common/compat.h>
17#include <common/config.h>
Willy Tarreaue9f49e72012-11-11 17:42:00 +010018#include <common/debug.h>
Willy Tarreau43d8fb22011-08-22 17:12:02 +020019#include <common/epoll.h>
Willy Tarreau58094f22007-04-10 01:33:20 +020020#include <common/standard.h>
Willy Tarreau0c303ee2008-07-07 00:09:58 +020021#include <common/ticks.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020022#include <common/time.h>
Willy Tarreau1db37712007-06-03 17:16:49 +020023#include <common/tools.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020024
Willy Tarreau4f60f162007-04-08 16:39:58 +020025#include <types/global.h>
26
Willy Tarreaue9f49e72012-11-11 17:42:00 +010027#include <proto/fd.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020028
Willy Tarreau58094f22007-04-10 01:33:20 +020029
Willy Tarreau4f60f162007-04-08 16:39:58 +020030/* private data */
Christopher Fauletd4604ad2017-05-29 10:40:41 +020031static THREAD_LOCAL struct epoll_event *epoll_events = NULL;
Willy Tarreaud9e7e362018-01-18 19:16:02 +010032static int epoll_fd[MAX_THREADS]; // per-thread epoll_fd
Willy Tarreau4f60f162007-04-08 16:39:58 +020033
Willy Tarreau58094f22007-04-10 01:33:20 +020034/* This structure may be used for any purpose. Warning! do not use it in
35 * recursive functions !
36 */
Christopher Fauletd4604ad2017-05-29 10:40:41 +020037static THREAD_LOCAL struct epoll_event ev;
Willy Tarreau58094f22007-04-10 01:33:20 +020038
Willy Tarreau1c07b072013-01-07 16:19:18 +010039#ifndef EPOLLRDHUP
40/* EPOLLRDHUP was defined late in libc, and it appeared in kernel 2.6.17 */
41#define EPOLLRDHUP 0x2000
42#endif
43
Willy Tarreau4f60f162007-04-08 16:39:58 +020044/*
Conrad Hoffmann041751c2014-05-20 14:28:24 +020045 * Immediately remove file descriptor from epoll set upon close.
46 * Since we forked, some fds share inodes with the other process, and epoll may
47 * send us events even though this process closed the fd (see man 7 epoll,
48 * "Questions and answers", Q 6).
49 */
50REGPRM1 static void __fd_clo(int fd)
51{
Willy Tarreaud9e7e362018-01-18 19:16:02 +010052 if (unlikely(fdtab[fd].cloned)) {
53 unsigned long m = fdtab[fd].thread_mask;
54 int i;
55
56 for (i = global.nbthread - 1; i >= 0; i--)
57 if (m & (1UL << i))
58 epoll_ctl(epoll_fd[i], EPOLL_CTL_DEL, fd, &ev);
59 }
Conrad Hoffmann041751c2014-05-20 14:28:24 +020060}
61
62/*
Willy Tarreauf817e9f2014-01-10 16:58:45 +010063 * Linux epoll() poller
Willy Tarreau4f60f162007-04-08 16:39:58 +020064 */
Willy Tarreaue9f49e72012-11-11 17:42:00 +010065REGPRM2 static void _do_poll(struct poller *p, int exp)
Willy Tarreau4f60f162007-04-08 16:39:58 +020066{
Willy Tarreaue9f49e72012-11-11 17:42:00 +010067 int status, eo, en;
68 int fd, opcode;
69 int count;
70 int updt_idx;
71 int wait_time;
Willy Tarreau58094f22007-04-10 01:33:20 +020072
Willy Tarreau5be2f352014-11-19 19:43:05 +010073 /* first, scan the update list to find polling changes */
Willy Tarreaue9f49e72012-11-11 17:42:00 +010074 for (updt_idx = 0; updt_idx < fd_nbupdt; updt_idx++) {
75 fd = fd_updt[updt_idx];
Willy Tarreau58094f22007-04-10 01:33:20 +020076
Willy Tarreaud80cb4e2018-01-20 19:30:13 +010077 if (!fdtab[fd].owner) {
78 activity[tid].poll_drop++;
Willy Tarreauf817e9f2014-01-10 16:58:45 +010079 continue;
Willy Tarreaud80cb4e2018-01-20 19:30:13 +010080 }
Willy Tarreau58094f22007-04-10 01:33:20 +020081
Christopher Faulet2a944ee2017-11-07 10:42:54 +010082 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreauebc78d72018-01-20 23:53:50 +010083 fdtab[fd].update_mask &= ~tid_bit;
Christopher Fauletd4604ad2017-05-29 10:40:41 +020084 fdtab[fd].new = 0;
85
Willy Tarreau25002d22014-01-25 10:32:56 +010086 eo = fdtab[fd].state;
87 en = fd_compute_new_polled_status(eo);
Christopher Fauletd4604ad2017-05-29 10:40:41 +020088 fdtab[fd].state = en;
Christopher Faulet2a944ee2017-11-07 10:42:54 +010089 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau58094f22007-04-10 01:33:20 +020090
Willy Tarreaud9e7e362018-01-18 19:16:02 +010091 if (fdtab[fd].polled_mask & tid_bit) {
92 if (!(fdtab[fd].thread_mask & tid_bit) || !(en & FD_EV_POLLED_RW)) {
Willy Tarreauf817e9f2014-01-10 16:58:45 +010093 /* fd removed from poll list */
94 opcode = EPOLL_CTL_DEL;
Willy Tarreaud9e7e362018-01-18 19:16:02 +010095 HA_ATOMIC_AND(&fdtab[fd].polled_mask, ~tid_bit);
Willy Tarreaue9f49e72012-11-11 17:42:00 +010096 }
Willy Tarreauf817e9f2014-01-10 16:58:45 +010097 else {
98 /* fd status changed */
99 opcode = EPOLL_CTL_MOD;
100 }
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100101 }
102 else if ((fdtab[fd].thread_mask & tid_bit) && (en & FD_EV_POLLED_RW)) {
103 /* new fd in the poll list */
104 opcode = EPOLL_CTL_ADD;
105 HA_ATOMIC_OR(&fdtab[fd].polled_mask, tid_bit);
106 }
107 else {
108 continue;
109 }
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100110
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100111 /* construct the epoll events based on new state */
112 ev.events = 0;
113 if (en & FD_EV_POLLED_R)
114 ev.events |= EPOLLIN | EPOLLRDHUP;
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100115
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100116 if (en & FD_EV_POLLED_W)
117 ev.events |= EPOLLOUT;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200118
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100119 ev.data.fd = fd;
120 epoll_ctl(epoll_fd[tid], opcode, fd, &ev);
Willy Tarreau58094f22007-04-10 01:33:20 +0200121 }
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100122 fd_nbupdt = 0;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200123
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100124 /* compute the epoll_wait() timeout */
Willy Tarreau10146c92015-04-13 20:44:19 +0200125 if (!exp)
126 wait_time = MAX_DELAY_MS;
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100127 else if (tick_is_expired(exp, now_ms)) {
128 activity[tid].poll_exp++;
Willy Tarreaubdefc512007-05-14 02:02:04 +0200129 wait_time = 0;
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100130 }
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200131 else {
Willy Tarreau10146c92015-04-13 20:44:19 +0200132 wait_time = TICKS_TO_MS(tick_remain(now_ms, exp)) + 1;
133 if (wait_time > MAX_DELAY_MS)
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200134 wait_time = MAX_DELAY_MS;
135 }
Willy Tarreaud825eef2007-05-12 22:35:00 +0200136
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100137 /* now let's wait for polled events */
138
Willy Tarreau45a12512011-09-10 16:56:42 +0200139 gettimeofday(&before_poll, NULL);
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100140 status = epoll_wait(epoll_fd[tid], epoll_events, global.tune.maxpollevents, wait_time);
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200141 tv_update_date(wait_time, status);
Willy Tarreau45a12512011-09-10 16:56:42 +0200142 measure_idle();
Willy Tarreau4f60f162007-04-08 16:39:58 +0200143
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100144 /* process polled events */
145
Willy Tarreau4f60f162007-04-08 16:39:58 +0200146 for (count = 0; count < status; count++) {
Willy Tarreau1c07b072013-01-07 16:19:18 +0100147 unsigned int n;
148 unsigned int e = epoll_events[count].events;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200149 fd = epoll_events[count].data.fd;
150
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100151 if (!fdtab[fd].owner) {
152 activity[tid].poll_dead++;
153 continue;
154 }
155
156 if (!(fdtab[fd].thread_mask & tid_bit)) {
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100157 /* FD has been migrated */
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100158 activity[tid].poll_skip++;
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100159 epoll_ctl(epoll_fd[tid], EPOLL_CTL_DEL, fd, &ev);
160 HA_ATOMIC_AND(&fdtab[fd].polled_mask, ~tid_bit);
Willy Tarreau076be252012-07-06 16:02:29 +0200161 continue;
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100162 }
Willy Tarreau076be252012-07-06 16:02:29 +0200163
Willy Tarreau491c4982012-07-06 11:16:01 +0200164 /* it looks complicated but gcc can optimize it away when constants
Willy Tarreau462c7202012-12-13 22:26:37 +0100165 * have same values... In fact it depends on gcc :-(
Willy Tarreau491c4982012-07-06 11:16:01 +0200166 */
Willy Tarreau462c7202012-12-13 22:26:37 +0100167 if (EPOLLIN == FD_POLL_IN && EPOLLOUT == FD_POLL_OUT &&
168 EPOLLPRI == FD_POLL_PRI && EPOLLERR == FD_POLL_ERR &&
169 EPOLLHUP == FD_POLL_HUP) {
Willy Tarreau6320c3c2012-12-13 23:52:58 +0100170 n = e & (EPOLLIN|EPOLLOUT|EPOLLPRI|EPOLLERR|EPOLLHUP);
Willy Tarreau462c7202012-12-13 22:26:37 +0100171 }
172 else {
Willy Tarreau6320c3c2012-12-13 23:52:58 +0100173 n = ((e & EPOLLIN ) ? FD_POLL_IN : 0) |
Willy Tarreau462c7202012-12-13 22:26:37 +0100174 ((e & EPOLLPRI) ? FD_POLL_PRI : 0) |
175 ((e & EPOLLOUT) ? FD_POLL_OUT : 0) |
176 ((e & EPOLLERR) ? FD_POLL_ERR : 0) |
177 ((e & EPOLLHUP) ? FD_POLL_HUP : 0);
178 }
Willy Tarreau491c4982012-07-06 11:16:01 +0200179
Willy Tarreau1c07b072013-01-07 16:19:18 +0100180 /* always remap RDHUP to HUP as they're used similarly */
Willy Tarreau5a767692017-03-13 11:38:28 +0100181 if (e & EPOLLRDHUP) {
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200182 HA_ATOMIC_OR(&cur_poller.flags, HAP_POLL_F_RDHUP);
Willy Tarreau1c07b072013-01-07 16:19:18 +0100183 n |= FD_POLL_HUP;
Willy Tarreau5a767692017-03-13 11:38:28 +0100184 }
Christopher Fauletab62f512017-08-30 10:34:36 +0200185 fd_update_events(fd, n);
Willy Tarreau4f60f162007-04-08 16:39:58 +0200186 }
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100187 /* the caller will take care of cached events */
Willy Tarreau4f60f162007-04-08 16:39:58 +0200188}
189
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200190static int init_epoll_per_thread()
191{
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100192 int fd;
193
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200194 epoll_events = calloc(1, sizeof(struct epoll_event) * global.tune.maxpollevents);
195 if (epoll_events == NULL)
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100196 goto fail_alloc;
197
198 if (tid) {
199 epoll_fd[tid] = epoll_create(global.maxsock + 1);
200 if (epoll_fd[tid] < 0)
201 goto fail_fd;
202 }
203
204 /* we may have to unregister some events initially registered on the
205 * original fd when it was alone, and/or to register events on the new
206 * fd for this thread. Let's just mark them as updated, the poller will
207 * do the rest.
208 */
209 for (fd = 0; fd < maxfd; fd++)
210 updt_fd_polling(fd);
211
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200212 return 1;
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100213 fail_fd:
214 free(epoll_events);
215 fail_alloc:
216 return 0;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200217}
218
219static void deinit_epoll_per_thread()
220{
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100221 if (tid)
222 close(epoll_fd[tid]);
223
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200224 free(epoll_events);
Christopher Fauletcd7879a2017-10-27 13:53:47 +0200225 epoll_events = NULL;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200226}
227
Willy Tarreau4f60f162007-04-08 16:39:58 +0200228/*
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100229 * Initialization of the epoll() poller.
Willy Tarreaue54e9172007-04-09 09:23:31 +0200230 * Returns 0 in case of failure, non-zero in case of success. If it fails, it
231 * disables the poller by setting its pref to 0.
232 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200233REGPRM1 static int _do_init(struct poller *p)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200234{
Willy Tarreaue54e9172007-04-09 09:23:31 +0200235 p->private = NULL;
Willy Tarreaue54e9172007-04-09 09:23:31 +0200236
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100237 epoll_fd[tid] = epoll_create(global.maxsock + 1);
238 if (epoll_fd[tid] < 0)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200239 goto fail_fd;
240
Christopher Fauletcd7879a2017-10-27 13:53:47 +0200241 hap_register_per_thread_init(init_epoll_per_thread);
242 hap_register_per_thread_deinit(deinit_epoll_per_thread);
Willy Tarreaue54e9172007-04-09 09:23:31 +0200243
Willy Tarreaue54e9172007-04-09 09:23:31 +0200244 return 1;
245
Willy Tarreaue54e9172007-04-09 09:23:31 +0200246 fail_fd:
247 p->pref = 0;
248 return 0;
249}
250
251/*
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100252 * Termination of the epoll() poller.
Willy Tarreaue54e9172007-04-09 09:23:31 +0200253 * Memory is released and the poller is marked as unselectable.
254 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200255REGPRM1 static void _do_term(struct poller *p)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200256{
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100257 if (epoll_fd[tid] >= 0) {
258 close(epoll_fd[tid]);
259 epoll_fd[tid] = -1;
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200260 }
Willy Tarreaue54e9172007-04-09 09:23:31 +0200261
262 p->private = NULL;
263 p->pref = 0;
264}
265
266/*
Willy Tarreau2ff76222007-04-09 19:29:56 +0200267 * Check that the poller works.
268 * Returns 1 if OK, otherwise 0.
269 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200270REGPRM1 static int _do_test(struct poller *p)
Willy Tarreau2ff76222007-04-09 19:29:56 +0200271{
272 int fd;
273
274 fd = epoll_create(global.maxsock + 1);
275 if (fd < 0)
276 return 0;
277 close(fd);
278 return 1;
279}
280
281/*
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200282 * Recreate the epoll file descriptor after a fork(). Returns 1 if OK,
283 * otherwise 0. It will ensure that all processes will not share their
284 * epoll_fd. Some side effects were encountered because of this, such
285 * as epoll_wait() returning an FD which was previously deleted.
286 */
287REGPRM1 static int _do_fork(struct poller *p)
288{
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100289 if (epoll_fd[tid] >= 0)
290 close(epoll_fd[tid]);
291 epoll_fd[tid] = epoll_create(global.maxsock + 1);
292 if (epoll_fd[tid] < 0)
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200293 return 0;
294 return 1;
295}
296
297/*
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200298 * It is a constructor, which means that it will automatically be called before
299 * main(). This is GCC-specific but it works at least since 2.95.
300 * Special care must be taken so that it does not need any uninitialized data.
Willy Tarreau4f60f162007-04-08 16:39:58 +0200301 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200302__attribute__((constructor))
303static void _do_register(void)
Willy Tarreau4f60f162007-04-08 16:39:58 +0200304{
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200305 struct poller *p;
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100306 int i;
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200307
308 if (nbpollers >= MAX_POLLERS)
309 return;
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200310
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100311 for (i = 0; i < MAX_THREADS; i++)
312 epoll_fd[i] = -1;
313
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200314 p = &pollers[nbpollers++];
315
Willy Tarreau4f60f162007-04-08 16:39:58 +0200316 p->name = "epoll";
317 p->pref = 300;
Willy Tarreau5a767692017-03-13 11:38:28 +0100318 p->flags = 0;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200319 p->private = NULL;
320
Conrad Hoffmann041751c2014-05-20 14:28:24 +0200321 p->clo = __fd_clo;
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200322 p->test = _do_test;
323 p->init = _do_init;
324 p->term = _do_term;
325 p->poll = _do_poll;
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200326 p->fork = _do_fork;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200327}
328
329
330/*
331 * Local variables:
332 * c-indent-level: 8
333 * c-basic-offset: 8
334 * End:
335 */