blob: d26c3a21f12f13eded54e61fc2cf7e69bc30c703 [file] [log] [blame]
Willy Tarreau4f60f162007-04-08 16:39:58 +02001/*
Willy Tarreaue9f49e72012-11-11 17:42:00 +01002 * FD polling functions for Linux epoll
Willy Tarreau4f60f162007-04-08 16:39:58 +02003 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +01004 * Copyright 2000-2014 Willy Tarreau <w@1wt.eu>
Willy Tarreau4f60f162007-04-08 16:39:58 +02005 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
Willy Tarreau4f60f162007-04-08 16:39:58 +020010 */
11
12#include <unistd.h>
13#include <sys/time.h>
14#include <sys/types.h>
15
16#include <common/compat.h>
17#include <common/config.h>
Willy Tarreaue9f49e72012-11-11 17:42:00 +010018#include <common/debug.h>
Willy Tarreau43d8fb22011-08-22 17:12:02 +020019#include <common/epoll.h>
Willy Tarreau58094f22007-04-10 01:33:20 +020020#include <common/standard.h>
Willy Tarreau0c303ee2008-07-07 00:09:58 +020021#include <common/ticks.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020022#include <common/time.h>
Willy Tarreau1db37712007-06-03 17:16:49 +020023#include <common/tools.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020024
Willy Tarreau4f60f162007-04-08 16:39:58 +020025#include <types/global.h>
26
Willy Tarreaue9f49e72012-11-11 17:42:00 +010027#include <proto/fd.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020028
Willy Tarreau58094f22007-04-10 01:33:20 +020029
Willy Tarreau4f60f162007-04-08 16:39:58 +020030/* private data */
Christopher Fauletd4604ad2017-05-29 10:40:41 +020031static THREAD_LOCAL struct epoll_event *epoll_events = NULL;
Willy Tarreau4f60f162007-04-08 16:39:58 +020032static int epoll_fd;
33
Willy Tarreau58094f22007-04-10 01:33:20 +020034/* This structure may be used for any purpose. Warning! do not use it in
35 * recursive functions !
36 */
Christopher Fauletd4604ad2017-05-29 10:40:41 +020037static THREAD_LOCAL struct epoll_event ev;
Willy Tarreau58094f22007-04-10 01:33:20 +020038
Willy Tarreau1c07b072013-01-07 16:19:18 +010039#ifndef EPOLLRDHUP
40/* EPOLLRDHUP was defined late in libc, and it appeared in kernel 2.6.17 */
41#define EPOLLRDHUP 0x2000
42#endif
43
Willy Tarreau4f60f162007-04-08 16:39:58 +020044/*
Conrad Hoffmann041751c2014-05-20 14:28:24 +020045 * Immediately remove file descriptor from epoll set upon close.
46 * Since we forked, some fds share inodes with the other process, and epoll may
47 * send us events even though this process closed the fd (see man 7 epoll,
48 * "Questions and answers", Q 6).
49 */
50REGPRM1 static void __fd_clo(int fd)
51{
Christopher Fauletd4604ad2017-05-29 10:40:41 +020052 if (unlikely(fdtab[fd].cloned))
Conrad Hoffmann041751c2014-05-20 14:28:24 +020053 epoll_ctl(epoll_fd, EPOLL_CTL_DEL, fd, &ev);
Conrad Hoffmann041751c2014-05-20 14:28:24 +020054}
55
56/*
Willy Tarreauf817e9f2014-01-10 16:58:45 +010057 * Linux epoll() poller
Willy Tarreau4f60f162007-04-08 16:39:58 +020058 */
Willy Tarreaue9f49e72012-11-11 17:42:00 +010059REGPRM2 static void _do_poll(struct poller *p, int exp)
Willy Tarreau4f60f162007-04-08 16:39:58 +020060{
Willy Tarreaue9f49e72012-11-11 17:42:00 +010061 int status, eo, en;
62 int fd, opcode;
63 int count;
64 int updt_idx;
65 int wait_time;
Willy Tarreau58094f22007-04-10 01:33:20 +020066
Willy Tarreau5be2f352014-11-19 19:43:05 +010067 /* first, scan the update list to find polling changes */
Willy Tarreaue9f49e72012-11-11 17:42:00 +010068 for (updt_idx = 0; updt_idx < fd_nbupdt; updt_idx++) {
69 fd = fd_updt[updt_idx];
Willy Tarreau58094f22007-04-10 01:33:20 +020070
Willy Tarreauf817e9f2014-01-10 16:58:45 +010071 if (!fdtab[fd].owner)
72 continue;
Willy Tarreau58094f22007-04-10 01:33:20 +020073
Christopher Fauletd4604ad2017-05-29 10:40:41 +020074 SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
75 fdtab[fd].updated = 0;
76 fdtab[fd].new = 0;
77
Willy Tarreau25002d22014-01-25 10:32:56 +010078 eo = fdtab[fd].state;
79 en = fd_compute_new_polled_status(eo);
Christopher Fauletd4604ad2017-05-29 10:40:41 +020080 fdtab[fd].state = en;
81 SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau58094f22007-04-10 01:33:20 +020082
Willy Tarreauf817e9f2014-01-10 16:58:45 +010083 if ((eo ^ en) & FD_EV_POLLED_RW) {
84 /* poll status changed */
Willy Tarreauf8cfa442012-10-04 21:54:41 +020085
Willy Tarreauf817e9f2014-01-10 16:58:45 +010086 if ((en & FD_EV_POLLED_RW) == 0) {
87 /* fd removed from poll list */
88 opcode = EPOLL_CTL_DEL;
Willy Tarreaue9f49e72012-11-11 17:42:00 +010089 }
Willy Tarreauf817e9f2014-01-10 16:58:45 +010090 else if ((eo & FD_EV_POLLED_RW) == 0) {
91 /* new fd in the poll list */
92 opcode = EPOLL_CTL_ADD;
Willy Tarreaue9f49e72012-11-11 17:42:00 +010093 }
Willy Tarreauf817e9f2014-01-10 16:58:45 +010094 else {
95 /* fd status changed */
96 opcode = EPOLL_CTL_MOD;
97 }
98
99 /* construct the epoll events based on new state */
100 ev.events = 0;
101 if (en & FD_EV_POLLED_R)
102 ev.events |= EPOLLIN | EPOLLRDHUP;
103
104 if (en & FD_EV_POLLED_W)
105 ev.events |= EPOLLOUT;
106
107 ev.data.fd = fd;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200108
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100109 epoll_ctl(epoll_fd, opcode, fd, &ev);
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100110 }
Willy Tarreau58094f22007-04-10 01:33:20 +0200111 }
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100112 fd_nbupdt = 0;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200113
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100114 /* compute the epoll_wait() timeout */
Willy Tarreau10146c92015-04-13 20:44:19 +0200115 if (!exp)
116 wait_time = MAX_DELAY_MS;
117 else if (tick_is_expired(exp, now_ms))
Willy Tarreaubdefc512007-05-14 02:02:04 +0200118 wait_time = 0;
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200119 else {
Willy Tarreau10146c92015-04-13 20:44:19 +0200120 wait_time = TICKS_TO_MS(tick_remain(now_ms, exp)) + 1;
121 if (wait_time > MAX_DELAY_MS)
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200122 wait_time = MAX_DELAY_MS;
123 }
Willy Tarreaud825eef2007-05-12 22:35:00 +0200124
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100125 /* now let's wait for polled events */
126
Willy Tarreau45a12512011-09-10 16:56:42 +0200127 gettimeofday(&before_poll, NULL);
Willy Tarreaucf181c92013-01-18 15:22:41 +0100128 status = epoll_wait(epoll_fd, epoll_events, global.tune.maxpollevents, wait_time);
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200129 tv_update_date(wait_time, status);
Willy Tarreau45a12512011-09-10 16:56:42 +0200130 measure_idle();
Willy Tarreau4f60f162007-04-08 16:39:58 +0200131
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100132 /* process polled events */
133
Willy Tarreau4f60f162007-04-08 16:39:58 +0200134 for (count = 0; count < status; count++) {
Willy Tarreau1c07b072013-01-07 16:19:18 +0100135 unsigned int n;
136 unsigned int e = epoll_events[count].events;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200137 fd = epoll_events[count].data.fd;
138
Christopher Faulet63e2ce62017-06-02 14:36:39 +0200139 if (!fdtab[fd].owner || !(fdtab[fd].process_mask & tid_bit))
Willy Tarreau076be252012-07-06 16:02:29 +0200140 continue;
141
Willy Tarreau491c4982012-07-06 11:16:01 +0200142 /* it looks complicated but gcc can optimize it away when constants
Willy Tarreau462c7202012-12-13 22:26:37 +0100143 * have same values... In fact it depends on gcc :-(
Willy Tarreau491c4982012-07-06 11:16:01 +0200144 */
Willy Tarreau462c7202012-12-13 22:26:37 +0100145 if (EPOLLIN == FD_POLL_IN && EPOLLOUT == FD_POLL_OUT &&
146 EPOLLPRI == FD_POLL_PRI && EPOLLERR == FD_POLL_ERR &&
147 EPOLLHUP == FD_POLL_HUP) {
Willy Tarreau6320c3c2012-12-13 23:52:58 +0100148 n = e & (EPOLLIN|EPOLLOUT|EPOLLPRI|EPOLLERR|EPOLLHUP);
Willy Tarreau462c7202012-12-13 22:26:37 +0100149 }
150 else {
Willy Tarreau6320c3c2012-12-13 23:52:58 +0100151 n = ((e & EPOLLIN ) ? FD_POLL_IN : 0) |
Willy Tarreau462c7202012-12-13 22:26:37 +0100152 ((e & EPOLLPRI) ? FD_POLL_PRI : 0) |
153 ((e & EPOLLOUT) ? FD_POLL_OUT : 0) |
154 ((e & EPOLLERR) ? FD_POLL_ERR : 0) |
155 ((e & EPOLLHUP) ? FD_POLL_HUP : 0);
156 }
Willy Tarreau491c4982012-07-06 11:16:01 +0200157
Willy Tarreau1c07b072013-01-07 16:19:18 +0100158 /* always remap RDHUP to HUP as they're used similarly */
Willy Tarreau5a767692017-03-13 11:38:28 +0100159 if (e & EPOLLRDHUP) {
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200160 HA_ATOMIC_OR(&cur_poller.flags, HAP_POLL_F_RDHUP);
Willy Tarreau1c07b072013-01-07 16:19:18 +0100161 n |= FD_POLL_HUP;
Willy Tarreau5a767692017-03-13 11:38:28 +0100162 }
Christopher Fauletab62f512017-08-30 10:34:36 +0200163 fd_update_events(fd, n);
Willy Tarreau4f60f162007-04-08 16:39:58 +0200164 }
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100165 /* the caller will take care of cached events */
Willy Tarreau4f60f162007-04-08 16:39:58 +0200166}
167
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200168static int init_epoll_per_thread()
169{
170 epoll_events = calloc(1, sizeof(struct epoll_event) * global.tune.maxpollevents);
171 if (epoll_events == NULL)
172 return 0;
173 return 1;
174}
175
176static void deinit_epoll_per_thread()
177{
178 free(epoll_events);
Christopher Fauletcd7879a2017-10-27 13:53:47 +0200179 epoll_events = NULL;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200180}
181
Willy Tarreau4f60f162007-04-08 16:39:58 +0200182/*
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100183 * Initialization of the epoll() poller.
Willy Tarreaue54e9172007-04-09 09:23:31 +0200184 * Returns 0 in case of failure, non-zero in case of success. If it fails, it
185 * disables the poller by setting its pref to 0.
186 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200187REGPRM1 static int _do_init(struct poller *p)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200188{
Willy Tarreaue54e9172007-04-09 09:23:31 +0200189 p->private = NULL;
Willy Tarreaue54e9172007-04-09 09:23:31 +0200190
191 epoll_fd = epoll_create(global.maxsock + 1);
192 if (epoll_fd < 0)
193 goto fail_fd;
194
Christopher Fauletcd7879a2017-10-27 13:53:47 +0200195 hap_register_per_thread_init(init_epoll_per_thread);
196 hap_register_per_thread_deinit(deinit_epoll_per_thread);
Willy Tarreaue54e9172007-04-09 09:23:31 +0200197
Willy Tarreaue54e9172007-04-09 09:23:31 +0200198 return 1;
199
Willy Tarreaue54e9172007-04-09 09:23:31 +0200200 fail_fd:
201 p->pref = 0;
202 return 0;
203}
204
205/*
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100206 * Termination of the epoll() poller.
Willy Tarreaue54e9172007-04-09 09:23:31 +0200207 * Memory is released and the poller is marked as unselectable.
208 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200209REGPRM1 static void _do_term(struct poller *p)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200210{
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200211 if (epoll_fd >= 0) {
212 close(epoll_fd);
213 epoll_fd = -1;
214 }
Willy Tarreaue54e9172007-04-09 09:23:31 +0200215
216 p->private = NULL;
217 p->pref = 0;
218}
219
220/*
Willy Tarreau2ff76222007-04-09 19:29:56 +0200221 * Check that the poller works.
222 * Returns 1 if OK, otherwise 0.
223 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200224REGPRM1 static int _do_test(struct poller *p)
Willy Tarreau2ff76222007-04-09 19:29:56 +0200225{
226 int fd;
227
228 fd = epoll_create(global.maxsock + 1);
229 if (fd < 0)
230 return 0;
231 close(fd);
232 return 1;
233}
234
235/*
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200236 * Recreate the epoll file descriptor after a fork(). Returns 1 if OK,
237 * otherwise 0. It will ensure that all processes will not share their
238 * epoll_fd. Some side effects were encountered because of this, such
239 * as epoll_wait() returning an FD which was previously deleted.
240 */
241REGPRM1 static int _do_fork(struct poller *p)
242{
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200243 if (epoll_fd >= 0)
244 close(epoll_fd);
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200245 epoll_fd = epoll_create(global.maxsock + 1);
246 if (epoll_fd < 0)
247 return 0;
248 return 1;
249}
250
251/*
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200252 * It is a constructor, which means that it will automatically be called before
253 * main(). This is GCC-specific but it works at least since 2.95.
254 * Special care must be taken so that it does not need any uninitialized data.
Willy Tarreau4f60f162007-04-08 16:39:58 +0200255 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200256__attribute__((constructor))
257static void _do_register(void)
Willy Tarreau4f60f162007-04-08 16:39:58 +0200258{
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200259 struct poller *p;
260
261 if (nbpollers >= MAX_POLLERS)
262 return;
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200263
264 epoll_fd = -1;
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200265 p = &pollers[nbpollers++];
266
Willy Tarreau4f60f162007-04-08 16:39:58 +0200267 p->name = "epoll";
268 p->pref = 300;
Willy Tarreau5a767692017-03-13 11:38:28 +0100269 p->flags = 0;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200270 p->private = NULL;
271
Conrad Hoffmann041751c2014-05-20 14:28:24 +0200272 p->clo = __fd_clo;
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200273 p->test = _do_test;
274 p->init = _do_init;
275 p->term = _do_term;
276 p->poll = _do_poll;
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200277 p->fork = _do_fork;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200278}
279
280
281/*
282 * Local variables:
283 * c-indent-level: 8
284 * c-basic-offset: 8
285 * End:
286 */