blob: 6c09c0498f7725b713dc1b37a57213e4cfbdb596 [file] [log] [blame]
Willy Tarreau4f60f162007-04-08 16:39:58 +02001/*
Willy Tarreaue9f49e72012-11-11 17:42:00 +01002 * FD polling functions for Linux epoll
Willy Tarreau4f60f162007-04-08 16:39:58 +02003 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +01004 * Copyright 2000-2014 Willy Tarreau <w@1wt.eu>
Willy Tarreau4f60f162007-04-08 16:39:58 +02005 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
Willy Tarreau4f60f162007-04-08 16:39:58 +020010 */
11
12#include <unistd.h>
13#include <sys/time.h>
14#include <sys/types.h>
15
16#include <common/compat.h>
17#include <common/config.h>
Willy Tarreaue9f49e72012-11-11 17:42:00 +010018#include <common/debug.h>
Willy Tarreau43d8fb22011-08-22 17:12:02 +020019#include <common/epoll.h>
Willy Tarreau60b639c2018-08-02 10:16:17 +020020#include <common/hathreads.h>
Willy Tarreau58094f22007-04-10 01:33:20 +020021#include <common/standard.h>
Willy Tarreau0c303ee2008-07-07 00:09:58 +020022#include <common/ticks.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020023#include <common/time.h>
Willy Tarreau1db37712007-06-03 17:16:49 +020024#include <common/tools.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020025
Willy Tarreau4f60f162007-04-08 16:39:58 +020026#include <types/global.h>
27
Willy Tarreau609aad92018-11-22 08:31:09 +010028#include <proto/activity.h>
Willy Tarreaue9f49e72012-11-11 17:42:00 +010029#include <proto/fd.h>
Willy Tarreaubeb859a2018-11-22 18:07:59 +010030#include <proto/signal.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020031
Willy Tarreau58094f22007-04-10 01:33:20 +020032
Willy Tarreau4f60f162007-04-08 16:39:58 +020033/* private data */
Christopher Fauletd4604ad2017-05-29 10:40:41 +020034static THREAD_LOCAL struct epoll_event *epoll_events = NULL;
Willy Tarreaud9e7e362018-01-18 19:16:02 +010035static int epoll_fd[MAX_THREADS]; // per-thread epoll_fd
Willy Tarreau4f60f162007-04-08 16:39:58 +020036
Willy Tarreau58094f22007-04-10 01:33:20 +020037/* This structure may be used for any purpose. Warning! do not use it in
38 * recursive functions !
39 */
Christopher Fauletd4604ad2017-05-29 10:40:41 +020040static THREAD_LOCAL struct epoll_event ev;
Willy Tarreau58094f22007-04-10 01:33:20 +020041
Willy Tarreau1c07b072013-01-07 16:19:18 +010042#ifndef EPOLLRDHUP
43/* EPOLLRDHUP was defined late in libc, and it appeared in kernel 2.6.17 */
44#define EPOLLRDHUP 0x2000
45#endif
46
Willy Tarreau4f60f162007-04-08 16:39:58 +020047/*
Conrad Hoffmann041751c2014-05-20 14:28:24 +020048 * Immediately remove file descriptor from epoll set upon close.
49 * Since we forked, some fds share inodes with the other process, and epoll may
50 * send us events even though this process closed the fd (see man 7 epoll,
51 * "Questions and answers", Q 6).
52 */
53REGPRM1 static void __fd_clo(int fd)
54{
Willy Tarreaud9e7e362018-01-18 19:16:02 +010055 if (unlikely(fdtab[fd].cloned)) {
Olivier Houchardcb92f5c2018-04-26 14:23:07 +020056 unsigned long m = polled_mask[fd];
Willy Tarreaud9e7e362018-01-18 19:16:02 +010057 int i;
58
59 for (i = global.nbthread - 1; i >= 0; i--)
60 if (m & (1UL << i))
61 epoll_ctl(epoll_fd[i], EPOLL_CTL_DEL, fd, &ev);
62 }
Conrad Hoffmann041751c2014-05-20 14:28:24 +020063}
64
Olivier Houchard6b96f722018-04-25 16:58:25 +020065static void _update_fd(int fd)
66{
67 int en, opcode;
68
69 en = fdtab[fd].state;
70
Olivier Houchardcb92f5c2018-04-26 14:23:07 +020071 if (polled_mask[fd] & tid_bit) {
Olivier Houchard6b96f722018-04-25 16:58:25 +020072 if (!(fdtab[fd].thread_mask & tid_bit) || !(en & FD_EV_POLLED_RW)) {
73 /* fd removed from poll list */
74 opcode = EPOLL_CTL_DEL;
Olivier Houchardcb6c9272019-03-08 18:49:54 +010075 _HA_ATOMIC_AND(&polled_mask[fd], ~tid_bit);
Olivier Houchard6b96f722018-04-25 16:58:25 +020076 }
77 else {
78 /* fd status changed */
79 opcode = EPOLL_CTL_MOD;
80 }
81 }
82 else if ((fdtab[fd].thread_mask & tid_bit) && (en & FD_EV_POLLED_RW)) {
83 /* new fd in the poll list */
84 opcode = EPOLL_CTL_ADD;
Olivier Houchardcb6c9272019-03-08 18:49:54 +010085 _HA_ATOMIC_OR(&polled_mask[fd], tid_bit);
Olivier Houchard6b96f722018-04-25 16:58:25 +020086 }
87 else {
88 return;
89 }
90
91 /* construct the epoll events based on new state */
92 ev.events = 0;
93 if (en & FD_EV_POLLED_R)
94 ev.events |= EPOLLIN | EPOLLRDHUP;
95
96 if (en & FD_EV_POLLED_W)
97 ev.events |= EPOLLOUT;
98
99 ev.data.fd = fd;
100 epoll_ctl(epoll_fd[tid], opcode, fd, &ev);
101}
102
Conrad Hoffmann041751c2014-05-20 14:28:24 +0200103/*
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100104 * Linux epoll() poller
Willy Tarreau4f60f162007-04-08 16:39:58 +0200105 */
Willy Tarreau2ae84e42019-05-28 16:44:05 +0200106REGPRM3 static void _do_poll(struct poller *p, int exp, int wake)
Willy Tarreau4f60f162007-04-08 16:39:58 +0200107{
Olivier Houchard6b96f722018-04-25 16:58:25 +0200108 int status;
109 int fd;
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100110 int count;
111 int updt_idx;
112 int wait_time;
Olivier Houchard6b96f722018-04-25 16:58:25 +0200113 int old_fd;
Willy Tarreau58094f22007-04-10 01:33:20 +0200114
Willy Tarreau5be2f352014-11-19 19:43:05 +0100115 /* first, scan the update list to find polling changes */
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100116 for (updt_idx = 0; updt_idx < fd_nbupdt; updt_idx++) {
117 fd = fd_updt[updt_idx];
Willy Tarreau58094f22007-04-10 01:33:20 +0200118
Olivier Houchardcb6c9272019-03-08 18:49:54 +0100119 _HA_ATOMIC_AND(&fdtab[fd].update_mask, ~tid_bit);
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100120 if (!fdtab[fd].owner) {
121 activity[tid].poll_drop++;
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100122 continue;
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100123 }
Willy Tarreau58094f22007-04-10 01:33:20 +0200124
Olivier Houchard6b96f722018-04-25 16:58:25 +0200125 _update_fd(fd);
126 }
127 fd_nbupdt = 0;
128 /* Scan the global update list */
129 for (old_fd = fd = update_list.first; fd != -1; fd = fdtab[fd].update.next) {
130 if (fd == -2) {
131 fd = old_fd;
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100132 continue;
133 }
Olivier Houchard6b96f722018-04-25 16:58:25 +0200134 else if (fd <= -3)
135 fd = -fd -4;
136 if (fd == -1)
137 break;
138 if (fdtab[fd].update_mask & tid_bit)
139 done_update_polling(fd);
140 else
141 continue;
142 if (!fdtab[fd].owner)
143 continue;
144 _update_fd(fd);
Willy Tarreau58094f22007-04-10 01:33:20 +0200145 }
Willy Tarreau4f60f162007-04-08 16:39:58 +0200146
Willy Tarreau60b639c2018-08-02 10:16:17 +0200147 thread_harmless_now();
148
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100149 /* now let's wait for polled events */
Willy Tarreau2ae84e42019-05-28 16:44:05 +0200150 wait_time = wake ? 0 : compute_poll_timeout(exp);
Willy Tarreau7e9c4ae2018-10-17 14:31:19 +0200151 tv_entering_poll();
Willy Tarreau609aad92018-11-22 08:31:09 +0100152 activity_count_runtime();
Willy Tarreaubeb859a2018-11-22 18:07:59 +0100153 do {
154 int timeout = (global.tune.options & GTUNE_BUSY_POLLING) ? 0 : wait_time;
155
156 status = epoll_wait(epoll_fd[tid], epoll_events, global.tune.maxpollevents, timeout);
157 tv_update_date(timeout, status);
158
159 if (status)
160 break;
161 if (timeout || !wait_time)
162 break;
Willy Tarreau2ae84e42019-05-28 16:44:05 +0200163 if (signal_queue_len || wake)
Willy Tarreaubeb859a2018-11-22 18:07:59 +0100164 break;
165 if (tick_isset(exp) && tick_is_expired(exp, now_ms))
166 break;
167 } while (1);
168
Willy Tarreau7e9c4ae2018-10-17 14:31:19 +0200169 tv_leaving_poll(wait_time, status);
Willy Tarreau4f60f162007-04-08 16:39:58 +0200170
Willy Tarreau60b639c2018-08-02 10:16:17 +0200171 thread_harmless_end();
172
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100173 /* process polled events */
174
Willy Tarreau4f60f162007-04-08 16:39:58 +0200175 for (count = 0; count < status; count++) {
Willy Tarreau1c07b072013-01-07 16:19:18 +0100176 unsigned int n;
177 unsigned int e = epoll_events[count].events;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200178 fd = epoll_events[count].data.fd;
179
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100180 if (!fdtab[fd].owner) {
181 activity[tid].poll_dead++;
182 continue;
183 }
184
185 if (!(fdtab[fd].thread_mask & tid_bit)) {
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100186 /* FD has been migrated */
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100187 activity[tid].poll_skip++;
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100188 epoll_ctl(epoll_fd[tid], EPOLL_CTL_DEL, fd, &ev);
Olivier Houchardcb6c9272019-03-08 18:49:54 +0100189 _HA_ATOMIC_AND(&polled_mask[fd], ~tid_bit);
Willy Tarreau076be252012-07-06 16:02:29 +0200190 continue;
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100191 }
Willy Tarreau076be252012-07-06 16:02:29 +0200192
Willy Tarreau491c4982012-07-06 11:16:01 +0200193 /* it looks complicated but gcc can optimize it away when constants
Willy Tarreau462c7202012-12-13 22:26:37 +0100194 * have same values... In fact it depends on gcc :-(
Willy Tarreau491c4982012-07-06 11:16:01 +0200195 */
Willy Tarreau462c7202012-12-13 22:26:37 +0100196 if (EPOLLIN == FD_POLL_IN && EPOLLOUT == FD_POLL_OUT &&
197 EPOLLPRI == FD_POLL_PRI && EPOLLERR == FD_POLL_ERR &&
198 EPOLLHUP == FD_POLL_HUP) {
Willy Tarreau6320c3c2012-12-13 23:52:58 +0100199 n = e & (EPOLLIN|EPOLLOUT|EPOLLPRI|EPOLLERR|EPOLLHUP);
Willy Tarreau462c7202012-12-13 22:26:37 +0100200 }
201 else {
Willy Tarreau6320c3c2012-12-13 23:52:58 +0100202 n = ((e & EPOLLIN ) ? FD_POLL_IN : 0) |
Willy Tarreau462c7202012-12-13 22:26:37 +0100203 ((e & EPOLLPRI) ? FD_POLL_PRI : 0) |
204 ((e & EPOLLOUT) ? FD_POLL_OUT : 0) |
205 ((e & EPOLLERR) ? FD_POLL_ERR : 0) |
206 ((e & EPOLLHUP) ? FD_POLL_HUP : 0);
207 }
Willy Tarreau491c4982012-07-06 11:16:01 +0200208
Willy Tarreau1c07b072013-01-07 16:19:18 +0100209 /* always remap RDHUP to HUP as they're used similarly */
Willy Tarreau5a767692017-03-13 11:38:28 +0100210 if (e & EPOLLRDHUP) {
Olivier Houchardcb6c9272019-03-08 18:49:54 +0100211 _HA_ATOMIC_OR(&cur_poller.flags, HAP_POLL_F_RDHUP);
Willy Tarreau1c07b072013-01-07 16:19:18 +0100212 n |= FD_POLL_HUP;
Willy Tarreau5a767692017-03-13 11:38:28 +0100213 }
Christopher Fauletab62f512017-08-30 10:34:36 +0200214 fd_update_events(fd, n);
Willy Tarreau4f60f162007-04-08 16:39:58 +0200215 }
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100216 /* the caller will take care of cached events */
Willy Tarreau4f60f162007-04-08 16:39:58 +0200217}
218
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200219static int init_epoll_per_thread()
220{
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100221 int fd;
222
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200223 epoll_events = calloc(1, sizeof(struct epoll_event) * global.tune.maxpollevents);
224 if (epoll_events == NULL)
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100225 goto fail_alloc;
226
Christopher Faulet3e805ed2018-01-25 16:18:09 +0100227 if (MAX_THREADS > 1 && tid) {
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100228 epoll_fd[tid] = epoll_create(global.maxsock + 1);
229 if (epoll_fd[tid] < 0)
230 goto fail_fd;
231 }
232
233 /* we may have to unregister some events initially registered on the
234 * original fd when it was alone, and/or to register events on the new
235 * fd for this thread. Let's just mark them as updated, the poller will
236 * do the rest.
237 */
Willy Tarreauce036bc2018-01-29 14:58:02 +0100238 for (fd = 0; fd < global.maxsock; fd++)
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100239 updt_fd_polling(fd);
240
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200241 return 1;
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100242 fail_fd:
243 free(epoll_events);
244 fail_alloc:
245 return 0;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200246}
247
248static void deinit_epoll_per_thread()
249{
Christopher Faulet3e805ed2018-01-25 16:18:09 +0100250 if (MAX_THREADS > 1 && tid)
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100251 close(epoll_fd[tid]);
252
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200253 free(epoll_events);
Christopher Fauletcd7879a2017-10-27 13:53:47 +0200254 epoll_events = NULL;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200255}
256
Willy Tarreau4f60f162007-04-08 16:39:58 +0200257/*
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100258 * Initialization of the epoll() poller.
Willy Tarreaue54e9172007-04-09 09:23:31 +0200259 * Returns 0 in case of failure, non-zero in case of success. If it fails, it
260 * disables the poller by setting its pref to 0.
261 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200262REGPRM1 static int _do_init(struct poller *p)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200263{
Willy Tarreaue54e9172007-04-09 09:23:31 +0200264 p->private = NULL;
Willy Tarreaue54e9172007-04-09 09:23:31 +0200265
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100266 epoll_fd[tid] = epoll_create(global.maxsock + 1);
267 if (epoll_fd[tid] < 0)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200268 goto fail_fd;
269
Christopher Fauletcd7879a2017-10-27 13:53:47 +0200270 hap_register_per_thread_init(init_epoll_per_thread);
271 hap_register_per_thread_deinit(deinit_epoll_per_thread);
Willy Tarreaue54e9172007-04-09 09:23:31 +0200272
Willy Tarreaue54e9172007-04-09 09:23:31 +0200273 return 1;
274
Willy Tarreaue54e9172007-04-09 09:23:31 +0200275 fail_fd:
276 p->pref = 0;
277 return 0;
278}
279
280/*
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100281 * Termination of the epoll() poller.
Willy Tarreaue54e9172007-04-09 09:23:31 +0200282 * Memory is released and the poller is marked as unselectable.
283 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200284REGPRM1 static void _do_term(struct poller *p)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200285{
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100286 if (epoll_fd[tid] >= 0) {
287 close(epoll_fd[tid]);
288 epoll_fd[tid] = -1;
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200289 }
Willy Tarreaue54e9172007-04-09 09:23:31 +0200290
291 p->private = NULL;
292 p->pref = 0;
293}
294
295/*
Willy Tarreau2ff76222007-04-09 19:29:56 +0200296 * Check that the poller works.
297 * Returns 1 if OK, otherwise 0.
298 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200299REGPRM1 static int _do_test(struct poller *p)
Willy Tarreau2ff76222007-04-09 19:29:56 +0200300{
301 int fd;
302
303 fd = epoll_create(global.maxsock + 1);
304 if (fd < 0)
305 return 0;
306 close(fd);
307 return 1;
308}
309
310/*
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200311 * Recreate the epoll file descriptor after a fork(). Returns 1 if OK,
312 * otherwise 0. It will ensure that all processes will not share their
313 * epoll_fd. Some side effects were encountered because of this, such
314 * as epoll_wait() returning an FD which was previously deleted.
315 */
316REGPRM1 static int _do_fork(struct poller *p)
317{
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100318 if (epoll_fd[tid] >= 0)
319 close(epoll_fd[tid]);
320 epoll_fd[tid] = epoll_create(global.maxsock + 1);
321 if (epoll_fd[tid] < 0)
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200322 return 0;
323 return 1;
324}
325
326/*
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200327 * It is a constructor, which means that it will automatically be called before
328 * main(). This is GCC-specific but it works at least since 2.95.
329 * Special care must be taken so that it does not need any uninitialized data.
Willy Tarreau4f60f162007-04-08 16:39:58 +0200330 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200331__attribute__((constructor))
332static void _do_register(void)
Willy Tarreau4f60f162007-04-08 16:39:58 +0200333{
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200334 struct poller *p;
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100335 int i;
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200336
337 if (nbpollers >= MAX_POLLERS)
338 return;
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200339
Willy Tarreaud9e7e362018-01-18 19:16:02 +0100340 for (i = 0; i < MAX_THREADS; i++)
341 epoll_fd[i] = -1;
342
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200343 p = &pollers[nbpollers++];
344
Willy Tarreau4f60f162007-04-08 16:39:58 +0200345 p->name = "epoll";
346 p->pref = 300;
Willy Tarreau5a767692017-03-13 11:38:28 +0100347 p->flags = 0;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200348 p->private = NULL;
349
Conrad Hoffmann041751c2014-05-20 14:28:24 +0200350 p->clo = __fd_clo;
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200351 p->test = _do_test;
352 p->init = _do_init;
353 p->term = _do_term;
354 p->poll = _do_poll;
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200355 p->fork = _do_fork;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200356}
357
358
359/*
360 * Local variables:
361 * c-indent-level: 8
362 * c-basic-offset: 8
363 * End:
364 */