blob: 538f0705723c90cac122cf68a014ced0e2f29038 [file] [log] [blame]
Willy Tarreau4f60f162007-04-08 16:39:58 +02001/*
2 * FD polling functions for generic poll()
3 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +01004 * Copyright 2000-2014 Willy Tarreau <w@1wt.eu>
Willy Tarreau4f60f162007-04-08 16:39:58 +02005 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
12
Willy Tarreau3c8a8962017-03-13 17:14:51 +010013#define _GNU_SOURCE // for POLLRDHUP on Linux
14
Willy Tarreau4f60f162007-04-08 16:39:58 +020015#include <unistd.h>
Willy Tarreau3c8a8962017-03-13 17:14:51 +010016#include <poll.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020017#include <sys/time.h>
18#include <sys/types.h>
19
20#include <common/compat.h>
21#include <common/config.h>
Willy Tarreau60b639c2018-08-02 10:16:17 +020022#include <common/hathreads.h>
Willy Tarreau0c303ee2008-07-07 00:09:58 +020023#include <common/ticks.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020024#include <common/time.h>
25
Willy Tarreau4f60f162007-04-08 16:39:58 +020026#include <types/global.h>
27
Willy Tarreau609aad92018-11-22 08:31:09 +010028#include <proto/activity.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020029#include <proto/fd.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020030
Willy Tarreau3c8a8962017-03-13 17:14:51 +010031
32#ifndef POLLRDHUP
33/* POLLRDHUP was defined late in libc, and it appeared in kernel 2.6.17 */
34#define POLLRDHUP 0
35#endif
Willy Tarreau4f60f162007-04-08 16:39:58 +020036
Willy Tarreau173d9952018-01-26 21:48:23 +010037static int maxfd; /* # of the highest fd + 1 */
Willy Tarreau80da05a2013-03-31 14:06:57 +020038static unsigned int *fd_evts[2];
Willy Tarreau4f60f162007-04-08 16:39:58 +020039
40/* private data */
Christopher Fauletd4604ad2017-05-29 10:40:41 +020041static THREAD_LOCAL int nbfd = 0;
42static THREAD_LOCAL struct pollfd *poll_events = NULL;
Willy Tarreau4f60f162007-04-08 16:39:58 +020043
Willy Tarreau03e78532020-02-25 07:38:05 +010044static void __fd_clo(int fd)
Willy Tarreau80da05a2013-03-31 14:06:57 +020045{
46 hap_fd_clr(fd, fd_evts[DIR_RD]);
47 hap_fd_clr(fd, fd_evts[DIR_WR]);
48}
49
Olivier Houchard6b96f722018-04-25 16:58:25 +020050static void _update_fd(int fd, int *max_add_fd)
51{
52 int en;
53
54 en = fdtab[fd].state;
55
56 /* we have a single state for all threads, which is why we
57 * don't check the tid_bit. First thread to see the update
58 * takes it for every other one.
59 */
Willy Tarreau5bee3e22019-09-04 09:52:57 +020060 if (!(en & FD_EV_ACTIVE_RW)) {
Olivier Houchard53055052019-07-25 14:00:18 +000061 if (!(polled_mask[fd].poll_recv | polled_mask[fd].poll_send)) {
Olivier Houchard6b96f722018-04-25 16:58:25 +020062 /* fd was not watched, it's still not */
63 return;
64 }
65 /* fd totally removed from poll list */
66 hap_fd_clr(fd, fd_evts[DIR_RD]);
67 hap_fd_clr(fd, fd_evts[DIR_WR]);
Olivier Houchard53055052019-07-25 14:00:18 +000068 _HA_ATOMIC_AND(&polled_mask[fd].poll_recv, 0);
69 _HA_ATOMIC_AND(&polled_mask[fd].poll_send, 0);
Olivier Houchard6b96f722018-04-25 16:58:25 +020070 }
71 else {
72 /* OK fd has to be monitored, it was either added or changed */
Willy Tarreau5bee3e22019-09-04 09:52:57 +020073 if (!(en & FD_EV_ACTIVE_R)) {
Olivier Houchard6b96f722018-04-25 16:58:25 +020074 hap_fd_clr(fd, fd_evts[DIR_RD]);
Olivier Houchard53055052019-07-25 14:00:18 +000075 if (polled_mask[fd].poll_recv & tid_bit)
76 _HA_ATOMIC_AND(&polled_mask[fd].poll_recv, ~tid_bit);
77 } else {
Olivier Houchard6b96f722018-04-25 16:58:25 +020078 hap_fd_set(fd, fd_evts[DIR_RD]);
Olivier Houchard53055052019-07-25 14:00:18 +000079 if (!(polled_mask[fd].poll_recv & tid_bit))
80 _HA_ATOMIC_OR(&polled_mask[fd].poll_recv, tid_bit);
81 }
Olivier Houchard6b96f722018-04-25 16:58:25 +020082
Willy Tarreau5bee3e22019-09-04 09:52:57 +020083 if (!(en & FD_EV_ACTIVE_W)) {
Olivier Houchard6b96f722018-04-25 16:58:25 +020084 hap_fd_clr(fd, fd_evts[DIR_WR]);
Olivier Houchard53055052019-07-25 14:00:18 +000085 if (polled_mask[fd].poll_send & tid_bit)
86 _HA_ATOMIC_AND(&polled_mask[fd].poll_send, ~tid_bit);
87 }else {
Olivier Houchard6b96f722018-04-25 16:58:25 +020088 hap_fd_set(fd, fd_evts[DIR_WR]);
Olivier Houchard53055052019-07-25 14:00:18 +000089 if (!(polled_mask[fd].poll_send & tid_bit))
90 _HA_ATOMIC_OR(&polled_mask[fd].poll_send, tid_bit);
91 }
Olivier Houchard6b96f722018-04-25 16:58:25 +020092
Olivier Houchard6b96f722018-04-25 16:58:25 +020093 if (fd > *max_add_fd)
94 *max_add_fd = fd;
95 }
96}
97
Willy Tarreau4f60f162007-04-08 16:39:58 +020098/*
99 * Poll() poller
100 */
Willy Tarreau03e78532020-02-25 07:38:05 +0100101static void _do_poll(struct poller *p, int exp, int wake)
Willy Tarreau4f60f162007-04-08 16:39:58 +0200102{
103 int status;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200104 int fd;
Willy Tarreaud825eef2007-05-12 22:35:00 +0200105 int wait_time;
Olivier Houchard6b96f722018-04-25 16:58:25 +0200106 int updt_idx;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200107 int fds, count;
108 int sr, sw;
Willy Tarreau173d9952018-01-26 21:48:23 +0100109 int old_maxfd, new_maxfd, max_add_fd;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200110 unsigned rn, wn; /* read new, write new */
Olivier Houchard6b96f722018-04-25 16:58:25 +0200111 int old_fd;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200112
Willy Tarreau173d9952018-01-26 21:48:23 +0100113 max_add_fd = -1;
114
Willy Tarreaucc7e3f72012-11-11 17:25:15 +0100115 /* first, scan the update list to find changes */
116 for (updt_idx = 0; updt_idx < fd_nbupdt; updt_idx++) {
117 fd = fd_updt[updt_idx];
Willy Tarreaucc7e3f72012-11-11 17:25:15 +0100118
Olivier Houchardcb6c9272019-03-08 18:49:54 +0100119 _HA_ATOMIC_AND(&fdtab[fd].update_mask, ~tid_bit);
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100120 if (!fdtab[fd].owner) {
121 activity[tid].poll_drop++;
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100122 continue;
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100123 }
Olivier Houchard6b96f722018-04-25 16:58:25 +0200124 _update_fd(fd, &max_add_fd);
125 }
Willy Tarreaucc7e3f72012-11-11 17:25:15 +0100126
Olivier Houchard6b96f722018-04-25 16:58:25 +0200127 /* Now scan the global update list */
128 for (old_fd = fd = update_list.first; fd != -1; fd = fdtab[fd].update.next) {
129 if (fd == -2) {
130 fd = old_fd;
131 continue;
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100132 }
Olivier Houchard6b96f722018-04-25 16:58:25 +0200133 else if (fd <= -3)
134 fd = -fd -4;
135 if (fd == -1)
136 break;
137 if (fdtab[fd].update_mask & tid_bit) {
138 /* Cheat a bit, as the state is global to all pollers
139 * we don't need every thread ot take care of the
140 * update.
141 */
Olivier Houchardcb6c9272019-03-08 18:49:54 +0100142 _HA_ATOMIC_AND(&fdtab[fd].update_mask, ~all_threads_mask);
Olivier Houchard6b96f722018-04-25 16:58:25 +0200143 done_update_polling(fd);
144 } else
145 continue;
146 if (!fdtab[fd].owner)
147 continue;
148 _update_fd(fd, &max_add_fd);
Willy Tarreaucc7e3f72012-11-11 17:25:15 +0100149 }
Willy Tarreau173d9952018-01-26 21:48:23 +0100150
151 /* maybe we added at least one fd larger than maxfd */
152 for (old_maxfd = maxfd; old_maxfd <= max_add_fd; ) {
Olivier Houchardcb6c9272019-03-08 18:49:54 +0100153 if (_HA_ATOMIC_CAS(&maxfd, &old_maxfd, max_add_fd + 1))
Willy Tarreau173d9952018-01-26 21:48:23 +0100154 break;
155 }
156
157 /* maxfd doesn't need to be precise but it needs to cover *all* active
158 * FDs. Thus we only shrink it if we have such an opportunity. The algo
159 * is simple : look for the previous used place, try to update maxfd to
160 * point to it, abort if maxfd changed in the mean time.
161 */
162 old_maxfd = maxfd;
163 do {
164 new_maxfd = old_maxfd;
165 while (new_maxfd - 1 >= 0 && !fdtab[new_maxfd - 1].owner)
166 new_maxfd--;
167 if (new_maxfd >= old_maxfd)
168 break;
Olivier Houchardcb6c9272019-03-08 18:49:54 +0100169 } while (!_HA_ATOMIC_CAS(&maxfd, &old_maxfd, new_maxfd));
Willy Tarreau173d9952018-01-26 21:48:23 +0100170
Willy Tarreau60b639c2018-08-02 10:16:17 +0200171 thread_harmless_now();
Olivier Houchard305d5ab2019-07-24 18:07:06 +0200172 if (sleeping_thread_mask & tid_bit)
173 _HA_ATOMIC_AND(&sleeping_thread_mask, ~tid_bit);
Willy Tarreau60b639c2018-08-02 10:16:17 +0200174
Willy Tarreaucc7e3f72012-11-11 17:25:15 +0100175 fd_nbupdt = 0;
176
Willy Tarreau4f60f162007-04-08 16:39:58 +0200177 nbfd = 0;
Willy Tarreau80da05a2013-03-31 14:06:57 +0200178 for (fds = 0; (fds * 8*sizeof(**fd_evts)) < maxfd; fds++) {
179 rn = fd_evts[DIR_RD][fds];
180 wn = fd_evts[DIR_WR][fds];
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200181
Willy Tarreau80da05a2013-03-31 14:06:57 +0200182 if (!(rn|wn))
183 continue;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200184
Willy Tarreau80da05a2013-03-31 14:06:57 +0200185 for (count = 0, fd = fds * 8*sizeof(**fd_evts); count < 8*sizeof(**fd_evts) && fd < maxfd; count++, fd++) {
186 sr = (rn >> count) & 1;
187 sw = (wn >> count) & 1;
188 if ((sr|sw)) {
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100189 if (!fdtab[fd].owner) {
190 /* should normally not happen here except
191 * due to rare thread concurrency
192 */
193 continue;
194 }
195
196 if (!(fdtab[fd].thread_mask & tid_bit)) {
197 activity[tid].poll_skip++;
198 continue;
199 }
200
Willy Tarreau80da05a2013-03-31 14:06:57 +0200201 poll_events[nbfd].fd = fd;
Willy Tarreau3c8a8962017-03-13 17:14:51 +0100202 poll_events[nbfd].events = (sr ? (POLLIN | POLLRDHUP) : 0) | (sw ? POLLOUT : 0);
Willy Tarreau80da05a2013-03-31 14:06:57 +0200203 nbfd++;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200204 }
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200205 }
Willy Tarreau4f60f162007-04-08 16:39:58 +0200206 }
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200207
Willy Tarreau4f60f162007-04-08 16:39:58 +0200208 /* now let's wait for events */
Willy Tarreau2ae84e42019-05-28 16:44:05 +0200209 wait_time = wake ? 0 : compute_poll_timeout(exp);
Willy Tarreau7e9c4ae2018-10-17 14:31:19 +0200210 tv_entering_poll();
Willy Tarreau609aad92018-11-22 08:31:09 +0100211 activity_count_runtime();
Willy Tarreau4f60f162007-04-08 16:39:58 +0200212 status = poll(poll_events, nbfd, wait_time);
Willy Tarreau48f8bc12018-11-22 18:57:37 +0100213 tv_update_date(wait_time, status);
Willy Tarreau7e9c4ae2018-10-17 14:31:19 +0200214 tv_leaving_poll(wait_time, status);
Willy Tarreau4f60f162007-04-08 16:39:58 +0200215
Willy Tarreau60b639c2018-08-02 10:16:17 +0200216 thread_harmless_end();
217
Willy Tarreau4f60f162007-04-08 16:39:58 +0200218 for (count = 0; status > 0 && count < nbfd; count++) {
Christopher Fauletab62f512017-08-30 10:34:36 +0200219 unsigned int n;
Willy Tarreau491c4982012-07-06 11:16:01 +0200220 int e = poll_events[count].revents;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200221 fd = poll_events[count].fd;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200222
Willy Tarreau3c8a8962017-03-13 17:14:51 +0100223 if (!(e & ( POLLOUT | POLLIN | POLLERR | POLLHUP | POLLRDHUP )))
Willy Tarreau4f60f162007-04-08 16:39:58 +0200224 continue;
225
Willy Tarreau076be252012-07-06 16:02:29 +0200226 /* ok, we found one active fd */
227 status--;
228
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100229 if (!fdtab[fd].owner) {
230 activity[tid].poll_dead++;
Willy Tarreau076be252012-07-06 16:02:29 +0200231 continue;
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100232 }
Willy Tarreau076be252012-07-06 16:02:29 +0200233
Willy Tarreau6b308982019-09-06 19:05:50 +0200234 n = ((e & POLLIN) ? FD_EV_READY_R : 0) |
235 ((e & POLLOUT) ? FD_EV_READY_W : 0) |
236 ((e & POLLRDHUP) ? FD_EV_SHUT_R : 0) |
237 ((e & POLLHUP) ? FD_EV_SHUT_RW : 0) |
238 ((e & POLLERR) ? FD_EV_ERR_RW : 0);
Willy Tarreau491c4982012-07-06 11:16:01 +0200239
Willy Tarreau6b308982019-09-06 19:05:50 +0200240 if ((e & POLLRDHUP) && !(cur_poller.flags & HAP_POLL_F_RDHUP))
Olivier Houchardcb6c9272019-03-08 18:49:54 +0100241 _HA_ATOMIC_OR(&cur_poller.flags, HAP_POLL_F_RDHUP);
Willy Tarreau6b308982019-09-06 19:05:50 +0200242
Christopher Fauletab62f512017-08-30 10:34:36 +0200243 fd_update_events(fd, n);
Willy Tarreau4f60f162007-04-08 16:39:58 +0200244 }
245
Willy Tarreaue54e9172007-04-09 09:23:31 +0200246}
247
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200248
249static int init_poll_per_thread()
250{
251 poll_events = calloc(1, sizeof(struct pollfd) * global.maxsock);
252 if (poll_events == NULL)
253 return 0;
254 return 1;
255}
256
257static void deinit_poll_per_thread()
258{
259 free(poll_events);
Christopher Fauletcd7879a2017-10-27 13:53:47 +0200260 poll_events = NULL;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200261}
262
Willy Tarreaue54e9172007-04-09 09:23:31 +0200263/*
264 * Initialization of the poll() poller.
265 * Returns 0 in case of failure, non-zero in case of success. If it fails, it
266 * disables the poller by setting its pref to 0.
267 */
Willy Tarreau03e78532020-02-25 07:38:05 +0100268static int _do_init(struct poller *p)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200269{
Christopher Fauletcd7879a2017-10-27 13:53:47 +0200270 __label__ fail_swevt, fail_srevt;
Willy Tarreau80da05a2013-03-31 14:06:57 +0200271 int fd_evts_bytes;
Willy Tarreaue54e9172007-04-09 09:23:31 +0200272
273 p->private = NULL;
Willy Tarreaucc359232018-01-17 15:48:53 +0100274 fd_evts_bytes = (global.maxsock + sizeof(**fd_evts) * 8 - 1) / (sizeof(**fd_evts) * 8) * sizeof(**fd_evts);
Willy Tarreaue54e9172007-04-09 09:23:31 +0200275
Willy Tarreau80da05a2013-03-31 14:06:57 +0200276 if ((fd_evts[DIR_RD] = calloc(1, fd_evts_bytes)) == NULL)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200277 goto fail_srevt;
Willy Tarreau80da05a2013-03-31 14:06:57 +0200278 if ((fd_evts[DIR_WR] = calloc(1, fd_evts_bytes)) == NULL)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200279 goto fail_swevt;
280
Christopher Fauletcd7879a2017-10-27 13:53:47 +0200281 hap_register_per_thread_init(init_poll_per_thread);
282 hap_register_per_thread_deinit(deinit_poll_per_thread);
283
Willy Tarreaue54e9172007-04-09 09:23:31 +0200284 return 1;
285
286 fail_swevt:
287 free(fd_evts[DIR_RD]);
288 fail_srevt:
289 free(poll_events);
Willy Tarreaue54e9172007-04-09 09:23:31 +0200290 p->pref = 0;
291 return 0;
292}
293
294/*
295 * Termination of the poll() poller.
296 * Memory is released and the poller is marked as unselectable.
297 */
Willy Tarreau03e78532020-02-25 07:38:05 +0100298static void _do_term(struct poller *p)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200299{
Willy Tarreaua534fea2008-08-03 12:19:50 +0200300 free(fd_evts[DIR_WR]);
301 free(fd_evts[DIR_RD]);
Willy Tarreaue54e9172007-04-09 09:23:31 +0200302 p->private = NULL;
303 p->pref = 0;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200304}
305
306/*
Willy Tarreau2ff76222007-04-09 19:29:56 +0200307 * Check that the poller works.
308 * Returns 1 if OK, otherwise 0.
309 */
Willy Tarreau03e78532020-02-25 07:38:05 +0100310static int _do_test(struct poller *p)
Willy Tarreau2ff76222007-04-09 19:29:56 +0200311{
312 return 1;
313}
314
315/*
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200316 * It is a constructor, which means that it will automatically be called before
317 * main(). This is GCC-specific but it works at least since 2.95.
318 * Special care must be taken so that it does not need any uninitialized data.
Willy Tarreau4f60f162007-04-08 16:39:58 +0200319 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200320__attribute__((constructor))
321static void _do_register(void)
Willy Tarreau4f60f162007-04-08 16:39:58 +0200322{
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200323 struct poller *p;
324
325 if (nbpollers >= MAX_POLLERS)
326 return;
327 p = &pollers[nbpollers++];
328
Willy Tarreau4f60f162007-04-08 16:39:58 +0200329 p->name = "poll";
330 p->pref = 200;
Willy Tarreau11ef0832019-11-28 18:17:33 +0100331 p->flags = HAP_POLL_F_ERRHUP;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200332 p->private = NULL;
333
Willy Tarreau70c6fd82012-11-11 21:02:34 +0100334 p->clo = __fd_clo;
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200335 p->test = _do_test;
336 p->init = _do_init;
337 p->term = _do_term;
338 p->poll = _do_poll;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200339}
340
341
342/*
343 * Local variables:
344 * c-indent-level: 8
345 * c-basic-offset: 8
346 * End:
347 */