blob: e63334e14ad881ef16e91d0f7829889f47f5503b [file] [log] [blame]
Willy Tarreau4f60f162007-04-08 16:39:58 +02001/*
2 * FD polling functions for generic poll()
3 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +01004 * Copyright 2000-2014 Willy Tarreau <w@1wt.eu>
Willy Tarreau4f60f162007-04-08 16:39:58 +02005 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
12
Willy Tarreau3c8a8962017-03-13 17:14:51 +010013#define _GNU_SOURCE // for POLLRDHUP on Linux
14
Willy Tarreau4f60f162007-04-08 16:39:58 +020015#include <unistd.h>
Willy Tarreau3c8a8962017-03-13 17:14:51 +010016#include <poll.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020017#include <sys/time.h>
18#include <sys/types.h>
19
Willy Tarreaub2551052020-06-09 09:07:15 +020020#include <haproxy/activity.h>
Willy Tarreau4c7e4b72020-05-27 12:58:42 +020021#include <haproxy/api.h>
Willy Tarreaub2551052020-06-09 09:07:15 +020022#include <haproxy/fd.h>
23#include <haproxy/global.h>
Willy Tarreauc2f7c582020-06-02 18:15:32 +020024#include <haproxy/ticks.h>
Willy Tarreau92b4f132020-06-01 11:05:15 +020025#include <haproxy/time.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020026
Willy Tarreau3c8a8962017-03-13 17:14:51 +010027
28#ifndef POLLRDHUP
29/* POLLRDHUP was defined late in libc, and it appeared in kernel 2.6.17 */
30#define POLLRDHUP 0
31#endif
Willy Tarreau4f60f162007-04-08 16:39:58 +020032
Willy Tarreau173d9952018-01-26 21:48:23 +010033static int maxfd; /* # of the highest fd + 1 */
Willy Tarreau80da05a2013-03-31 14:06:57 +020034static unsigned int *fd_evts[2];
Willy Tarreau4f60f162007-04-08 16:39:58 +020035
36/* private data */
Christopher Fauletd4604ad2017-05-29 10:40:41 +020037static THREAD_LOCAL int nbfd = 0;
38static THREAD_LOCAL struct pollfd *poll_events = NULL;
Willy Tarreau4f60f162007-04-08 16:39:58 +020039
Willy Tarreau03e78532020-02-25 07:38:05 +010040static void __fd_clo(int fd)
Willy Tarreau80da05a2013-03-31 14:06:57 +020041{
42 hap_fd_clr(fd, fd_evts[DIR_RD]);
43 hap_fd_clr(fd, fd_evts[DIR_WR]);
44}
45
Olivier Houchard6b96f722018-04-25 16:58:25 +020046static void _update_fd(int fd, int *max_add_fd)
47{
48 int en;
49
50 en = fdtab[fd].state;
51
52 /* we have a single state for all threads, which is why we
53 * don't check the tid_bit. First thread to see the update
54 * takes it for every other one.
55 */
Willy Tarreau5bee3e22019-09-04 09:52:57 +020056 if (!(en & FD_EV_ACTIVE_RW)) {
Olivier Houchard53055052019-07-25 14:00:18 +000057 if (!(polled_mask[fd].poll_recv | polled_mask[fd].poll_send)) {
Olivier Houchard6b96f722018-04-25 16:58:25 +020058 /* fd was not watched, it's still not */
59 return;
60 }
61 /* fd totally removed from poll list */
62 hap_fd_clr(fd, fd_evts[DIR_RD]);
63 hap_fd_clr(fd, fd_evts[DIR_WR]);
Olivier Houchard53055052019-07-25 14:00:18 +000064 _HA_ATOMIC_AND(&polled_mask[fd].poll_recv, 0);
65 _HA_ATOMIC_AND(&polled_mask[fd].poll_send, 0);
Olivier Houchard6b96f722018-04-25 16:58:25 +020066 }
67 else {
68 /* OK fd has to be monitored, it was either added or changed */
Willy Tarreau5bee3e22019-09-04 09:52:57 +020069 if (!(en & FD_EV_ACTIVE_R)) {
Olivier Houchard6b96f722018-04-25 16:58:25 +020070 hap_fd_clr(fd, fd_evts[DIR_RD]);
Olivier Houchard53055052019-07-25 14:00:18 +000071 if (polled_mask[fd].poll_recv & tid_bit)
72 _HA_ATOMIC_AND(&polled_mask[fd].poll_recv, ~tid_bit);
73 } else {
Olivier Houchard6b96f722018-04-25 16:58:25 +020074 hap_fd_set(fd, fd_evts[DIR_RD]);
Olivier Houchard53055052019-07-25 14:00:18 +000075 if (!(polled_mask[fd].poll_recv & tid_bit))
76 _HA_ATOMIC_OR(&polled_mask[fd].poll_recv, tid_bit);
77 }
Olivier Houchard6b96f722018-04-25 16:58:25 +020078
Willy Tarreau5bee3e22019-09-04 09:52:57 +020079 if (!(en & FD_EV_ACTIVE_W)) {
Olivier Houchard6b96f722018-04-25 16:58:25 +020080 hap_fd_clr(fd, fd_evts[DIR_WR]);
Olivier Houchard53055052019-07-25 14:00:18 +000081 if (polled_mask[fd].poll_send & tid_bit)
82 _HA_ATOMIC_AND(&polled_mask[fd].poll_send, ~tid_bit);
83 }else {
Olivier Houchard6b96f722018-04-25 16:58:25 +020084 hap_fd_set(fd, fd_evts[DIR_WR]);
Olivier Houchard53055052019-07-25 14:00:18 +000085 if (!(polled_mask[fd].poll_send & tid_bit))
86 _HA_ATOMIC_OR(&polled_mask[fd].poll_send, tid_bit);
87 }
Olivier Houchard6b96f722018-04-25 16:58:25 +020088
Olivier Houchard6b96f722018-04-25 16:58:25 +020089 if (fd > *max_add_fd)
90 *max_add_fd = fd;
91 }
92}
93
Willy Tarreau4f60f162007-04-08 16:39:58 +020094/*
95 * Poll() poller
96 */
Willy Tarreau03e78532020-02-25 07:38:05 +010097static void _do_poll(struct poller *p, int exp, int wake)
Willy Tarreau4f60f162007-04-08 16:39:58 +020098{
99 int status;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200100 int fd;
Willy Tarreaud825eef2007-05-12 22:35:00 +0200101 int wait_time;
Olivier Houchard6b96f722018-04-25 16:58:25 +0200102 int updt_idx;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200103 int fds, count;
104 int sr, sw;
Willy Tarreau173d9952018-01-26 21:48:23 +0100105 int old_maxfd, new_maxfd, max_add_fd;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200106 unsigned rn, wn; /* read new, write new */
Olivier Houchard6b96f722018-04-25 16:58:25 +0200107 int old_fd;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200108
Willy Tarreau173d9952018-01-26 21:48:23 +0100109 max_add_fd = -1;
110
Willy Tarreaucc7e3f72012-11-11 17:25:15 +0100111 /* first, scan the update list to find changes */
112 for (updt_idx = 0; updt_idx < fd_nbupdt; updt_idx++) {
113 fd = fd_updt[updt_idx];
Willy Tarreaucc7e3f72012-11-11 17:25:15 +0100114
Olivier Houchardcb6c9272019-03-08 18:49:54 +0100115 _HA_ATOMIC_AND(&fdtab[fd].update_mask, ~tid_bit);
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100116 if (!fdtab[fd].owner) {
Willy Tarreaue4063862020-06-17 20:35:33 +0200117 activity[tid].poll_drop_fd++;
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100118 continue;
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100119 }
Olivier Houchard6b96f722018-04-25 16:58:25 +0200120 _update_fd(fd, &max_add_fd);
121 }
Willy Tarreaucc7e3f72012-11-11 17:25:15 +0100122
Olivier Houchard6b96f722018-04-25 16:58:25 +0200123 /* Now scan the global update list */
124 for (old_fd = fd = update_list.first; fd != -1; fd = fdtab[fd].update.next) {
125 if (fd == -2) {
126 fd = old_fd;
127 continue;
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100128 }
Olivier Houchard6b96f722018-04-25 16:58:25 +0200129 else if (fd <= -3)
130 fd = -fd -4;
131 if (fd == -1)
132 break;
133 if (fdtab[fd].update_mask & tid_bit) {
134 /* Cheat a bit, as the state is global to all pollers
135 * we don't need every thread ot take care of the
136 * update.
137 */
Olivier Houchardcb6c9272019-03-08 18:49:54 +0100138 _HA_ATOMIC_AND(&fdtab[fd].update_mask, ~all_threads_mask);
Olivier Houchard6b96f722018-04-25 16:58:25 +0200139 done_update_polling(fd);
140 } else
141 continue;
142 if (!fdtab[fd].owner)
143 continue;
144 _update_fd(fd, &max_add_fd);
Willy Tarreaucc7e3f72012-11-11 17:25:15 +0100145 }
Willy Tarreau173d9952018-01-26 21:48:23 +0100146
147 /* maybe we added at least one fd larger than maxfd */
148 for (old_maxfd = maxfd; old_maxfd <= max_add_fd; ) {
Olivier Houchardcb6c9272019-03-08 18:49:54 +0100149 if (_HA_ATOMIC_CAS(&maxfd, &old_maxfd, max_add_fd + 1))
Willy Tarreau173d9952018-01-26 21:48:23 +0100150 break;
151 }
152
153 /* maxfd doesn't need to be precise but it needs to cover *all* active
154 * FDs. Thus we only shrink it if we have such an opportunity. The algo
155 * is simple : look for the previous used place, try to update maxfd to
156 * point to it, abort if maxfd changed in the mean time.
157 */
158 old_maxfd = maxfd;
159 do {
160 new_maxfd = old_maxfd;
161 while (new_maxfd - 1 >= 0 && !fdtab[new_maxfd - 1].owner)
162 new_maxfd--;
163 if (new_maxfd >= old_maxfd)
164 break;
Olivier Houchardcb6c9272019-03-08 18:49:54 +0100165 } while (!_HA_ATOMIC_CAS(&maxfd, &old_maxfd, new_maxfd));
Willy Tarreau173d9952018-01-26 21:48:23 +0100166
Willy Tarreau60b639c2018-08-02 10:16:17 +0200167 thread_harmless_now();
Olivier Houchard305d5ab2019-07-24 18:07:06 +0200168 if (sleeping_thread_mask & tid_bit)
169 _HA_ATOMIC_AND(&sleeping_thread_mask, ~tid_bit);
Willy Tarreau60b639c2018-08-02 10:16:17 +0200170
Willy Tarreaucc7e3f72012-11-11 17:25:15 +0100171 fd_nbupdt = 0;
172
Willy Tarreau4f60f162007-04-08 16:39:58 +0200173 nbfd = 0;
Willy Tarreau80da05a2013-03-31 14:06:57 +0200174 for (fds = 0; (fds * 8*sizeof(**fd_evts)) < maxfd; fds++) {
175 rn = fd_evts[DIR_RD][fds];
176 wn = fd_evts[DIR_WR][fds];
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200177
Willy Tarreau80da05a2013-03-31 14:06:57 +0200178 if (!(rn|wn))
179 continue;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200180
Willy Tarreau80da05a2013-03-31 14:06:57 +0200181 for (count = 0, fd = fds * 8*sizeof(**fd_evts); count < 8*sizeof(**fd_evts) && fd < maxfd; count++, fd++) {
182 sr = (rn >> count) & 1;
183 sw = (wn >> count) & 1;
184 if ((sr|sw)) {
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100185 if (!fdtab[fd].owner) {
186 /* should normally not happen here except
187 * due to rare thread concurrency
188 */
189 continue;
190 }
191
192 if (!(fdtab[fd].thread_mask & tid_bit)) {
Willy Tarreaue4063862020-06-17 20:35:33 +0200193 activity[tid].poll_skip_fd++;
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100194 continue;
195 }
196
Willy Tarreau80da05a2013-03-31 14:06:57 +0200197 poll_events[nbfd].fd = fd;
Willy Tarreau3c8a8962017-03-13 17:14:51 +0100198 poll_events[nbfd].events = (sr ? (POLLIN | POLLRDHUP) : 0) | (sw ? POLLOUT : 0);
Willy Tarreau80da05a2013-03-31 14:06:57 +0200199 nbfd++;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200200 }
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200201 }
Willy Tarreau4f60f162007-04-08 16:39:58 +0200202 }
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200203
Willy Tarreau4f60f162007-04-08 16:39:58 +0200204 /* now let's wait for events */
Willy Tarreau2ae84e42019-05-28 16:44:05 +0200205 wait_time = wake ? 0 : compute_poll_timeout(exp);
Willy Tarreau7e9c4ae2018-10-17 14:31:19 +0200206 tv_entering_poll();
Willy Tarreau609aad92018-11-22 08:31:09 +0100207 activity_count_runtime();
Willy Tarreau4f60f162007-04-08 16:39:58 +0200208 status = poll(poll_events, nbfd, wait_time);
Willy Tarreau48f8bc12018-11-22 18:57:37 +0100209 tv_update_date(wait_time, status);
Willy Tarreau7e9c4ae2018-10-17 14:31:19 +0200210 tv_leaving_poll(wait_time, status);
Willy Tarreau4f60f162007-04-08 16:39:58 +0200211
Willy Tarreau60b639c2018-08-02 10:16:17 +0200212 thread_harmless_end();
213
Willy Tarreaue5451532020-06-17 20:25:18 +0200214 if (status > 0)
215 activity[tid].poll_io++;
216
Willy Tarreau4f60f162007-04-08 16:39:58 +0200217 for (count = 0; status > 0 && count < nbfd; count++) {
Christopher Fauletab62f512017-08-30 10:34:36 +0200218 unsigned int n;
Willy Tarreau491c4982012-07-06 11:16:01 +0200219 int e = poll_events[count].revents;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200220 fd = poll_events[count].fd;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200221
Willy Tarreau3c8a8962017-03-13 17:14:51 +0100222 if (!(e & ( POLLOUT | POLLIN | POLLERR | POLLHUP | POLLRDHUP )))
Willy Tarreau4f60f162007-04-08 16:39:58 +0200223 continue;
224
Willy Tarreau076be252012-07-06 16:02:29 +0200225 /* ok, we found one active fd */
226 status--;
227
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100228 if (!fdtab[fd].owner) {
Willy Tarreaue4063862020-06-17 20:35:33 +0200229 activity[tid].poll_dead_fd++;
Willy Tarreau076be252012-07-06 16:02:29 +0200230 continue;
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100231 }
Willy Tarreau076be252012-07-06 16:02:29 +0200232
Willy Tarreau6b308982019-09-06 19:05:50 +0200233 n = ((e & POLLIN) ? FD_EV_READY_R : 0) |
234 ((e & POLLOUT) ? FD_EV_READY_W : 0) |
235 ((e & POLLRDHUP) ? FD_EV_SHUT_R : 0) |
236 ((e & POLLHUP) ? FD_EV_SHUT_RW : 0) |
237 ((e & POLLERR) ? FD_EV_ERR_RW : 0);
Willy Tarreau491c4982012-07-06 11:16:01 +0200238
Willy Tarreau6b308982019-09-06 19:05:50 +0200239 if ((e & POLLRDHUP) && !(cur_poller.flags & HAP_POLL_F_RDHUP))
Olivier Houchardcb6c9272019-03-08 18:49:54 +0100240 _HA_ATOMIC_OR(&cur_poller.flags, HAP_POLL_F_RDHUP);
Willy Tarreau6b308982019-09-06 19:05:50 +0200241
Christopher Fauletab62f512017-08-30 10:34:36 +0200242 fd_update_events(fd, n);
Willy Tarreau4f60f162007-04-08 16:39:58 +0200243 }
244
Willy Tarreaue54e9172007-04-09 09:23:31 +0200245}
246
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200247
248static int init_poll_per_thread()
249{
250 poll_events = calloc(1, sizeof(struct pollfd) * global.maxsock);
251 if (poll_events == NULL)
252 return 0;
253 return 1;
254}
255
256static void deinit_poll_per_thread()
257{
258 free(poll_events);
Christopher Fauletcd7879a2017-10-27 13:53:47 +0200259 poll_events = NULL;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200260}
261
Willy Tarreaue54e9172007-04-09 09:23:31 +0200262/*
263 * Initialization of the poll() poller.
264 * Returns 0 in case of failure, non-zero in case of success. If it fails, it
265 * disables the poller by setting its pref to 0.
266 */
Willy Tarreau03e78532020-02-25 07:38:05 +0100267static int _do_init(struct poller *p)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200268{
Christopher Fauletcd7879a2017-10-27 13:53:47 +0200269 __label__ fail_swevt, fail_srevt;
Willy Tarreau80da05a2013-03-31 14:06:57 +0200270 int fd_evts_bytes;
Willy Tarreaue54e9172007-04-09 09:23:31 +0200271
272 p->private = NULL;
Willy Tarreaucc359232018-01-17 15:48:53 +0100273 fd_evts_bytes = (global.maxsock + sizeof(**fd_evts) * 8 - 1) / (sizeof(**fd_evts) * 8) * sizeof(**fd_evts);
Willy Tarreaue54e9172007-04-09 09:23:31 +0200274
Willy Tarreau80da05a2013-03-31 14:06:57 +0200275 if ((fd_evts[DIR_RD] = calloc(1, fd_evts_bytes)) == NULL)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200276 goto fail_srevt;
Willy Tarreau80da05a2013-03-31 14:06:57 +0200277 if ((fd_evts[DIR_WR] = calloc(1, fd_evts_bytes)) == NULL)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200278 goto fail_swevt;
279
Christopher Fauletcd7879a2017-10-27 13:53:47 +0200280 hap_register_per_thread_init(init_poll_per_thread);
281 hap_register_per_thread_deinit(deinit_poll_per_thread);
282
Willy Tarreaue54e9172007-04-09 09:23:31 +0200283 return 1;
284
285 fail_swevt:
286 free(fd_evts[DIR_RD]);
287 fail_srevt:
Willy Tarreaue54e9172007-04-09 09:23:31 +0200288 p->pref = 0;
289 return 0;
290}
291
292/*
293 * Termination of the poll() poller.
294 * Memory is released and the poller is marked as unselectable.
295 */
Willy Tarreau03e78532020-02-25 07:38:05 +0100296static void _do_term(struct poller *p)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200297{
Willy Tarreaua534fea2008-08-03 12:19:50 +0200298 free(fd_evts[DIR_WR]);
299 free(fd_evts[DIR_RD]);
Willy Tarreaue54e9172007-04-09 09:23:31 +0200300 p->private = NULL;
301 p->pref = 0;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200302}
303
304/*
Willy Tarreau2ff76222007-04-09 19:29:56 +0200305 * Check that the poller works.
306 * Returns 1 if OK, otherwise 0.
307 */
Willy Tarreau03e78532020-02-25 07:38:05 +0100308static int _do_test(struct poller *p)
Willy Tarreau2ff76222007-04-09 19:29:56 +0200309{
310 return 1;
311}
312
313/*
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200314 * It is a constructor, which means that it will automatically be called before
315 * main(). This is GCC-specific but it works at least since 2.95.
316 * Special care must be taken so that it does not need any uninitialized data.
Willy Tarreau4f60f162007-04-08 16:39:58 +0200317 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200318__attribute__((constructor))
319static void _do_register(void)
Willy Tarreau4f60f162007-04-08 16:39:58 +0200320{
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200321 struct poller *p;
322
323 if (nbpollers >= MAX_POLLERS)
324 return;
325 p = &pollers[nbpollers++];
326
Willy Tarreau4f60f162007-04-08 16:39:58 +0200327 p->name = "poll";
328 p->pref = 200;
Willy Tarreau11ef0832019-11-28 18:17:33 +0100329 p->flags = HAP_POLL_F_ERRHUP;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200330 p->private = NULL;
331
Willy Tarreau70c6fd82012-11-11 21:02:34 +0100332 p->clo = __fd_clo;
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200333 p->test = _do_test;
334 p->init = _do_init;
335 p->term = _do_term;
336 p->poll = _do_poll;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200337}
338
339
340/*
341 * Local variables:
342 * c-indent-level: 8
343 * c-basic-offset: 8
344 * End:
345 */