blob: 836e0afefa31f0756796ea0bdd4d8e87e4ab71de [file] [log] [blame]
Willy Tarreau4f60f162007-04-08 16:39:58 +02001/*
2 * FD polling functions for generic poll()
3 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +01004 * Copyright 2000-2014 Willy Tarreau <w@1wt.eu>
Willy Tarreau4f60f162007-04-08 16:39:58 +02005 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
12
Willy Tarreau3c8a8962017-03-13 17:14:51 +010013#define _GNU_SOURCE // for POLLRDHUP on Linux
14
Willy Tarreau4f60f162007-04-08 16:39:58 +020015#include <unistd.h>
Willy Tarreau3c8a8962017-03-13 17:14:51 +010016#include <poll.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020017#include <sys/time.h>
18#include <sys/types.h>
19
Willy Tarreaub2551052020-06-09 09:07:15 +020020#include <haproxy/activity.h>
Willy Tarreau4c7e4b72020-05-27 12:58:42 +020021#include <haproxy/api.h>
Willy Tarreaub2551052020-06-09 09:07:15 +020022#include <haproxy/fd.h>
23#include <haproxy/global.h>
Willy Tarreauc2f7c582020-06-02 18:15:32 +020024#include <haproxy/ticks.h>
Willy Tarreau92b4f132020-06-01 11:05:15 +020025#include <haproxy/time.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020026
Willy Tarreau3c8a8962017-03-13 17:14:51 +010027
28#ifndef POLLRDHUP
29/* POLLRDHUP was defined late in libc, and it appeared in kernel 2.6.17 */
30#define POLLRDHUP 0
31#endif
Willy Tarreau4f60f162007-04-08 16:39:58 +020032
Willy Tarreau173d9952018-01-26 21:48:23 +010033static int maxfd; /* # of the highest fd + 1 */
Willy Tarreau80da05a2013-03-31 14:06:57 +020034static unsigned int *fd_evts[2];
Willy Tarreau4f60f162007-04-08 16:39:58 +020035
36/* private data */
Christopher Fauletd4604ad2017-05-29 10:40:41 +020037static THREAD_LOCAL int nbfd = 0;
38static THREAD_LOCAL struct pollfd *poll_events = NULL;
Willy Tarreau4f60f162007-04-08 16:39:58 +020039
Willy Tarreau03e78532020-02-25 07:38:05 +010040static void __fd_clo(int fd)
Willy Tarreau80da05a2013-03-31 14:06:57 +020041{
42 hap_fd_clr(fd, fd_evts[DIR_RD]);
43 hap_fd_clr(fd, fd_evts[DIR_WR]);
44}
45
Olivier Houchard6b96f722018-04-25 16:58:25 +020046static void _update_fd(int fd, int *max_add_fd)
47{
48 int en;
49
50 en = fdtab[fd].state;
51
52 /* we have a single state for all threads, which is why we
53 * don't check the tid_bit. First thread to see the update
54 * takes it for every other one.
55 */
Willy Tarreau5bee3e22019-09-04 09:52:57 +020056 if (!(en & FD_EV_ACTIVE_RW)) {
Olivier Houchard53055052019-07-25 14:00:18 +000057 if (!(polled_mask[fd].poll_recv | polled_mask[fd].poll_send)) {
Olivier Houchard6b96f722018-04-25 16:58:25 +020058 /* fd was not watched, it's still not */
59 return;
60 }
61 /* fd totally removed from poll list */
62 hap_fd_clr(fd, fd_evts[DIR_RD]);
63 hap_fd_clr(fd, fd_evts[DIR_WR]);
Olivier Houchard53055052019-07-25 14:00:18 +000064 _HA_ATOMIC_AND(&polled_mask[fd].poll_recv, 0);
65 _HA_ATOMIC_AND(&polled_mask[fd].poll_send, 0);
Olivier Houchard6b96f722018-04-25 16:58:25 +020066 }
67 else {
68 /* OK fd has to be monitored, it was either added or changed */
Willy Tarreau5bee3e22019-09-04 09:52:57 +020069 if (!(en & FD_EV_ACTIVE_R)) {
Olivier Houchard6b96f722018-04-25 16:58:25 +020070 hap_fd_clr(fd, fd_evts[DIR_RD]);
Olivier Houchard53055052019-07-25 14:00:18 +000071 if (polled_mask[fd].poll_recv & tid_bit)
72 _HA_ATOMIC_AND(&polled_mask[fd].poll_recv, ~tid_bit);
73 } else {
Olivier Houchard6b96f722018-04-25 16:58:25 +020074 hap_fd_set(fd, fd_evts[DIR_RD]);
Olivier Houchard53055052019-07-25 14:00:18 +000075 if (!(polled_mask[fd].poll_recv & tid_bit))
76 _HA_ATOMIC_OR(&polled_mask[fd].poll_recv, tid_bit);
77 }
Olivier Houchard6b96f722018-04-25 16:58:25 +020078
Willy Tarreau5bee3e22019-09-04 09:52:57 +020079 if (!(en & FD_EV_ACTIVE_W)) {
Olivier Houchard6b96f722018-04-25 16:58:25 +020080 hap_fd_clr(fd, fd_evts[DIR_WR]);
Olivier Houchard53055052019-07-25 14:00:18 +000081 if (polled_mask[fd].poll_send & tid_bit)
82 _HA_ATOMIC_AND(&polled_mask[fd].poll_send, ~tid_bit);
83 }else {
Olivier Houchard6b96f722018-04-25 16:58:25 +020084 hap_fd_set(fd, fd_evts[DIR_WR]);
Olivier Houchard53055052019-07-25 14:00:18 +000085 if (!(polled_mask[fd].poll_send & tid_bit))
86 _HA_ATOMIC_OR(&polled_mask[fd].poll_send, tid_bit);
87 }
Olivier Houchard6b96f722018-04-25 16:58:25 +020088
Olivier Houchard6b96f722018-04-25 16:58:25 +020089 if (fd > *max_add_fd)
90 *max_add_fd = fd;
91 }
92}
93
Willy Tarreau4f60f162007-04-08 16:39:58 +020094/*
95 * Poll() poller
96 */
Willy Tarreau03e78532020-02-25 07:38:05 +010097static void _do_poll(struct poller *p, int exp, int wake)
Willy Tarreau4f60f162007-04-08 16:39:58 +020098{
99 int status;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200100 int fd;
Willy Tarreaud825eef2007-05-12 22:35:00 +0200101 int wait_time;
Olivier Houchard6b96f722018-04-25 16:58:25 +0200102 int updt_idx;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200103 int fds, count;
104 int sr, sw;
Willy Tarreau173d9952018-01-26 21:48:23 +0100105 int old_maxfd, new_maxfd, max_add_fd;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200106 unsigned rn, wn; /* read new, write new */
Olivier Houchard6b96f722018-04-25 16:58:25 +0200107 int old_fd;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200108
Willy Tarreau173d9952018-01-26 21:48:23 +0100109 max_add_fd = -1;
110
Willy Tarreaucc7e3f72012-11-11 17:25:15 +0100111 /* first, scan the update list to find changes */
112 for (updt_idx = 0; updt_idx < fd_nbupdt; updt_idx++) {
113 fd = fd_updt[updt_idx];
Willy Tarreaucc7e3f72012-11-11 17:25:15 +0100114
Olivier Houchardcb6c9272019-03-08 18:49:54 +0100115 _HA_ATOMIC_AND(&fdtab[fd].update_mask, ~tid_bit);
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100116 if (!fdtab[fd].owner) {
Willy Tarreaue4063862020-06-17 20:35:33 +0200117 activity[tid].poll_drop_fd++;
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100118 continue;
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100119 }
Olivier Houchard6b96f722018-04-25 16:58:25 +0200120 _update_fd(fd, &max_add_fd);
121 }
Willy Tarreaucc7e3f72012-11-11 17:25:15 +0100122
Olivier Houchard6b96f722018-04-25 16:58:25 +0200123 /* Now scan the global update list */
124 for (old_fd = fd = update_list.first; fd != -1; fd = fdtab[fd].update.next) {
125 if (fd == -2) {
126 fd = old_fd;
127 continue;
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100128 }
Olivier Houchard6b96f722018-04-25 16:58:25 +0200129 else if (fd <= -3)
130 fd = -fd -4;
131 if (fd == -1)
132 break;
133 if (fdtab[fd].update_mask & tid_bit) {
134 /* Cheat a bit, as the state is global to all pollers
135 * we don't need every thread ot take care of the
136 * update.
137 */
Olivier Houchardcb6c9272019-03-08 18:49:54 +0100138 _HA_ATOMIC_AND(&fdtab[fd].update_mask, ~all_threads_mask);
Olivier Houchard6b96f722018-04-25 16:58:25 +0200139 done_update_polling(fd);
140 } else
141 continue;
142 if (!fdtab[fd].owner)
143 continue;
144 _update_fd(fd, &max_add_fd);
Willy Tarreaucc7e3f72012-11-11 17:25:15 +0100145 }
Willy Tarreau173d9952018-01-26 21:48:23 +0100146
147 /* maybe we added at least one fd larger than maxfd */
148 for (old_maxfd = maxfd; old_maxfd <= max_add_fd; ) {
Olivier Houchardcb6c9272019-03-08 18:49:54 +0100149 if (_HA_ATOMIC_CAS(&maxfd, &old_maxfd, max_add_fd + 1))
Willy Tarreau173d9952018-01-26 21:48:23 +0100150 break;
151 }
152
153 /* maxfd doesn't need to be precise but it needs to cover *all* active
154 * FDs. Thus we only shrink it if we have such an opportunity. The algo
155 * is simple : look for the previous used place, try to update maxfd to
156 * point to it, abort if maxfd changed in the mean time.
157 */
158 old_maxfd = maxfd;
159 do {
160 new_maxfd = old_maxfd;
161 while (new_maxfd - 1 >= 0 && !fdtab[new_maxfd - 1].owner)
162 new_maxfd--;
163 if (new_maxfd >= old_maxfd)
164 break;
Olivier Houchardcb6c9272019-03-08 18:49:54 +0100165 } while (!_HA_ATOMIC_CAS(&maxfd, &old_maxfd, new_maxfd));
Willy Tarreau173d9952018-01-26 21:48:23 +0100166
Willy Tarreau60b639c2018-08-02 10:16:17 +0200167 thread_harmless_now();
168
Willy Tarreaucc7e3f72012-11-11 17:25:15 +0100169 fd_nbupdt = 0;
170
Willy Tarreau4f60f162007-04-08 16:39:58 +0200171 nbfd = 0;
Willy Tarreau80da05a2013-03-31 14:06:57 +0200172 for (fds = 0; (fds * 8*sizeof(**fd_evts)) < maxfd; fds++) {
173 rn = fd_evts[DIR_RD][fds];
174 wn = fd_evts[DIR_WR][fds];
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200175
Willy Tarreau80da05a2013-03-31 14:06:57 +0200176 if (!(rn|wn))
177 continue;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200178
Willy Tarreau80da05a2013-03-31 14:06:57 +0200179 for (count = 0, fd = fds * 8*sizeof(**fd_evts); count < 8*sizeof(**fd_evts) && fd < maxfd; count++, fd++) {
180 sr = (rn >> count) & 1;
181 sw = (wn >> count) & 1;
182 if ((sr|sw)) {
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100183 if (!fdtab[fd].owner) {
184 /* should normally not happen here except
185 * due to rare thread concurrency
186 */
187 continue;
188 }
189
190 if (!(fdtab[fd].thread_mask & tid_bit)) {
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100191 continue;
192 }
193
Willy Tarreau80da05a2013-03-31 14:06:57 +0200194 poll_events[nbfd].fd = fd;
Willy Tarreau3c8a8962017-03-13 17:14:51 +0100195 poll_events[nbfd].events = (sr ? (POLLIN | POLLRDHUP) : 0) | (sw ? POLLOUT : 0);
Willy Tarreau80da05a2013-03-31 14:06:57 +0200196 nbfd++;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200197 }
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200198 }
Willy Tarreau4f60f162007-04-08 16:39:58 +0200199 }
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200200
Willy Tarreau4f60f162007-04-08 16:39:58 +0200201 /* now let's wait for events */
Willy Tarreau2ae84e42019-05-28 16:44:05 +0200202 wait_time = wake ? 0 : compute_poll_timeout(exp);
Willy Tarreau7e9c4ae2018-10-17 14:31:19 +0200203 tv_entering_poll();
Willy Tarreau609aad92018-11-22 08:31:09 +0100204 activity_count_runtime();
Willy Tarreau4f60f162007-04-08 16:39:58 +0200205 status = poll(poll_events, nbfd, wait_time);
Willy Tarreau48f8bc12018-11-22 18:57:37 +0100206 tv_update_date(wait_time, status);
Willy Tarreau7e9c4ae2018-10-17 14:31:19 +0200207 tv_leaving_poll(wait_time, status);
Willy Tarreau4f60f162007-04-08 16:39:58 +0200208
Willy Tarreau60b639c2018-08-02 10:16:17 +0200209 thread_harmless_end();
Willy Tarreau54d6add2021-07-30 10:57:09 +0200210 if (sleeping_thread_mask & tid_bit)
211 _HA_ATOMIC_AND(&sleeping_thread_mask, ~tid_bit);
Willy Tarreau60b639c2018-08-02 10:16:17 +0200212
Willy Tarreaue5451532020-06-17 20:25:18 +0200213 if (status > 0)
214 activity[tid].poll_io++;
215
Willy Tarreau4f60f162007-04-08 16:39:58 +0200216 for (count = 0; status > 0 && count < nbfd; count++) {
Christopher Fauletab62f512017-08-30 10:34:36 +0200217 unsigned int n;
Willy Tarreau491c4982012-07-06 11:16:01 +0200218 int e = poll_events[count].revents;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200219 fd = poll_events[count].fd;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200220
Willy Tarreau38e8a1c2020-06-23 10:04:54 +0200221#ifdef DEBUG_FD
222 _HA_ATOMIC_ADD(&fdtab[fd].event_count, 1);
223#endif
Willy Tarreau3c8a8962017-03-13 17:14:51 +0100224 if (!(e & ( POLLOUT | POLLIN | POLLERR | POLLHUP | POLLRDHUP )))
Willy Tarreau4f60f162007-04-08 16:39:58 +0200225 continue;
226
Willy Tarreau076be252012-07-06 16:02:29 +0200227 /* ok, we found one active fd */
228 status--;
229
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100230 if (!fdtab[fd].owner) {
Willy Tarreaue4063862020-06-17 20:35:33 +0200231 activity[tid].poll_dead_fd++;
Willy Tarreau076be252012-07-06 16:02:29 +0200232 continue;
Willy Tarreaud80cb4e2018-01-20 19:30:13 +0100233 }
Willy Tarreau076be252012-07-06 16:02:29 +0200234
Willy Tarreaub7d186d2021-07-30 14:04:28 +0200235 if (!(fdtab[fd].thread_mask & tid_bit)) {
236 activity[tid].poll_skip_fd++;
Willy Tarreaubc547162021-07-30 14:18:49 +0200237 if (!HA_ATOMIC_BTS(&fdtab[fd].update_mask, tid))
238 fd_updt[fd_nbupdt++] = fd;
Willy Tarreaub7d186d2021-07-30 14:04:28 +0200239 continue;
240 }
241
Willy Tarreau6b308982019-09-06 19:05:50 +0200242 n = ((e & POLLIN) ? FD_EV_READY_R : 0) |
243 ((e & POLLOUT) ? FD_EV_READY_W : 0) |
244 ((e & POLLRDHUP) ? FD_EV_SHUT_R : 0) |
245 ((e & POLLHUP) ? FD_EV_SHUT_RW : 0) |
246 ((e & POLLERR) ? FD_EV_ERR_RW : 0);
Willy Tarreau491c4982012-07-06 11:16:01 +0200247
Willy Tarreau6b308982019-09-06 19:05:50 +0200248 if ((e & POLLRDHUP) && !(cur_poller.flags & HAP_POLL_F_RDHUP))
Olivier Houchardcb6c9272019-03-08 18:49:54 +0100249 _HA_ATOMIC_OR(&cur_poller.flags, HAP_POLL_F_RDHUP);
Willy Tarreau6b308982019-09-06 19:05:50 +0200250
Christopher Fauletab62f512017-08-30 10:34:36 +0200251 fd_update_events(fd, n);
Willy Tarreau4f60f162007-04-08 16:39:58 +0200252 }
253
Willy Tarreaue54e9172007-04-09 09:23:31 +0200254}
255
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200256
257static int init_poll_per_thread()
258{
259 poll_events = calloc(1, sizeof(struct pollfd) * global.maxsock);
260 if (poll_events == NULL)
261 return 0;
262 return 1;
263}
264
265static void deinit_poll_per_thread()
266{
267 free(poll_events);
Christopher Fauletcd7879a2017-10-27 13:53:47 +0200268 poll_events = NULL;
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200269}
270
Willy Tarreaue54e9172007-04-09 09:23:31 +0200271/*
272 * Initialization of the poll() poller.
273 * Returns 0 in case of failure, non-zero in case of success. If it fails, it
274 * disables the poller by setting its pref to 0.
275 */
Willy Tarreau03e78532020-02-25 07:38:05 +0100276static int _do_init(struct poller *p)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200277{
Christopher Fauletcd7879a2017-10-27 13:53:47 +0200278 __label__ fail_swevt, fail_srevt;
Willy Tarreau80da05a2013-03-31 14:06:57 +0200279 int fd_evts_bytes;
Willy Tarreaue54e9172007-04-09 09:23:31 +0200280
281 p->private = NULL;
Willy Tarreaucc359232018-01-17 15:48:53 +0100282 fd_evts_bytes = (global.maxsock + sizeof(**fd_evts) * 8 - 1) / (sizeof(**fd_evts) * 8) * sizeof(**fd_evts);
Willy Tarreaue54e9172007-04-09 09:23:31 +0200283
Willy Tarreau80da05a2013-03-31 14:06:57 +0200284 if ((fd_evts[DIR_RD] = calloc(1, fd_evts_bytes)) == NULL)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200285 goto fail_srevt;
Willy Tarreau80da05a2013-03-31 14:06:57 +0200286 if ((fd_evts[DIR_WR] = calloc(1, fd_evts_bytes)) == NULL)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200287 goto fail_swevt;
288
Christopher Fauletcd7879a2017-10-27 13:53:47 +0200289 hap_register_per_thread_init(init_poll_per_thread);
290 hap_register_per_thread_deinit(deinit_poll_per_thread);
291
Willy Tarreaue54e9172007-04-09 09:23:31 +0200292 return 1;
293
294 fail_swevt:
295 free(fd_evts[DIR_RD]);
296 fail_srevt:
Willy Tarreaue54e9172007-04-09 09:23:31 +0200297 p->pref = 0;
298 return 0;
299}
300
301/*
302 * Termination of the poll() poller.
303 * Memory is released and the poller is marked as unselectable.
304 */
Willy Tarreau03e78532020-02-25 07:38:05 +0100305static void _do_term(struct poller *p)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200306{
Willy Tarreaua534fea2008-08-03 12:19:50 +0200307 free(fd_evts[DIR_WR]);
308 free(fd_evts[DIR_RD]);
Willy Tarreaue54e9172007-04-09 09:23:31 +0200309 p->private = NULL;
310 p->pref = 0;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200311}
312
313/*
Willy Tarreau2ff76222007-04-09 19:29:56 +0200314 * Check that the poller works.
315 * Returns 1 if OK, otherwise 0.
316 */
Willy Tarreau03e78532020-02-25 07:38:05 +0100317static int _do_test(struct poller *p)
Willy Tarreau2ff76222007-04-09 19:29:56 +0200318{
319 return 1;
320}
321
322/*
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200323 * It is a constructor, which means that it will automatically be called before
324 * main(). This is GCC-specific but it works at least since 2.95.
325 * Special care must be taken so that it does not need any uninitialized data.
Willy Tarreau4f60f162007-04-08 16:39:58 +0200326 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200327__attribute__((constructor))
328static void _do_register(void)
Willy Tarreau4f60f162007-04-08 16:39:58 +0200329{
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200330 struct poller *p;
331
332 if (nbpollers >= MAX_POLLERS)
333 return;
334 p = &pollers[nbpollers++];
335
Willy Tarreau4f60f162007-04-08 16:39:58 +0200336 p->name = "poll";
337 p->pref = 200;
Willy Tarreau11ef0832019-11-28 18:17:33 +0100338 p->flags = HAP_POLL_F_ERRHUP;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200339 p->private = NULL;
340
Willy Tarreau70c6fd82012-11-11 21:02:34 +0100341 p->clo = __fd_clo;
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200342 p->test = _do_test;
343 p->init = _do_init;
344 p->term = _do_term;
345 p->poll = _do_poll;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200346}
347
348
349/*
350 * Local variables:
351 * c-indent-level: 8
352 * c-basic-offset: 8
353 * End:
354 */