Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 1 | /* |
Willy Tarreau | e9f49e7 | 2012-11-11 17:42:00 +0100 | [diff] [blame] | 2 | * FD polling functions for Linux epoll |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 3 | * |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 4 | * Copyright 2000-2014 Willy Tarreau <w@1wt.eu> |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 5 | * |
| 6 | * This program is free software; you can redistribute it and/or |
| 7 | * modify it under the terms of the GNU General Public License |
| 8 | * as published by the Free Software Foundation; either version |
| 9 | * 2 of the License, or (at your option) any later version. |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 10 | */ |
| 11 | |
| 12 | #include <unistd.h> |
Willy Tarreau | 3858b12 | 2020-03-10 07:08:10 +0100 | [diff] [blame] | 13 | #include <sys/epoll.h> |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 14 | #include <sys/time.h> |
| 15 | #include <sys/types.h> |
| 16 | |
Willy Tarreau | 4c7e4b7 | 2020-05-27 12:58:42 +0200 | [diff] [blame] | 17 | #include <haproxy/api.h> |
Willy Tarreau | f268ee8 | 2020-06-04 17:05:57 +0200 | [diff] [blame] | 18 | #include <haproxy/global.h> |
Willy Tarreau | 3f567e4 | 2020-05-28 15:29:19 +0200 | [diff] [blame] | 19 | #include <haproxy/thread-t.h> |
Willy Tarreau | 48fbcae | 2020-06-03 18:09:46 +0200 | [diff] [blame] | 20 | #include <haproxy/tools.h> |
Willy Tarreau | c2f7c58 | 2020-06-02 18:15:32 +0200 | [diff] [blame] | 21 | #include <haproxy/ticks.h> |
Willy Tarreau | 92b4f13 | 2020-06-01 11:05:15 +0200 | [diff] [blame] | 22 | #include <haproxy/time.h> |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 23 | |
Willy Tarreau | a04ded5 | 2020-06-02 10:29:48 +0200 | [diff] [blame] | 24 | #include <haproxy/activity.h> |
Willy Tarreau | 0f6ffd6 | 2020-06-03 19:33:00 +0200 | [diff] [blame] | 25 | #include <haproxy/fd.h> |
Willy Tarreau | beb859a | 2018-11-22 18:07:59 +0100 | [diff] [blame] | 26 | #include <proto/signal.h> |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 27 | |
Willy Tarreau | 58094f2 | 2007-04-10 01:33:20 +0200 | [diff] [blame] | 28 | |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 29 | /* private data */ |
Christopher Faulet | d4604ad | 2017-05-29 10:40:41 +0200 | [diff] [blame] | 30 | static THREAD_LOCAL struct epoll_event *epoll_events = NULL; |
Willy Tarreau | d9e7e36 | 2018-01-18 19:16:02 +0100 | [diff] [blame] | 31 | static int epoll_fd[MAX_THREADS]; // per-thread epoll_fd |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 32 | |
Willy Tarreau | 1c07b07 | 2013-01-07 16:19:18 +0100 | [diff] [blame] | 33 | #ifndef EPOLLRDHUP |
| 34 | /* EPOLLRDHUP was defined late in libc, and it appeared in kernel 2.6.17 */ |
| 35 | #define EPOLLRDHUP 0x2000 |
| 36 | #endif |
| 37 | |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 38 | /* |
Conrad Hoffmann | 041751c | 2014-05-20 14:28:24 +0200 | [diff] [blame] | 39 | * Immediately remove file descriptor from epoll set upon close. |
| 40 | * Since we forked, some fds share inodes with the other process, and epoll may |
| 41 | * send us events even though this process closed the fd (see man 7 epoll, |
| 42 | * "Questions and answers", Q 6). |
| 43 | */ |
Willy Tarreau | 03e7853 | 2020-02-25 07:38:05 +0100 | [diff] [blame] | 44 | static void __fd_clo(int fd) |
Conrad Hoffmann | 041751c | 2014-05-20 14:28:24 +0200 | [diff] [blame] | 45 | { |
Willy Tarreau | d9e7e36 | 2018-01-18 19:16:02 +0100 | [diff] [blame] | 46 | if (unlikely(fdtab[fd].cloned)) { |
Olivier Houchard | 5305505 | 2019-07-25 14:00:18 +0000 | [diff] [blame] | 47 | unsigned long m = polled_mask[fd].poll_recv | polled_mask[fd].poll_send; |
Willy Tarreau | 902871d | 2020-02-20 11:23:43 +0100 | [diff] [blame] | 48 | struct epoll_event ev; |
Willy Tarreau | d9e7e36 | 2018-01-18 19:16:02 +0100 | [diff] [blame] | 49 | int i; |
| 50 | |
| 51 | for (i = global.nbthread - 1; i >= 0; i--) |
| 52 | if (m & (1UL << i)) |
| 53 | epoll_ctl(epoll_fd[i], EPOLL_CTL_DEL, fd, &ev); |
| 54 | } |
Conrad Hoffmann | 041751c | 2014-05-20 14:28:24 +0200 | [diff] [blame] | 55 | } |
| 56 | |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 57 | static void _update_fd(int fd) |
| 58 | { |
| 59 | int en, opcode; |
Willy Tarreau | 55c5399 | 2020-02-26 14:32:53 +0100 | [diff] [blame] | 60 | struct epoll_event ev = { }; |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 61 | |
| 62 | en = fdtab[fd].state; |
| 63 | |
Willy Tarreau | 5d7dcc2 | 2019-12-26 16:40:24 +0100 | [diff] [blame] | 64 | /* if we're already polling or are going to poll for this FD and it's |
| 65 | * neither active nor ready, force it to be active so that we don't |
| 66 | * needlessly unsubscribe then re-subscribe it. |
| 67 | */ |
| 68 | if (!(en & FD_EV_READY_R) && |
| 69 | ((en & FD_EV_ACTIVE_W) || |
| 70 | ((polled_mask[fd].poll_send | polled_mask[fd].poll_recv) & tid_bit))) |
| 71 | en |= FD_EV_ACTIVE_R; |
| 72 | |
Olivier Houchard | 5305505 | 2019-07-25 14:00:18 +0000 | [diff] [blame] | 73 | if ((polled_mask[fd].poll_send | polled_mask[fd].poll_recv) & tid_bit) { |
Willy Tarreau | 5bee3e2 | 2019-09-04 09:52:57 +0200 | [diff] [blame] | 74 | if (!(fdtab[fd].thread_mask & tid_bit) || !(en & FD_EV_ACTIVE_RW)) { |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 75 | /* fd removed from poll list */ |
| 76 | opcode = EPOLL_CTL_DEL; |
Olivier Houchard | 5305505 | 2019-07-25 14:00:18 +0000 | [diff] [blame] | 77 | if (polled_mask[fd].poll_recv & tid_bit) |
| 78 | _HA_ATOMIC_AND(&polled_mask[fd].poll_recv, ~tid_bit); |
| 79 | if (polled_mask[fd].poll_send & tid_bit) |
| 80 | _HA_ATOMIC_AND(&polled_mask[fd].poll_send, ~tid_bit); |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 81 | } |
| 82 | else { |
Willy Tarreau | 5bee3e2 | 2019-09-04 09:52:57 +0200 | [diff] [blame] | 83 | if (((en & FD_EV_ACTIVE_R) != 0) == |
Olivier Houchard | 5305505 | 2019-07-25 14:00:18 +0000 | [diff] [blame] | 84 | ((polled_mask[fd].poll_recv & tid_bit) != 0) && |
Willy Tarreau | 5bee3e2 | 2019-09-04 09:52:57 +0200 | [diff] [blame] | 85 | ((en & FD_EV_ACTIVE_W) != 0) == |
Olivier Houchard | 5305505 | 2019-07-25 14:00:18 +0000 | [diff] [blame] | 86 | ((polled_mask[fd].poll_send & tid_bit) != 0)) |
| 87 | return; |
Willy Tarreau | 5bee3e2 | 2019-09-04 09:52:57 +0200 | [diff] [blame] | 88 | if (en & FD_EV_ACTIVE_R) { |
Olivier Houchard | 5305505 | 2019-07-25 14:00:18 +0000 | [diff] [blame] | 89 | if (!(polled_mask[fd].poll_recv & tid_bit)) |
| 90 | _HA_ATOMIC_OR(&polled_mask[fd].poll_recv, tid_bit); |
| 91 | } else { |
| 92 | if (polled_mask[fd].poll_recv & tid_bit) |
| 93 | _HA_ATOMIC_AND(&polled_mask[fd].poll_recv, ~tid_bit); |
| 94 | } |
Willy Tarreau | 5bee3e2 | 2019-09-04 09:52:57 +0200 | [diff] [blame] | 95 | if (en & FD_EV_ACTIVE_W) { |
Olivier Houchard | 5305505 | 2019-07-25 14:00:18 +0000 | [diff] [blame] | 96 | if (!(polled_mask[fd].poll_send & tid_bit)) |
| 97 | _HA_ATOMIC_OR(&polled_mask[fd].poll_send, tid_bit); |
| 98 | } else { |
| 99 | if (polled_mask[fd].poll_send & tid_bit) |
| 100 | _HA_ATOMIC_AND(&polled_mask[fd].poll_send, ~tid_bit); |
| 101 | } |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 102 | /* fd status changed */ |
| 103 | opcode = EPOLL_CTL_MOD; |
| 104 | } |
| 105 | } |
Willy Tarreau | 5bee3e2 | 2019-09-04 09:52:57 +0200 | [diff] [blame] | 106 | else if ((fdtab[fd].thread_mask & tid_bit) && (en & FD_EV_ACTIVE_RW)) { |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 107 | /* new fd in the poll list */ |
| 108 | opcode = EPOLL_CTL_ADD; |
Willy Tarreau | 5bee3e2 | 2019-09-04 09:52:57 +0200 | [diff] [blame] | 109 | if (en & FD_EV_ACTIVE_R) |
Olivier Houchard | 5305505 | 2019-07-25 14:00:18 +0000 | [diff] [blame] | 110 | _HA_ATOMIC_OR(&polled_mask[fd].poll_recv, tid_bit); |
Willy Tarreau | 5bee3e2 | 2019-09-04 09:52:57 +0200 | [diff] [blame] | 111 | if (en & FD_EV_ACTIVE_W) |
Olivier Houchard | 5305505 | 2019-07-25 14:00:18 +0000 | [diff] [blame] | 112 | _HA_ATOMIC_OR(&polled_mask[fd].poll_send, tid_bit); |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 113 | } |
| 114 | else { |
| 115 | return; |
| 116 | } |
| 117 | |
| 118 | /* construct the epoll events based on new state */ |
Willy Tarreau | 5bee3e2 | 2019-09-04 09:52:57 +0200 | [diff] [blame] | 119 | if (en & FD_EV_ACTIVE_R) |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 120 | ev.events |= EPOLLIN | EPOLLRDHUP; |
| 121 | |
Willy Tarreau | 5bee3e2 | 2019-09-04 09:52:57 +0200 | [diff] [blame] | 122 | if (en & FD_EV_ACTIVE_W) |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 123 | ev.events |= EPOLLOUT; |
| 124 | |
| 125 | ev.data.fd = fd; |
| 126 | epoll_ctl(epoll_fd[tid], opcode, fd, &ev); |
| 127 | } |
| 128 | |
Conrad Hoffmann | 041751c | 2014-05-20 14:28:24 +0200 | [diff] [blame] | 129 | /* |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 130 | * Linux epoll() poller |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 131 | */ |
Willy Tarreau | 03e7853 | 2020-02-25 07:38:05 +0100 | [diff] [blame] | 132 | static void _do_poll(struct poller *p, int exp, int wake) |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 133 | { |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 134 | int status; |
| 135 | int fd; |
Willy Tarreau | e9f49e7 | 2012-11-11 17:42:00 +0100 | [diff] [blame] | 136 | int count; |
| 137 | int updt_idx; |
| 138 | int wait_time; |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 139 | int old_fd; |
Willy Tarreau | 58094f2 | 2007-04-10 01:33:20 +0200 | [diff] [blame] | 140 | |
Willy Tarreau | 5be2f35 | 2014-11-19 19:43:05 +0100 | [diff] [blame] | 141 | /* first, scan the update list to find polling changes */ |
Willy Tarreau | e9f49e7 | 2012-11-11 17:42:00 +0100 | [diff] [blame] | 142 | for (updt_idx = 0; updt_idx < fd_nbupdt; updt_idx++) { |
| 143 | fd = fd_updt[updt_idx]; |
Willy Tarreau | 58094f2 | 2007-04-10 01:33:20 +0200 | [diff] [blame] | 144 | |
Olivier Houchard | cb6c927 | 2019-03-08 18:49:54 +0100 | [diff] [blame] | 145 | _HA_ATOMIC_AND(&fdtab[fd].update_mask, ~tid_bit); |
Willy Tarreau | d80cb4e | 2018-01-20 19:30:13 +0100 | [diff] [blame] | 146 | if (!fdtab[fd].owner) { |
| 147 | activity[tid].poll_drop++; |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 148 | continue; |
Willy Tarreau | d80cb4e | 2018-01-20 19:30:13 +0100 | [diff] [blame] | 149 | } |
Willy Tarreau | 58094f2 | 2007-04-10 01:33:20 +0200 | [diff] [blame] | 150 | |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 151 | _update_fd(fd); |
| 152 | } |
| 153 | fd_nbupdt = 0; |
| 154 | /* Scan the global update list */ |
| 155 | for (old_fd = fd = update_list.first; fd != -1; fd = fdtab[fd].update.next) { |
| 156 | if (fd == -2) { |
| 157 | fd = old_fd; |
Willy Tarreau | d9e7e36 | 2018-01-18 19:16:02 +0100 | [diff] [blame] | 158 | continue; |
| 159 | } |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 160 | else if (fd <= -3) |
| 161 | fd = -fd -4; |
| 162 | if (fd == -1) |
| 163 | break; |
| 164 | if (fdtab[fd].update_mask & tid_bit) |
| 165 | done_update_polling(fd); |
| 166 | else |
| 167 | continue; |
| 168 | if (!fdtab[fd].owner) |
| 169 | continue; |
| 170 | _update_fd(fd); |
Willy Tarreau | 58094f2 | 2007-04-10 01:33:20 +0200 | [diff] [blame] | 171 | } |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 172 | |
Willy Tarreau | 60b639c | 2018-08-02 10:16:17 +0200 | [diff] [blame] | 173 | thread_harmless_now(); |
| 174 | |
Willy Tarreau | e9f49e7 | 2012-11-11 17:42:00 +0100 | [diff] [blame] | 175 | /* now let's wait for polled events */ |
Willy Tarreau | 2ae84e4 | 2019-05-28 16:44:05 +0200 | [diff] [blame] | 176 | wait_time = wake ? 0 : compute_poll_timeout(exp); |
Willy Tarreau | 7e9c4ae | 2018-10-17 14:31:19 +0200 | [diff] [blame] | 177 | tv_entering_poll(); |
Willy Tarreau | 609aad9 | 2018-11-22 08:31:09 +0100 | [diff] [blame] | 178 | activity_count_runtime(); |
Willy Tarreau | beb859a | 2018-11-22 18:07:59 +0100 | [diff] [blame] | 179 | do { |
| 180 | int timeout = (global.tune.options & GTUNE_BUSY_POLLING) ? 0 : wait_time; |
| 181 | |
| 182 | status = epoll_wait(epoll_fd[tid], epoll_events, global.tune.maxpollevents, timeout); |
| 183 | tv_update_date(timeout, status); |
| 184 | |
| 185 | if (status) |
| 186 | break; |
| 187 | if (timeout || !wait_time) |
| 188 | break; |
Willy Tarreau | 2ae84e4 | 2019-05-28 16:44:05 +0200 | [diff] [blame] | 189 | if (signal_queue_len || wake) |
Willy Tarreau | beb859a | 2018-11-22 18:07:59 +0100 | [diff] [blame] | 190 | break; |
| 191 | if (tick_isset(exp) && tick_is_expired(exp, now_ms)) |
| 192 | break; |
| 193 | } while (1); |
| 194 | |
Willy Tarreau | 7e9c4ae | 2018-10-17 14:31:19 +0200 | [diff] [blame] | 195 | tv_leaving_poll(wait_time, status); |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 196 | |
Willy Tarreau | 60b639c | 2018-08-02 10:16:17 +0200 | [diff] [blame] | 197 | thread_harmless_end(); |
Olivier Houchard | 305d5ab | 2019-07-24 18:07:06 +0200 | [diff] [blame] | 198 | if (sleeping_thread_mask & tid_bit) |
| 199 | _HA_ATOMIC_AND(&sleeping_thread_mask, ~tid_bit); |
Willy Tarreau | 60b639c | 2018-08-02 10:16:17 +0200 | [diff] [blame] | 200 | |
Willy Tarreau | e9f49e7 | 2012-11-11 17:42:00 +0100 | [diff] [blame] | 201 | /* process polled events */ |
| 202 | |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 203 | for (count = 0; count < status; count++) { |
Willy Tarreau | 902871d | 2020-02-20 11:23:43 +0100 | [diff] [blame] | 204 | struct epoll_event ev; |
| 205 | unsigned int n, e; |
| 206 | |
| 207 | e = epoll_events[count].events; |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 208 | fd = epoll_events[count].data.fd; |
| 209 | |
Willy Tarreau | d80cb4e | 2018-01-20 19:30:13 +0100 | [diff] [blame] | 210 | if (!fdtab[fd].owner) { |
| 211 | activity[tid].poll_dead++; |
| 212 | continue; |
| 213 | } |
| 214 | |
| 215 | if (!(fdtab[fd].thread_mask & tid_bit)) { |
Willy Tarreau | d9e7e36 | 2018-01-18 19:16:02 +0100 | [diff] [blame] | 216 | /* FD has been migrated */ |
Willy Tarreau | d80cb4e | 2018-01-20 19:30:13 +0100 | [diff] [blame] | 217 | activity[tid].poll_skip++; |
Willy Tarreau | d9e7e36 | 2018-01-18 19:16:02 +0100 | [diff] [blame] | 218 | epoll_ctl(epoll_fd[tid], EPOLL_CTL_DEL, fd, &ev); |
Olivier Houchard | 5305505 | 2019-07-25 14:00:18 +0000 | [diff] [blame] | 219 | _HA_ATOMIC_AND(&polled_mask[fd].poll_recv, ~tid_bit); |
| 220 | _HA_ATOMIC_AND(&polled_mask[fd].poll_send, ~tid_bit); |
Willy Tarreau | 076be25 | 2012-07-06 16:02:29 +0200 | [diff] [blame] | 221 | continue; |
Willy Tarreau | d80cb4e | 2018-01-20 19:30:13 +0100 | [diff] [blame] | 222 | } |
Willy Tarreau | 076be25 | 2012-07-06 16:02:29 +0200 | [diff] [blame] | 223 | |
Willy Tarreau | 6b30898 | 2019-09-06 19:05:50 +0200 | [diff] [blame] | 224 | n = ((e & EPOLLIN) ? FD_EV_READY_R : 0) | |
| 225 | ((e & EPOLLOUT) ? FD_EV_READY_W : 0) | |
| 226 | ((e & EPOLLRDHUP) ? FD_EV_SHUT_R : 0) | |
| 227 | ((e & EPOLLHUP) ? FD_EV_SHUT_RW : 0) | |
| 228 | ((e & EPOLLERR) ? FD_EV_ERR_RW : 0); |
Willy Tarreau | 491c498 | 2012-07-06 11:16:01 +0200 | [diff] [blame] | 229 | |
Willy Tarreau | 6b30898 | 2019-09-06 19:05:50 +0200 | [diff] [blame] | 230 | if ((e & EPOLLRDHUP) && !(cur_poller.flags & HAP_POLL_F_RDHUP)) |
Olivier Houchard | cb6c927 | 2019-03-08 18:49:54 +0100 | [diff] [blame] | 231 | _HA_ATOMIC_OR(&cur_poller.flags, HAP_POLL_F_RDHUP); |
Willy Tarreau | 6b30898 | 2019-09-06 19:05:50 +0200 | [diff] [blame] | 232 | |
Christopher Faulet | ab62f51 | 2017-08-30 10:34:36 +0200 | [diff] [blame] | 233 | fd_update_events(fd, n); |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 234 | } |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 235 | /* the caller will take care of cached events */ |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 236 | } |
| 237 | |
Christopher Faulet | d4604ad | 2017-05-29 10:40:41 +0200 | [diff] [blame] | 238 | static int init_epoll_per_thread() |
| 239 | { |
Willy Tarreau | d9e7e36 | 2018-01-18 19:16:02 +0100 | [diff] [blame] | 240 | int fd; |
| 241 | |
Christopher Faulet | d4604ad | 2017-05-29 10:40:41 +0200 | [diff] [blame] | 242 | epoll_events = calloc(1, sizeof(struct epoll_event) * global.tune.maxpollevents); |
| 243 | if (epoll_events == NULL) |
Willy Tarreau | d9e7e36 | 2018-01-18 19:16:02 +0100 | [diff] [blame] | 244 | goto fail_alloc; |
| 245 | |
Christopher Faulet | 3e805ed | 2018-01-25 16:18:09 +0100 | [diff] [blame] | 246 | if (MAX_THREADS > 1 && tid) { |
Willy Tarreau | d9e7e36 | 2018-01-18 19:16:02 +0100 | [diff] [blame] | 247 | epoll_fd[tid] = epoll_create(global.maxsock + 1); |
| 248 | if (epoll_fd[tid] < 0) |
| 249 | goto fail_fd; |
| 250 | } |
| 251 | |
| 252 | /* we may have to unregister some events initially registered on the |
| 253 | * original fd when it was alone, and/or to register events on the new |
| 254 | * fd for this thread. Let's just mark them as updated, the poller will |
| 255 | * do the rest. |
| 256 | */ |
Willy Tarreau | ce036bc | 2018-01-29 14:58:02 +0100 | [diff] [blame] | 257 | for (fd = 0; fd < global.maxsock; fd++) |
Willy Tarreau | d9e7e36 | 2018-01-18 19:16:02 +0100 | [diff] [blame] | 258 | updt_fd_polling(fd); |
| 259 | |
Christopher Faulet | d4604ad | 2017-05-29 10:40:41 +0200 | [diff] [blame] | 260 | return 1; |
Willy Tarreau | d9e7e36 | 2018-01-18 19:16:02 +0100 | [diff] [blame] | 261 | fail_fd: |
| 262 | free(epoll_events); |
| 263 | fail_alloc: |
| 264 | return 0; |
Christopher Faulet | d4604ad | 2017-05-29 10:40:41 +0200 | [diff] [blame] | 265 | } |
| 266 | |
| 267 | static void deinit_epoll_per_thread() |
| 268 | { |
Christopher Faulet | 3e805ed | 2018-01-25 16:18:09 +0100 | [diff] [blame] | 269 | if (MAX_THREADS > 1 && tid) |
Willy Tarreau | d9e7e36 | 2018-01-18 19:16:02 +0100 | [diff] [blame] | 270 | close(epoll_fd[tid]); |
| 271 | |
Christopher Faulet | d4604ad | 2017-05-29 10:40:41 +0200 | [diff] [blame] | 272 | free(epoll_events); |
Christopher Faulet | cd7879a | 2017-10-27 13:53:47 +0200 | [diff] [blame] | 273 | epoll_events = NULL; |
Christopher Faulet | d4604ad | 2017-05-29 10:40:41 +0200 | [diff] [blame] | 274 | } |
| 275 | |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 276 | /* |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 277 | * Initialization of the epoll() poller. |
Willy Tarreau | e54e917 | 2007-04-09 09:23:31 +0200 | [diff] [blame] | 278 | * Returns 0 in case of failure, non-zero in case of success. If it fails, it |
| 279 | * disables the poller by setting its pref to 0. |
| 280 | */ |
Willy Tarreau | 03e7853 | 2020-02-25 07:38:05 +0100 | [diff] [blame] | 281 | static int _do_init(struct poller *p) |
Willy Tarreau | e54e917 | 2007-04-09 09:23:31 +0200 | [diff] [blame] | 282 | { |
Willy Tarreau | e54e917 | 2007-04-09 09:23:31 +0200 | [diff] [blame] | 283 | p->private = NULL; |
Willy Tarreau | e54e917 | 2007-04-09 09:23:31 +0200 | [diff] [blame] | 284 | |
Willy Tarreau | d9e7e36 | 2018-01-18 19:16:02 +0100 | [diff] [blame] | 285 | epoll_fd[tid] = epoll_create(global.maxsock + 1); |
| 286 | if (epoll_fd[tid] < 0) |
Willy Tarreau | e54e917 | 2007-04-09 09:23:31 +0200 | [diff] [blame] | 287 | goto fail_fd; |
| 288 | |
Christopher Faulet | cd7879a | 2017-10-27 13:53:47 +0200 | [diff] [blame] | 289 | hap_register_per_thread_init(init_epoll_per_thread); |
| 290 | hap_register_per_thread_deinit(deinit_epoll_per_thread); |
Willy Tarreau | e54e917 | 2007-04-09 09:23:31 +0200 | [diff] [blame] | 291 | |
Willy Tarreau | e54e917 | 2007-04-09 09:23:31 +0200 | [diff] [blame] | 292 | return 1; |
| 293 | |
Willy Tarreau | e54e917 | 2007-04-09 09:23:31 +0200 | [diff] [blame] | 294 | fail_fd: |
| 295 | p->pref = 0; |
| 296 | return 0; |
| 297 | } |
| 298 | |
| 299 | /* |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 300 | * Termination of the epoll() poller. |
Willy Tarreau | e54e917 | 2007-04-09 09:23:31 +0200 | [diff] [blame] | 301 | * Memory is released and the poller is marked as unselectable. |
| 302 | */ |
Willy Tarreau | 03e7853 | 2020-02-25 07:38:05 +0100 | [diff] [blame] | 303 | static void _do_term(struct poller *p) |
Willy Tarreau | e54e917 | 2007-04-09 09:23:31 +0200 | [diff] [blame] | 304 | { |
Willy Tarreau | d9e7e36 | 2018-01-18 19:16:02 +0100 | [diff] [blame] | 305 | if (epoll_fd[tid] >= 0) { |
| 306 | close(epoll_fd[tid]); |
| 307 | epoll_fd[tid] = -1; |
Willy Tarreau | d79e79b | 2009-05-10 10:18:54 +0200 | [diff] [blame] | 308 | } |
Willy Tarreau | e54e917 | 2007-04-09 09:23:31 +0200 | [diff] [blame] | 309 | |
| 310 | p->private = NULL; |
| 311 | p->pref = 0; |
| 312 | } |
| 313 | |
| 314 | /* |
Willy Tarreau | 2ff7622 | 2007-04-09 19:29:56 +0200 | [diff] [blame] | 315 | * Check that the poller works. |
| 316 | * Returns 1 if OK, otherwise 0. |
| 317 | */ |
Willy Tarreau | 03e7853 | 2020-02-25 07:38:05 +0100 | [diff] [blame] | 318 | static int _do_test(struct poller *p) |
Willy Tarreau | 2ff7622 | 2007-04-09 19:29:56 +0200 | [diff] [blame] | 319 | { |
| 320 | int fd; |
| 321 | |
| 322 | fd = epoll_create(global.maxsock + 1); |
| 323 | if (fd < 0) |
| 324 | return 0; |
| 325 | close(fd); |
| 326 | return 1; |
| 327 | } |
| 328 | |
| 329 | /* |
Willy Tarreau | fb8983f | 2007-06-03 16:40:44 +0200 | [diff] [blame] | 330 | * Recreate the epoll file descriptor after a fork(). Returns 1 if OK, |
| 331 | * otherwise 0. It will ensure that all processes will not share their |
| 332 | * epoll_fd. Some side effects were encountered because of this, such |
| 333 | * as epoll_wait() returning an FD which was previously deleted. |
| 334 | */ |
Willy Tarreau | 03e7853 | 2020-02-25 07:38:05 +0100 | [diff] [blame] | 335 | static int _do_fork(struct poller *p) |
Willy Tarreau | fb8983f | 2007-06-03 16:40:44 +0200 | [diff] [blame] | 336 | { |
Willy Tarreau | d9e7e36 | 2018-01-18 19:16:02 +0100 | [diff] [blame] | 337 | if (epoll_fd[tid] >= 0) |
| 338 | close(epoll_fd[tid]); |
| 339 | epoll_fd[tid] = epoll_create(global.maxsock + 1); |
| 340 | if (epoll_fd[tid] < 0) |
Willy Tarreau | fb8983f | 2007-06-03 16:40:44 +0200 | [diff] [blame] | 341 | return 0; |
| 342 | return 1; |
| 343 | } |
| 344 | |
| 345 | /* |
Willy Tarreau | ef1d1f8 | 2007-04-16 00:25:25 +0200 | [diff] [blame] | 346 | * It is a constructor, which means that it will automatically be called before |
| 347 | * main(). This is GCC-specific but it works at least since 2.95. |
| 348 | * Special care must be taken so that it does not need any uninitialized data. |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 349 | */ |
Willy Tarreau | ef1d1f8 | 2007-04-16 00:25:25 +0200 | [diff] [blame] | 350 | __attribute__((constructor)) |
| 351 | static void _do_register(void) |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 352 | { |
Willy Tarreau | ef1d1f8 | 2007-04-16 00:25:25 +0200 | [diff] [blame] | 353 | struct poller *p; |
Willy Tarreau | d9e7e36 | 2018-01-18 19:16:02 +0100 | [diff] [blame] | 354 | int i; |
Willy Tarreau | ef1d1f8 | 2007-04-16 00:25:25 +0200 | [diff] [blame] | 355 | |
| 356 | if (nbpollers >= MAX_POLLERS) |
| 357 | return; |
Willy Tarreau | d79e79b | 2009-05-10 10:18:54 +0200 | [diff] [blame] | 358 | |
Willy Tarreau | d9e7e36 | 2018-01-18 19:16:02 +0100 | [diff] [blame] | 359 | for (i = 0; i < MAX_THREADS; i++) |
| 360 | epoll_fd[i] = -1; |
| 361 | |
Willy Tarreau | ef1d1f8 | 2007-04-16 00:25:25 +0200 | [diff] [blame] | 362 | p = &pollers[nbpollers++]; |
| 363 | |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 364 | p->name = "epoll"; |
| 365 | p->pref = 300; |
Willy Tarreau | 11ef083 | 2019-11-28 18:17:33 +0100 | [diff] [blame] | 366 | p->flags = HAP_POLL_F_ERRHUP; // note: RDHUP might be dynamically added |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 367 | p->private = NULL; |
| 368 | |
Conrad Hoffmann | 041751c | 2014-05-20 14:28:24 +0200 | [diff] [blame] | 369 | p->clo = __fd_clo; |
Willy Tarreau | ef1d1f8 | 2007-04-16 00:25:25 +0200 | [diff] [blame] | 370 | p->test = _do_test; |
| 371 | p->init = _do_init; |
| 372 | p->term = _do_term; |
| 373 | p->poll = _do_poll; |
Willy Tarreau | fb8983f | 2007-06-03 16:40:44 +0200 | [diff] [blame] | 374 | p->fork = _do_fork; |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 375 | } |
| 376 | |
| 377 | |
| 378 | /* |
| 379 | * Local variables: |
| 380 | * c-indent-level: 8 |
| 381 | * c-basic-offset: 8 |
| 382 | * End: |
| 383 | */ |