Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 1 | /* |
Willy Tarreau | 49b046d | 2012-08-09 12:11:58 +0200 | [diff] [blame] | 2 | * include/proto/fd.h |
| 3 | * File descriptors states. |
| 4 | * |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 5 | * Copyright (C) 2000-2014 Willy Tarreau - w@1wt.eu |
Willy Tarreau | 49b046d | 2012-08-09 12:11:58 +0200 | [diff] [blame] | 6 | * |
| 7 | * This library is free software; you can redistribute it and/or |
| 8 | * modify it under the terms of the GNU Lesser General Public |
| 9 | * License as published by the Free Software Foundation, version 2.1 |
| 10 | * exclusively. |
| 11 | * |
| 12 | * This library is distributed in the hope that it will be useful, |
| 13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 15 | * Lesser General Public License for more details. |
| 16 | * |
| 17 | * You should have received a copy of the GNU Lesser General Public |
| 18 | * License along with this library; if not, write to the Free Software |
| 19 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
| 20 | */ |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 21 | |
| 22 | #ifndef _PROTO_FD_H |
| 23 | #define _PROTO_FD_H |
| 24 | |
Willy Tarreau | 2ff7622 | 2007-04-09 19:29:56 +0200 | [diff] [blame] | 25 | #include <stdio.h> |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 26 | #include <sys/time.h> |
| 27 | #include <sys/types.h> |
| 28 | #include <unistd.h> |
| 29 | |
Willy Tarreau | e3ba5f0 | 2006-06-29 18:54:54 +0200 | [diff] [blame] | 30 | #include <common/config.h> |
Willy Tarreau | f37ba94 | 2018-10-17 11:25:54 +0200 | [diff] [blame] | 31 | #include <common/ticks.h> |
| 32 | #include <common/time.h> |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 33 | #include <types/fd.h> |
Willy Tarreau | 609aad9 | 2018-11-22 08:31:09 +0100 | [diff] [blame] | 34 | #include <proto/activity.h> |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 35 | |
Willy Tarreau | 7be79a4 | 2012-11-11 15:02:54 +0100 | [diff] [blame] | 36 | /* public variables */ |
Christopher Faulet | d4604ad | 2017-05-29 10:40:41 +0200 | [diff] [blame] | 37 | |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 38 | extern volatile struct fdlist update_list; |
| 39 | |
Olivier Houchard | cb92f5c | 2018-04-26 14:23:07 +0200 | [diff] [blame] | 40 | extern unsigned long *polled_mask; |
| 41 | |
Christopher Faulet | d4604ad | 2017-05-29 10:40:41 +0200 | [diff] [blame] | 42 | extern THREAD_LOCAL int *fd_updt; // FD updates list |
| 43 | extern THREAD_LOCAL int fd_nbupdt; // number of updates in the list |
| 44 | |
Olivier Houchard | 79321b9 | 2018-07-26 17:55:11 +0200 | [diff] [blame] | 45 | extern int poller_wr_pipe[MAX_THREADS]; |
| 46 | |
Olivier Houchard | 7c49d2e | 2019-04-16 18:37:05 +0200 | [diff] [blame] | 47 | extern volatile int ha_used_fds; // Number of FDs we're currently using |
| 48 | |
Willy Tarreau | 173d995 | 2018-01-26 21:48:23 +0100 | [diff] [blame] | 49 | /* Deletes an FD from the fdsets. |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 50 | * The file descriptor is also closed. |
| 51 | */ |
| 52 | void fd_delete(int fd); |
| 53 | |
Willy Tarreau | 173d995 | 2018-01-26 21:48:23 +0100 | [diff] [blame] | 54 | /* Deletes an FD from the fdsets. |
Olivier Houchard | 1fc0516 | 2017-04-06 01:05:05 +0200 | [diff] [blame] | 55 | * The file descriptor is kept open. |
| 56 | */ |
| 57 | void fd_remove(int fd); |
| 58 | |
Willy Tarreau | 2d7f81b | 2019-02-21 22:19:17 +0100 | [diff] [blame] | 59 | /* close all FDs starting from <start> */ |
| 60 | void my_closefrom(int start); |
| 61 | |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 62 | /* disable the specified poller */ |
| 63 | void disable_poller(const char *poller_name); |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 64 | |
Olivier Houchard | 79321b9 | 2018-07-26 17:55:11 +0200 | [diff] [blame] | 65 | void poller_pipe_io_handler(int fd); |
| 66 | |
Willy Tarreau | 2a42950 | 2006-10-15 14:52:29 +0200 | [diff] [blame] | 67 | /* |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 68 | * Initialize the pollers till the best one is found. |
| 69 | * If none works, returns 0, otherwise 1. |
Willy Tarreau | ef1d1f8 | 2007-04-16 00:25:25 +0200 | [diff] [blame] | 70 | * The pollers register themselves just before main() is called. |
Willy Tarreau | 2a42950 | 2006-10-15 14:52:29 +0200 | [diff] [blame] | 71 | */ |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 72 | int init_pollers(); |
Willy Tarreau | 2a42950 | 2006-10-15 14:52:29 +0200 | [diff] [blame] | 73 | |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 74 | /* |
Krzysztof Piotr Oledzki | a643baf | 2008-05-29 23:53:44 +0200 | [diff] [blame] | 75 | * Deinitialize the pollers. |
| 76 | */ |
| 77 | void deinit_pollers(); |
| 78 | |
| 79 | /* |
Willy Tarreau | 2ff7622 | 2007-04-09 19:29:56 +0200 | [diff] [blame] | 80 | * Some pollers may lose their connection after a fork(). It may be necessary |
| 81 | * to create initialize part of them again. Returns 0 in case of failure, |
| 82 | * otherwise 1. The fork() function may be NULL if unused. In case of error, |
| 83 | * the the current poller is destroyed and the caller is responsible for trying |
| 84 | * another one by calling init_pollers() again. |
| 85 | */ |
| 86 | int fork_poller(); |
| 87 | |
| 88 | /* |
| 89 | * Lists the known pollers on <out>. |
| 90 | * Should be performed only before initialization. |
| 91 | */ |
| 92 | int list_pollers(FILE *out); |
| 93 | |
| 94 | /* |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 95 | * Runs the polling loop |
| 96 | */ |
| 97 | void run_poller(); |
Willy Tarreau | 2a42950 | 2006-10-15 14:52:29 +0200 | [diff] [blame] | 98 | |
Olivier Houchard | 6a2cf87 | 2018-04-25 15:10:30 +0200 | [diff] [blame] | 99 | void fd_add_to_fd_list(volatile struct fdlist *list, int fd, int off); |
| 100 | void fd_rm_from_fd_list(volatile struct fdlist *list, int fd, int off); |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 101 | |
Willy Tarreau | 5be2f35 | 2014-11-19 19:43:05 +0100 | [diff] [blame] | 102 | /* Mark fd <fd> as updated for polling and allocate an entry in the update list |
| 103 | * for this if it was not already there. This can be done at any time. |
Willy Tarreau | e852545 | 2014-01-25 09:58:06 +0100 | [diff] [blame] | 104 | */ |
Willy Tarreau | 5be2f35 | 2014-11-19 19:43:05 +0100 | [diff] [blame] | 105 | static inline void updt_fd_polling(const int fd) |
Willy Tarreau | 7be79a4 | 2012-11-11 15:02:54 +0100 | [diff] [blame] | 106 | { |
Olivier Houchard | 6aab737 | 2018-08-17 13:37:59 +0200 | [diff] [blame] | 107 | if ((fdtab[fd].thread_mask & all_threads_mask) == tid_bit) { |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 108 | |
| 109 | /* note: we don't have a test-and-set yet in hathreads */ |
Willy Tarreau | 4d84186 | 2018-01-17 22:57:54 +0100 | [diff] [blame] | 110 | |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 111 | if (HA_ATOMIC_BTS(&fdtab[fd].update_mask, tid)) |
| 112 | return; |
| 113 | |
Olivier Houchard | a0fdce3 | 2019-06-12 14:31:08 +0200 | [diff] [blame] | 114 | fd_updt[fd_nbupdt++] = fd; |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 115 | } else { |
| 116 | unsigned long update_mask = fdtab[fd].update_mask; |
| 117 | do { |
| 118 | if (update_mask == fdtab[fd].thread_mask) |
| 119 | return; |
Olivier Houchard | d360879 | 2019-03-08 18:47:42 +0100 | [diff] [blame] | 120 | } while (!_HA_ATOMIC_CAS(&fdtab[fd].update_mask, &update_mask, |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 121 | fdtab[fd].thread_mask)); |
| 122 | fd_add_to_fd_list(&update_list, fd, offsetof(struct fdtab, update)); |
| 123 | } |
| 124 | |
| 125 | } |
Willy Tarreau | 4d84186 | 2018-01-17 22:57:54 +0100 | [diff] [blame] | 126 | |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 127 | /* Called from the poller to acknoledge we read an entry from the global |
| 128 | * update list, to remove our bit from the update_mask, and remove it from |
| 129 | * the list if we were the last one. |
| 130 | */ |
| 131 | static inline void done_update_polling(int fd) |
| 132 | { |
| 133 | unsigned long update_mask; |
| 134 | |
Olivier Houchard | d360879 | 2019-03-08 18:47:42 +0100 | [diff] [blame] | 135 | update_mask = _HA_ATOMIC_AND(&fdtab[fd].update_mask, ~tid_bit); |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 136 | while ((update_mask & all_threads_mask)== 0) { |
| 137 | /* If we were the last one that had to update that entry, remove it from the list */ |
| 138 | fd_rm_from_fd_list(&update_list, fd, offsetof(struct fdtab, update)); |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 139 | update_mask = (volatile unsigned long)fdtab[fd].update_mask; |
| 140 | if ((update_mask & all_threads_mask) != 0) { |
| 141 | /* Maybe it's been re-updated in the meanwhile, and we |
| 142 | * wrongly removed it from the list, if so, re-add it |
| 143 | */ |
| 144 | fd_add_to_fd_list(&update_list, fd, offsetof(struct fdtab, update)); |
| 145 | update_mask = (volatile unsigned long)(fdtab[fd].update_mask); |
| 146 | /* And then check again, just in case after all it |
| 147 | * should be removed, even if it's very unlikely, given |
| 148 | * the current thread wouldn't have been able to take |
| 149 | * care of it yet */ |
| 150 | } else |
| 151 | break; |
Willy Tarreau | 4d84186 | 2018-01-17 22:57:54 +0100 | [diff] [blame] | 152 | |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 153 | } |
Willy Tarreau | 7be79a4 | 2012-11-11 15:02:54 +0100 | [diff] [blame] | 154 | } |
| 155 | |
Willy Tarreau | 6ea20b1 | 2012-11-11 16:05:19 +0100 | [diff] [blame] | 156 | /* |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 157 | * returns the FD's recv state (FD_EV_*) |
Willy Tarreau | 6ea20b1 | 2012-11-11 16:05:19 +0100 | [diff] [blame] | 158 | */ |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 159 | static inline int fd_recv_state(const int fd) |
Willy Tarreau | 6ea20b1 | 2012-11-11 16:05:19 +0100 | [diff] [blame] | 160 | { |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 161 | return ((unsigned)fdtab[fd].state >> (4 * DIR_RD)) & FD_EV_STATUS; |
Willy Tarreau | 6ea20b1 | 2012-11-11 16:05:19 +0100 | [diff] [blame] | 162 | } |
| 163 | |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 164 | /* |
| 165 | * returns true if the FD is active for recv |
Willy Tarreau | 6ea20b1 | 2012-11-11 16:05:19 +0100 | [diff] [blame] | 166 | */ |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 167 | static inline int fd_recv_active(const int fd) |
Willy Tarreau | 6ea20b1 | 2012-11-11 16:05:19 +0100 | [diff] [blame] | 168 | { |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 169 | return (unsigned)fdtab[fd].state & FD_EV_ACTIVE_R; |
Willy Tarreau | 6ea20b1 | 2012-11-11 16:05:19 +0100 | [diff] [blame] | 170 | } |
| 171 | |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 172 | /* |
| 173 | * returns true if the FD is ready for recv |
Willy Tarreau | 6ea20b1 | 2012-11-11 16:05:19 +0100 | [diff] [blame] | 174 | */ |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 175 | static inline int fd_recv_ready(const int fd) |
Willy Tarreau | 6ea20b1 | 2012-11-11 16:05:19 +0100 | [diff] [blame] | 176 | { |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 177 | return (unsigned)fdtab[fd].state & FD_EV_READY_R; |
| 178 | } |
| 179 | |
| 180 | /* |
| 181 | * returns true if the FD is polled for recv |
| 182 | */ |
| 183 | static inline int fd_recv_polled(const int fd) |
| 184 | { |
| 185 | return (unsigned)fdtab[fd].state & FD_EV_POLLED_R; |
| 186 | } |
| 187 | |
| 188 | /* |
| 189 | * returns the FD's send state (FD_EV_*) |
| 190 | */ |
| 191 | static inline int fd_send_state(const int fd) |
| 192 | { |
| 193 | return ((unsigned)fdtab[fd].state >> (4 * DIR_WR)) & FD_EV_STATUS; |
| 194 | } |
| 195 | |
| 196 | /* |
| 197 | * returns true if the FD is active for send |
| 198 | */ |
| 199 | static inline int fd_send_active(const int fd) |
| 200 | { |
| 201 | return (unsigned)fdtab[fd].state & FD_EV_ACTIVE_W; |
Willy Tarreau | 6ea20b1 | 2012-11-11 16:05:19 +0100 | [diff] [blame] | 202 | } |
| 203 | |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 204 | /* |
| 205 | * returns true if the FD is ready for send |
Willy Tarreau | 6ea20b1 | 2012-11-11 16:05:19 +0100 | [diff] [blame] | 206 | */ |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 207 | static inline int fd_send_ready(const int fd) |
Willy Tarreau | 6ea20b1 | 2012-11-11 16:05:19 +0100 | [diff] [blame] | 208 | { |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 209 | return (unsigned)fdtab[fd].state & FD_EV_READY_W; |
| 210 | } |
Willy Tarreau | 6ea20b1 | 2012-11-11 16:05:19 +0100 | [diff] [blame] | 211 | |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 212 | /* |
| 213 | * returns true if the FD is polled for send |
| 214 | */ |
| 215 | static inline int fd_send_polled(const int fd) |
| 216 | { |
| 217 | return (unsigned)fdtab[fd].state & FD_EV_POLLED_W; |
| 218 | } |
| 219 | |
Christopher Faulet | 8db2fdf | 2017-08-30 09:59:38 +0200 | [diff] [blame] | 220 | /* |
| 221 | * returns true if the FD is active for recv or send |
| 222 | */ |
| 223 | static inline int fd_active(const int fd) |
| 224 | { |
| 225 | return (unsigned)fdtab[fd].state & FD_EV_ACTIVE_RW; |
| 226 | } |
| 227 | |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 228 | /* Disable processing recv events on fd <fd> */ |
| 229 | static inline void fd_stop_recv(int fd) |
| 230 | { |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 231 | unsigned char old, new; |
| 232 | |
| 233 | old = fdtab[fd].state; |
| 234 | do { |
| 235 | if (!(old & FD_EV_ACTIVE_R)) |
| 236 | return; |
| 237 | new = old & ~FD_EV_ACTIVE_R; |
| 238 | new &= ~FD_EV_POLLED_R; |
Olivier Houchard | d360879 | 2019-03-08 18:47:42 +0100 | [diff] [blame] | 239 | } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new))); |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 240 | |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 241 | if ((old ^ new) & FD_EV_POLLED_R) |
| 242 | updt_fd_polling(fd); |
Willy Tarreau | 6ea20b1 | 2012-11-11 16:05:19 +0100 | [diff] [blame] | 243 | } |
| 244 | |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 245 | /* Disable processing send events on fd <fd> */ |
| 246 | static inline void fd_stop_send(int fd) |
Willy Tarreau | 6ea20b1 | 2012-11-11 16:05:19 +0100 | [diff] [blame] | 247 | { |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 248 | unsigned char old, new; |
| 249 | |
| 250 | old = fdtab[fd].state; |
| 251 | do { |
| 252 | if (!(old & FD_EV_ACTIVE_W)) |
| 253 | return; |
| 254 | new = old & ~FD_EV_ACTIVE_W; |
| 255 | new &= ~FD_EV_POLLED_W; |
Olivier Houchard | d360879 | 2019-03-08 18:47:42 +0100 | [diff] [blame] | 256 | } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new))); |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 257 | |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 258 | if ((old ^ new) & FD_EV_POLLED_W) |
| 259 | updt_fd_polling(fd); |
Willy Tarreau | 6ea20b1 | 2012-11-11 16:05:19 +0100 | [diff] [blame] | 260 | } |
| 261 | |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 262 | /* Disable processing of events on fd <fd> for both directions. */ |
| 263 | static inline void fd_stop_both(int fd) |
Willy Tarreau | 49b046d | 2012-08-09 12:11:58 +0200 | [diff] [blame] | 264 | { |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 265 | unsigned char old, new; |
| 266 | |
| 267 | old = fdtab[fd].state; |
| 268 | do { |
| 269 | if (!(old & FD_EV_ACTIVE_RW)) |
| 270 | return; |
| 271 | new = old & ~FD_EV_ACTIVE_RW; |
| 272 | new &= ~FD_EV_POLLED_RW; |
Olivier Houchard | d360879 | 2019-03-08 18:47:42 +0100 | [diff] [blame] | 273 | } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new))); |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 274 | |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 275 | if ((old ^ new) & FD_EV_POLLED_RW) |
| 276 | updt_fd_polling(fd); |
Willy Tarreau | 49b046d | 2012-08-09 12:11:58 +0200 | [diff] [blame] | 277 | } |
| 278 | |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 279 | /* Report that FD <fd> cannot receive anymore without polling (EAGAIN detected). */ |
| 280 | static inline void fd_cant_recv(const int fd) |
Willy Tarreau | 49b046d | 2012-08-09 12:11:58 +0200 | [diff] [blame] | 281 | { |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 282 | unsigned char old, new; |
| 283 | |
| 284 | old = fdtab[fd].state; |
| 285 | do { |
| 286 | if (!(old & FD_EV_READY_R)) |
| 287 | return; |
| 288 | new = old & ~FD_EV_READY_R; |
| 289 | if (new & FD_EV_ACTIVE_R) |
| 290 | new |= FD_EV_POLLED_R; |
Olivier Houchard | d360879 | 2019-03-08 18:47:42 +0100 | [diff] [blame] | 291 | } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new))); |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 292 | |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 293 | if ((old ^ new) & FD_EV_POLLED_R) |
| 294 | updt_fd_polling(fd); |
Willy Tarreau | 49b046d | 2012-08-09 12:11:58 +0200 | [diff] [blame] | 295 | } |
| 296 | |
Willy Tarreau | 1dad384 | 2019-07-08 23:09:03 +0200 | [diff] [blame] | 297 | /* Report that FD <fd> may receive again without polling. */ |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 298 | static inline void fd_may_recv(const int fd) |
Willy Tarreau | babd05a | 2012-08-09 12:14:03 +0200 | [diff] [blame] | 299 | { |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 300 | /* marking ready never changes polled status */ |
Willy Tarreau | 1dad384 | 2019-07-08 23:09:03 +0200 | [diff] [blame] | 301 | if ((fdtab[fd].state & FD_EV_READY_R) || |
| 302 | HA_ATOMIC_BTS(&fdtab[fd].state, FD_EV_READY_R_BIT)) |
| 303 | return; |
Willy Tarreau | babd05a | 2012-08-09 12:14:03 +0200 | [diff] [blame] | 304 | } |
| 305 | |
Willy Tarreau | 6c11bd2 | 2014-01-24 00:54:27 +0100 | [diff] [blame] | 306 | /* Disable readiness when polled. This is useful to interrupt reading when it |
| 307 | * is suspected that the end of data might have been reached (eg: short read). |
| 308 | * This can only be done using level-triggered pollers, so if any edge-triggered |
| 309 | * is ever implemented, a test will have to be added here. |
| 310 | */ |
| 311 | static inline void fd_done_recv(const int fd) |
| 312 | { |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 313 | unsigned char old, new; |
| 314 | |
| 315 | old = fdtab[fd].state; |
| 316 | do { |
| 317 | if ((old & (FD_EV_POLLED_R|FD_EV_READY_R)) != (FD_EV_POLLED_R|FD_EV_READY_R)) |
| 318 | return; |
| 319 | new = old & ~FD_EV_READY_R; |
| 320 | if (new & FD_EV_ACTIVE_R) |
| 321 | new |= FD_EV_POLLED_R; |
Olivier Houchard | d360879 | 2019-03-08 18:47:42 +0100 | [diff] [blame] | 322 | } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new))); |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 323 | |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 324 | if ((old ^ new) & FD_EV_POLLED_R) |
| 325 | updt_fd_polling(fd); |
Willy Tarreau | 6c11bd2 | 2014-01-24 00:54:27 +0100 | [diff] [blame] | 326 | } |
| 327 | |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 328 | /* Report that FD <fd> cannot send anymore without polling (EAGAIN detected). */ |
| 329 | static inline void fd_cant_send(const int fd) |
Willy Tarreau | 49b046d | 2012-08-09 12:11:58 +0200 | [diff] [blame] | 330 | { |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 331 | unsigned char old, new; |
| 332 | |
| 333 | old = fdtab[fd].state; |
| 334 | do { |
| 335 | if (!(old & FD_EV_READY_W)) |
| 336 | return; |
| 337 | new = old & ~FD_EV_READY_W; |
| 338 | if (new & FD_EV_ACTIVE_W) |
| 339 | new |= FD_EV_POLLED_W; |
Olivier Houchard | d360879 | 2019-03-08 18:47:42 +0100 | [diff] [blame] | 340 | } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new))); |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 341 | |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 342 | if ((old ^ new) & FD_EV_POLLED_W) |
| 343 | updt_fd_polling(fd); |
Willy Tarreau | 49b046d | 2012-08-09 12:11:58 +0200 | [diff] [blame] | 344 | } |
| 345 | |
Willy Tarreau | 1dad384 | 2019-07-08 23:09:03 +0200 | [diff] [blame] | 346 | /* Report that FD <fd> may send again without polling (EAGAIN not detected). */ |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 347 | static inline void fd_may_send(const int fd) |
Willy Tarreau | 49b046d | 2012-08-09 12:11:58 +0200 | [diff] [blame] | 348 | { |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 349 | /* marking ready never changes polled status */ |
Willy Tarreau | 1dad384 | 2019-07-08 23:09:03 +0200 | [diff] [blame] | 350 | if ((fdtab[fd].state & FD_EV_READY_W) || |
| 351 | HA_ATOMIC_BTS(&fdtab[fd].state, FD_EV_READY_W_BIT)) |
| 352 | return; |
Willy Tarreau | 49b046d | 2012-08-09 12:11:58 +0200 | [diff] [blame] | 353 | } |
Willy Tarreau | 2a42950 | 2006-10-15 14:52:29 +0200 | [diff] [blame] | 354 | |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 355 | /* Prepare FD <fd> to try to receive */ |
| 356 | static inline void fd_want_recv(int fd) |
Willy Tarreau | babd05a | 2012-08-09 12:14:03 +0200 | [diff] [blame] | 357 | { |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 358 | unsigned char old, new; |
| 359 | |
| 360 | old = fdtab[fd].state; |
| 361 | do { |
| 362 | if (old & FD_EV_ACTIVE_R) |
| 363 | return; |
Olivier Houchard | 305d5ab | 2019-07-24 18:07:06 +0200 | [diff] [blame^] | 364 | new = old | FD_EV_ACTIVE_R | FD_EV_POLLED_R; |
Olivier Houchard | d360879 | 2019-03-08 18:47:42 +0100 | [diff] [blame] | 365 | } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new))); |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 366 | |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 367 | if ((old ^ new) & FD_EV_POLLED_R) |
| 368 | updt_fd_polling(fd); |
Willy Tarreau | babd05a | 2012-08-09 12:14:03 +0200 | [diff] [blame] | 369 | } |
| 370 | |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 371 | /* Prepare FD <fd> to try to send */ |
| 372 | static inline void fd_want_send(int fd) |
Willy Tarreau | 49b046d | 2012-08-09 12:11:58 +0200 | [diff] [blame] | 373 | { |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 374 | unsigned char old, new; |
| 375 | |
| 376 | old = fdtab[fd].state; |
| 377 | do { |
| 378 | if (old & FD_EV_ACTIVE_W) |
| 379 | return; |
Olivier Houchard | 305d5ab | 2019-07-24 18:07:06 +0200 | [diff] [blame^] | 380 | new = old | FD_EV_ACTIVE_W | FD_EV_POLLED_W; |
Olivier Houchard | d360879 | 2019-03-08 18:47:42 +0100 | [diff] [blame] | 381 | } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new))); |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 382 | |
Willy Tarreau | 7ac0e35 | 2018-01-17 21:25:57 +0100 | [diff] [blame] | 383 | if ((old ^ new) & FD_EV_POLLED_W) |
| 384 | updt_fd_polling(fd); |
Willy Tarreau | 49b046d | 2012-08-09 12:11:58 +0200 | [diff] [blame] | 385 | } |
Willy Tarreau | 2a42950 | 2006-10-15 14:52:29 +0200 | [diff] [blame] | 386 | |
Christopher Faulet | 21e9267 | 2017-08-30 10:30:04 +0200 | [diff] [blame] | 387 | /* Update events seen for FD <fd> and its state if needed. This should be called |
| 388 | * by the poller to set FD_POLL_* flags. */ |
| 389 | static inline void fd_update_events(int fd, int evts) |
| 390 | { |
Richard Russo | bc9d984 | 2019-02-20 12:43:45 -0800 | [diff] [blame] | 391 | unsigned long locked = atleast2(fdtab[fd].thread_mask); |
Willy Tarreau | 1dad384 | 2019-07-08 23:09:03 +0200 | [diff] [blame] | 392 | unsigned char old, new; |
Richard Russo | bc9d984 | 2019-02-20 12:43:45 -0800 | [diff] [blame] | 393 | |
Willy Tarreau | 1dad384 | 2019-07-08 23:09:03 +0200 | [diff] [blame] | 394 | old = fdtab[fd].ev; |
| 395 | new = (old & FD_POLL_STICKY) | evts; |
| 396 | |
| 397 | if (unlikely(locked)) { |
| 398 | /* Locked FDs (those with more than 2 threads) are atomically updated */ |
| 399 | while (unlikely(new != old && !_HA_ATOMIC_CAS(&fdtab[fd].ev, &old, new))) |
| 400 | new = (old & FD_POLL_STICKY) | evts; |
| 401 | } else { |
| 402 | if (new != old) |
| 403 | fdtab[fd].ev = new; |
| 404 | } |
Christopher Faulet | 21e9267 | 2017-08-30 10:30:04 +0200 | [diff] [blame] | 405 | |
| 406 | if (fdtab[fd].ev & (FD_POLL_IN | FD_POLL_HUP | FD_POLL_ERR)) |
| 407 | fd_may_recv(fd); |
| 408 | |
| 409 | if (fdtab[fd].ev & (FD_POLL_OUT | FD_POLL_ERR)) |
| 410 | fd_may_send(fd); |
Olivier Houchard | 305d5ab | 2019-07-24 18:07:06 +0200 | [diff] [blame^] | 411 | |
| 412 | if (fdtab[fd].iocb) |
| 413 | fdtab[fd].iocb(fd); |
Christopher Faulet | 21e9267 | 2017-08-30 10:30:04 +0200 | [diff] [blame] | 414 | } |
| 415 | |
Willy Tarreau | d6f087e | 2008-01-18 17:20:13 +0100 | [diff] [blame] | 416 | /* Prepares <fd> for being polled */ |
Willy Tarreau | a9786b6 | 2018-01-25 07:22:13 +0100 | [diff] [blame] | 417 | static inline void fd_insert(int fd, void *owner, void (*iocb)(int fd), unsigned long thread_mask) |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 418 | { |
Richard Russo | bc9d984 | 2019-02-20 12:43:45 -0800 | [diff] [blame] | 419 | unsigned long locked = atleast2(thread_mask); |
| 420 | |
| 421 | if (locked) |
Willy Tarreau | 87d54a9 | 2018-10-15 09:44:46 +0200 | [diff] [blame] | 422 | HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock); |
Willy Tarreau | a9786b6 | 2018-01-25 07:22:13 +0100 | [diff] [blame] | 423 | fdtab[fd].owner = owner; |
| 424 | fdtab[fd].iocb = iocb; |
Willy Tarreau | d6f087e | 2008-01-18 17:20:13 +0100 | [diff] [blame] | 425 | fdtab[fd].ev = 0; |
Willy Tarreau | ad38ace | 2013-12-15 14:19:38 +0100 | [diff] [blame] | 426 | fdtab[fd].linger_risk = 0; |
Conrad Hoffmann | 041751c | 2014-05-20 14:28:24 +0200 | [diff] [blame] | 427 | fdtab[fd].cloned = 0; |
Willy Tarreau | f65610a | 2017-10-31 16:06:06 +0100 | [diff] [blame] | 428 | fdtab[fd].thread_mask = thread_mask; |
Willy Tarreau | c9c8378 | 2018-01-17 18:44:46 +0100 | [diff] [blame] | 429 | /* note: do not reset polled_mask here as it indicates which poller |
| 430 | * still knows this FD from a possible previous round. |
| 431 | */ |
Richard Russo | bc9d984 | 2019-02-20 12:43:45 -0800 | [diff] [blame] | 432 | if (locked) |
Willy Tarreau | 87d54a9 | 2018-10-15 09:44:46 +0200 | [diff] [blame] | 433 | HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock); |
Olivier Houchard | 7c49d2e | 2019-04-16 18:37:05 +0200 | [diff] [blame] | 434 | _HA_ATOMIC_ADD(&ha_used_fds, 1); |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 435 | } |
| 436 | |
Willy Tarreau | f37ba94 | 2018-10-17 11:25:54 +0200 | [diff] [blame] | 437 | /* Computes the bounded poll() timeout based on the next expiration timer <next> |
| 438 | * by bounding it to MAX_DELAY_MS. <next> may equal TICK_ETERNITY. The pollers |
| 439 | * just needs to call this function right before polling to get their timeout |
| 440 | * value. Timeouts that are already expired (possibly due to a pending event) |
| 441 | * are accounted for in activity.poll_exp. |
| 442 | */ |
| 443 | static inline int compute_poll_timeout(int next) |
| 444 | { |
| 445 | int wait_time; |
| 446 | |
| 447 | if (!tick_isset(next)) |
| 448 | wait_time = MAX_DELAY_MS; |
| 449 | else if (tick_is_expired(next, now_ms)) { |
| 450 | activity[tid].poll_exp++; |
| 451 | wait_time = 0; |
| 452 | } |
| 453 | else { |
| 454 | wait_time = TICKS_TO_MS(tick_remain(now_ms, next)) + 1; |
| 455 | if (wait_time > MAX_DELAY_MS) |
| 456 | wait_time = MAX_DELAY_MS; |
| 457 | } |
| 458 | return wait_time; |
| 459 | } |
| 460 | |
Willy Tarreau | 322e6c7 | 2018-01-25 16:37:04 +0100 | [diff] [blame] | 461 | /* These are replacements for FD_SET, FD_CLR, FD_ISSET, working on uints */ |
| 462 | static inline void hap_fd_set(int fd, unsigned int *evts) |
| 463 | { |
Olivier Houchard | d360879 | 2019-03-08 18:47:42 +0100 | [diff] [blame] | 464 | _HA_ATOMIC_OR(&evts[fd / (8*sizeof(*evts))], 1U << (fd & (8*sizeof(*evts) - 1))); |
Willy Tarreau | 322e6c7 | 2018-01-25 16:37:04 +0100 | [diff] [blame] | 465 | } |
| 466 | |
| 467 | static inline void hap_fd_clr(int fd, unsigned int *evts) |
| 468 | { |
Olivier Houchard | d360879 | 2019-03-08 18:47:42 +0100 | [diff] [blame] | 469 | _HA_ATOMIC_AND(&evts[fd / (8*sizeof(*evts))], ~(1U << (fd & (8*sizeof(*evts) - 1)))); |
Willy Tarreau | 322e6c7 | 2018-01-25 16:37:04 +0100 | [diff] [blame] | 470 | } |
| 471 | |
| 472 | static inline unsigned int hap_fd_isset(int fd, unsigned int *evts) |
| 473 | { |
| 474 | return evts[fd / (8*sizeof(*evts))] & (1U << (fd & (8*sizeof(*evts) - 1))); |
| 475 | } |
| 476 | |
Olivier Houchard | 79321b9 | 2018-07-26 17:55:11 +0200 | [diff] [blame] | 477 | static inline void wake_thread(int tid) |
| 478 | { |
| 479 | char c = 'c'; |
| 480 | |
| 481 | shut_your_big_mouth_gcc(write(poller_wr_pipe[tid], &c, 1)); |
| 482 | } |
| 483 | |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 484 | |
| 485 | #endif /* _PROTO_FD_H */ |
| 486 | |
| 487 | /* |
| 488 | * Local variables: |
| 489 | * c-indent-level: 8 |
| 490 | * c-basic-offset: 8 |
| 491 | * End: |
| 492 | */ |