Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 1 | /* |
| 2 | * File descriptors management functions. |
| 3 | * |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 4 | * Copyright 2000-2014 Willy Tarreau <w@1wt.eu> |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 5 | * |
| 6 | * This program is free software; you can redistribute it and/or |
| 7 | * modify it under the terms of the GNU General Public License |
| 8 | * as published by the Free Software Foundation; either version |
| 9 | * 2 of the License, or (at your option) any later version. |
| 10 | * |
Willy Tarreau | 7be79a4 | 2012-11-11 15:02:54 +0100 | [diff] [blame] | 11 | * There is no direct link between the FD and the updates list. There is only a |
| 12 | * bit in the fdtab[] to indicate than a file descriptor is already present in |
| 13 | * the updates list. Once an fd is present in the updates list, it will have to |
| 14 | * be considered even if its changes are reverted in the middle or if the fd is |
| 15 | * replaced. |
| 16 | * |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 17 | * The event state for an FD, as found in fdtab[].state, is maintained for each |
| 18 | * direction. The state field is built this way, with R bits in the low nibble |
| 19 | * and W bits in the high nibble for ease of access and debugging : |
Willy Tarreau | 7be79a4 | 2012-11-11 15:02:54 +0100 | [diff] [blame] | 20 | * |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 21 | * 7 6 5 4 3 2 1 0 |
Willy Tarreau | 5bee3e2 | 2019-09-04 09:52:57 +0200 | [diff] [blame] | 22 | * [ 0 | 0 | RW | AW | 0 | 0 | RR | AR ] |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 23 | * |
| 24 | * A* = active *R = read |
Willy Tarreau | 5bee3e2 | 2019-09-04 09:52:57 +0200 | [diff] [blame] | 25 | * R* = ready *W = write |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 26 | * |
| 27 | * An FD is marked "active" when there is a desire to use it. |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 28 | * An FD is marked "ready" when it has not faced a new EAGAIN since last wake-up |
Willy Tarreau | 5bee3e2 | 2019-09-04 09:52:57 +0200 | [diff] [blame] | 29 | * (it is a cache of the last EAGAIN regardless of polling changes). Each poller |
| 30 | * has its own "polled" state for the same fd, as stored in the polled_mask. |
Willy Tarreau | 7be79a4 | 2012-11-11 15:02:54 +0100 | [diff] [blame] | 31 | * |
Willy Tarreau | 5bee3e2 | 2019-09-04 09:52:57 +0200 | [diff] [blame] | 32 | * We have 4 possible states for each direction based on these 2 flags : |
Willy Tarreau | 7be79a4 | 2012-11-11 15:02:54 +0100 | [diff] [blame] | 33 | * |
Willy Tarreau | 5bee3e2 | 2019-09-04 09:52:57 +0200 | [diff] [blame] | 34 | * +---+---+----------+---------------------------------------------+ |
| 35 | * | R | A | State | Description | |
| 36 | * +---+---+----------+---------------------------------------------+ |
| 37 | * | 0 | 0 | DISABLED | No activity desired, not ready. | |
| 38 | * | 0 | 1 | ACTIVE | Activity desired. | |
| 39 | * | 1 | 0 | STOPPED | End of activity. | |
| 40 | * | 1 | 1 | READY | Activity desired and reported. | |
| 41 | * +---+---+----------+---------------------------------------------+ |
Willy Tarreau | 7be79a4 | 2012-11-11 15:02:54 +0100 | [diff] [blame] | 42 | * |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 43 | * The transitions are pretty simple : |
| 44 | * - fd_want_*() : set flag A |
| 45 | * - fd_stop_*() : clear flag A |
| 46 | * - fd_cant_*() : clear flag R (when facing EAGAIN) |
| 47 | * - fd_may_*() : set flag R (upon return from poll()) |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 48 | * |
Willy Tarreau | 5bee3e2 | 2019-09-04 09:52:57 +0200 | [diff] [blame] | 49 | * Each poller then computes its own polled state : |
| 50 | * if (A) { if (!R) P := 1 } else { P := 0 } |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 51 | * |
Willy Tarreau | 5bee3e2 | 2019-09-04 09:52:57 +0200 | [diff] [blame] | 52 | * The state transitions look like the diagram below. |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 53 | * |
Willy Tarreau | 5bee3e2 | 2019-09-04 09:52:57 +0200 | [diff] [blame] | 54 | * may +----------+ |
| 55 | * ,----| DISABLED | (READY=0, ACTIVE=0) |
| 56 | * | +----------+ |
| 57 | * | want | ^ |
| 58 | * | | | |
| 59 | * | v | stop |
| 60 | * | +----------+ |
| 61 | * | | ACTIVE | (READY=0, ACTIVE=1) |
| 62 | * | +----------+ |
| 63 | * | | ^ |
| 64 | * | may | | |
Thayne McCombs | 8f0cc5c | 2021-01-07 21:35:52 -0700 | [diff] [blame] | 65 | * | v | EAGAIN (can't) |
Willy Tarreau | 5bee3e2 | 2019-09-04 09:52:57 +0200 | [diff] [blame] | 66 | * | +--------+ |
| 67 | * | | READY | (READY=1, ACTIVE=1) |
| 68 | * | +--------+ |
| 69 | * | stop | ^ |
| 70 | * | | | |
| 71 | * | v | want |
| 72 | * | +---------+ |
| 73 | * `--->| STOPPED | (READY=1, ACTIVE=0) |
| 74 | * +---------+ |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 75 | */ |
| 76 | |
Willy Tarreau | 2ff7622 | 2007-04-09 19:29:56 +0200 | [diff] [blame] | 77 | #include <stdio.h> |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 78 | #include <string.h> |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 79 | #include <unistd.h> |
Olivier Houchard | 79321b9 | 2018-07-26 17:55:11 +0200 | [diff] [blame] | 80 | #include <fcntl.h> |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 81 | #include <sys/types.h> |
Willy Tarreau | 2d7f81b | 2019-02-21 22:19:17 +0100 | [diff] [blame] | 82 | #include <sys/resource.h> |
Willy Tarreau | 931d8b7 | 2019-08-27 11:08:17 +0200 | [diff] [blame] | 83 | #include <sys/uio.h> |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 84 | |
Willy Tarreau | e573323 | 2019-05-22 19:24:06 +0200 | [diff] [blame] | 85 | #if defined(USE_POLL) |
Willy Tarreau | 9188ac6 | 2019-02-21 22:12:47 +0100 | [diff] [blame] | 86 | #include <poll.h> |
| 87 | #include <errno.h> |
| 88 | #endif |
| 89 | |
Willy Tarreau | 4c7e4b7 | 2020-05-27 12:58:42 +0200 | [diff] [blame] | 90 | #include <haproxy/api.h> |
Willy Tarreau | bc52bec | 2020-06-18 08:58:47 +0200 | [diff] [blame] | 91 | #include <haproxy/cfgparse.h> |
Willy Tarreau | 0f6ffd6 | 2020-06-03 19:33:00 +0200 | [diff] [blame] | 92 | #include <haproxy/fd.h> |
Willy Tarreau | b255105 | 2020-06-09 09:07:15 +0200 | [diff] [blame] | 93 | #include <haproxy/global.h> |
Willy Tarreau | fc8f6a8 | 2020-06-03 19:20:59 +0200 | [diff] [blame] | 94 | #include <haproxy/port_range.h> |
Willy Tarreau | bc52bec | 2020-06-18 08:58:47 +0200 | [diff] [blame] | 95 | #include <haproxy/tools.h> |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 96 | |
Willy Tarreau | b255105 | 2020-06-09 09:07:15 +0200 | [diff] [blame] | 97 | |
Willy Tarreau | a1090a5 | 2021-04-10 16:58:13 +0200 | [diff] [blame] | 98 | struct fdtab *fdtab __read_mostly = NULL; /* array of all the file descriptors */ |
| 99 | struct polled_mask *polled_mask __read_mostly = NULL; /* Array for the polled_mask of each fd */ |
| 100 | struct fdinfo *fdinfo __read_mostly = NULL; /* less-often used infos for file descriptors */ |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 101 | int totalconn; /* total # of terminated sessions */ |
| 102 | int actconn; /* # of active sessions */ |
| 103 | |
Willy Tarreau | a1090a5 | 2021-04-10 16:58:13 +0200 | [diff] [blame] | 104 | struct poller pollers[MAX_POLLERS] __read_mostly; |
| 105 | struct poller cur_poller __read_mostly; |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 106 | int nbpollers = 0; |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 107 | |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 108 | volatile struct fdlist update_list; // Global update list |
Olivier Houchard | 4815c8c | 2018-01-24 18:17:56 +0100 | [diff] [blame] | 109 | |
Christopher Faulet | d4604ad | 2017-05-29 10:40:41 +0200 | [diff] [blame] | 110 | THREAD_LOCAL int *fd_updt = NULL; // FD updates list |
| 111 | THREAD_LOCAL int fd_nbupdt = 0; // number of updates in the list |
Olivier Houchard | 79321b9 | 2018-07-26 17:55:11 +0200 | [diff] [blame] | 112 | THREAD_LOCAL int poller_rd_pipe = -1; // Pipe to wake the thread |
Willy Tarreau | a1090a5 | 2021-04-10 16:58:13 +0200 | [diff] [blame] | 113 | int poller_wr_pipe[MAX_THREADS] __read_mostly; // Pipe to wake the threads |
Christopher Faulet | d4604ad | 2017-05-29 10:40:41 +0200 | [diff] [blame] | 114 | |
Olivier Houchard | 7c49d2e | 2019-04-16 18:37:05 +0200 | [diff] [blame] | 115 | volatile int ha_used_fds = 0; // Number of FD we're currently using |
| 116 | |
Willy Tarreau | 337fb71 | 2019-12-20 07:20:00 +0100 | [diff] [blame] | 117 | #define _GET_NEXT(fd, off) ((volatile struct fdlist_entry *)(void *)((char *)(&fdtab[fd]) + off))->next |
| 118 | #define _GET_PREV(fd, off) ((volatile struct fdlist_entry *)(void *)((char *)(&fdtab[fd]) + off))->prev |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 119 | /* adds fd <fd> to fd list <list> if it was not yet in it */ |
Olivier Houchard | 6a2cf87 | 2018-04-25 15:10:30 +0200 | [diff] [blame] | 120 | void fd_add_to_fd_list(volatile struct fdlist *list, int fd, int off) |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 121 | { |
| 122 | int next; |
| 123 | int new; |
| 124 | int old; |
| 125 | int last; |
| 126 | |
| 127 | redo_next: |
Olivier Houchard | 6a2cf87 | 2018-04-25 15:10:30 +0200 | [diff] [blame] | 128 | next = _GET_NEXT(fd, off); |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 129 | /* Check that we're not already in the cache, and if not, lock us. */ |
Olivier Houchard | fc51f0f | 2019-12-19 18:33:08 +0100 | [diff] [blame] | 130 | if (next > -2) |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 131 | goto done; |
Olivier Houchard | fc51f0f | 2019-12-19 18:33:08 +0100 | [diff] [blame] | 132 | if (next == -2) |
| 133 | goto redo_next; |
Olivier Houchard | d360879 | 2019-03-08 18:47:42 +0100 | [diff] [blame] | 134 | if (!_HA_ATOMIC_CAS(&_GET_NEXT(fd, off), &next, -2)) |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 135 | goto redo_next; |
Olivier Houchard | d2b5d16 | 2019-03-08 13:47:21 +0100 | [diff] [blame] | 136 | __ha_barrier_atomic_store(); |
Willy Tarreau | 11559a7 | 2018-02-05 17:52:24 +0100 | [diff] [blame] | 137 | |
| 138 | new = fd; |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 139 | redo_last: |
| 140 | /* First, insert in the linked list */ |
| 141 | last = list->last; |
| 142 | old = -1; |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 143 | |
Olivier Houchard | 6a2cf87 | 2018-04-25 15:10:30 +0200 | [diff] [blame] | 144 | _GET_PREV(fd, off) = -2; |
Willy Tarreau | 11559a7 | 2018-02-05 17:52:24 +0100 | [diff] [blame] | 145 | /* Make sure the "prev" store is visible before we update the last entry */ |
| 146 | __ha_barrier_store(); |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 147 | |
Willy Tarreau | 11559a7 | 2018-02-05 17:52:24 +0100 | [diff] [blame] | 148 | if (unlikely(last == -1)) { |
| 149 | /* list is empty, try to add ourselves alone so that list->last=fd */ |
Olivier Houchard | d360879 | 2019-03-08 18:47:42 +0100 | [diff] [blame] | 150 | if (unlikely(!_HA_ATOMIC_CAS(&list->last, &old, new))) |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 151 | goto redo_last; |
| 152 | |
| 153 | /* list->first was necessary -1, we're guaranteed to be alone here */ |
| 154 | list->first = fd; |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 155 | } else { |
Willy Tarreau | 11559a7 | 2018-02-05 17:52:24 +0100 | [diff] [blame] | 156 | /* adding ourselves past the last element |
| 157 | * The CAS will only succeed if its next is -1, |
| 158 | * which means it's in the cache, and the last element. |
| 159 | */ |
Olivier Houchard | d360879 | 2019-03-08 18:47:42 +0100 | [diff] [blame] | 160 | if (unlikely(!_HA_ATOMIC_CAS(&_GET_NEXT(last, off), &old, new))) |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 161 | goto redo_last; |
Willy Tarreau | 11559a7 | 2018-02-05 17:52:24 +0100 | [diff] [blame] | 162 | |
| 163 | /* Then, update the last entry */ |
| 164 | list->last = fd; |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 165 | } |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 166 | __ha_barrier_store(); |
Willy Tarreau | 11559a7 | 2018-02-05 17:52:24 +0100 | [diff] [blame] | 167 | /* since we're alone at the end of the list and still locked(-2), |
Ilya Shipitsin | b8888ab | 2021-01-06 21:20:16 +0500 | [diff] [blame] | 168 | * we know no one tried to add past us. Mark the end of list. |
Willy Tarreau | 11559a7 | 2018-02-05 17:52:24 +0100 | [diff] [blame] | 169 | */ |
Olivier Houchard | 6a2cf87 | 2018-04-25 15:10:30 +0200 | [diff] [blame] | 170 | _GET_PREV(fd, off) = last; |
| 171 | _GET_NEXT(fd, off) = -1; |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 172 | __ha_barrier_store(); |
| 173 | done: |
| 174 | return; |
| 175 | } |
| 176 | |
| 177 | /* removes fd <fd> from fd list <list> */ |
Olivier Houchard | 6a2cf87 | 2018-04-25 15:10:30 +0200 | [diff] [blame] | 178 | void fd_rm_from_fd_list(volatile struct fdlist *list, int fd, int off) |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 179 | { |
| 180 | #if defined(HA_HAVE_CAS_DW) || defined(HA_CAS_IS_8B) |
Willy Tarreau | 2b9f066 | 2020-02-25 09:25:53 +0100 | [diff] [blame] | 181 | volatile union { |
| 182 | struct fdlist_entry ent; |
| 183 | uint64_t u64; |
| 184 | uint32_t u32[2]; |
| 185 | } cur_list, next_list; |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 186 | #endif |
| 187 | int old; |
| 188 | int new = -2; |
| 189 | int prev; |
| 190 | int next; |
| 191 | int last; |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 192 | lock_self: |
| 193 | #if (defined(HA_CAS_IS_8B) || defined(HA_HAVE_CAS_DW)) |
Willy Tarreau | 2b9f066 | 2020-02-25 09:25:53 +0100 | [diff] [blame] | 194 | next_list.ent.next = next_list.ent.prev = -2; |
| 195 | cur_list.ent = *(volatile struct fdlist_entry *)(((char *)&fdtab[fd]) + off); |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 196 | /* First, attempt to lock our own entries */ |
| 197 | do { |
| 198 | /* The FD is not in the FD cache, give up */ |
Willy Tarreau | 2b9f066 | 2020-02-25 09:25:53 +0100 | [diff] [blame] | 199 | if (unlikely(cur_list.ent.next <= -3)) |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 200 | return; |
Willy Tarreau | 2b9f066 | 2020-02-25 09:25:53 +0100 | [diff] [blame] | 201 | if (unlikely(cur_list.ent.prev == -2 || cur_list.ent.next == -2)) |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 202 | goto lock_self; |
| 203 | } while ( |
| 204 | #ifdef HA_CAS_IS_8B |
Willy Tarreau | 2b9f066 | 2020-02-25 09:25:53 +0100 | [diff] [blame] | 205 | unlikely(!_HA_ATOMIC_CAS(((uint64_t *)&_GET_NEXT(fd, off)), (uint64_t *)&cur_list.u64, next_list.u64)) |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 206 | #else |
Willy Tarreau | 2b9f066 | 2020-02-25 09:25:53 +0100 | [diff] [blame] | 207 | unlikely(!_HA_ATOMIC_DWCAS(((long *)&_GET_NEXT(fd, off)), (uint32_t *)&cur_list.u32, &next_list.u32)) |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 208 | #endif |
Willy Tarreau | 2b9f066 | 2020-02-25 09:25:53 +0100 | [diff] [blame] | 209 | ); |
| 210 | next = cur_list.ent.next; |
| 211 | prev = cur_list.ent.prev; |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 212 | |
| 213 | #else |
| 214 | lock_self_next: |
Willy Tarreau | 337fb71 | 2019-12-20 07:20:00 +0100 | [diff] [blame] | 215 | next = _GET_NEXT(fd, off); |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 216 | if (next == -2) |
| 217 | goto lock_self_next; |
| 218 | if (next <= -3) |
| 219 | goto done; |
Olivier Houchard | d360879 | 2019-03-08 18:47:42 +0100 | [diff] [blame] | 220 | if (unlikely(!_HA_ATOMIC_CAS(&_GET_NEXT(fd, off), &next, -2))) |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 221 | goto lock_self_next; |
| 222 | lock_self_prev: |
Willy Tarreau | 337fb71 | 2019-12-20 07:20:00 +0100 | [diff] [blame] | 223 | prev = _GET_PREV(fd, off); |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 224 | if (prev == -2) |
| 225 | goto lock_self_prev; |
Olivier Houchard | d360879 | 2019-03-08 18:47:42 +0100 | [diff] [blame] | 226 | if (unlikely(!_HA_ATOMIC_CAS(&_GET_PREV(fd, off), &prev, -2))) |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 227 | goto lock_self_prev; |
| 228 | #endif |
Olivier Houchard | d2b5d16 | 2019-03-08 13:47:21 +0100 | [diff] [blame] | 229 | __ha_barrier_atomic_store(); |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 230 | |
| 231 | /* Now, lock the entries of our neighbours */ |
| 232 | if (likely(prev != -1)) { |
| 233 | redo_prev: |
| 234 | old = fd; |
| 235 | |
Olivier Houchard | d360879 | 2019-03-08 18:47:42 +0100 | [diff] [blame] | 236 | if (unlikely(!_HA_ATOMIC_CAS(&_GET_NEXT(prev, off), &old, new))) { |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 237 | if (unlikely(old == -2)) { |
| 238 | /* Neighbour already locked, give up and |
| 239 | * retry again once he's done |
| 240 | */ |
Olivier Houchard | 6a2cf87 | 2018-04-25 15:10:30 +0200 | [diff] [blame] | 241 | _GET_PREV(fd, off) = prev; |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 242 | __ha_barrier_store(); |
Olivier Houchard | 6a2cf87 | 2018-04-25 15:10:30 +0200 | [diff] [blame] | 243 | _GET_NEXT(fd, off) = next; |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 244 | __ha_barrier_store(); |
| 245 | goto lock_self; |
| 246 | } |
| 247 | goto redo_prev; |
| 248 | } |
| 249 | } |
| 250 | if (likely(next != -1)) { |
| 251 | redo_next: |
| 252 | old = fd; |
Olivier Houchard | d360879 | 2019-03-08 18:47:42 +0100 | [diff] [blame] | 253 | if (unlikely(!_HA_ATOMIC_CAS(&_GET_PREV(next, off), &old, new))) { |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 254 | if (unlikely(old == -2)) { |
| 255 | /* Neighbour already locked, give up and |
| 256 | * retry again once he's done |
| 257 | */ |
| 258 | if (prev != -1) { |
Olivier Houchard | 6a2cf87 | 2018-04-25 15:10:30 +0200 | [diff] [blame] | 259 | _GET_NEXT(prev, off) = fd; |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 260 | __ha_barrier_store(); |
| 261 | } |
Olivier Houchard | 6a2cf87 | 2018-04-25 15:10:30 +0200 | [diff] [blame] | 262 | _GET_PREV(fd, off) = prev; |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 263 | __ha_barrier_store(); |
Olivier Houchard | 6a2cf87 | 2018-04-25 15:10:30 +0200 | [diff] [blame] | 264 | _GET_NEXT(fd, off) = next; |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 265 | __ha_barrier_store(); |
| 266 | goto lock_self; |
| 267 | } |
| 268 | goto redo_next; |
| 269 | } |
| 270 | } |
| 271 | if (list->first == fd) |
| 272 | list->first = next; |
| 273 | __ha_barrier_store(); |
| 274 | last = list->last; |
Olivier Houchard | d360879 | 2019-03-08 18:47:42 +0100 | [diff] [blame] | 275 | while (unlikely(last == fd && (!_HA_ATOMIC_CAS(&list->last, &last, prev)))) |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 276 | __ha_compiler_barrier(); |
| 277 | /* Make sure we let other threads know we're no longer in cache, |
| 278 | * before releasing our neighbours. |
| 279 | */ |
| 280 | __ha_barrier_store(); |
| 281 | if (likely(prev != -1)) |
Olivier Houchard | 6a2cf87 | 2018-04-25 15:10:30 +0200 | [diff] [blame] | 282 | _GET_NEXT(prev, off) = next; |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 283 | __ha_barrier_store(); |
| 284 | if (likely(next != -1)) |
Olivier Houchard | 6a2cf87 | 2018-04-25 15:10:30 +0200 | [diff] [blame] | 285 | _GET_PREV(next, off) = prev; |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 286 | __ha_barrier_store(); |
| 287 | /* Ok, now we're out of the fd cache */ |
Olivier Houchard | 6a2cf87 | 2018-04-25 15:10:30 +0200 | [diff] [blame] | 288 | _GET_NEXT(fd, off) = -(next + 4); |
Willy Tarreau | 4cc67a2 | 2018-02-05 17:14:55 +0100 | [diff] [blame] | 289 | __ha_barrier_store(); |
| 290 | done: |
| 291 | return; |
| 292 | } |
| 293 | |
Olivier Houchard | 6a2cf87 | 2018-04-25 15:10:30 +0200 | [diff] [blame] | 294 | #undef _GET_NEXT |
| 295 | #undef _GET_PREV |
| 296 | |
Willy Tarreau | 2c3f981 | 2021-03-24 10:51:32 +0100 | [diff] [blame] | 297 | /* deletes the FD once nobody uses it anymore, as detected by the caller by its |
| 298 | * thread_mask being zero and its running mask turning to zero. There is no |
| 299 | * protection against concurrent accesses, it's up to the caller to make sure |
| 300 | * only the last thread will call it. This is only for internal use, please use |
| 301 | * fd_delete() instead. |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 302 | */ |
Willy Tarreau | 2c3f981 | 2021-03-24 10:51:32 +0100 | [diff] [blame] | 303 | void _fd_delete_orphan(int fd) |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 304 | { |
Willy Tarreau | b41a6e9 | 2021-04-06 17:49:19 +0200 | [diff] [blame] | 305 | if (fdtab[fd].state & FD_LINGER_RISK) { |
Willy Tarreau | ad38ace | 2013-12-15 14:19:38 +0100 | [diff] [blame] | 306 | /* this is generally set when connecting to servers */ |
Ilya Shipitsin | b7e43f0 | 2020-04-02 15:02:08 +0500 | [diff] [blame] | 307 | DISGUISE(setsockopt(fd, SOL_SOCKET, SO_LINGER, |
| 308 | (struct linger *) &nolinger, sizeof(struct linger))); |
Willy Tarreau | ad38ace | 2013-12-15 14:19:38 +0100 | [diff] [blame] | 309 | } |
Willy Tarreau | 6ea20b1 | 2012-11-11 16:05:19 +0100 | [diff] [blame] | 310 | if (cur_poller.clo) |
| 311 | cur_poller.clo(fd); |
Willy Tarreau | 2d42329 | 2021-03-24 15:34:25 +0100 | [diff] [blame] | 312 | |
| 313 | port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port); |
Olivier Houchard | c22580c | 2019-08-05 18:51:52 +0200 | [diff] [blame] | 314 | polled_mask[fd].poll_recv = polled_mask[fd].poll_send = 0; |
Willy Tarreau | 6ea20b1 | 2012-11-11 16:05:19 +0100 | [diff] [blame] | 315 | |
Willy Tarreau | f817e9f | 2014-01-10 16:58:45 +0100 | [diff] [blame] | 316 | fdtab[fd].state = 0; |
Willy Tarreau | 6ea20b1 | 2012-11-11 16:05:19 +0100 | [diff] [blame] | 317 | |
Willy Tarreau | 38e8a1c | 2020-06-23 10:04:54 +0200 | [diff] [blame] | 318 | #ifdef DEBUG_FD |
| 319 | fdtab[fd].event_count = 0; |
| 320 | #endif |
Willy Tarreau | 8d5d77e | 2009-10-18 07:25:52 +0200 | [diff] [blame] | 321 | fdinfo[fd].port_range = NULL; |
Willy Tarreau | db3b326 | 2012-07-05 23:19:22 +0200 | [diff] [blame] | 322 | fdtab[fd].owner = NULL; |
Willy Tarreau | 2c3f981 | 2021-03-24 10:51:32 +0100 | [diff] [blame] | 323 | /* perform the close() call last as it's what unlocks the instant reuse |
| 324 | * of this FD by any other thread. |
| 325 | */ |
Willy Tarreau | 63d8b60 | 2020-08-26 11:54:06 +0200 | [diff] [blame] | 326 | close(fd); |
Willy Tarreau | 4781b15 | 2021-04-06 13:53:36 +0200 | [diff] [blame] | 327 | _HA_ATOMIC_DEC(&ha_used_fds); |
Willy Tarreau | 2c3f981 | 2021-03-24 10:51:32 +0100 | [diff] [blame] | 328 | } |
| 329 | |
Olivier Houchard | c23b337 | 2021-03-25 01:38:54 +0100 | [diff] [blame] | 330 | #ifndef HA_HAVE_CAS_DW |
| 331 | __decl_thread(__decl_rwlock(fd_mig_lock)); |
| 332 | #endif |
| 333 | |
Willy Tarreau | 2c3f981 | 2021-03-24 10:51:32 +0100 | [diff] [blame] | 334 | /* Deletes an FD from the fdsets. The file descriptor is also closed, possibly |
| 335 | * asynchronously. Only the owning thread may do this. |
| 336 | */ |
| 337 | void fd_delete(int fd) |
| 338 | { |
| 339 | /* we must postpone removal of an FD that may currently be in use |
Ilya Shipitsin | b2be9a1 | 2021-04-24 13:25:42 +0500 | [diff] [blame] | 340 | * by another thread. This can happen in the following two situations: |
Willy Tarreau | 2c3f981 | 2021-03-24 10:51:32 +0100 | [diff] [blame] | 341 | * - after a takeover, the owning thread closes the connection but |
| 342 | * the previous one just woke up from the poller and entered |
| 343 | * the FD handler iocb. That thread holds an entry in running_mask |
| 344 | * and requires removal protection. |
| 345 | * - multiple threads are accepting connections on a listener, and |
| 346 | * one of them (or even an separate one) decides to unbind the |
| 347 | * listener under the listener's lock while other ones still hold |
| 348 | * the running bit. |
| 349 | * In both situations the FD is marked as unused (thread_mask = 0) and |
| 350 | * will not take new bits in its running_mask so we have the guarantee |
| 351 | * that the last thread eliminating running_mask is the one allowed to |
| 352 | * safely delete the FD. Most of the time it will be the current thread. |
| 353 | */ |
| 354 | |
| 355 | HA_ATOMIC_OR(&fdtab[fd].running_mask, tid_bit); |
Olivier Houchard | c23b337 | 2021-03-25 01:38:54 +0100 | [diff] [blame] | 356 | #ifndef HA_HAVE_CAS_DW |
| 357 | HA_RWLOCK_WRLOCK(OTHER_LOCK, &fd_mig_lock); |
| 358 | #endif |
Willy Tarreau | 2c3f981 | 2021-03-24 10:51:32 +0100 | [diff] [blame] | 359 | HA_ATOMIC_STORE(&fdtab[fd].thread_mask, 0); |
Olivier Houchard | c23b337 | 2021-03-25 01:38:54 +0100 | [diff] [blame] | 360 | #ifndef HA_HAVE_CAS_DW |
| 361 | HA_RWLOCK_WRUNLOCK(OTHER_LOCK, &fd_mig_lock); |
| 362 | #endif |
Willy Tarreau | 2c3f981 | 2021-03-24 10:51:32 +0100 | [diff] [blame] | 363 | if (fd_clr_running(fd) == 0) |
| 364 | _fd_delete_orphan(fd); |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 365 | } |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 366 | |
Olivier Houchard | 8851664 | 2020-03-05 18:10:51 +0100 | [diff] [blame] | 367 | /* |
| 368 | * Take over a FD belonging to another thread. |
| 369 | * unexpected_conn is the expected owner of the fd. |
| 370 | * Returns 0 on success, and -1 on failure. |
| 371 | */ |
| 372 | int fd_takeover(int fd, void *expected_owner) |
| 373 | { |
Willy Tarreau | c460c91 | 2020-06-18 07:28:09 +0200 | [diff] [blame] | 374 | int ret = -1; |
Olivier Houchard | 8851664 | 2020-03-05 18:10:51 +0100 | [diff] [blame] | 375 | |
Willy Tarreau | f1cad38 | 2020-06-18 08:14:59 +0200 | [diff] [blame] | 376 | #ifndef HA_HAVE_CAS_DW |
Willy Tarreau | 1db4273 | 2021-04-06 11:44:07 +0200 | [diff] [blame] | 377 | if (_HA_ATOMIC_OR_FETCH(&fdtab[fd].running_mask, tid_bit) == tid_bit) { |
Willy Tarreau | c460c91 | 2020-06-18 07:28:09 +0200 | [diff] [blame] | 378 | HA_RWLOCK_WRLOCK(OTHER_LOCK, &fd_mig_lock); |
| 379 | if (fdtab[fd].owner == expected_owner) { |
| 380 | fdtab[fd].thread_mask = tid_bit; |
| 381 | ret = 0; |
| 382 | } |
| 383 | HA_RWLOCK_WRUNLOCK(OTHER_LOCK, &fd_mig_lock); |
Olivier Houchard | 8851664 | 2020-03-05 18:10:51 +0100 | [diff] [blame] | 384 | } |
Olivier Houchard | 8851664 | 2020-03-05 18:10:51 +0100 | [diff] [blame] | 385 | #else |
| 386 | unsigned long old_masks[2]; |
| 387 | unsigned long new_masks[2]; |
| 388 | |
Olivier Houchard | 8851664 | 2020-03-05 18:10:51 +0100 | [diff] [blame] | 389 | new_masks[0] = new_masks[1] = tid_bit; |
Willy Tarreau | 4297363 | 2020-06-18 08:05:15 +0200 | [diff] [blame] | 390 | |
Willy Tarreau | 1db4273 | 2021-04-06 11:44:07 +0200 | [diff] [blame] | 391 | old_masks[0] = _HA_ATOMIC_OR_FETCH(&fdtab[fd].running_mask, tid_bit); |
Willy Tarreau | 4297363 | 2020-06-18 08:05:15 +0200 | [diff] [blame] | 392 | old_masks[1] = fdtab[fd].thread_mask; |
| 393 | |
Olivier Houchard | 8851664 | 2020-03-05 18:10:51 +0100 | [diff] [blame] | 394 | /* protect ourself against a delete then an insert for the same fd, |
| 395 | * if it happens, then the owner will no longer be the expected |
| 396 | * connection. |
| 397 | */ |
Willy Tarreau | f1cad38 | 2020-06-18 08:14:59 +0200 | [diff] [blame] | 398 | if (fdtab[fd].owner == expected_owner) { |
| 399 | while (old_masks[0] == tid_bit && old_masks[1]) { |
| 400 | if (_HA_ATOMIC_DWCAS(&fdtab[fd].running_mask, &old_masks, &new_masks)) { |
| 401 | ret = 0; |
| 402 | break; |
| 403 | } |
Olivier Houchard | 8851664 | 2020-03-05 18:10:51 +0100 | [diff] [blame] | 404 | } |
Willy Tarreau | f1cad38 | 2020-06-18 08:14:59 +0200 | [diff] [blame] | 405 | } |
| 406 | #endif /* HW_HAVE_CAS_DW */ |
| 407 | |
Olivier Houchard | 8851664 | 2020-03-05 18:10:51 +0100 | [diff] [blame] | 408 | _HA_ATOMIC_AND(&fdtab[fd].running_mask, ~tid_bit); |
Willy Tarreau | f1cad38 | 2020-06-18 08:14:59 +0200 | [diff] [blame] | 409 | |
Olivier Houchard | ddc874c | 2020-06-17 20:34:05 +0200 | [diff] [blame] | 410 | /* Make sure the FD doesn't have the active bit. It is possible that |
| 411 | * the fd is polled by the thread that used to own it, the new thread |
| 412 | * is supposed to call subscribe() later, to activate polling. |
| 413 | */ |
Willy Tarreau | f1cad38 | 2020-06-18 08:14:59 +0200 | [diff] [blame] | 414 | if (likely(ret == 0)) |
| 415 | fd_stop_recv(fd); |
| 416 | return ret; |
Olivier Houchard | 8851664 | 2020-03-05 18:10:51 +0100 | [diff] [blame] | 417 | } |
| 418 | |
Willy Tarreau | dbe3060 | 2019-09-04 13:25:41 +0200 | [diff] [blame] | 419 | void updt_fd_polling(const int fd) |
| 420 | { |
Willy Tarreau | 5a7d6eb | 2020-11-26 22:25:10 +0100 | [diff] [blame] | 421 | if (all_threads_mask == 1UL || (fdtab[fd].thread_mask & all_threads_mask) == tid_bit) { |
Willy Tarreau | dbe3060 | 2019-09-04 13:25:41 +0200 | [diff] [blame] | 422 | if (HA_ATOMIC_BTS(&fdtab[fd].update_mask, tid)) |
| 423 | return; |
| 424 | |
| 425 | fd_updt[fd_nbupdt++] = fd; |
| 426 | } else { |
| 427 | unsigned long update_mask = fdtab[fd].update_mask; |
| 428 | do { |
| 429 | if (update_mask == fdtab[fd].thread_mask) |
| 430 | return; |
Willy Tarreau | f015887 | 2020-09-25 12:18:53 +0200 | [diff] [blame] | 431 | } while (!_HA_ATOMIC_CAS(&fdtab[fd].update_mask, &update_mask, fdtab[fd].thread_mask)); |
| 432 | |
Willy Tarreau | dbe3060 | 2019-09-04 13:25:41 +0200 | [diff] [blame] | 433 | fd_add_to_fd_list(&update_list, fd, offsetof(struct fdtab, update)); |
Willy Tarreau | f015887 | 2020-09-25 12:18:53 +0200 | [diff] [blame] | 434 | |
| 435 | if (fd_active(fd) && |
| 436 | !(fdtab[fd].thread_mask & tid_bit) && |
| 437 | (fdtab[fd].thread_mask & ~tid_bit & all_threads_mask & ~sleeping_thread_mask) == 0) { |
| 438 | /* we need to wake up one thread to handle it immediately */ |
| 439 | int thr = my_ffsl(fdtab[fd].thread_mask & ~tid_bit & all_threads_mask) - 1; |
| 440 | |
| 441 | _HA_ATOMIC_AND(&sleeping_thread_mask, ~(1UL << thr)); |
| 442 | wake_thread(thr); |
| 443 | } |
Willy Tarreau | dbe3060 | 2019-09-04 13:25:41 +0200 | [diff] [blame] | 444 | } |
| 445 | } |
| 446 | |
Willy Tarreau | 931d8b7 | 2019-08-27 11:08:17 +0200 | [diff] [blame] | 447 | /* Tries to send <npfx> parts from <prefix> followed by <nmsg> parts from <msg> |
| 448 | * optionally followed by a newline if <nl> is non-null, to file descriptor |
| 449 | * <fd>. The message is sent atomically using writev(). It may be truncated to |
| 450 | * <maxlen> bytes if <maxlen> is non-null. There is no distinction between the |
| 451 | * two lists, it's just a convenience to help the caller prepend some prefixes |
| 452 | * when necessary. It takes the fd's lock to make sure no other thread will |
| 453 | * write to the same fd in parallel. Returns the number of bytes sent, or <=0 |
| 454 | * on failure. A limit to 31 total non-empty segments is enforced. The caller |
| 455 | * is responsible for taking care of making the fd non-blocking. |
| 456 | */ |
| 457 | ssize_t fd_write_frag_line(int fd, size_t maxlen, const struct ist pfx[], size_t npfx, const struct ist msg[], size_t nmsg, int nl) |
| 458 | { |
| 459 | struct iovec iovec[32]; |
| 460 | size_t totlen = 0; |
| 461 | size_t sent = 0; |
| 462 | int vec = 0; |
Willy Tarreau | df18787 | 2020-06-11 14:25:47 +0200 | [diff] [blame] | 463 | int attempts = 0; |
Willy Tarreau | 931d8b7 | 2019-08-27 11:08:17 +0200 | [diff] [blame] | 464 | |
| 465 | if (!maxlen) |
| 466 | maxlen = ~0; |
| 467 | |
| 468 | /* keep one char for a possible trailing '\n' in any case */ |
| 469 | maxlen--; |
| 470 | |
| 471 | /* make an iovec from the concatenation of all parts of the original |
| 472 | * message. Skip empty fields and truncate the whole message to maxlen, |
| 473 | * leaving one spare iovec for the '\n'. |
| 474 | */ |
| 475 | while (vec < (sizeof(iovec) / sizeof(iovec[0]) - 1)) { |
| 476 | if (!npfx) { |
| 477 | pfx = msg; |
| 478 | npfx = nmsg; |
| 479 | nmsg = 0; |
| 480 | if (!npfx) |
| 481 | break; |
| 482 | } |
| 483 | |
| 484 | iovec[vec].iov_base = pfx->ptr; |
| 485 | iovec[vec].iov_len = MIN(maxlen, pfx->len); |
| 486 | maxlen -= iovec[vec].iov_len; |
| 487 | totlen += iovec[vec].iov_len; |
| 488 | if (iovec[vec].iov_len) |
| 489 | vec++; |
| 490 | pfx++; npfx--; |
| 491 | }; |
| 492 | |
| 493 | if (nl) { |
| 494 | iovec[vec].iov_base = "\n"; |
| 495 | iovec[vec].iov_len = 1; |
| 496 | vec++; |
| 497 | } |
| 498 | |
Willy Tarreau | df18787 | 2020-06-11 14:25:47 +0200 | [diff] [blame] | 499 | /* make sure we never interleave writes and we never block. This means |
| 500 | * we prefer to fail on collision than to block. But we don't want to |
| 501 | * lose too many logs so we just perform a few lock attempts then give |
| 502 | * up. |
| 503 | */ |
| 504 | |
Willy Tarreau | 1673c4a | 2021-04-07 17:36:57 +0200 | [diff] [blame] | 505 | while (HA_ATOMIC_BTS(&fdtab[fd].state, FD_EXCL_SYSCALL_BIT)) { |
Willy Tarreau | df18787 | 2020-06-11 14:25:47 +0200 | [diff] [blame] | 506 | if (++attempts >= 200) { |
| 507 | /* so that the caller knows the message couldn't be delivered */ |
| 508 | sent = -1; |
| 509 | errno = EAGAIN; |
| 510 | goto leave; |
| 511 | } |
| 512 | ha_thread_relax(); |
| 513 | } |
| 514 | |
Willy Tarreau | 0cc6128 | 2021-04-06 17:57:12 +0200 | [diff] [blame] | 515 | if (unlikely(!(fdtab[fd].state & FD_INITIALIZED))) { |
| 516 | HA_ATOMIC_OR(&fdtab[fd].state, FD_INITIALIZED); |
Willy Tarreau | 7e9776a | 2019-08-30 14:41:47 +0200 | [diff] [blame] | 517 | if (!isatty(fd)) |
| 518 | fcntl(fd, F_SETFL, O_NONBLOCK); |
| 519 | } |
Willy Tarreau | 931d8b7 | 2019-08-27 11:08:17 +0200 | [diff] [blame] | 520 | sent = writev(fd, iovec, vec); |
Willy Tarreau | 1673c4a | 2021-04-07 17:36:57 +0200 | [diff] [blame] | 521 | HA_ATOMIC_BTR(&fdtab[fd].state, FD_EXCL_SYSCALL_BIT); |
Willy Tarreau | 931d8b7 | 2019-08-27 11:08:17 +0200 | [diff] [blame] | 522 | |
Willy Tarreau | df18787 | 2020-06-11 14:25:47 +0200 | [diff] [blame] | 523 | leave: |
Willy Tarreau | 931d8b7 | 2019-08-27 11:08:17 +0200 | [diff] [blame] | 524 | /* sent > 0 if the message was delivered */ |
| 525 | return sent; |
| 526 | } |
| 527 | |
Olivier Houchard | 2292edf | 2019-02-25 14:26:54 +0100 | [diff] [blame] | 528 | #if defined(USE_CLOSEFROM) |
| 529 | void my_closefrom(int start) |
| 530 | { |
| 531 | closefrom(start); |
| 532 | } |
| 533 | |
Willy Tarreau | e573323 | 2019-05-22 19:24:06 +0200 | [diff] [blame] | 534 | #elif defined(USE_POLL) |
Willy Tarreau | 9188ac6 | 2019-02-21 22:12:47 +0100 | [diff] [blame] | 535 | /* This is a portable implementation of closefrom(). It closes all open file |
| 536 | * descriptors starting at <start> and above. It relies on the fact that poll() |
| 537 | * will return POLLNVAL for each invalid (hence close) file descriptor passed |
| 538 | * in argument in order to skip them. It acts with batches of FDs and will |
| 539 | * typically perform one poll() call per 1024 FDs so the overhead is low in |
| 540 | * case all FDs have to be closed. |
| 541 | */ |
| 542 | void my_closefrom(int start) |
| 543 | { |
| 544 | struct pollfd poll_events[1024]; |
| 545 | struct rlimit limit; |
| 546 | int nbfds, fd, ret, idx; |
| 547 | int step, next; |
| 548 | |
| 549 | if (getrlimit(RLIMIT_NOFILE, &limit) == 0) |
| 550 | step = nbfds = limit.rlim_cur; |
| 551 | else |
| 552 | step = nbfds = 0; |
| 553 | |
| 554 | if (nbfds <= 0) { |
| 555 | /* set safe limit */ |
| 556 | nbfds = 1024; |
| 557 | step = 256; |
| 558 | } |
| 559 | |
| 560 | if (step > sizeof(poll_events) / sizeof(poll_events[0])) |
| 561 | step = sizeof(poll_events) / sizeof(poll_events[0]); |
| 562 | |
| 563 | while (start < nbfds) { |
| 564 | next = (start / step + 1) * step; |
| 565 | |
| 566 | for (fd = start; fd < next && fd < nbfds; fd++) { |
| 567 | poll_events[fd - start].fd = fd; |
| 568 | poll_events[fd - start].events = 0; |
| 569 | } |
| 570 | |
| 571 | do { |
| 572 | ret = poll(poll_events, fd - start, 0); |
| 573 | if (ret >= 0) |
| 574 | break; |
| 575 | } while (errno == EAGAIN || errno == EINTR || errno == ENOMEM); |
| 576 | |
Willy Tarreau | b8e602c | 2019-02-22 09:07:42 +0100 | [diff] [blame] | 577 | if (ret) |
| 578 | ret = fd - start; |
| 579 | |
Willy Tarreau | 9188ac6 | 2019-02-21 22:12:47 +0100 | [diff] [blame] | 580 | for (idx = 0; idx < ret; idx++) { |
| 581 | if (poll_events[idx].revents & POLLNVAL) |
| 582 | continue; /* already closed */ |
| 583 | |
| 584 | fd = poll_events[idx].fd; |
| 585 | close(fd); |
| 586 | } |
| 587 | start = next; |
| 588 | } |
| 589 | } |
| 590 | |
Willy Tarreau | e573323 | 2019-05-22 19:24:06 +0200 | [diff] [blame] | 591 | #else // defined(USE_POLL) |
Willy Tarreau | 9188ac6 | 2019-02-21 22:12:47 +0100 | [diff] [blame] | 592 | |
Willy Tarreau | 2d7f81b | 2019-02-21 22:19:17 +0100 | [diff] [blame] | 593 | /* This is a portable implementation of closefrom(). It closes all open file |
| 594 | * descriptors starting at <start> and above. This is a naive version for use |
| 595 | * when the operating system provides no alternative. |
| 596 | */ |
| 597 | void my_closefrom(int start) |
| 598 | { |
| 599 | struct rlimit limit; |
| 600 | int nbfds; |
| 601 | |
| 602 | if (getrlimit(RLIMIT_NOFILE, &limit) == 0) |
| 603 | nbfds = limit.rlim_cur; |
| 604 | else |
| 605 | nbfds = 0; |
| 606 | |
| 607 | if (nbfds <= 0) |
| 608 | nbfds = 1024; /* safe limit */ |
| 609 | |
| 610 | while (start < nbfds) |
| 611 | close(start++); |
| 612 | } |
Willy Tarreau | e573323 | 2019-05-22 19:24:06 +0200 | [diff] [blame] | 613 | #endif // defined(USE_POLL) |
Willy Tarreau | 2d7f81b | 2019-02-21 22:19:17 +0100 | [diff] [blame] | 614 | |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 615 | /* disable the specified poller */ |
| 616 | void disable_poller(const char *poller_name) |
| 617 | { |
| 618 | int p; |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 619 | |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 620 | for (p = 0; p < nbpollers; p++) |
| 621 | if (strcmp(pollers[p].name, poller_name) == 0) |
| 622 | pollers[p].pref = 0; |
| 623 | } |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 624 | |
Olivier Houchard | 79321b9 | 2018-07-26 17:55:11 +0200 | [diff] [blame] | 625 | void poller_pipe_io_handler(int fd) |
| 626 | { |
| 627 | char buf[1024]; |
| 628 | /* Flush the pipe */ |
| 629 | while (read(fd, buf, sizeof(buf)) > 0); |
| 630 | fd_cant_recv(fd); |
| 631 | } |
| 632 | |
Willy Tarreau | 082b628 | 2019-05-22 14:42:12 +0200 | [diff] [blame] | 633 | /* allocate the per-thread fd_updt thus needs to be called early after |
| 634 | * thread creation. |
| 635 | */ |
| 636 | static int alloc_pollers_per_thread() |
| 637 | { |
| 638 | fd_updt = calloc(global.maxsock, sizeof(*fd_updt)); |
| 639 | return fd_updt != NULL; |
| 640 | } |
| 641 | |
| 642 | /* Initialize the pollers per thread.*/ |
Christopher Faulet | d4604ad | 2017-05-29 10:40:41 +0200 | [diff] [blame] | 643 | static int init_pollers_per_thread() |
| 644 | { |
Olivier Houchard | 79321b9 | 2018-07-26 17:55:11 +0200 | [diff] [blame] | 645 | int mypipe[2]; |
Willy Tarreau | 082b628 | 2019-05-22 14:42:12 +0200 | [diff] [blame] | 646 | |
| 647 | if (pipe(mypipe) < 0) |
Olivier Houchard | 79321b9 | 2018-07-26 17:55:11 +0200 | [diff] [blame] | 648 | return 0; |
Willy Tarreau | 082b628 | 2019-05-22 14:42:12 +0200 | [diff] [blame] | 649 | |
Olivier Houchard | 79321b9 | 2018-07-26 17:55:11 +0200 | [diff] [blame] | 650 | poller_rd_pipe = mypipe[0]; |
| 651 | poller_wr_pipe[tid] = mypipe[1]; |
| 652 | fcntl(poller_rd_pipe, F_SETFL, O_NONBLOCK); |
| 653 | fd_insert(poller_rd_pipe, poller_pipe_io_handler, poller_pipe_io_handler, |
| 654 | tid_bit); |
| 655 | fd_want_recv(poller_rd_pipe); |
Christopher Faulet | d4604ad | 2017-05-29 10:40:41 +0200 | [diff] [blame] | 656 | return 1; |
| 657 | } |
| 658 | |
| 659 | /* Deinitialize the pollers per thread */ |
| 660 | static void deinit_pollers_per_thread() |
| 661 | { |
William Lallemand | 808e1b7 | 2018-12-15 22:34:31 +0100 | [diff] [blame] | 662 | /* rd and wr are init at the same place, but only rd is init to -1, so |
| 663 | we rely to rd to close. */ |
| 664 | if (poller_rd_pipe > -1) { |
| 665 | close(poller_rd_pipe); |
| 666 | poller_rd_pipe = -1; |
| 667 | close(poller_wr_pipe[tid]); |
| 668 | poller_wr_pipe[tid] = -1; |
| 669 | } |
Christopher Faulet | d4604ad | 2017-05-29 10:40:41 +0200 | [diff] [blame] | 670 | } |
| 671 | |
Willy Tarreau | 082b628 | 2019-05-22 14:42:12 +0200 | [diff] [blame] | 672 | /* Release the pollers per thread, to be called late */ |
| 673 | static void free_pollers_per_thread() |
| 674 | { |
Willy Tarreau | 61cfdf4 | 2021-02-20 10:46:51 +0100 | [diff] [blame] | 675 | ha_free(&fd_updt); |
Willy Tarreau | 082b628 | 2019-05-22 14:42:12 +0200 | [diff] [blame] | 676 | } |
| 677 | |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 678 | /* |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 679 | * Initialize the pollers till the best one is found. |
| 680 | * If none works, returns 0, otherwise 1. |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 681 | */ |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 682 | int init_pollers() |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 683 | { |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 684 | int p; |
| 685 | struct poller *bp; |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 686 | |
Willy Tarreau | 7c9f756 | 2020-10-13 15:45:07 +0200 | [diff] [blame] | 687 | if ((fdtab = calloc(global.maxsock, sizeof(*fdtab))) == NULL) { |
| 688 | ha_alert("Not enough memory to allocate %d entries for fdtab!\n", global.maxsock); |
Christopher Faulet | 63fe652 | 2017-08-31 17:52:09 +0200 | [diff] [blame] | 689 | goto fail_tab; |
Willy Tarreau | 7c9f756 | 2020-10-13 15:45:07 +0200 | [diff] [blame] | 690 | } |
Christopher Faulet | 63fe652 | 2017-08-31 17:52:09 +0200 | [diff] [blame] | 691 | |
Willy Tarreau | 7c9f756 | 2020-10-13 15:45:07 +0200 | [diff] [blame] | 692 | if ((polled_mask = calloc(global.maxsock, sizeof(*polled_mask))) == NULL) { |
| 693 | ha_alert("Not enough memory to allocate %d entries for polled_mask!\n", global.maxsock); |
Olivier Houchard | cb92f5c | 2018-04-26 14:23:07 +0200 | [diff] [blame] | 694 | goto fail_polledmask; |
Willy Tarreau | 7c9f756 | 2020-10-13 15:45:07 +0200 | [diff] [blame] | 695 | } |
Uman Shahzad | da7eeed | 2019-01-17 08:21:39 +0000 | [diff] [blame] | 696 | |
Willy Tarreau | 7c9f756 | 2020-10-13 15:45:07 +0200 | [diff] [blame] | 697 | if ((fdinfo = calloc(global.maxsock, sizeof(*fdinfo))) == NULL) { |
| 698 | ha_alert("Not enough memory to allocate %d entries for fdinfo!\n", global.maxsock); |
Christopher Faulet | 63fe652 | 2017-08-31 17:52:09 +0200 | [diff] [blame] | 699 | goto fail_info; |
Willy Tarreau | 7c9f756 | 2020-10-13 15:45:07 +0200 | [diff] [blame] | 700 | } |
Christopher Faulet | 63fe652 | 2017-08-31 17:52:09 +0200 | [diff] [blame] | 701 | |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 702 | update_list.first = update_list.last = -1; |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 703 | |
Olivier Houchard | 4815c8c | 2018-01-24 18:17:56 +0100 | [diff] [blame] | 704 | for (p = 0; p < global.maxsock; p++) { |
Olivier Houchard | 4815c8c | 2018-01-24 18:17:56 +0100 | [diff] [blame] | 705 | /* Mark the fd as out of the fd cache */ |
Olivier Houchard | 6b96f72 | 2018-04-25 16:58:25 +0200 | [diff] [blame] | 706 | fdtab[p].update.next = -3; |
Olivier Houchard | 4815c8c | 2018-01-24 18:17:56 +0100 | [diff] [blame] | 707 | } |
Christopher Faulet | d4604ad | 2017-05-29 10:40:41 +0200 | [diff] [blame] | 708 | |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 709 | do { |
| 710 | bp = NULL; |
| 711 | for (p = 0; p < nbpollers; p++) |
| 712 | if (!bp || (pollers[p].pref > bp->pref)) |
| 713 | bp = &pollers[p]; |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 714 | |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 715 | if (!bp || bp->pref == 0) |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 716 | break; |
| 717 | |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 718 | if (bp->init(bp)) { |
| 719 | memcpy(&cur_poller, bp, sizeof(*bp)); |
| 720 | return 1; |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 721 | } |
Willy Tarreau | 4f60f16 | 2007-04-08 16:39:58 +0200 | [diff] [blame] | 722 | } while (!bp || bp->pref == 0); |
Willy Tarreau | 7be79a4 | 2012-11-11 15:02:54 +0100 | [diff] [blame] | 723 | |
Christopher Faulet | 63fe652 | 2017-08-31 17:52:09 +0200 | [diff] [blame] | 724 | free(fdinfo); |
| 725 | fail_info: |
Olivier Houchard | cb92f5c | 2018-04-26 14:23:07 +0200 | [diff] [blame] | 726 | free(polled_mask); |
| 727 | fail_polledmask: |
Uman Shahzad | da7eeed | 2019-01-17 08:21:39 +0000 | [diff] [blame] | 728 | free(fdtab); |
| 729 | fail_tab: |
Willy Tarreau | 7be79a4 | 2012-11-11 15:02:54 +0100 | [diff] [blame] | 730 | return 0; |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 731 | } |
| 732 | |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 733 | /* |
Krzysztof Piotr Oledzki | a643baf | 2008-05-29 23:53:44 +0200 | [diff] [blame] | 734 | * Deinitialize the pollers. |
| 735 | */ |
| 736 | void deinit_pollers() { |
| 737 | |
| 738 | struct poller *bp; |
| 739 | int p; |
| 740 | |
| 741 | for (p = 0; p < nbpollers; p++) { |
| 742 | bp = &pollers[p]; |
| 743 | |
| 744 | if (bp && bp->pref) |
| 745 | bp->term(bp); |
| 746 | } |
Christopher Faulet | d4604ad | 2017-05-29 10:40:41 +0200 | [diff] [blame] | 747 | |
Willy Tarreau | 61cfdf4 | 2021-02-20 10:46:51 +0100 | [diff] [blame] | 748 | ha_free(&fdinfo); |
| 749 | ha_free(&fdtab); |
| 750 | ha_free(&polled_mask); |
Krzysztof Piotr Oledzki | a643baf | 2008-05-29 23:53:44 +0200 | [diff] [blame] | 751 | } |
| 752 | |
| 753 | /* |
Willy Tarreau | 2ff7622 | 2007-04-09 19:29:56 +0200 | [diff] [blame] | 754 | * Lists the known pollers on <out>. |
| 755 | * Should be performed only before initialization. |
| 756 | */ |
| 757 | int list_pollers(FILE *out) |
| 758 | { |
| 759 | int p; |
| 760 | int last, next; |
| 761 | int usable; |
| 762 | struct poller *bp; |
| 763 | |
| 764 | fprintf(out, "Available polling systems :\n"); |
| 765 | |
| 766 | usable = 0; |
| 767 | bp = NULL; |
| 768 | last = next = -1; |
| 769 | while (1) { |
| 770 | for (p = 0; p < nbpollers; p++) { |
Willy Tarreau | 2ff7622 | 2007-04-09 19:29:56 +0200 | [diff] [blame] | 771 | if ((next < 0 || pollers[p].pref > next) |
Willy Tarreau | e79c3b2 | 2010-11-19 10:20:36 +0100 | [diff] [blame] | 772 | && (last < 0 || pollers[p].pref < last)) { |
Willy Tarreau | 2ff7622 | 2007-04-09 19:29:56 +0200 | [diff] [blame] | 773 | next = pollers[p].pref; |
Willy Tarreau | e79c3b2 | 2010-11-19 10:20:36 +0100 | [diff] [blame] | 774 | if (!bp || (pollers[p].pref > bp->pref)) |
| 775 | bp = &pollers[p]; |
| 776 | } |
Willy Tarreau | 2ff7622 | 2007-04-09 19:29:56 +0200 | [diff] [blame] | 777 | } |
| 778 | |
| 779 | if (next == -1) |
| 780 | break; |
| 781 | |
| 782 | for (p = 0; p < nbpollers; p++) { |
| 783 | if (pollers[p].pref == next) { |
| 784 | fprintf(out, " %10s : ", pollers[p].name); |
| 785 | if (pollers[p].pref == 0) |
| 786 | fprintf(out, "disabled, "); |
| 787 | else |
| 788 | fprintf(out, "pref=%3d, ", pollers[p].pref); |
| 789 | if (pollers[p].test(&pollers[p])) { |
| 790 | fprintf(out, " test result OK"); |
| 791 | if (next > 0) |
| 792 | usable++; |
Willy Tarreau | e79c3b2 | 2010-11-19 10:20:36 +0100 | [diff] [blame] | 793 | } else { |
Willy Tarreau | 2ff7622 | 2007-04-09 19:29:56 +0200 | [diff] [blame] | 794 | fprintf(out, " test result FAILED"); |
Willy Tarreau | e79c3b2 | 2010-11-19 10:20:36 +0100 | [diff] [blame] | 795 | if (bp == &pollers[p]) |
| 796 | bp = NULL; |
| 797 | } |
Willy Tarreau | 2ff7622 | 2007-04-09 19:29:56 +0200 | [diff] [blame] | 798 | fprintf(out, "\n"); |
| 799 | } |
| 800 | } |
| 801 | last = next; |
| 802 | next = -1; |
| 803 | }; |
| 804 | fprintf(out, "Total: %d (%d usable), will use %s.\n", nbpollers, usable, bp ? bp->name : "none"); |
| 805 | return 0; |
| 806 | } |
| 807 | |
| 808 | /* |
| 809 | * Some pollers may lose their connection after a fork(). It may be necessary |
| 810 | * to create initialize part of them again. Returns 0 in case of failure, |
| 811 | * otherwise 1. The fork() function may be NULL if unused. In case of error, |
| 812 | * the the current poller is destroyed and the caller is responsible for trying |
| 813 | * another one by calling init_pollers() again. |
| 814 | */ |
| 815 | int fork_poller() |
| 816 | { |
Conrad Hoffmann | 041751c | 2014-05-20 14:28:24 +0200 | [diff] [blame] | 817 | int fd; |
Willy Tarreau | ce036bc | 2018-01-29 14:58:02 +0100 | [diff] [blame] | 818 | for (fd = 0; fd < global.maxsock; fd++) { |
Conrad Hoffmann | 041751c | 2014-05-20 14:28:24 +0200 | [diff] [blame] | 819 | if (fdtab[fd].owner) { |
Willy Tarreau | 030dae1 | 2021-04-06 17:53:33 +0200 | [diff] [blame] | 820 | HA_ATOMIC_OR(&fdtab[fd].state, FD_CLONED); |
Conrad Hoffmann | 041751c | 2014-05-20 14:28:24 +0200 | [diff] [blame] | 821 | } |
| 822 | } |
| 823 | |
Willy Tarreau | 2ff7622 | 2007-04-09 19:29:56 +0200 | [diff] [blame] | 824 | if (cur_poller.fork) { |
| 825 | if (cur_poller.fork(&cur_poller)) |
| 826 | return 1; |
| 827 | cur_poller.term(&cur_poller); |
| 828 | return 0; |
| 829 | } |
| 830 | return 1; |
| 831 | } |
| 832 | |
Willy Tarreau | bc52bec | 2020-06-18 08:58:47 +0200 | [diff] [blame] | 833 | /* config parser for global "tune.fd.edge-triggered", accepts "on" or "off" */ |
| 834 | static int cfg_parse_tune_fd_edge_triggered(char **args, int section_type, struct proxy *curpx, |
Willy Tarreau | 0182516 | 2021-03-09 09:53:46 +0100 | [diff] [blame] | 835 | const struct proxy *defpx, const char *file, int line, |
Willy Tarreau | bc52bec | 2020-06-18 08:58:47 +0200 | [diff] [blame] | 836 | char **err) |
| 837 | { |
| 838 | if (too_many_args(1, args, err, NULL)) |
| 839 | return -1; |
| 840 | |
| 841 | if (strcmp(args[1], "on") == 0) |
| 842 | global.tune.options |= GTUNE_FD_ET; |
| 843 | else if (strcmp(args[1], "off") == 0) |
| 844 | global.tune.options &= ~GTUNE_FD_ET; |
| 845 | else { |
| 846 | memprintf(err, "'%s' expects either 'on' or 'off' but got '%s'.", args[0], args[1]); |
| 847 | return -1; |
| 848 | } |
| 849 | return 0; |
| 850 | } |
| 851 | |
| 852 | /* config keyword parsers */ |
| 853 | static struct cfg_kw_list cfg_kws = {ILH, { |
Willy Tarreau | 9eec7e2 | 2021-05-08 11:06:32 +0200 | [diff] [blame] | 854 | { CFG_GLOBAL, "tune.fd.edge-triggered", cfg_parse_tune_fd_edge_triggered, KWF_EXPERIMENTAL }, |
Willy Tarreau | bc52bec | 2020-06-18 08:58:47 +0200 | [diff] [blame] | 855 | { 0, NULL, NULL } |
| 856 | }}; |
| 857 | |
| 858 | INITCALL1(STG_REGISTER, cfg_register_keywords, &cfg_kws); |
| 859 | |
Willy Tarreau | 082b628 | 2019-05-22 14:42:12 +0200 | [diff] [blame] | 860 | REGISTER_PER_THREAD_ALLOC(alloc_pollers_per_thread); |
Willy Tarreau | 172f5ce | 2018-11-26 11:21:50 +0100 | [diff] [blame] | 861 | REGISTER_PER_THREAD_INIT(init_pollers_per_thread); |
| 862 | REGISTER_PER_THREAD_DEINIT(deinit_pollers_per_thread); |
Willy Tarreau | 082b628 | 2019-05-22 14:42:12 +0200 | [diff] [blame] | 863 | REGISTER_PER_THREAD_FREE(free_pollers_per_thread); |
Willy Tarreau | 172f5ce | 2018-11-26 11:21:50 +0100 | [diff] [blame] | 864 | |
Willy Tarreau | 2ff7622 | 2007-04-09 19:29:56 +0200 | [diff] [blame] | 865 | /* |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 866 | * Local variables: |
| 867 | * c-indent-level: 8 |
| 868 | * c-basic-offset: 8 |
| 869 | * End: |
| 870 | */ |