blob: bf2383e4d7e630c0f5f9d9c70ff2be370f1f385c [file] [log] [blame]
Willy Tarreaubaaee002006-06-26 02:48:02 +02001/*
2 * File descriptors management functions.
3 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +01004 * Copyright 2000-2014 Willy Tarreau <w@1wt.eu>
Willy Tarreaubaaee002006-06-26 02:48:02 +02005 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
Willy Tarreau7be79a42012-11-11 15:02:54 +010011 * There is no direct link between the FD and the updates list. There is only a
12 * bit in the fdtab[] to indicate than a file descriptor is already present in
13 * the updates list. Once an fd is present in the updates list, it will have to
14 * be considered even if its changes are reverted in the middle or if the fd is
15 * replaced.
16 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010017 * The event state for an FD, as found in fdtab[].state, is maintained for each
18 * direction. The state field is built this way, with R bits in the low nibble
19 * and W bits in the high nibble for ease of access and debugging :
Willy Tarreau7be79a42012-11-11 15:02:54 +010020 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010021 * 7 6 5 4 3 2 1 0
Willy Tarreau5bee3e22019-09-04 09:52:57 +020022 * [ 0 | 0 | RW | AW | 0 | 0 | RR | AR ]
Willy Tarreauf817e9f2014-01-10 16:58:45 +010023 *
24 * A* = active *R = read
Willy Tarreau5bee3e22019-09-04 09:52:57 +020025 * R* = ready *W = write
Willy Tarreauf817e9f2014-01-10 16:58:45 +010026 *
27 * An FD is marked "active" when there is a desire to use it.
Willy Tarreauf817e9f2014-01-10 16:58:45 +010028 * An FD is marked "ready" when it has not faced a new EAGAIN since last wake-up
Willy Tarreau5bee3e22019-09-04 09:52:57 +020029 * (it is a cache of the last EAGAIN regardless of polling changes). Each poller
30 * has its own "polled" state for the same fd, as stored in the polled_mask.
Willy Tarreau7be79a42012-11-11 15:02:54 +010031 *
Willy Tarreau5bee3e22019-09-04 09:52:57 +020032 * We have 4 possible states for each direction based on these 2 flags :
Willy Tarreau7be79a42012-11-11 15:02:54 +010033 *
Willy Tarreau5bee3e22019-09-04 09:52:57 +020034 * +---+---+----------+---------------------------------------------+
35 * | R | A | State | Description |
36 * +---+---+----------+---------------------------------------------+
37 * | 0 | 0 | DISABLED | No activity desired, not ready. |
38 * | 0 | 1 | ACTIVE | Activity desired. |
39 * | 1 | 0 | STOPPED | End of activity. |
40 * | 1 | 1 | READY | Activity desired and reported. |
41 * +---+---+----------+---------------------------------------------+
Willy Tarreau7be79a42012-11-11 15:02:54 +010042 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010043 * The transitions are pretty simple :
44 * - fd_want_*() : set flag A
45 * - fd_stop_*() : clear flag A
46 * - fd_cant_*() : clear flag R (when facing EAGAIN)
47 * - fd_may_*() : set flag R (upon return from poll())
Willy Tarreauf817e9f2014-01-10 16:58:45 +010048 *
Willy Tarreau5bee3e22019-09-04 09:52:57 +020049 * Each poller then computes its own polled state :
50 * if (A) { if (!R) P := 1 } else { P := 0 }
Willy Tarreauf817e9f2014-01-10 16:58:45 +010051 *
Willy Tarreau5bee3e22019-09-04 09:52:57 +020052 * The state transitions look like the diagram below.
Willy Tarreauf817e9f2014-01-10 16:58:45 +010053 *
Willy Tarreau5bee3e22019-09-04 09:52:57 +020054 * may +----------+
55 * ,----| DISABLED | (READY=0, ACTIVE=0)
56 * | +----------+
57 * | want | ^
58 * | | |
59 * | v | stop
60 * | +----------+
61 * | | ACTIVE | (READY=0, ACTIVE=1)
62 * | +----------+
63 * | | ^
64 * | may | |
65 * | v | EAGAIN (cant)
66 * | +--------+
67 * | | READY | (READY=1, ACTIVE=1)
68 * | +--------+
69 * | stop | ^
70 * | | |
71 * | v | want
72 * | +---------+
73 * `--->| STOPPED | (READY=1, ACTIVE=0)
74 * +---------+
Willy Tarreaubaaee002006-06-26 02:48:02 +020075 */
76
Willy Tarreau2ff76222007-04-09 19:29:56 +020077#include <stdio.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020078#include <string.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020079#include <unistd.h>
Olivier Houchard79321b92018-07-26 17:55:11 +020080#include <fcntl.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020081#include <sys/types.h>
Willy Tarreau2d7f81b2019-02-21 22:19:17 +010082#include <sys/resource.h>
Willy Tarreau931d8b72019-08-27 11:08:17 +020083#include <sys/uio.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020084
Willy Tarreaue5733232019-05-22 19:24:06 +020085#if defined(USE_POLL)
Willy Tarreau9188ac62019-02-21 22:12:47 +010086#include <poll.h>
87#include <errno.h>
88#endif
89
Willy Tarreau4c7e4b72020-05-27 12:58:42 +020090#include <haproxy/api.h>
Willy Tarreaubc52bec2020-06-18 08:58:47 +020091#include <haproxy/cfgparse.h>
Willy Tarreau0f6ffd62020-06-03 19:33:00 +020092#include <haproxy/fd.h>
Willy Tarreaub2551052020-06-09 09:07:15 +020093#include <haproxy/global.h>
Willy Tarreaufc8f6a82020-06-03 19:20:59 +020094#include <haproxy/port_range.h>
Willy Tarreaubc52bec2020-06-18 08:58:47 +020095#include <haproxy/tools.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020096
Willy Tarreaub2551052020-06-09 09:07:15 +020097
Willy Tarreaubaaee002006-06-26 02:48:02 +020098struct fdtab *fdtab = NULL; /* array of all the file descriptors */
Olivier Houchard53055052019-07-25 14:00:18 +000099struct polled_mask *polled_mask = NULL; /* Array for the polled_mask of each fd */
Willy Tarreau8d5d77e2009-10-18 07:25:52 +0200100struct fdinfo *fdinfo = NULL; /* less-often used infos for file descriptors */
Willy Tarreaubaaee002006-06-26 02:48:02 +0200101int totalconn; /* total # of terminated sessions */
102int actconn; /* # of active sessions */
103
Willy Tarreau4f60f162007-04-08 16:39:58 +0200104struct poller pollers[MAX_POLLERS];
105struct poller cur_poller;
106int nbpollers = 0;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200107
Olivier Houchard6b96f722018-04-25 16:58:25 +0200108volatile struct fdlist update_list; // Global update list
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100109
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200110THREAD_LOCAL int *fd_updt = NULL; // FD updates list
111THREAD_LOCAL int fd_nbupdt = 0; // number of updates in the list
Olivier Houchard79321b92018-07-26 17:55:11 +0200112THREAD_LOCAL int poller_rd_pipe = -1; // Pipe to wake the thread
113int poller_wr_pipe[MAX_THREADS]; // Pipe to wake the threads
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200114
Olivier Houchard7c49d2e2019-04-16 18:37:05 +0200115volatile int ha_used_fds = 0; // Number of FD we're currently using
116
Willy Tarreau337fb712019-12-20 07:20:00 +0100117#define _GET_NEXT(fd, off) ((volatile struct fdlist_entry *)(void *)((char *)(&fdtab[fd]) + off))->next
118#define _GET_PREV(fd, off) ((volatile struct fdlist_entry *)(void *)((char *)(&fdtab[fd]) + off))->prev
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100119/* adds fd <fd> to fd list <list> if it was not yet in it */
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200120void fd_add_to_fd_list(volatile struct fdlist *list, int fd, int off)
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100121{
122 int next;
123 int new;
124 int old;
125 int last;
126
127redo_next:
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200128 next = _GET_NEXT(fd, off);
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100129 /* Check that we're not already in the cache, and if not, lock us. */
Olivier Houchardfc51f0f52019-12-19 18:33:08 +0100130 if (next > -2)
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100131 goto done;
Olivier Houchardfc51f0f52019-12-19 18:33:08 +0100132 if (next == -2)
133 goto redo_next;
Olivier Houchardd3608792019-03-08 18:47:42 +0100134 if (!_HA_ATOMIC_CAS(&_GET_NEXT(fd, off), &next, -2))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100135 goto redo_next;
Olivier Houchardd2b5d162019-03-08 13:47:21 +0100136 __ha_barrier_atomic_store();
Willy Tarreau11559a72018-02-05 17:52:24 +0100137
138 new = fd;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100139redo_last:
140 /* First, insert in the linked list */
141 last = list->last;
142 old = -1;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100143
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200144 _GET_PREV(fd, off) = -2;
Willy Tarreau11559a72018-02-05 17:52:24 +0100145 /* Make sure the "prev" store is visible before we update the last entry */
146 __ha_barrier_store();
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100147
Willy Tarreau11559a72018-02-05 17:52:24 +0100148 if (unlikely(last == -1)) {
149 /* list is empty, try to add ourselves alone so that list->last=fd */
Olivier Houchardd3608792019-03-08 18:47:42 +0100150 if (unlikely(!_HA_ATOMIC_CAS(&list->last, &old, new)))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100151 goto redo_last;
152
153 /* list->first was necessary -1, we're guaranteed to be alone here */
154 list->first = fd;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100155 } else {
Willy Tarreau11559a72018-02-05 17:52:24 +0100156 /* adding ourselves past the last element
157 * The CAS will only succeed if its next is -1,
158 * which means it's in the cache, and the last element.
159 */
Olivier Houchardd3608792019-03-08 18:47:42 +0100160 if (unlikely(!_HA_ATOMIC_CAS(&_GET_NEXT(last, off), &old, new)))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100161 goto redo_last;
Willy Tarreau11559a72018-02-05 17:52:24 +0100162
163 /* Then, update the last entry */
164 list->last = fd;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100165 }
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100166 __ha_barrier_store();
Willy Tarreau11559a72018-02-05 17:52:24 +0100167 /* since we're alone at the end of the list and still locked(-2),
168 * we know noone tried to add past us. Mark the end of list.
169 */
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200170 _GET_PREV(fd, off) = last;
171 _GET_NEXT(fd, off) = -1;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100172 __ha_barrier_store();
173done:
174 return;
175}
176
177/* removes fd <fd> from fd list <list> */
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200178void fd_rm_from_fd_list(volatile struct fdlist *list, int fd, int off)
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100179{
180#if defined(HA_HAVE_CAS_DW) || defined(HA_CAS_IS_8B)
Willy Tarreau2b9f0662020-02-25 09:25:53 +0100181 volatile union {
182 struct fdlist_entry ent;
183 uint64_t u64;
184 uint32_t u32[2];
185 } cur_list, next_list;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100186#endif
187 int old;
188 int new = -2;
189 int prev;
190 int next;
191 int last;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100192lock_self:
193#if (defined(HA_CAS_IS_8B) || defined(HA_HAVE_CAS_DW))
Willy Tarreau2b9f0662020-02-25 09:25:53 +0100194 next_list.ent.next = next_list.ent.prev = -2;
195 cur_list.ent = *(volatile struct fdlist_entry *)(((char *)&fdtab[fd]) + off);
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100196 /* First, attempt to lock our own entries */
197 do {
198 /* The FD is not in the FD cache, give up */
Willy Tarreau2b9f0662020-02-25 09:25:53 +0100199 if (unlikely(cur_list.ent.next <= -3))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100200 return;
Willy Tarreau2b9f0662020-02-25 09:25:53 +0100201 if (unlikely(cur_list.ent.prev == -2 || cur_list.ent.next == -2))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100202 goto lock_self;
203 } while (
204#ifdef HA_CAS_IS_8B
Willy Tarreau2b9f0662020-02-25 09:25:53 +0100205 unlikely(!_HA_ATOMIC_CAS(((uint64_t *)&_GET_NEXT(fd, off)), (uint64_t *)&cur_list.u64, next_list.u64))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100206#else
Willy Tarreau2b9f0662020-02-25 09:25:53 +0100207 unlikely(!_HA_ATOMIC_DWCAS(((long *)&_GET_NEXT(fd, off)), (uint32_t *)&cur_list.u32, &next_list.u32))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100208#endif
Willy Tarreau2b9f0662020-02-25 09:25:53 +0100209 );
210 next = cur_list.ent.next;
211 prev = cur_list.ent.prev;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100212
213#else
214lock_self_next:
Willy Tarreau337fb712019-12-20 07:20:00 +0100215 next = _GET_NEXT(fd, off);
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100216 if (next == -2)
217 goto lock_self_next;
218 if (next <= -3)
219 goto done;
Olivier Houchardd3608792019-03-08 18:47:42 +0100220 if (unlikely(!_HA_ATOMIC_CAS(&_GET_NEXT(fd, off), &next, -2)))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100221 goto lock_self_next;
222lock_self_prev:
Willy Tarreau337fb712019-12-20 07:20:00 +0100223 prev = _GET_PREV(fd, off);
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100224 if (prev == -2)
225 goto lock_self_prev;
Olivier Houchardd3608792019-03-08 18:47:42 +0100226 if (unlikely(!_HA_ATOMIC_CAS(&_GET_PREV(fd, off), &prev, -2)))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100227 goto lock_self_prev;
228#endif
Olivier Houchardd2b5d162019-03-08 13:47:21 +0100229 __ha_barrier_atomic_store();
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100230
231 /* Now, lock the entries of our neighbours */
232 if (likely(prev != -1)) {
233redo_prev:
234 old = fd;
235
Olivier Houchardd3608792019-03-08 18:47:42 +0100236 if (unlikely(!_HA_ATOMIC_CAS(&_GET_NEXT(prev, off), &old, new))) {
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100237 if (unlikely(old == -2)) {
238 /* Neighbour already locked, give up and
239 * retry again once he's done
240 */
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200241 _GET_PREV(fd, off) = prev;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100242 __ha_barrier_store();
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200243 _GET_NEXT(fd, off) = next;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100244 __ha_barrier_store();
245 goto lock_self;
246 }
247 goto redo_prev;
248 }
249 }
250 if (likely(next != -1)) {
251redo_next:
252 old = fd;
Olivier Houchardd3608792019-03-08 18:47:42 +0100253 if (unlikely(!_HA_ATOMIC_CAS(&_GET_PREV(next, off), &old, new))) {
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100254 if (unlikely(old == -2)) {
255 /* Neighbour already locked, give up and
256 * retry again once he's done
257 */
258 if (prev != -1) {
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200259 _GET_NEXT(prev, off) = fd;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100260 __ha_barrier_store();
261 }
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200262 _GET_PREV(fd, off) = prev;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100263 __ha_barrier_store();
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200264 _GET_NEXT(fd, off) = next;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100265 __ha_barrier_store();
266 goto lock_self;
267 }
268 goto redo_next;
269 }
270 }
271 if (list->first == fd)
272 list->first = next;
273 __ha_barrier_store();
274 last = list->last;
Olivier Houchardd3608792019-03-08 18:47:42 +0100275 while (unlikely(last == fd && (!_HA_ATOMIC_CAS(&list->last, &last, prev))))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100276 __ha_compiler_barrier();
277 /* Make sure we let other threads know we're no longer in cache,
278 * before releasing our neighbours.
279 */
280 __ha_barrier_store();
281 if (likely(prev != -1))
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200282 _GET_NEXT(prev, off) = next;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100283 __ha_barrier_store();
284 if (likely(next != -1))
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200285 _GET_PREV(next, off) = prev;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100286 __ha_barrier_store();
287 /* Ok, now we're out of the fd cache */
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200288 _GET_NEXT(fd, off) = -(next + 4);
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100289 __ha_barrier_store();
290done:
291 return;
292}
293
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200294#undef _GET_NEXT
295#undef _GET_PREV
296
Willy Tarreau173d9952018-01-26 21:48:23 +0100297/* Deletes an FD from the fdsets.
Willy Tarreau4f60f162007-04-08 16:39:58 +0200298 * The file descriptor is also closed.
299 */
Olivier Houchard1fc05162017-04-06 01:05:05 +0200300static void fd_dodelete(int fd, int do_close)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200301{
Olivier Houcharda7bf5732020-02-27 17:26:13 +0100302 int locked = fdtab[fd].running_mask != tid_bit;
Willy Tarreau87d54a92018-10-15 09:44:46 +0200303
Olivier Houchard88516642020-03-05 18:10:51 +0100304 /* We're just trying to protect against a concurrent fd_insert()
Olivier Houchardf86a1062020-06-17 20:28:27 +0200305 * here, not against fd_takeover(), because either we're called
Olivier Houchard88516642020-03-05 18:10:51 +0100306 * directly from the iocb(), and we're already locked, or we're
307 * called from the mux tasklet, but then the mux is responsible for
308 * making sure the tasklet does nothing, and the connection is never
309 * destroyed.
310 */
Willy Tarreau87d54a92018-10-15 09:44:46 +0200311 if (locked)
Olivier Houcharda7bf5732020-02-27 17:26:13 +0100312 fd_set_running_excl(fd);
313
Willy Tarreauad38ace2013-12-15 14:19:38 +0100314 if (fdtab[fd].linger_risk) {
315 /* this is generally set when connecting to servers */
Ilya Shipitsinb7e43f02020-04-02 15:02:08 +0500316 DISGUISE(setsockopt(fd, SOL_SOCKET, SO_LINGER,
317 (struct linger *) &nolinger, sizeof(struct linger)));
Willy Tarreauad38ace2013-12-15 14:19:38 +0100318 }
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100319 if (cur_poller.clo)
320 cur_poller.clo(fd);
Olivier Houchardc22580c2019-08-05 18:51:52 +0200321 polled_mask[fd].poll_recv = polled_mask[fd].poll_send = 0;
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100322
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100323 fdtab[fd].state = 0;
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100324
Willy Tarreau38e8a1c2020-06-23 10:04:54 +0200325#ifdef DEBUG_FD
326 fdtab[fd].event_count = 0;
327#endif
Willy Tarreau8d5d77e2009-10-18 07:25:52 +0200328 port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port);
329 fdinfo[fd].port_range = NULL;
Willy Tarreaudb3b3262012-07-05 23:19:22 +0200330 fdtab[fd].owner = NULL;
Willy Tarreauf65610a2017-10-31 16:06:06 +0100331 fdtab[fd].thread_mask = 0;
Willy Tarreauc9c83782018-01-17 18:44:46 +0100332 if (do_close) {
Christopher Fauletd531f882017-06-01 16:55:03 +0200333 close(fd);
Olivier Houchard7c49d2e2019-04-16 18:37:05 +0200334 _HA_ATOMIC_SUB(&ha_used_fds, 1);
Willy Tarreauc9c83782018-01-17 18:44:46 +0100335 }
Willy Tarreau87d54a92018-10-15 09:44:46 +0200336 if (locked)
Olivier Houcharda7bf5732020-02-27 17:26:13 +0100337 fd_clr_running(fd);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200338}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200339
Olivier Houchard88516642020-03-05 18:10:51 +0100340#ifndef HA_HAVE_CAS_DW
Willy Tarreauaf613e82020-06-05 08:40:51 +0200341__decl_thread(__decl_rwlock(fd_mig_lock));
Olivier Houchard88516642020-03-05 18:10:51 +0100342#endif
343
344/*
345 * Take over a FD belonging to another thread.
346 * unexpected_conn is the expected owner of the fd.
347 * Returns 0 on success, and -1 on failure.
348 */
349int fd_takeover(int fd, void *expected_owner)
350{
Willy Tarreauc460c912020-06-18 07:28:09 +0200351 int ret = -1;
Olivier Houchard88516642020-03-05 18:10:51 +0100352
Willy Tarreauf1cad382020-06-18 08:14:59 +0200353#ifndef HA_HAVE_CAS_DW
Willy Tarreauc460c912020-06-18 07:28:09 +0200354 if (_HA_ATOMIC_OR(&fdtab[fd].running_mask, tid_bit) == tid_bit) {
355 HA_RWLOCK_WRLOCK(OTHER_LOCK, &fd_mig_lock);
356 if (fdtab[fd].owner == expected_owner) {
357 fdtab[fd].thread_mask = tid_bit;
358 ret = 0;
359 }
360 HA_RWLOCK_WRUNLOCK(OTHER_LOCK, &fd_mig_lock);
Olivier Houchard88516642020-03-05 18:10:51 +0100361 }
Olivier Houchard88516642020-03-05 18:10:51 +0100362#else
363 unsigned long old_masks[2];
364 unsigned long new_masks[2];
365
Olivier Houchard88516642020-03-05 18:10:51 +0100366 new_masks[0] = new_masks[1] = tid_bit;
Willy Tarreau42973632020-06-18 08:05:15 +0200367
368 old_masks[0] = _HA_ATOMIC_OR(&fdtab[fd].running_mask, tid_bit);
369 old_masks[1] = fdtab[fd].thread_mask;
370
Olivier Houchard88516642020-03-05 18:10:51 +0100371 /* protect ourself against a delete then an insert for the same fd,
372 * if it happens, then the owner will no longer be the expected
373 * connection.
374 */
Willy Tarreauf1cad382020-06-18 08:14:59 +0200375 if (fdtab[fd].owner == expected_owner) {
376 while (old_masks[0] == tid_bit && old_masks[1]) {
377 if (_HA_ATOMIC_DWCAS(&fdtab[fd].running_mask, &old_masks, &new_masks)) {
378 ret = 0;
379 break;
380 }
Olivier Houchard88516642020-03-05 18:10:51 +0100381 }
Willy Tarreauf1cad382020-06-18 08:14:59 +0200382 }
383#endif /* HW_HAVE_CAS_DW */
384
Olivier Houchard88516642020-03-05 18:10:51 +0100385 _HA_ATOMIC_AND(&fdtab[fd].running_mask, ~tid_bit);
Willy Tarreauf1cad382020-06-18 08:14:59 +0200386
Olivier Houchardddc874c2020-06-17 20:34:05 +0200387 /* Make sure the FD doesn't have the active bit. It is possible that
388 * the fd is polled by the thread that used to own it, the new thread
389 * is supposed to call subscribe() later, to activate polling.
390 */
Willy Tarreauf1cad382020-06-18 08:14:59 +0200391 if (likely(ret == 0))
392 fd_stop_recv(fd);
393 return ret;
Olivier Houchard88516642020-03-05 18:10:51 +0100394}
395
Willy Tarreau173d9952018-01-26 21:48:23 +0100396/* Deletes an FD from the fdsets.
Olivier Houchard1fc05162017-04-06 01:05:05 +0200397 * The file descriptor is also closed.
398 */
399void fd_delete(int fd)
400{
401 fd_dodelete(fd, 1);
402}
403
Willy Tarreau173d9952018-01-26 21:48:23 +0100404/* Deletes an FD from the fdsets.
Olivier Houchard1fc05162017-04-06 01:05:05 +0200405 * The file descriptor is kept open.
406 */
407void fd_remove(int fd)
408{
409 fd_dodelete(fd, 0);
410}
411
Willy Tarreaudbe30602019-09-04 13:25:41 +0200412void updt_fd_polling(const int fd)
413{
414 if ((fdtab[fd].thread_mask & all_threads_mask) == tid_bit) {
415
416 /* note: we don't have a test-and-set yet in hathreads */
417
418 if (HA_ATOMIC_BTS(&fdtab[fd].update_mask, tid))
419 return;
420
421 fd_updt[fd_nbupdt++] = fd;
422 } else {
423 unsigned long update_mask = fdtab[fd].update_mask;
424 do {
425 if (update_mask == fdtab[fd].thread_mask)
426 return;
427 } while (!_HA_ATOMIC_CAS(&fdtab[fd].update_mask, &update_mask,
428 fdtab[fd].thread_mask));
429 fd_add_to_fd_list(&update_list, fd, offsetof(struct fdtab, update));
430 }
431}
432
Olivier Houchard7fa55622020-02-27 17:25:43 +0100433__decl_spinlock(log_lock);
Willy Tarreau931d8b72019-08-27 11:08:17 +0200434/* Tries to send <npfx> parts from <prefix> followed by <nmsg> parts from <msg>
435 * optionally followed by a newline if <nl> is non-null, to file descriptor
436 * <fd>. The message is sent atomically using writev(). It may be truncated to
437 * <maxlen> bytes if <maxlen> is non-null. There is no distinction between the
438 * two lists, it's just a convenience to help the caller prepend some prefixes
439 * when necessary. It takes the fd's lock to make sure no other thread will
440 * write to the same fd in parallel. Returns the number of bytes sent, or <=0
441 * on failure. A limit to 31 total non-empty segments is enforced. The caller
442 * is responsible for taking care of making the fd non-blocking.
443 */
444ssize_t fd_write_frag_line(int fd, size_t maxlen, const struct ist pfx[], size_t npfx, const struct ist msg[], size_t nmsg, int nl)
445{
446 struct iovec iovec[32];
447 size_t totlen = 0;
448 size_t sent = 0;
449 int vec = 0;
Willy Tarreaudf187872020-06-11 14:25:47 +0200450 int attempts = 0;
Willy Tarreau931d8b72019-08-27 11:08:17 +0200451
452 if (!maxlen)
453 maxlen = ~0;
454
455 /* keep one char for a possible trailing '\n' in any case */
456 maxlen--;
457
458 /* make an iovec from the concatenation of all parts of the original
459 * message. Skip empty fields and truncate the whole message to maxlen,
460 * leaving one spare iovec for the '\n'.
461 */
462 while (vec < (sizeof(iovec) / sizeof(iovec[0]) - 1)) {
463 if (!npfx) {
464 pfx = msg;
465 npfx = nmsg;
466 nmsg = 0;
467 if (!npfx)
468 break;
469 }
470
471 iovec[vec].iov_base = pfx->ptr;
472 iovec[vec].iov_len = MIN(maxlen, pfx->len);
473 maxlen -= iovec[vec].iov_len;
474 totlen += iovec[vec].iov_len;
475 if (iovec[vec].iov_len)
476 vec++;
477 pfx++; npfx--;
478 };
479
480 if (nl) {
481 iovec[vec].iov_base = "\n";
482 iovec[vec].iov_len = 1;
483 vec++;
484 }
485
Willy Tarreaudf187872020-06-11 14:25:47 +0200486 /* make sure we never interleave writes and we never block. This means
487 * we prefer to fail on collision than to block. But we don't want to
488 * lose too many logs so we just perform a few lock attempts then give
489 * up.
490 */
491
492 while (HA_SPIN_TRYLOCK(OTHER_LOCK, &log_lock) != 0) {
493 if (++attempts >= 200) {
494 /* so that the caller knows the message couldn't be delivered */
495 sent = -1;
496 errno = EAGAIN;
497 goto leave;
498 }
499 ha_thread_relax();
500 }
501
Willy Tarreau7e9776a2019-08-30 14:41:47 +0200502 if (unlikely(!fdtab[fd].initialized)) {
503 fdtab[fd].initialized = 1;
504 if (!isatty(fd))
505 fcntl(fd, F_SETFL, O_NONBLOCK);
506 }
Willy Tarreau931d8b72019-08-27 11:08:17 +0200507 sent = writev(fd, iovec, vec);
Olivier Houchard7fa55622020-02-27 17:25:43 +0100508 HA_SPIN_UNLOCK(OTHER_LOCK, &log_lock);
Willy Tarreau931d8b72019-08-27 11:08:17 +0200509
Willy Tarreaudf187872020-06-11 14:25:47 +0200510 leave:
Willy Tarreau931d8b72019-08-27 11:08:17 +0200511 /* sent > 0 if the message was delivered */
512 return sent;
513}
514
Olivier Houchard2292edf2019-02-25 14:26:54 +0100515#if defined(USE_CLOSEFROM)
516void my_closefrom(int start)
517{
518 closefrom(start);
519}
520
Willy Tarreaue5733232019-05-22 19:24:06 +0200521#elif defined(USE_POLL)
Willy Tarreau9188ac62019-02-21 22:12:47 +0100522/* This is a portable implementation of closefrom(). It closes all open file
523 * descriptors starting at <start> and above. It relies on the fact that poll()
524 * will return POLLNVAL for each invalid (hence close) file descriptor passed
525 * in argument in order to skip them. It acts with batches of FDs and will
526 * typically perform one poll() call per 1024 FDs so the overhead is low in
527 * case all FDs have to be closed.
528 */
529void my_closefrom(int start)
530{
531 struct pollfd poll_events[1024];
532 struct rlimit limit;
533 int nbfds, fd, ret, idx;
534 int step, next;
535
536 if (getrlimit(RLIMIT_NOFILE, &limit) == 0)
537 step = nbfds = limit.rlim_cur;
538 else
539 step = nbfds = 0;
540
541 if (nbfds <= 0) {
542 /* set safe limit */
543 nbfds = 1024;
544 step = 256;
545 }
546
547 if (step > sizeof(poll_events) / sizeof(poll_events[0]))
548 step = sizeof(poll_events) / sizeof(poll_events[0]);
549
550 while (start < nbfds) {
551 next = (start / step + 1) * step;
552
553 for (fd = start; fd < next && fd < nbfds; fd++) {
554 poll_events[fd - start].fd = fd;
555 poll_events[fd - start].events = 0;
556 }
557
558 do {
559 ret = poll(poll_events, fd - start, 0);
560 if (ret >= 0)
561 break;
562 } while (errno == EAGAIN || errno == EINTR || errno == ENOMEM);
563
Willy Tarreaub8e602c2019-02-22 09:07:42 +0100564 if (ret)
565 ret = fd - start;
566
Willy Tarreau9188ac62019-02-21 22:12:47 +0100567 for (idx = 0; idx < ret; idx++) {
568 if (poll_events[idx].revents & POLLNVAL)
569 continue; /* already closed */
570
571 fd = poll_events[idx].fd;
572 close(fd);
573 }
574 start = next;
575 }
576}
577
Willy Tarreaue5733232019-05-22 19:24:06 +0200578#else // defined(USE_POLL)
Willy Tarreau9188ac62019-02-21 22:12:47 +0100579
Willy Tarreau2d7f81b2019-02-21 22:19:17 +0100580/* This is a portable implementation of closefrom(). It closes all open file
581 * descriptors starting at <start> and above. This is a naive version for use
582 * when the operating system provides no alternative.
583 */
584void my_closefrom(int start)
585{
586 struct rlimit limit;
587 int nbfds;
588
589 if (getrlimit(RLIMIT_NOFILE, &limit) == 0)
590 nbfds = limit.rlim_cur;
591 else
592 nbfds = 0;
593
594 if (nbfds <= 0)
595 nbfds = 1024; /* safe limit */
596
597 while (start < nbfds)
598 close(start++);
599}
Willy Tarreaue5733232019-05-22 19:24:06 +0200600#endif // defined(USE_POLL)
Willy Tarreau2d7f81b2019-02-21 22:19:17 +0100601
Willy Tarreau4f60f162007-04-08 16:39:58 +0200602/* disable the specified poller */
603void disable_poller(const char *poller_name)
604{
605 int p;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200606
Willy Tarreau4f60f162007-04-08 16:39:58 +0200607 for (p = 0; p < nbpollers; p++)
608 if (strcmp(pollers[p].name, poller_name) == 0)
609 pollers[p].pref = 0;
610}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200611
Olivier Houchard79321b92018-07-26 17:55:11 +0200612void poller_pipe_io_handler(int fd)
613{
614 char buf[1024];
615 /* Flush the pipe */
616 while (read(fd, buf, sizeof(buf)) > 0);
617 fd_cant_recv(fd);
618}
619
Willy Tarreau082b6282019-05-22 14:42:12 +0200620/* allocate the per-thread fd_updt thus needs to be called early after
621 * thread creation.
622 */
623static int alloc_pollers_per_thread()
624{
625 fd_updt = calloc(global.maxsock, sizeof(*fd_updt));
626 return fd_updt != NULL;
627}
628
629/* Initialize the pollers per thread.*/
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200630static int init_pollers_per_thread()
631{
Olivier Houchard79321b92018-07-26 17:55:11 +0200632 int mypipe[2];
Willy Tarreau082b6282019-05-22 14:42:12 +0200633
634 if (pipe(mypipe) < 0)
Olivier Houchard79321b92018-07-26 17:55:11 +0200635 return 0;
Willy Tarreau082b6282019-05-22 14:42:12 +0200636
Olivier Houchard79321b92018-07-26 17:55:11 +0200637 poller_rd_pipe = mypipe[0];
638 poller_wr_pipe[tid] = mypipe[1];
639 fcntl(poller_rd_pipe, F_SETFL, O_NONBLOCK);
640 fd_insert(poller_rd_pipe, poller_pipe_io_handler, poller_pipe_io_handler,
641 tid_bit);
642 fd_want_recv(poller_rd_pipe);
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200643 return 1;
644}
645
646/* Deinitialize the pollers per thread */
647static void deinit_pollers_per_thread()
648{
William Lallemand808e1b72018-12-15 22:34:31 +0100649 /* rd and wr are init at the same place, but only rd is init to -1, so
650 we rely to rd to close. */
651 if (poller_rd_pipe > -1) {
652 close(poller_rd_pipe);
653 poller_rd_pipe = -1;
654 close(poller_wr_pipe[tid]);
655 poller_wr_pipe[tid] = -1;
656 }
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200657}
658
Willy Tarreau082b6282019-05-22 14:42:12 +0200659/* Release the pollers per thread, to be called late */
660static void free_pollers_per_thread()
661{
662 free(fd_updt);
663 fd_updt = NULL;
664}
665
Willy Tarreaubaaee002006-06-26 02:48:02 +0200666/*
Willy Tarreau4f60f162007-04-08 16:39:58 +0200667 * Initialize the pollers till the best one is found.
668 * If none works, returns 0, otherwise 1.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200669 */
Willy Tarreau4f60f162007-04-08 16:39:58 +0200670int init_pollers()
Willy Tarreaubaaee002006-06-26 02:48:02 +0200671{
Willy Tarreau4f60f162007-04-08 16:39:58 +0200672 int p;
673 struct poller *bp;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200674
Christopher Faulet63fe6522017-08-31 17:52:09 +0200675 if ((fdtab = calloc(global.maxsock, sizeof(struct fdtab))) == NULL)
676 goto fail_tab;
677
Olivier Houchard53055052019-07-25 14:00:18 +0000678 if ((polled_mask = calloc(global.maxsock, sizeof(*polled_mask))) == NULL)
Olivier Houchardcb92f5c2018-04-26 14:23:07 +0200679 goto fail_polledmask;
Uman Shahzadda7eeed2019-01-17 08:21:39 +0000680
Christopher Faulet63fe6522017-08-31 17:52:09 +0200681 if ((fdinfo = calloc(global.maxsock, sizeof(struct fdinfo))) == NULL)
682 goto fail_info;
683
Olivier Houchard6b96f722018-04-25 16:58:25 +0200684 update_list.first = update_list.last = -1;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200685
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100686 for (p = 0; p < global.maxsock; p++) {
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100687 /* Mark the fd as out of the fd cache */
Olivier Houchard6b96f722018-04-25 16:58:25 +0200688 fdtab[p].update.next = -3;
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100689 }
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200690
Willy Tarreau4f60f162007-04-08 16:39:58 +0200691 do {
692 bp = NULL;
693 for (p = 0; p < nbpollers; p++)
694 if (!bp || (pollers[p].pref > bp->pref))
695 bp = &pollers[p];
Willy Tarreaubaaee002006-06-26 02:48:02 +0200696
Willy Tarreau4f60f162007-04-08 16:39:58 +0200697 if (!bp || bp->pref == 0)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200698 break;
699
Willy Tarreau4f60f162007-04-08 16:39:58 +0200700 if (bp->init(bp)) {
701 memcpy(&cur_poller, bp, sizeof(*bp));
702 return 1;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200703 }
Willy Tarreau4f60f162007-04-08 16:39:58 +0200704 } while (!bp || bp->pref == 0);
Willy Tarreau7be79a42012-11-11 15:02:54 +0100705
Christopher Faulet63fe6522017-08-31 17:52:09 +0200706 free(fdinfo);
707 fail_info:
Olivier Houchardcb92f5c2018-04-26 14:23:07 +0200708 free(polled_mask);
709 fail_polledmask:
Uman Shahzadda7eeed2019-01-17 08:21:39 +0000710 free(fdtab);
711 fail_tab:
Willy Tarreau7be79a42012-11-11 15:02:54 +0100712 return 0;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200713}
714
Willy Tarreaubaaee002006-06-26 02:48:02 +0200715/*
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +0200716 * Deinitialize the pollers.
717 */
718void deinit_pollers() {
719
720 struct poller *bp;
721 int p;
722
723 for (p = 0; p < nbpollers; p++) {
724 bp = &pollers[p];
725
726 if (bp && bp->pref)
727 bp->term(bp);
728 }
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200729
Christopher Faulet63fe6522017-08-31 17:52:09 +0200730 free(fdinfo); fdinfo = NULL;
731 free(fdtab); fdtab = NULL;
Olivier Houchardcb92f5c2018-04-26 14:23:07 +0200732 free(polled_mask); polled_mask = NULL;
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +0200733}
734
735/*
Willy Tarreau2ff76222007-04-09 19:29:56 +0200736 * Lists the known pollers on <out>.
737 * Should be performed only before initialization.
738 */
739int list_pollers(FILE *out)
740{
741 int p;
742 int last, next;
743 int usable;
744 struct poller *bp;
745
746 fprintf(out, "Available polling systems :\n");
747
748 usable = 0;
749 bp = NULL;
750 last = next = -1;
751 while (1) {
752 for (p = 0; p < nbpollers; p++) {
Willy Tarreau2ff76222007-04-09 19:29:56 +0200753 if ((next < 0 || pollers[p].pref > next)
Willy Tarreaue79c3b22010-11-19 10:20:36 +0100754 && (last < 0 || pollers[p].pref < last)) {
Willy Tarreau2ff76222007-04-09 19:29:56 +0200755 next = pollers[p].pref;
Willy Tarreaue79c3b22010-11-19 10:20:36 +0100756 if (!bp || (pollers[p].pref > bp->pref))
757 bp = &pollers[p];
758 }
Willy Tarreau2ff76222007-04-09 19:29:56 +0200759 }
760
761 if (next == -1)
762 break;
763
764 for (p = 0; p < nbpollers; p++) {
765 if (pollers[p].pref == next) {
766 fprintf(out, " %10s : ", pollers[p].name);
767 if (pollers[p].pref == 0)
768 fprintf(out, "disabled, ");
769 else
770 fprintf(out, "pref=%3d, ", pollers[p].pref);
771 if (pollers[p].test(&pollers[p])) {
772 fprintf(out, " test result OK");
773 if (next > 0)
774 usable++;
Willy Tarreaue79c3b22010-11-19 10:20:36 +0100775 } else {
Willy Tarreau2ff76222007-04-09 19:29:56 +0200776 fprintf(out, " test result FAILED");
Willy Tarreaue79c3b22010-11-19 10:20:36 +0100777 if (bp == &pollers[p])
778 bp = NULL;
779 }
Willy Tarreau2ff76222007-04-09 19:29:56 +0200780 fprintf(out, "\n");
781 }
782 }
783 last = next;
784 next = -1;
785 };
786 fprintf(out, "Total: %d (%d usable), will use %s.\n", nbpollers, usable, bp ? bp->name : "none");
787 return 0;
788}
789
790/*
791 * Some pollers may lose their connection after a fork(). It may be necessary
792 * to create initialize part of them again. Returns 0 in case of failure,
793 * otherwise 1. The fork() function may be NULL if unused. In case of error,
794 * the the current poller is destroyed and the caller is responsible for trying
795 * another one by calling init_pollers() again.
796 */
797int fork_poller()
798{
Conrad Hoffmann041751c2014-05-20 14:28:24 +0200799 int fd;
Willy Tarreauce036bc2018-01-29 14:58:02 +0100800 for (fd = 0; fd < global.maxsock; fd++) {
Conrad Hoffmann041751c2014-05-20 14:28:24 +0200801 if (fdtab[fd].owner) {
802 fdtab[fd].cloned = 1;
803 }
804 }
805
Willy Tarreau2ff76222007-04-09 19:29:56 +0200806 if (cur_poller.fork) {
807 if (cur_poller.fork(&cur_poller))
808 return 1;
809 cur_poller.term(&cur_poller);
810 return 0;
811 }
812 return 1;
813}
814
Willy Tarreaubc52bec2020-06-18 08:58:47 +0200815/* config parser for global "tune.fd.edge-triggered", accepts "on" or "off" */
816static int cfg_parse_tune_fd_edge_triggered(char **args, int section_type, struct proxy *curpx,
817 struct proxy *defpx, const char *file, int line,
818 char **err)
819{
820 if (too_many_args(1, args, err, NULL))
821 return -1;
822
823 if (strcmp(args[1], "on") == 0)
824 global.tune.options |= GTUNE_FD_ET;
825 else if (strcmp(args[1], "off") == 0)
826 global.tune.options &= ~GTUNE_FD_ET;
827 else {
828 memprintf(err, "'%s' expects either 'on' or 'off' but got '%s'.", args[0], args[1]);
829 return -1;
830 }
831 return 0;
832}
833
834/* config keyword parsers */
835static struct cfg_kw_list cfg_kws = {ILH, {
836 { CFG_GLOBAL, "tune.fd.edge-triggered", cfg_parse_tune_fd_edge_triggered },
837 { 0, NULL, NULL }
838}};
839
840INITCALL1(STG_REGISTER, cfg_register_keywords, &cfg_kws);
841
Willy Tarreau082b6282019-05-22 14:42:12 +0200842REGISTER_PER_THREAD_ALLOC(alloc_pollers_per_thread);
Willy Tarreau172f5ce2018-11-26 11:21:50 +0100843REGISTER_PER_THREAD_INIT(init_pollers_per_thread);
844REGISTER_PER_THREAD_DEINIT(deinit_pollers_per_thread);
Willy Tarreau082b6282019-05-22 14:42:12 +0200845REGISTER_PER_THREAD_FREE(free_pollers_per_thread);
Willy Tarreau172f5ce2018-11-26 11:21:50 +0100846
Willy Tarreau2ff76222007-04-09 19:29:56 +0200847/*
Willy Tarreaubaaee002006-06-26 02:48:02 +0200848 * Local variables:
849 * c-indent-level: 8
850 * c-basic-offset: 8
851 * End:
852 */