blob: 5f432338ce17da6032a4dd6dc24900531d1b06ff [file] [log] [blame]
Willy Tarreaubaaee002006-06-26 02:48:02 +02001/*
2 * File descriptors management functions.
3 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +01004 * Copyright 2000-2014 Willy Tarreau <w@1wt.eu>
Willy Tarreaubaaee002006-06-26 02:48:02 +02005 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
Willy Tarreau7be79a42012-11-11 15:02:54 +010011 * There is no direct link between the FD and the updates list. There is only a
12 * bit in the fdtab[] to indicate than a file descriptor is already present in
13 * the updates list. Once an fd is present in the updates list, it will have to
14 * be considered even if its changes are reverted in the middle or if the fd is
15 * replaced.
16 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010017 * The event state for an FD, as found in fdtab[].state, is maintained for each
18 * direction. The state field is built this way, with R bits in the low nibble
19 * and W bits in the high nibble for ease of access and debugging :
Willy Tarreau7be79a42012-11-11 15:02:54 +010020 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010021 * 7 6 5 4 3 2 1 0
Willy Tarreau5bee3e22019-09-04 09:52:57 +020022 * [ 0 | 0 | RW | AW | 0 | 0 | RR | AR ]
Willy Tarreauf817e9f2014-01-10 16:58:45 +010023 *
24 * A* = active *R = read
Willy Tarreau5bee3e22019-09-04 09:52:57 +020025 * R* = ready *W = write
Willy Tarreauf817e9f2014-01-10 16:58:45 +010026 *
27 * An FD is marked "active" when there is a desire to use it.
Willy Tarreauf817e9f2014-01-10 16:58:45 +010028 * An FD is marked "ready" when it has not faced a new EAGAIN since last wake-up
Willy Tarreau5bee3e22019-09-04 09:52:57 +020029 * (it is a cache of the last EAGAIN regardless of polling changes). Each poller
30 * has its own "polled" state for the same fd, as stored in the polled_mask.
Willy Tarreau7be79a42012-11-11 15:02:54 +010031 *
Willy Tarreau5bee3e22019-09-04 09:52:57 +020032 * We have 4 possible states for each direction based on these 2 flags :
Willy Tarreau7be79a42012-11-11 15:02:54 +010033 *
Willy Tarreau5bee3e22019-09-04 09:52:57 +020034 * +---+---+----------+---------------------------------------------+
35 * | R | A | State | Description |
36 * +---+---+----------+---------------------------------------------+
37 * | 0 | 0 | DISABLED | No activity desired, not ready. |
38 * | 0 | 1 | ACTIVE | Activity desired. |
39 * | 1 | 0 | STOPPED | End of activity. |
40 * | 1 | 1 | READY | Activity desired and reported. |
41 * +---+---+----------+---------------------------------------------+
Willy Tarreau7be79a42012-11-11 15:02:54 +010042 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +010043 * The transitions are pretty simple :
44 * - fd_want_*() : set flag A
45 * - fd_stop_*() : clear flag A
46 * - fd_cant_*() : clear flag R (when facing EAGAIN)
47 * - fd_may_*() : set flag R (upon return from poll())
Willy Tarreauf817e9f2014-01-10 16:58:45 +010048 *
Willy Tarreau5bee3e22019-09-04 09:52:57 +020049 * Each poller then computes its own polled state :
50 * if (A) { if (!R) P := 1 } else { P := 0 }
Willy Tarreauf817e9f2014-01-10 16:58:45 +010051 *
Willy Tarreau5bee3e22019-09-04 09:52:57 +020052 * The state transitions look like the diagram below.
Willy Tarreauf817e9f2014-01-10 16:58:45 +010053 *
Willy Tarreau5bee3e22019-09-04 09:52:57 +020054 * may +----------+
55 * ,----| DISABLED | (READY=0, ACTIVE=0)
56 * | +----------+
57 * | want | ^
58 * | | |
59 * | v | stop
60 * | +----------+
61 * | | ACTIVE | (READY=0, ACTIVE=1)
62 * | +----------+
63 * | | ^
64 * | may | |
65 * | v | EAGAIN (cant)
66 * | +--------+
67 * | | READY | (READY=1, ACTIVE=1)
68 * | +--------+
69 * | stop | ^
70 * | | |
71 * | v | want
72 * | +---------+
73 * `--->| STOPPED | (READY=1, ACTIVE=0)
74 * +---------+
Willy Tarreaubaaee002006-06-26 02:48:02 +020075 */
76
Willy Tarreau2ff76222007-04-09 19:29:56 +020077#include <stdio.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020078#include <string.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020079#include <unistd.h>
Olivier Houchard79321b92018-07-26 17:55:11 +020080#include <fcntl.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020081#include <sys/types.h>
Willy Tarreau2d7f81b2019-02-21 22:19:17 +010082#include <sys/resource.h>
Willy Tarreau931d8b72019-08-27 11:08:17 +020083#include <sys/uio.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020084
Willy Tarreaue5733232019-05-22 19:24:06 +020085#if defined(USE_POLL)
Willy Tarreau9188ac62019-02-21 22:12:47 +010086#include <poll.h>
87#include <errno.h>
88#endif
89
Willy Tarreau2dd0d472006-06-29 17:53:05 +020090#include <common/compat.h>
91#include <common/config.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020092
Willy Tarreau7be79a42012-11-11 15:02:54 +010093#include <types/global.h>
94
Willy Tarreau2a429502006-10-15 14:52:29 +020095#include <proto/fd.h>
Christopher Fauletd4604ad2017-05-29 10:40:41 +020096#include <proto/log.h>
Willy Tarreauc6f4ce82009-06-10 11:09:37 +020097#include <proto/port_range.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020098
99struct fdtab *fdtab = NULL; /* array of all the file descriptors */
Olivier Houchard53055052019-07-25 14:00:18 +0000100struct polled_mask *polled_mask = NULL; /* Array for the polled_mask of each fd */
Willy Tarreau8d5d77e2009-10-18 07:25:52 +0200101struct fdinfo *fdinfo = NULL; /* less-often used infos for file descriptors */
Willy Tarreaubaaee002006-06-26 02:48:02 +0200102int totalconn; /* total # of terminated sessions */
103int actconn; /* # of active sessions */
104
Willy Tarreau4f60f162007-04-08 16:39:58 +0200105struct poller pollers[MAX_POLLERS];
106struct poller cur_poller;
107int nbpollers = 0;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200108
Olivier Houchard6b96f722018-04-25 16:58:25 +0200109volatile struct fdlist update_list; // Global update list
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100110
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200111THREAD_LOCAL int *fd_updt = NULL; // FD updates list
112THREAD_LOCAL int fd_nbupdt = 0; // number of updates in the list
Olivier Houchard79321b92018-07-26 17:55:11 +0200113THREAD_LOCAL int poller_rd_pipe = -1; // Pipe to wake the thread
114int poller_wr_pipe[MAX_THREADS]; // Pipe to wake the threads
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200115
Olivier Houchard7c49d2e2019-04-16 18:37:05 +0200116volatile int ha_used_fds = 0; // Number of FD we're currently using
117
Willy Tarreau337fb712019-12-20 07:20:00 +0100118#define _GET_NEXT(fd, off) ((volatile struct fdlist_entry *)(void *)((char *)(&fdtab[fd]) + off))->next
119#define _GET_PREV(fd, off) ((volatile struct fdlist_entry *)(void *)((char *)(&fdtab[fd]) + off))->prev
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100120/* adds fd <fd> to fd list <list> if it was not yet in it */
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200121void fd_add_to_fd_list(volatile struct fdlist *list, int fd, int off)
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100122{
123 int next;
124 int new;
125 int old;
126 int last;
127
128redo_next:
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200129 next = _GET_NEXT(fd, off);
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100130 /* Check that we're not already in the cache, and if not, lock us. */
Olivier Houchardfc51f0f52019-12-19 18:33:08 +0100131 if (next > -2)
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100132 goto done;
Olivier Houchardfc51f0f52019-12-19 18:33:08 +0100133 if (next == -2)
134 goto redo_next;
Olivier Houchardd3608792019-03-08 18:47:42 +0100135 if (!_HA_ATOMIC_CAS(&_GET_NEXT(fd, off), &next, -2))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100136 goto redo_next;
Olivier Houchardd2b5d162019-03-08 13:47:21 +0100137 __ha_barrier_atomic_store();
Willy Tarreau11559a72018-02-05 17:52:24 +0100138
139 new = fd;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100140redo_last:
141 /* First, insert in the linked list */
142 last = list->last;
143 old = -1;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100144
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200145 _GET_PREV(fd, off) = -2;
Willy Tarreau11559a72018-02-05 17:52:24 +0100146 /* Make sure the "prev" store is visible before we update the last entry */
147 __ha_barrier_store();
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100148
Willy Tarreau11559a72018-02-05 17:52:24 +0100149 if (unlikely(last == -1)) {
150 /* list is empty, try to add ourselves alone so that list->last=fd */
Olivier Houchardd3608792019-03-08 18:47:42 +0100151 if (unlikely(!_HA_ATOMIC_CAS(&list->last, &old, new)))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100152 goto redo_last;
153
154 /* list->first was necessary -1, we're guaranteed to be alone here */
155 list->first = fd;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100156 } else {
Willy Tarreau11559a72018-02-05 17:52:24 +0100157 /* adding ourselves past the last element
158 * The CAS will only succeed if its next is -1,
159 * which means it's in the cache, and the last element.
160 */
Olivier Houchardd3608792019-03-08 18:47:42 +0100161 if (unlikely(!_HA_ATOMIC_CAS(&_GET_NEXT(last, off), &old, new)))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100162 goto redo_last;
Willy Tarreau11559a72018-02-05 17:52:24 +0100163
164 /* Then, update the last entry */
165 list->last = fd;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100166 }
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100167 __ha_barrier_store();
Willy Tarreau11559a72018-02-05 17:52:24 +0100168 /* since we're alone at the end of the list and still locked(-2),
169 * we know noone tried to add past us. Mark the end of list.
170 */
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200171 _GET_PREV(fd, off) = last;
172 _GET_NEXT(fd, off) = -1;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100173 __ha_barrier_store();
174done:
175 return;
176}
177
178/* removes fd <fd> from fd list <list> */
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200179void fd_rm_from_fd_list(volatile struct fdlist *list, int fd, int off)
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100180{
181#if defined(HA_HAVE_CAS_DW) || defined(HA_CAS_IS_8B)
182 volatile struct fdlist_entry cur_list, next_list;
183#endif
184 int old;
185 int new = -2;
186 int prev;
187 int next;
188 int last;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100189lock_self:
190#if (defined(HA_CAS_IS_8B) || defined(HA_HAVE_CAS_DW))
191 next_list.next = next_list.prev = -2;
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200192 cur_list = *(volatile struct fdlist_entry *)(((char *)&fdtab[fd]) + off);
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100193 /* First, attempt to lock our own entries */
194 do {
195 /* The FD is not in the FD cache, give up */
196 if (unlikely(cur_list.next <= -3))
197 return;
198 if (unlikely(cur_list.prev == -2 || cur_list.next == -2))
199 goto lock_self;
200 } while (
201#ifdef HA_CAS_IS_8B
Olivier Houchardd3608792019-03-08 18:47:42 +0100202 unlikely(!_HA_ATOMIC_CAS(((void **)(void *)&_GET_NEXT(fd, off)), ((void **)(void *)&cur_list), (*(void **)(void *)&next_list))))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100203#else
Willy Tarreauc3b59582019-05-27 17:37:20 +0200204 unlikely(!_HA_ATOMIC_DWCAS(((void *)&_GET_NEXT(fd, off)), ((void *)&cur_list), ((void *)&next_list))))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100205#endif
206 ;
207 next = cur_list.next;
208 prev = cur_list.prev;
209
210#else
211lock_self_next:
Willy Tarreau337fb712019-12-20 07:20:00 +0100212 next = _GET_NEXT(fd, off);
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100213 if (next == -2)
214 goto lock_self_next;
215 if (next <= -3)
216 goto done;
Olivier Houchardd3608792019-03-08 18:47:42 +0100217 if (unlikely(!_HA_ATOMIC_CAS(&_GET_NEXT(fd, off), &next, -2)))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100218 goto lock_self_next;
219lock_self_prev:
Willy Tarreau337fb712019-12-20 07:20:00 +0100220 prev = _GET_PREV(fd, off);
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100221 if (prev == -2)
222 goto lock_self_prev;
Olivier Houchardd3608792019-03-08 18:47:42 +0100223 if (unlikely(!_HA_ATOMIC_CAS(&_GET_PREV(fd, off), &prev, -2)))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100224 goto lock_self_prev;
225#endif
Olivier Houchardd2b5d162019-03-08 13:47:21 +0100226 __ha_barrier_atomic_store();
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100227
228 /* Now, lock the entries of our neighbours */
229 if (likely(prev != -1)) {
230redo_prev:
231 old = fd;
232
Olivier Houchardd3608792019-03-08 18:47:42 +0100233 if (unlikely(!_HA_ATOMIC_CAS(&_GET_NEXT(prev, off), &old, new))) {
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100234 if (unlikely(old == -2)) {
235 /* Neighbour already locked, give up and
236 * retry again once he's done
237 */
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200238 _GET_PREV(fd, off) = prev;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100239 __ha_barrier_store();
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200240 _GET_NEXT(fd, off) = next;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100241 __ha_barrier_store();
242 goto lock_self;
243 }
244 goto redo_prev;
245 }
246 }
247 if (likely(next != -1)) {
248redo_next:
249 old = fd;
Olivier Houchardd3608792019-03-08 18:47:42 +0100250 if (unlikely(!_HA_ATOMIC_CAS(&_GET_PREV(next, off), &old, new))) {
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100251 if (unlikely(old == -2)) {
252 /* Neighbour already locked, give up and
253 * retry again once he's done
254 */
255 if (prev != -1) {
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200256 _GET_NEXT(prev, off) = fd;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100257 __ha_barrier_store();
258 }
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200259 _GET_PREV(fd, off) = prev;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100260 __ha_barrier_store();
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200261 _GET_NEXT(fd, off) = next;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100262 __ha_barrier_store();
263 goto lock_self;
264 }
265 goto redo_next;
266 }
267 }
268 if (list->first == fd)
269 list->first = next;
270 __ha_barrier_store();
271 last = list->last;
Olivier Houchardd3608792019-03-08 18:47:42 +0100272 while (unlikely(last == fd && (!_HA_ATOMIC_CAS(&list->last, &last, prev))))
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100273 __ha_compiler_barrier();
274 /* Make sure we let other threads know we're no longer in cache,
275 * before releasing our neighbours.
276 */
277 __ha_barrier_store();
278 if (likely(prev != -1))
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200279 _GET_NEXT(prev, off) = next;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100280 __ha_barrier_store();
281 if (likely(next != -1))
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200282 _GET_PREV(next, off) = prev;
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100283 __ha_barrier_store();
284 /* Ok, now we're out of the fd cache */
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200285 _GET_NEXT(fd, off) = -(next + 4);
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100286 __ha_barrier_store();
287done:
288 return;
289}
290
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200291#undef _GET_NEXT
292#undef _GET_PREV
293
Willy Tarreau173d9952018-01-26 21:48:23 +0100294/* Deletes an FD from the fdsets.
Willy Tarreau4f60f162007-04-08 16:39:58 +0200295 * The file descriptor is also closed.
296 */
Olivier Houchard1fc05162017-04-06 01:05:05 +0200297static void fd_dodelete(int fd, int do_close)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200298{
Willy Tarreau87d54a92018-10-15 09:44:46 +0200299 unsigned long locked = atleast2(fdtab[fd].thread_mask);
300
301 if (locked)
302 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreauad38ace2013-12-15 14:19:38 +0100303 if (fdtab[fd].linger_risk) {
304 /* this is generally set when connecting to servers */
305 setsockopt(fd, SOL_SOCKET, SO_LINGER,
306 (struct linger *) &nolinger, sizeof(struct linger));
307 }
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100308 if (cur_poller.clo)
309 cur_poller.clo(fd);
Olivier Houchardc22580c2019-08-05 18:51:52 +0200310 polled_mask[fd].poll_recv = polled_mask[fd].poll_send = 0;
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100311
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100312 fdtab[fd].state = 0;
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100313
Willy Tarreau8d5d77e2009-10-18 07:25:52 +0200314 port_range_release_port(fdinfo[fd].port_range, fdinfo[fd].local_port);
315 fdinfo[fd].port_range = NULL;
Willy Tarreaudb3b3262012-07-05 23:19:22 +0200316 fdtab[fd].owner = NULL;
Willy Tarreauf65610a2017-10-31 16:06:06 +0100317 fdtab[fd].thread_mask = 0;
Willy Tarreauc9c83782018-01-17 18:44:46 +0100318 if (do_close) {
Christopher Fauletd531f882017-06-01 16:55:03 +0200319 close(fd);
Olivier Houchard7c49d2e2019-04-16 18:37:05 +0200320 _HA_ATOMIC_SUB(&ha_used_fds, 1);
Willy Tarreauc9c83782018-01-17 18:44:46 +0100321 }
Willy Tarreau87d54a92018-10-15 09:44:46 +0200322 if (locked)
323 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200324}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200325
Willy Tarreau173d9952018-01-26 21:48:23 +0100326/* Deletes an FD from the fdsets.
Olivier Houchard1fc05162017-04-06 01:05:05 +0200327 * The file descriptor is also closed.
328 */
329void fd_delete(int fd)
330{
331 fd_dodelete(fd, 1);
332}
333
Willy Tarreau173d9952018-01-26 21:48:23 +0100334/* Deletes an FD from the fdsets.
Olivier Houchard1fc05162017-04-06 01:05:05 +0200335 * The file descriptor is kept open.
336 */
337void fd_remove(int fd)
338{
339 fd_dodelete(fd, 0);
340}
341
Willy Tarreaudbe30602019-09-04 13:25:41 +0200342void updt_fd_polling(const int fd)
343{
344 if ((fdtab[fd].thread_mask & all_threads_mask) == tid_bit) {
345
346 /* note: we don't have a test-and-set yet in hathreads */
347
348 if (HA_ATOMIC_BTS(&fdtab[fd].update_mask, tid))
349 return;
350
351 fd_updt[fd_nbupdt++] = fd;
352 } else {
353 unsigned long update_mask = fdtab[fd].update_mask;
354 do {
355 if (update_mask == fdtab[fd].thread_mask)
356 return;
357 } while (!_HA_ATOMIC_CAS(&fdtab[fd].update_mask, &update_mask,
358 fdtab[fd].thread_mask));
359 fd_add_to_fd_list(&update_list, fd, offsetof(struct fdtab, update));
360 }
361}
362
Willy Tarreau931d8b72019-08-27 11:08:17 +0200363/* Tries to send <npfx> parts from <prefix> followed by <nmsg> parts from <msg>
364 * optionally followed by a newline if <nl> is non-null, to file descriptor
365 * <fd>. The message is sent atomically using writev(). It may be truncated to
366 * <maxlen> bytes if <maxlen> is non-null. There is no distinction between the
367 * two lists, it's just a convenience to help the caller prepend some prefixes
368 * when necessary. It takes the fd's lock to make sure no other thread will
369 * write to the same fd in parallel. Returns the number of bytes sent, or <=0
370 * on failure. A limit to 31 total non-empty segments is enforced. The caller
371 * is responsible for taking care of making the fd non-blocking.
372 */
373ssize_t fd_write_frag_line(int fd, size_t maxlen, const struct ist pfx[], size_t npfx, const struct ist msg[], size_t nmsg, int nl)
374{
375 struct iovec iovec[32];
376 size_t totlen = 0;
377 size_t sent = 0;
378 int vec = 0;
379
380 if (!maxlen)
381 maxlen = ~0;
382
383 /* keep one char for a possible trailing '\n' in any case */
384 maxlen--;
385
386 /* make an iovec from the concatenation of all parts of the original
387 * message. Skip empty fields and truncate the whole message to maxlen,
388 * leaving one spare iovec for the '\n'.
389 */
390 while (vec < (sizeof(iovec) / sizeof(iovec[0]) - 1)) {
391 if (!npfx) {
392 pfx = msg;
393 npfx = nmsg;
394 nmsg = 0;
395 if (!npfx)
396 break;
397 }
398
399 iovec[vec].iov_base = pfx->ptr;
400 iovec[vec].iov_len = MIN(maxlen, pfx->len);
401 maxlen -= iovec[vec].iov_len;
402 totlen += iovec[vec].iov_len;
403 if (iovec[vec].iov_len)
404 vec++;
405 pfx++; npfx--;
406 };
407
408 if (nl) {
409 iovec[vec].iov_base = "\n";
410 iovec[vec].iov_len = 1;
411 vec++;
412 }
413
Willy Tarreau7e9776a2019-08-30 14:41:47 +0200414 if (unlikely(!fdtab[fd].initialized)) {
415 fdtab[fd].initialized = 1;
416 if (!isatty(fd))
417 fcntl(fd, F_SETFL, O_NONBLOCK);
418 }
419
Willy Tarreau931d8b72019-08-27 11:08:17 +0200420 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
421 sent = writev(fd, iovec, vec);
422 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
423
424 /* sent > 0 if the message was delivered */
425 return sent;
426}
427
Olivier Houchard2292edf2019-02-25 14:26:54 +0100428#if defined(USE_CLOSEFROM)
429void my_closefrom(int start)
430{
431 closefrom(start);
432}
433
Willy Tarreaue5733232019-05-22 19:24:06 +0200434#elif defined(USE_POLL)
Willy Tarreau9188ac62019-02-21 22:12:47 +0100435/* This is a portable implementation of closefrom(). It closes all open file
436 * descriptors starting at <start> and above. It relies on the fact that poll()
437 * will return POLLNVAL for each invalid (hence close) file descriptor passed
438 * in argument in order to skip them. It acts with batches of FDs and will
439 * typically perform one poll() call per 1024 FDs so the overhead is low in
440 * case all FDs have to be closed.
441 */
442void my_closefrom(int start)
443{
444 struct pollfd poll_events[1024];
445 struct rlimit limit;
446 int nbfds, fd, ret, idx;
447 int step, next;
448
449 if (getrlimit(RLIMIT_NOFILE, &limit) == 0)
450 step = nbfds = limit.rlim_cur;
451 else
452 step = nbfds = 0;
453
454 if (nbfds <= 0) {
455 /* set safe limit */
456 nbfds = 1024;
457 step = 256;
458 }
459
460 if (step > sizeof(poll_events) / sizeof(poll_events[0]))
461 step = sizeof(poll_events) / sizeof(poll_events[0]);
462
463 while (start < nbfds) {
464 next = (start / step + 1) * step;
465
466 for (fd = start; fd < next && fd < nbfds; fd++) {
467 poll_events[fd - start].fd = fd;
468 poll_events[fd - start].events = 0;
469 }
470
471 do {
472 ret = poll(poll_events, fd - start, 0);
473 if (ret >= 0)
474 break;
475 } while (errno == EAGAIN || errno == EINTR || errno == ENOMEM);
476
Willy Tarreaub8e602c2019-02-22 09:07:42 +0100477 if (ret)
478 ret = fd - start;
479
Willy Tarreau9188ac62019-02-21 22:12:47 +0100480 for (idx = 0; idx < ret; idx++) {
481 if (poll_events[idx].revents & POLLNVAL)
482 continue; /* already closed */
483
484 fd = poll_events[idx].fd;
485 close(fd);
486 }
487 start = next;
488 }
489}
490
Willy Tarreaue5733232019-05-22 19:24:06 +0200491#else // defined(USE_POLL)
Willy Tarreau9188ac62019-02-21 22:12:47 +0100492
Willy Tarreau2d7f81b2019-02-21 22:19:17 +0100493/* This is a portable implementation of closefrom(). It closes all open file
494 * descriptors starting at <start> and above. This is a naive version for use
495 * when the operating system provides no alternative.
496 */
497void my_closefrom(int start)
498{
499 struct rlimit limit;
500 int nbfds;
501
502 if (getrlimit(RLIMIT_NOFILE, &limit) == 0)
503 nbfds = limit.rlim_cur;
504 else
505 nbfds = 0;
506
507 if (nbfds <= 0)
508 nbfds = 1024; /* safe limit */
509
510 while (start < nbfds)
511 close(start++);
512}
Willy Tarreaue5733232019-05-22 19:24:06 +0200513#endif // defined(USE_POLL)
Willy Tarreau2d7f81b2019-02-21 22:19:17 +0100514
Willy Tarreau4f60f162007-04-08 16:39:58 +0200515/* disable the specified poller */
516void disable_poller(const char *poller_name)
517{
518 int p;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200519
Willy Tarreau4f60f162007-04-08 16:39:58 +0200520 for (p = 0; p < nbpollers; p++)
521 if (strcmp(pollers[p].name, poller_name) == 0)
522 pollers[p].pref = 0;
523}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200524
Olivier Houchard79321b92018-07-26 17:55:11 +0200525void poller_pipe_io_handler(int fd)
526{
527 char buf[1024];
528 /* Flush the pipe */
529 while (read(fd, buf, sizeof(buf)) > 0);
530 fd_cant_recv(fd);
531}
532
Willy Tarreau082b6282019-05-22 14:42:12 +0200533/* allocate the per-thread fd_updt thus needs to be called early after
534 * thread creation.
535 */
536static int alloc_pollers_per_thread()
537{
538 fd_updt = calloc(global.maxsock, sizeof(*fd_updt));
539 return fd_updt != NULL;
540}
541
542/* Initialize the pollers per thread.*/
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200543static int init_pollers_per_thread()
544{
Olivier Houchard79321b92018-07-26 17:55:11 +0200545 int mypipe[2];
Willy Tarreau082b6282019-05-22 14:42:12 +0200546
547 if (pipe(mypipe) < 0)
Olivier Houchard79321b92018-07-26 17:55:11 +0200548 return 0;
Willy Tarreau082b6282019-05-22 14:42:12 +0200549
Olivier Houchard79321b92018-07-26 17:55:11 +0200550 poller_rd_pipe = mypipe[0];
551 poller_wr_pipe[tid] = mypipe[1];
552 fcntl(poller_rd_pipe, F_SETFL, O_NONBLOCK);
553 fd_insert(poller_rd_pipe, poller_pipe_io_handler, poller_pipe_io_handler,
554 tid_bit);
555 fd_want_recv(poller_rd_pipe);
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200556 return 1;
557}
558
559/* Deinitialize the pollers per thread */
560static void deinit_pollers_per_thread()
561{
William Lallemand808e1b72018-12-15 22:34:31 +0100562 /* rd and wr are init at the same place, but only rd is init to -1, so
563 we rely to rd to close. */
564 if (poller_rd_pipe > -1) {
565 close(poller_rd_pipe);
566 poller_rd_pipe = -1;
567 close(poller_wr_pipe[tid]);
568 poller_wr_pipe[tid] = -1;
569 }
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200570}
571
Willy Tarreau082b6282019-05-22 14:42:12 +0200572/* Release the pollers per thread, to be called late */
573static void free_pollers_per_thread()
574{
575 free(fd_updt);
576 fd_updt = NULL;
577}
578
Willy Tarreaubaaee002006-06-26 02:48:02 +0200579/*
Willy Tarreau4f60f162007-04-08 16:39:58 +0200580 * Initialize the pollers till the best one is found.
581 * If none works, returns 0, otherwise 1.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200582 */
Willy Tarreau4f60f162007-04-08 16:39:58 +0200583int init_pollers()
Willy Tarreaubaaee002006-06-26 02:48:02 +0200584{
Willy Tarreau4f60f162007-04-08 16:39:58 +0200585 int p;
586 struct poller *bp;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200587
Christopher Faulet63fe6522017-08-31 17:52:09 +0200588 if ((fdtab = calloc(global.maxsock, sizeof(struct fdtab))) == NULL)
589 goto fail_tab;
590
Olivier Houchard53055052019-07-25 14:00:18 +0000591 if ((polled_mask = calloc(global.maxsock, sizeof(*polled_mask))) == NULL)
Olivier Houchardcb92f5c2018-04-26 14:23:07 +0200592 goto fail_polledmask;
Uman Shahzadda7eeed2019-01-17 08:21:39 +0000593
Christopher Faulet63fe6522017-08-31 17:52:09 +0200594 if ((fdinfo = calloc(global.maxsock, sizeof(struct fdinfo))) == NULL)
595 goto fail_info;
596
Olivier Houchard6b96f722018-04-25 16:58:25 +0200597 update_list.first = update_list.last = -1;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200598
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100599 for (p = 0; p < global.maxsock; p++) {
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100600 HA_SPIN_INIT(&fdtab[p].lock);
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100601 /* Mark the fd as out of the fd cache */
Olivier Houchard6b96f722018-04-25 16:58:25 +0200602 fdtab[p].update.next = -3;
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100603 }
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200604
Willy Tarreau4f60f162007-04-08 16:39:58 +0200605 do {
606 bp = NULL;
607 for (p = 0; p < nbpollers; p++)
608 if (!bp || (pollers[p].pref > bp->pref))
609 bp = &pollers[p];
Willy Tarreaubaaee002006-06-26 02:48:02 +0200610
Willy Tarreau4f60f162007-04-08 16:39:58 +0200611 if (!bp || bp->pref == 0)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200612 break;
613
Willy Tarreau4f60f162007-04-08 16:39:58 +0200614 if (bp->init(bp)) {
615 memcpy(&cur_poller, bp, sizeof(*bp));
616 return 1;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200617 }
Willy Tarreau4f60f162007-04-08 16:39:58 +0200618 } while (!bp || bp->pref == 0);
Willy Tarreau7be79a42012-11-11 15:02:54 +0100619
Christopher Faulet63fe6522017-08-31 17:52:09 +0200620 free(fdinfo);
621 fail_info:
Olivier Houchardcb92f5c2018-04-26 14:23:07 +0200622 free(polled_mask);
623 fail_polledmask:
Uman Shahzadda7eeed2019-01-17 08:21:39 +0000624 free(fdtab);
625 fail_tab:
Willy Tarreau7be79a42012-11-11 15:02:54 +0100626 return 0;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200627}
628
Willy Tarreaubaaee002006-06-26 02:48:02 +0200629/*
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +0200630 * Deinitialize the pollers.
631 */
632void deinit_pollers() {
633
634 struct poller *bp;
635 int p;
636
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200637 for (p = 0; p < global.maxsock; p++)
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100638 HA_SPIN_DESTROY(&fdtab[p].lock);
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200639
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +0200640 for (p = 0; p < nbpollers; p++) {
641 bp = &pollers[p];
642
643 if (bp && bp->pref)
644 bp->term(bp);
645 }
Christopher Fauletd4604ad2017-05-29 10:40:41 +0200646
Christopher Faulet63fe6522017-08-31 17:52:09 +0200647 free(fdinfo); fdinfo = NULL;
648 free(fdtab); fdtab = NULL;
Olivier Houchardcb92f5c2018-04-26 14:23:07 +0200649 free(polled_mask); polled_mask = NULL;
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +0200650}
651
652/*
Willy Tarreau2ff76222007-04-09 19:29:56 +0200653 * Lists the known pollers on <out>.
654 * Should be performed only before initialization.
655 */
656int list_pollers(FILE *out)
657{
658 int p;
659 int last, next;
660 int usable;
661 struct poller *bp;
662
663 fprintf(out, "Available polling systems :\n");
664
665 usable = 0;
666 bp = NULL;
667 last = next = -1;
668 while (1) {
669 for (p = 0; p < nbpollers; p++) {
Willy Tarreau2ff76222007-04-09 19:29:56 +0200670 if ((next < 0 || pollers[p].pref > next)
Willy Tarreaue79c3b22010-11-19 10:20:36 +0100671 && (last < 0 || pollers[p].pref < last)) {
Willy Tarreau2ff76222007-04-09 19:29:56 +0200672 next = pollers[p].pref;
Willy Tarreaue79c3b22010-11-19 10:20:36 +0100673 if (!bp || (pollers[p].pref > bp->pref))
674 bp = &pollers[p];
675 }
Willy Tarreau2ff76222007-04-09 19:29:56 +0200676 }
677
678 if (next == -1)
679 break;
680
681 for (p = 0; p < nbpollers; p++) {
682 if (pollers[p].pref == next) {
683 fprintf(out, " %10s : ", pollers[p].name);
684 if (pollers[p].pref == 0)
685 fprintf(out, "disabled, ");
686 else
687 fprintf(out, "pref=%3d, ", pollers[p].pref);
688 if (pollers[p].test(&pollers[p])) {
689 fprintf(out, " test result OK");
690 if (next > 0)
691 usable++;
Willy Tarreaue79c3b22010-11-19 10:20:36 +0100692 } else {
Willy Tarreau2ff76222007-04-09 19:29:56 +0200693 fprintf(out, " test result FAILED");
Willy Tarreaue79c3b22010-11-19 10:20:36 +0100694 if (bp == &pollers[p])
695 bp = NULL;
696 }
Willy Tarreau2ff76222007-04-09 19:29:56 +0200697 fprintf(out, "\n");
698 }
699 }
700 last = next;
701 next = -1;
702 };
703 fprintf(out, "Total: %d (%d usable), will use %s.\n", nbpollers, usable, bp ? bp->name : "none");
704 return 0;
705}
706
707/*
708 * Some pollers may lose their connection after a fork(). It may be necessary
709 * to create initialize part of them again. Returns 0 in case of failure,
710 * otherwise 1. The fork() function may be NULL if unused. In case of error,
711 * the the current poller is destroyed and the caller is responsible for trying
712 * another one by calling init_pollers() again.
713 */
714int fork_poller()
715{
Conrad Hoffmann041751c2014-05-20 14:28:24 +0200716 int fd;
Willy Tarreauce036bc2018-01-29 14:58:02 +0100717 for (fd = 0; fd < global.maxsock; fd++) {
Conrad Hoffmann041751c2014-05-20 14:28:24 +0200718 if (fdtab[fd].owner) {
719 fdtab[fd].cloned = 1;
720 }
721 }
722
Willy Tarreau2ff76222007-04-09 19:29:56 +0200723 if (cur_poller.fork) {
724 if (cur_poller.fork(&cur_poller))
725 return 1;
726 cur_poller.term(&cur_poller);
727 return 0;
728 }
729 return 1;
730}
731
Willy Tarreau082b6282019-05-22 14:42:12 +0200732REGISTER_PER_THREAD_ALLOC(alloc_pollers_per_thread);
Willy Tarreau172f5ce2018-11-26 11:21:50 +0100733REGISTER_PER_THREAD_INIT(init_pollers_per_thread);
734REGISTER_PER_THREAD_DEINIT(deinit_pollers_per_thread);
Willy Tarreau082b6282019-05-22 14:42:12 +0200735REGISTER_PER_THREAD_FREE(free_pollers_per_thread);
Willy Tarreau172f5ce2018-11-26 11:21:50 +0100736
Willy Tarreau2ff76222007-04-09 19:29:56 +0200737/*
Willy Tarreaubaaee002006-06-26 02:48:02 +0200738 * Local variables:
739 * c-indent-level: 8
740 * c-basic-offset: 8
741 * End:
742 */