blob: 444d1b011348ddc6c78683a4e2a0816e75743bd1 [file] [log] [blame]
Willy Tarreaubaaee002006-06-26 02:48:02 +02001/*
Willy Tarreau49b046d2012-08-09 12:11:58 +02002 * include/proto/fd.h
3 * File descriptors states.
4 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +01005 * Copyright (C) 2000-2014 Willy Tarreau - w@1wt.eu
Willy Tarreau49b046d2012-08-09 12:11:58 +02006 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation, version 2.1
10 * exclusively.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
Willy Tarreaubaaee002006-06-26 02:48:02 +020021
22#ifndef _PROTO_FD_H
23#define _PROTO_FD_H
24
Willy Tarreau2ff76222007-04-09 19:29:56 +020025#include <stdio.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020026#include <sys/time.h>
27#include <sys/types.h>
28#include <unistd.h>
29
Willy Tarreaue3ba5f02006-06-29 18:54:54 +020030#include <common/config.h>
Willy Tarreauf37ba942018-10-17 11:25:54 +020031#include <common/ticks.h>
32#include <common/time.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020033#include <types/fd.h>
Willy Tarreau609aad92018-11-22 08:31:09 +010034#include <proto/activity.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020035
Willy Tarreau7be79a42012-11-11 15:02:54 +010036/* public variables */
Christopher Fauletd4604ad2017-05-29 10:40:41 +020037
Olivier Houchard4815c8c2018-01-24 18:17:56 +010038extern volatile struct fdlist fd_cache;
39extern volatile struct fdlist fd_cache_local[MAX_THREADS];
40
Olivier Houchard6b96f722018-04-25 16:58:25 +020041extern volatile struct fdlist update_list;
42
Olivier Houchardcb92f5c2018-04-26 14:23:07 +020043extern unsigned long *polled_mask;
44
Christopher Faulet69553fe2018-01-15 11:57:03 +010045extern unsigned long fd_cache_mask; // Mask of threads with events in the cache
Christopher Fauletd4604ad2017-05-29 10:40:41 +020046
47extern THREAD_LOCAL int *fd_updt; // FD updates list
48extern THREAD_LOCAL int fd_nbupdt; // number of updates in the list
49
Olivier Houchard79321b92018-07-26 17:55:11 +020050extern int poller_wr_pipe[MAX_THREADS];
51
Olivier Houchard7c49d2e2019-04-16 18:37:05 +020052extern volatile int ha_used_fds; // Number of FDs we're currently using
53
Willy Tarreau8b949692017-11-26 11:07:34 +010054__decl_hathreads(extern HA_RWLOCK_T __attribute__((aligned(64))) fdcache_lock); /* global lock to protect fd_cache array */
Willy Tarreau7be79a42012-11-11 15:02:54 +010055
Willy Tarreau173d9952018-01-26 21:48:23 +010056/* Deletes an FD from the fdsets.
Willy Tarreaubaaee002006-06-26 02:48:02 +020057 * The file descriptor is also closed.
58 */
59void fd_delete(int fd);
60
Willy Tarreau173d9952018-01-26 21:48:23 +010061/* Deletes an FD from the fdsets.
Olivier Houchard1fc05162017-04-06 01:05:05 +020062 * The file descriptor is kept open.
63 */
64void fd_remove(int fd);
65
Willy Tarreau2d7f81b2019-02-21 22:19:17 +010066/* close all FDs starting from <start> */
67void my_closefrom(int start);
68
Willy Tarreau4f60f162007-04-08 16:39:58 +020069/* disable the specified poller */
70void disable_poller(const char *poller_name);
Willy Tarreaubaaee002006-06-26 02:48:02 +020071
Olivier Houchard79321b92018-07-26 17:55:11 +020072void poller_pipe_io_handler(int fd);
73
Willy Tarreau2a429502006-10-15 14:52:29 +020074/*
Willy Tarreau4f60f162007-04-08 16:39:58 +020075 * Initialize the pollers till the best one is found.
76 * If none works, returns 0, otherwise 1.
Willy Tarreauef1d1f82007-04-16 00:25:25 +020077 * The pollers register themselves just before main() is called.
Willy Tarreau2a429502006-10-15 14:52:29 +020078 */
Willy Tarreau4f60f162007-04-08 16:39:58 +020079int init_pollers();
Willy Tarreau2a429502006-10-15 14:52:29 +020080
Willy Tarreau4f60f162007-04-08 16:39:58 +020081/*
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +020082 * Deinitialize the pollers.
83 */
84void deinit_pollers();
85
86/*
Willy Tarreau2ff76222007-04-09 19:29:56 +020087 * Some pollers may lose their connection after a fork(). It may be necessary
88 * to create initialize part of them again. Returns 0 in case of failure,
89 * otherwise 1. The fork() function may be NULL if unused. In case of error,
90 * the the current poller is destroyed and the caller is responsible for trying
91 * another one by calling init_pollers() again.
92 */
93int fork_poller();
94
95/*
96 * Lists the known pollers on <out>.
97 * Should be performed only before initialization.
98 */
99int list_pollers(FILE *out);
100
101/*
Willy Tarreau4f60f162007-04-08 16:39:58 +0200102 * Runs the polling loop
103 */
104void run_poller();
Willy Tarreau2a429502006-10-15 14:52:29 +0200105
Willy Tarreau033cd9d2014-01-25 19:24:15 +0100106/* Scan and process the cached events. This should be called right after
Willy Tarreau09f24562012-11-11 16:43:45 +0100107 * the poller.
108 */
Willy Tarreau033cd9d2014-01-25 19:24:15 +0100109void fd_process_cached_events();
Willy Tarreau09f24562012-11-11 16:43:45 +0100110
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200111void fd_add_to_fd_list(volatile struct fdlist *list, int fd, int off);
112void fd_rm_from_fd_list(volatile struct fdlist *list, int fd, int off);
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100113
Willy Tarreau5be2f352014-11-19 19:43:05 +0100114/* Mark fd <fd> as updated for polling and allocate an entry in the update list
115 * for this if it was not already there. This can be done at any time.
Willy Tarreaue8525452014-01-25 09:58:06 +0100116 */
Willy Tarreau5be2f352014-11-19 19:43:05 +0100117static inline void updt_fd_polling(const int fd)
Willy Tarreau7be79a42012-11-11 15:02:54 +0100118{
Olivier Houchard6aab7372018-08-17 13:37:59 +0200119 if ((fdtab[fd].thread_mask & all_threads_mask) == tid_bit) {
Olivier Houchard6b96f722018-04-25 16:58:25 +0200120
121 /* note: we don't have a test-and-set yet in hathreads */
Willy Tarreau4d841862018-01-17 22:57:54 +0100122
Olivier Houchard6b96f722018-04-25 16:58:25 +0200123 if (HA_ATOMIC_BTS(&fdtab[fd].update_mask, tid))
124 return;
125
Olivier Houcharda0fdce32019-06-12 14:31:08 +0200126 fd_updt[fd_nbupdt++] = fd;
Olivier Houchard6b96f722018-04-25 16:58:25 +0200127 } else {
128 unsigned long update_mask = fdtab[fd].update_mask;
129 do {
130 if (update_mask == fdtab[fd].thread_mask)
131 return;
Olivier Houchardd3608792019-03-08 18:47:42 +0100132 } while (!_HA_ATOMIC_CAS(&fdtab[fd].update_mask, &update_mask,
Olivier Houchard6b96f722018-04-25 16:58:25 +0200133 fdtab[fd].thread_mask));
134 fd_add_to_fd_list(&update_list, fd, offsetof(struct fdtab, update));
135 }
136
137}
Willy Tarreau4d841862018-01-17 22:57:54 +0100138
Olivier Houchard6b96f722018-04-25 16:58:25 +0200139/* Called from the poller to acknoledge we read an entry from the global
140 * update list, to remove our bit from the update_mask, and remove it from
141 * the list if we were the last one.
142 */
143static inline void done_update_polling(int fd)
144{
145 unsigned long update_mask;
146
Olivier Houchardd3608792019-03-08 18:47:42 +0100147 update_mask = _HA_ATOMIC_AND(&fdtab[fd].update_mask, ~tid_bit);
Olivier Houchard6b96f722018-04-25 16:58:25 +0200148 while ((update_mask & all_threads_mask)== 0) {
149 /* If we were the last one that had to update that entry, remove it from the list */
150 fd_rm_from_fd_list(&update_list, fd, offsetof(struct fdtab, update));
Olivier Houchard6b96f722018-04-25 16:58:25 +0200151 update_mask = (volatile unsigned long)fdtab[fd].update_mask;
152 if ((update_mask & all_threads_mask) != 0) {
153 /* Maybe it's been re-updated in the meanwhile, and we
154 * wrongly removed it from the list, if so, re-add it
155 */
156 fd_add_to_fd_list(&update_list, fd, offsetof(struct fdtab, update));
157 update_mask = (volatile unsigned long)(fdtab[fd].update_mask);
158 /* And then check again, just in case after all it
159 * should be removed, even if it's very unlikely, given
160 * the current thread wouldn't have been able to take
161 * care of it yet */
162 } else
163 break;
Willy Tarreau4d841862018-01-17 22:57:54 +0100164
Olivier Houchard6b96f722018-04-25 16:58:25 +0200165 }
Willy Tarreau7be79a42012-11-11 15:02:54 +0100166}
167
Willy Tarreau899d9572014-01-25 19:20:35 +0100168/* Allocates a cache entry for a file descriptor if it does not yet have one.
169 * This can be done at any time.
170 */
171static inline void fd_alloc_cache_entry(const int fd)
Willy Tarreau7be79a42012-11-11 15:02:54 +0100172{
Olivier Houchardd3608792019-03-08 18:47:42 +0100173 _HA_ATOMIC_OR(&fd_cache_mask, fdtab[fd].thread_mask);
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100174 if (!(fdtab[fd].thread_mask & (fdtab[fd].thread_mask - 1)))
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200175 fd_add_to_fd_list(&fd_cache_local[my_ffsl(fdtab[fd].thread_mask) - 1], fd, offsetof(struct fdtab, cache));
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100176 else
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200177 fd_add_to_fd_list(&fd_cache, fd, offsetof(struct fdtab, cache));
Willy Tarreau7be79a42012-11-11 15:02:54 +0100178}
179
Willy Tarreau899d9572014-01-25 19:20:35 +0100180/* Removes entry used by fd <fd> from the FD cache and replaces it with the
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100181 * last one.
Willy Tarreau7be79a42012-11-11 15:02:54 +0100182 * If the fd has no entry assigned, return immediately.
183 */
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100184static inline void fd_release_cache_entry(const int fd)
Willy Tarreau7be79a42012-11-11 15:02:54 +0100185{
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100186 if (!(fdtab[fd].thread_mask & (fdtab[fd].thread_mask - 1)))
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200187 fd_rm_from_fd_list(&fd_cache_local[my_ffsl(fdtab[fd].thread_mask) - 1], fd, offsetof(struct fdtab, cache));
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100188 else
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200189 fd_rm_from_fd_list(&fd_cache, fd, offsetof(struct fdtab, cache));
Willy Tarreau7be79a42012-11-11 15:02:54 +0100190}
Willy Tarreau49b046d2012-08-09 12:11:58 +0200191
Willy Tarreau5be2f352014-11-19 19:43:05 +0100192/* This function automatically enables/disables caching for an entry depending
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100193 * on its state. It is only called on state changes.
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100194 */
Willy Tarreau5be2f352014-11-19 19:43:05 +0100195static inline void fd_update_cache(int fd)
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100196{
Willy Tarreau5be2f352014-11-19 19:43:05 +0100197 /* only READY and ACTIVE states (the two with both flags set) require a cache entry */
198 if (((fdtab[fd].state & (FD_EV_READY_R | FD_EV_ACTIVE_R)) == (FD_EV_READY_R | FD_EV_ACTIVE_R)) ||
199 ((fdtab[fd].state & (FD_EV_READY_W | FD_EV_ACTIVE_W)) == (FD_EV_READY_W | FD_EV_ACTIVE_W))) {
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100200 fd_alloc_cache_entry(fd);
201 }
202 else {
203 fd_release_cache_entry(fd);
204 }
205}
206
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100207/*
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100208 * returns the FD's recv state (FD_EV_*)
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100209 */
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100210static inline int fd_recv_state(const int fd)
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100211{
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100212 return ((unsigned)fdtab[fd].state >> (4 * DIR_RD)) & FD_EV_STATUS;
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100213}
214
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100215/*
216 * returns true if the FD is active for recv
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100217 */
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100218static inline int fd_recv_active(const int fd)
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100219{
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100220 return (unsigned)fdtab[fd].state & FD_EV_ACTIVE_R;
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100221}
222
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100223/*
224 * returns true if the FD is ready for recv
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100225 */
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100226static inline int fd_recv_ready(const int fd)
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100227{
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100228 return (unsigned)fdtab[fd].state & FD_EV_READY_R;
229}
230
231/*
232 * returns true if the FD is polled for recv
233 */
234static inline int fd_recv_polled(const int fd)
235{
236 return (unsigned)fdtab[fd].state & FD_EV_POLLED_R;
237}
238
239/*
240 * returns the FD's send state (FD_EV_*)
241 */
242static inline int fd_send_state(const int fd)
243{
244 return ((unsigned)fdtab[fd].state >> (4 * DIR_WR)) & FD_EV_STATUS;
245}
246
247/*
248 * returns true if the FD is active for send
249 */
250static inline int fd_send_active(const int fd)
251{
252 return (unsigned)fdtab[fd].state & FD_EV_ACTIVE_W;
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100253}
254
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100255/*
256 * returns true if the FD is ready for send
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100257 */
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100258static inline int fd_send_ready(const int fd)
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100259{
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100260 return (unsigned)fdtab[fd].state & FD_EV_READY_W;
261}
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100262
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100263/*
264 * returns true if the FD is polled for send
265 */
266static inline int fd_send_polled(const int fd)
267{
268 return (unsigned)fdtab[fd].state & FD_EV_POLLED_W;
269}
270
Christopher Faulet8db2fdf2017-08-30 09:59:38 +0200271/*
272 * returns true if the FD is active for recv or send
273 */
274static inline int fd_active(const int fd)
275{
276 return (unsigned)fdtab[fd].state & FD_EV_ACTIVE_RW;
277}
278
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100279/* Disable processing recv events on fd <fd> */
280static inline void fd_stop_recv(int fd)
281{
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100282 unsigned char old, new;
Richard Russobc9d9842019-02-20 12:43:45 -0800283 unsigned long locked;
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100284
285 old = fdtab[fd].state;
286 do {
287 if (!(old & FD_EV_ACTIVE_R))
288 return;
289 new = old & ~FD_EV_ACTIVE_R;
290 new &= ~FD_EV_POLLED_R;
Olivier Houchardd3608792019-03-08 18:47:42 +0100291 } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new)));
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100292
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100293 if ((old ^ new) & FD_EV_POLLED_R)
294 updt_fd_polling(fd);
295
Richard Russobc9d9842019-02-20 12:43:45 -0800296 locked = atleast2(fdtab[fd].thread_mask);
297 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200298 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100299 fd_update_cache(fd); /* need an update entry to change the state */
Richard Russobc9d9842019-02-20 12:43:45 -0800300 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200301 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100302}
303
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100304/* Disable processing send events on fd <fd> */
305static inline void fd_stop_send(int fd)
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100306{
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100307 unsigned char old, new;
Richard Russobc9d9842019-02-20 12:43:45 -0800308 unsigned long locked;
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100309
310 old = fdtab[fd].state;
311 do {
312 if (!(old & FD_EV_ACTIVE_W))
313 return;
314 new = old & ~FD_EV_ACTIVE_W;
315 new &= ~FD_EV_POLLED_W;
Olivier Houchardd3608792019-03-08 18:47:42 +0100316 } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new)));
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100317
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100318 if ((old ^ new) & FD_EV_POLLED_W)
319 updt_fd_polling(fd);
320
Richard Russobc9d9842019-02-20 12:43:45 -0800321 locked = atleast2(fdtab[fd].thread_mask);
322 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200323 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100324 fd_update_cache(fd); /* need an update entry to change the state */
Richard Russobc9d9842019-02-20 12:43:45 -0800325 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200326 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100327}
328
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100329/* Disable processing of events on fd <fd> for both directions. */
330static inline void fd_stop_both(int fd)
Willy Tarreau49b046d2012-08-09 12:11:58 +0200331{
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100332 unsigned char old, new;
Richard Russobc9d9842019-02-20 12:43:45 -0800333 unsigned long locked;
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100334
335 old = fdtab[fd].state;
336 do {
337 if (!(old & FD_EV_ACTIVE_RW))
338 return;
339 new = old & ~FD_EV_ACTIVE_RW;
340 new &= ~FD_EV_POLLED_RW;
Olivier Houchardd3608792019-03-08 18:47:42 +0100341 } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new)));
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100342
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100343 if ((old ^ new) & FD_EV_POLLED_RW)
344 updt_fd_polling(fd);
345
Richard Russobc9d9842019-02-20 12:43:45 -0800346 locked = atleast2(fdtab[fd].thread_mask);
347 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200348 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100349 fd_update_cache(fd); /* need an update entry to change the state */
Richard Russobc9d9842019-02-20 12:43:45 -0800350 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200351 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau49b046d2012-08-09 12:11:58 +0200352}
353
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100354/* Report that FD <fd> cannot receive anymore without polling (EAGAIN detected). */
355static inline void fd_cant_recv(const int fd)
Willy Tarreau49b046d2012-08-09 12:11:58 +0200356{
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100357 unsigned char old, new;
Richard Russobc9d9842019-02-20 12:43:45 -0800358 unsigned long locked;
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100359
360 old = fdtab[fd].state;
361 do {
362 if (!(old & FD_EV_READY_R))
363 return;
364 new = old & ~FD_EV_READY_R;
365 if (new & FD_EV_ACTIVE_R)
366 new |= FD_EV_POLLED_R;
Olivier Houchardd3608792019-03-08 18:47:42 +0100367 } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new)));
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100368
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100369 if ((old ^ new) & FD_EV_POLLED_R)
370 updt_fd_polling(fd);
371
Richard Russobc9d9842019-02-20 12:43:45 -0800372 locked = atleast2(fdtab[fd].thread_mask);
373 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200374 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100375 fd_update_cache(fd); /* need an update entry to change the state */
Richard Russobc9d9842019-02-20 12:43:45 -0800376 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200377 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau49b046d2012-08-09 12:11:58 +0200378}
379
Willy Tarreau1dad3842019-07-08 23:09:03 +0200380/* Report that FD <fd> may receive again without polling. */
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100381static inline void fd_may_recv(const int fd)
Willy Tarreaubabd05a2012-08-09 12:14:03 +0200382{
Richard Russobc9d9842019-02-20 12:43:45 -0800383 unsigned long locked;
384
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100385 /* marking ready never changes polled status */
Willy Tarreau1dad3842019-07-08 23:09:03 +0200386 if ((fdtab[fd].state & FD_EV_READY_R) ||
387 HA_ATOMIC_BTS(&fdtab[fd].state, FD_EV_READY_R_BIT))
388 return;
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100389
Richard Russobc9d9842019-02-20 12:43:45 -0800390 locked = atleast2(fdtab[fd].thread_mask);
391 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200392 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100393 fd_update_cache(fd); /* need an update entry to change the state */
Richard Russobc9d9842019-02-20 12:43:45 -0800394 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200395 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreaubabd05a2012-08-09 12:14:03 +0200396}
397
Willy Tarreau6c11bd22014-01-24 00:54:27 +0100398/* Disable readiness when polled. This is useful to interrupt reading when it
399 * is suspected that the end of data might have been reached (eg: short read).
400 * This can only be done using level-triggered pollers, so if any edge-triggered
401 * is ever implemented, a test will have to be added here.
402 */
403static inline void fd_done_recv(const int fd)
404{
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100405 unsigned char old, new;
Richard Russobc9d9842019-02-20 12:43:45 -0800406 unsigned long locked;
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100407
408 old = fdtab[fd].state;
409 do {
410 if ((old & (FD_EV_POLLED_R|FD_EV_READY_R)) != (FD_EV_POLLED_R|FD_EV_READY_R))
411 return;
412 new = old & ~FD_EV_READY_R;
413 if (new & FD_EV_ACTIVE_R)
414 new |= FD_EV_POLLED_R;
Olivier Houchardd3608792019-03-08 18:47:42 +0100415 } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new)));
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100416
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100417 if ((old ^ new) & FD_EV_POLLED_R)
418 updt_fd_polling(fd);
419
Richard Russobc9d9842019-02-20 12:43:45 -0800420 locked = atleast2(fdtab[fd].thread_mask);
421 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200422 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100423 fd_update_cache(fd); /* need an update entry to change the state */
Richard Russobc9d9842019-02-20 12:43:45 -0800424 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200425 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau6c11bd22014-01-24 00:54:27 +0100426}
427
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100428/* Report that FD <fd> cannot send anymore without polling (EAGAIN detected). */
429static inline void fd_cant_send(const int fd)
Willy Tarreau49b046d2012-08-09 12:11:58 +0200430{
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100431 unsigned char old, new;
Richard Russobc9d9842019-02-20 12:43:45 -0800432 unsigned long locked;
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100433
434 old = fdtab[fd].state;
435 do {
436 if (!(old & FD_EV_READY_W))
437 return;
438 new = old & ~FD_EV_READY_W;
439 if (new & FD_EV_ACTIVE_W)
440 new |= FD_EV_POLLED_W;
Olivier Houchardd3608792019-03-08 18:47:42 +0100441 } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new)));
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100442
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100443 if ((old ^ new) & FD_EV_POLLED_W)
444 updt_fd_polling(fd);
445
Richard Russobc9d9842019-02-20 12:43:45 -0800446 locked = atleast2(fdtab[fd].thread_mask);
447 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200448 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100449 fd_update_cache(fd); /* need an update entry to change the state */
Richard Russobc9d9842019-02-20 12:43:45 -0800450 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200451 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau49b046d2012-08-09 12:11:58 +0200452}
453
Willy Tarreau1dad3842019-07-08 23:09:03 +0200454/* Report that FD <fd> may send again without polling (EAGAIN not detected). */
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100455static inline void fd_may_send(const int fd)
Willy Tarreau49b046d2012-08-09 12:11:58 +0200456{
Richard Russobc9d9842019-02-20 12:43:45 -0800457 unsigned long locked;
458
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100459 /* marking ready never changes polled status */
Willy Tarreau1dad3842019-07-08 23:09:03 +0200460 if ((fdtab[fd].state & FD_EV_READY_W) ||
461 HA_ATOMIC_BTS(&fdtab[fd].state, FD_EV_READY_W_BIT))
462 return;
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100463
Richard Russobc9d9842019-02-20 12:43:45 -0800464 locked = atleast2(fdtab[fd].thread_mask);
465 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200466 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100467 fd_update_cache(fd); /* need an update entry to change the state */
Richard Russobc9d9842019-02-20 12:43:45 -0800468 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200469 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau49b046d2012-08-09 12:11:58 +0200470}
Willy Tarreau2a429502006-10-15 14:52:29 +0200471
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100472/* Prepare FD <fd> to try to receive */
473static inline void fd_want_recv(int fd)
Willy Tarreaubabd05a2012-08-09 12:14:03 +0200474{
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100475 unsigned char old, new;
Richard Russobc9d9842019-02-20 12:43:45 -0800476 unsigned long locked;
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100477
478 old = fdtab[fd].state;
479 do {
480 if (old & FD_EV_ACTIVE_R)
481 return;
482 new = old | FD_EV_ACTIVE_R;
483 if (!(new & FD_EV_READY_R))
484 new |= FD_EV_POLLED_R;
Olivier Houchardd3608792019-03-08 18:47:42 +0100485 } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new)));
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100486
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100487 if ((old ^ new) & FD_EV_POLLED_R)
488 updt_fd_polling(fd);
489
Richard Russobc9d9842019-02-20 12:43:45 -0800490 locked = atleast2(fdtab[fd].thread_mask);
491 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200492 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100493 fd_update_cache(fd); /* need an update entry to change the state */
Richard Russobc9d9842019-02-20 12:43:45 -0800494 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200495 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreaubabd05a2012-08-09 12:14:03 +0200496}
497
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100498/* Prepare FD <fd> to try to send */
499static inline void fd_want_send(int fd)
Willy Tarreau49b046d2012-08-09 12:11:58 +0200500{
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100501 unsigned char old, new;
Richard Russobc9d9842019-02-20 12:43:45 -0800502 unsigned long locked;
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100503
504 old = fdtab[fd].state;
505 do {
506 if (old & FD_EV_ACTIVE_W)
507 return;
508 new = old | FD_EV_ACTIVE_W;
509 if (!(new & FD_EV_READY_W))
510 new |= FD_EV_POLLED_W;
Olivier Houchardd3608792019-03-08 18:47:42 +0100511 } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new)));
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100512
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100513 if ((old ^ new) & FD_EV_POLLED_W)
514 updt_fd_polling(fd);
515
Richard Russobc9d9842019-02-20 12:43:45 -0800516 locked = atleast2(fdtab[fd].thread_mask);
517 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200518 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100519 fd_update_cache(fd); /* need an update entry to change the state */
Richard Russobc9d9842019-02-20 12:43:45 -0800520 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200521 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau49b046d2012-08-09 12:11:58 +0200522}
Willy Tarreau2a429502006-10-15 14:52:29 +0200523
Christopher Faulet21e92672017-08-30 10:30:04 +0200524/* Update events seen for FD <fd> and its state if needed. This should be called
525 * by the poller to set FD_POLL_* flags. */
526static inline void fd_update_events(int fd, int evts)
527{
Richard Russobc9d9842019-02-20 12:43:45 -0800528 unsigned long locked = atleast2(fdtab[fd].thread_mask);
Willy Tarreau1dad3842019-07-08 23:09:03 +0200529 unsigned char old, new;
Richard Russobc9d9842019-02-20 12:43:45 -0800530
Willy Tarreau1dad3842019-07-08 23:09:03 +0200531 old = fdtab[fd].ev;
532 new = (old & FD_POLL_STICKY) | evts;
533
534 if (unlikely(locked)) {
535 /* Locked FDs (those with more than 2 threads) are atomically updated */
536 while (unlikely(new != old && !_HA_ATOMIC_CAS(&fdtab[fd].ev, &old, new)))
537 new = (old & FD_POLL_STICKY) | evts;
538 } else {
539 if (new != old)
540 fdtab[fd].ev = new;
541 }
Christopher Faulet21e92672017-08-30 10:30:04 +0200542
543 if (fdtab[fd].ev & (FD_POLL_IN | FD_POLL_HUP | FD_POLL_ERR))
544 fd_may_recv(fd);
545
546 if (fdtab[fd].ev & (FD_POLL_OUT | FD_POLL_ERR))
547 fd_may_send(fd);
548}
549
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100550/* Prepares <fd> for being polled */
Willy Tarreaua9786b62018-01-25 07:22:13 +0100551static inline void fd_insert(int fd, void *owner, void (*iocb)(int fd), unsigned long thread_mask)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200552{
Richard Russobc9d9842019-02-20 12:43:45 -0800553 unsigned long locked = atleast2(thread_mask);
554
555 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200556 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreaua9786b62018-01-25 07:22:13 +0100557 fdtab[fd].owner = owner;
558 fdtab[fd].iocb = iocb;
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100559 fdtab[fd].ev = 0;
Willy Tarreauad38ace2013-12-15 14:19:38 +0100560 fdtab[fd].linger_risk = 0;
Conrad Hoffmann041751c2014-05-20 14:28:24 +0200561 fdtab[fd].cloned = 0;
Willy Tarreauf65610a2017-10-31 16:06:06 +0100562 fdtab[fd].thread_mask = thread_mask;
Willy Tarreauc9c83782018-01-17 18:44:46 +0100563 /* note: do not reset polled_mask here as it indicates which poller
564 * still knows this FD from a possible previous round.
565 */
Richard Russobc9d9842019-02-20 12:43:45 -0800566 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200567 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Olivier Houchard7c49d2e2019-04-16 18:37:05 +0200568 _HA_ATOMIC_ADD(&ha_used_fds, 1);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200569}
570
Willy Tarreauf37ba942018-10-17 11:25:54 +0200571/* Computes the bounded poll() timeout based on the next expiration timer <next>
572 * by bounding it to MAX_DELAY_MS. <next> may equal TICK_ETERNITY. The pollers
573 * just needs to call this function right before polling to get their timeout
574 * value. Timeouts that are already expired (possibly due to a pending event)
575 * are accounted for in activity.poll_exp.
576 */
577static inline int compute_poll_timeout(int next)
578{
579 int wait_time;
580
581 if (!tick_isset(next))
582 wait_time = MAX_DELAY_MS;
583 else if (tick_is_expired(next, now_ms)) {
584 activity[tid].poll_exp++;
585 wait_time = 0;
586 }
587 else {
588 wait_time = TICKS_TO_MS(tick_remain(now_ms, next)) + 1;
589 if (wait_time > MAX_DELAY_MS)
590 wait_time = MAX_DELAY_MS;
591 }
592 return wait_time;
593}
594
Willy Tarreau322e6c72018-01-25 16:37:04 +0100595/* These are replacements for FD_SET, FD_CLR, FD_ISSET, working on uints */
596static inline void hap_fd_set(int fd, unsigned int *evts)
597{
Olivier Houchardd3608792019-03-08 18:47:42 +0100598 _HA_ATOMIC_OR(&evts[fd / (8*sizeof(*evts))], 1U << (fd & (8*sizeof(*evts) - 1)));
Willy Tarreau322e6c72018-01-25 16:37:04 +0100599}
600
601static inline void hap_fd_clr(int fd, unsigned int *evts)
602{
Olivier Houchardd3608792019-03-08 18:47:42 +0100603 _HA_ATOMIC_AND(&evts[fd / (8*sizeof(*evts))], ~(1U << (fd & (8*sizeof(*evts) - 1))));
Willy Tarreau322e6c72018-01-25 16:37:04 +0100604}
605
606static inline unsigned int hap_fd_isset(int fd, unsigned int *evts)
607{
608 return evts[fd / (8*sizeof(*evts))] & (1U << (fd & (8*sizeof(*evts) - 1)));
609}
610
Olivier Houchard79321b92018-07-26 17:55:11 +0200611static inline void wake_thread(int tid)
612{
613 char c = 'c';
614
615 shut_your_big_mouth_gcc(write(poller_wr_pipe[tid], &c, 1));
616}
617
Willy Tarreaubaaee002006-06-26 02:48:02 +0200618
619#endif /* _PROTO_FD_H */
620
621/*
622 * Local variables:
623 * c-indent-level: 8
624 * c-basic-offset: 8
625 * End:
626 */