blob: 07bb20a05ca18cd3107ef8b8303a716b2e257122 [file] [log] [blame]
Willy Tarreaubaaee002006-06-26 02:48:02 +02001/*
Willy Tarreau49b046d2012-08-09 12:11:58 +02002 * include/proto/fd.h
3 * File descriptors states.
4 *
Willy Tarreauf817e9f2014-01-10 16:58:45 +01005 * Copyright (C) 2000-2014 Willy Tarreau - w@1wt.eu
Willy Tarreau49b046d2012-08-09 12:11:58 +02006 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation, version 2.1
10 * exclusively.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
Willy Tarreaubaaee002006-06-26 02:48:02 +020021
22#ifndef _PROTO_FD_H
23#define _PROTO_FD_H
24
Willy Tarreau2ff76222007-04-09 19:29:56 +020025#include <stdio.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020026#include <sys/time.h>
27#include <sys/types.h>
28#include <unistd.h>
29
Willy Tarreaue3ba5f02006-06-29 18:54:54 +020030#include <common/config.h>
Willy Tarreauf37ba942018-10-17 11:25:54 +020031#include <common/ticks.h>
32#include <common/time.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020033#include <types/fd.h>
Willy Tarreau609aad92018-11-22 08:31:09 +010034#include <proto/activity.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020035
Willy Tarreau7be79a42012-11-11 15:02:54 +010036/* public variables */
Christopher Fauletd4604ad2017-05-29 10:40:41 +020037
Olivier Houchard4815c8c2018-01-24 18:17:56 +010038extern volatile struct fdlist fd_cache;
39extern volatile struct fdlist fd_cache_local[MAX_THREADS];
40
Olivier Houchard6b96f722018-04-25 16:58:25 +020041extern volatile struct fdlist update_list;
42
Olivier Houchardcb92f5c2018-04-26 14:23:07 +020043extern unsigned long *polled_mask;
44
Christopher Faulet69553fe2018-01-15 11:57:03 +010045extern unsigned long fd_cache_mask; // Mask of threads with events in the cache
Christopher Fauletd4604ad2017-05-29 10:40:41 +020046
47extern THREAD_LOCAL int *fd_updt; // FD updates list
48extern THREAD_LOCAL int fd_nbupdt; // number of updates in the list
49
Olivier Houchard79321b92018-07-26 17:55:11 +020050extern int poller_wr_pipe[MAX_THREADS];
51
Willy Tarreau8b949692017-11-26 11:07:34 +010052__decl_hathreads(extern HA_RWLOCK_T __attribute__((aligned(64))) fdcache_lock); /* global lock to protect fd_cache array */
Willy Tarreau7be79a42012-11-11 15:02:54 +010053
Willy Tarreau173d9952018-01-26 21:48:23 +010054/* Deletes an FD from the fdsets.
Willy Tarreaubaaee002006-06-26 02:48:02 +020055 * The file descriptor is also closed.
56 */
57void fd_delete(int fd);
58
Willy Tarreau173d9952018-01-26 21:48:23 +010059/* Deletes an FD from the fdsets.
Olivier Houchard1fc05162017-04-06 01:05:05 +020060 * The file descriptor is kept open.
61 */
62void fd_remove(int fd);
63
Willy Tarreau2d7f81b2019-02-21 22:19:17 +010064/* close all FDs starting from <start> */
65void my_closefrom(int start);
66
Willy Tarreau4f60f162007-04-08 16:39:58 +020067/* disable the specified poller */
68void disable_poller(const char *poller_name);
Willy Tarreaubaaee002006-06-26 02:48:02 +020069
Olivier Houchard79321b92018-07-26 17:55:11 +020070void poller_pipe_io_handler(int fd);
71
Willy Tarreau2a429502006-10-15 14:52:29 +020072/*
Willy Tarreau4f60f162007-04-08 16:39:58 +020073 * Initialize the pollers till the best one is found.
74 * If none works, returns 0, otherwise 1.
Willy Tarreauef1d1f82007-04-16 00:25:25 +020075 * The pollers register themselves just before main() is called.
Willy Tarreau2a429502006-10-15 14:52:29 +020076 */
Willy Tarreau4f60f162007-04-08 16:39:58 +020077int init_pollers();
Willy Tarreau2a429502006-10-15 14:52:29 +020078
Willy Tarreau4f60f162007-04-08 16:39:58 +020079/*
Krzysztof Piotr Oledzkia643baf2008-05-29 23:53:44 +020080 * Deinitialize the pollers.
81 */
82void deinit_pollers();
83
84/*
Willy Tarreau2ff76222007-04-09 19:29:56 +020085 * Some pollers may lose their connection after a fork(). It may be necessary
86 * to create initialize part of them again. Returns 0 in case of failure,
87 * otherwise 1. The fork() function may be NULL if unused. In case of error,
88 * the the current poller is destroyed and the caller is responsible for trying
89 * another one by calling init_pollers() again.
90 */
91int fork_poller();
92
93/*
94 * Lists the known pollers on <out>.
95 * Should be performed only before initialization.
96 */
97int list_pollers(FILE *out);
98
99/*
Willy Tarreau4f60f162007-04-08 16:39:58 +0200100 * Runs the polling loop
101 */
102void run_poller();
Willy Tarreau2a429502006-10-15 14:52:29 +0200103
Willy Tarreau033cd9d2014-01-25 19:24:15 +0100104/* Scan and process the cached events. This should be called right after
Willy Tarreau09f24562012-11-11 16:43:45 +0100105 * the poller.
106 */
Willy Tarreau033cd9d2014-01-25 19:24:15 +0100107void fd_process_cached_events();
Willy Tarreau09f24562012-11-11 16:43:45 +0100108
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200109void fd_add_to_fd_list(volatile struct fdlist *list, int fd, int off);
110void fd_rm_from_fd_list(volatile struct fdlist *list, int fd, int off);
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100111
Willy Tarreau5be2f352014-11-19 19:43:05 +0100112/* Mark fd <fd> as updated for polling and allocate an entry in the update list
113 * for this if it was not already there. This can be done at any time.
Willy Tarreaue8525452014-01-25 09:58:06 +0100114 */
Willy Tarreau5be2f352014-11-19 19:43:05 +0100115static inline void updt_fd_polling(const int fd)
Willy Tarreau7be79a42012-11-11 15:02:54 +0100116{
Olivier Houchard6aab7372018-08-17 13:37:59 +0200117 if ((fdtab[fd].thread_mask & all_threads_mask) == tid_bit) {
Olivier Houchard6b96f722018-04-25 16:58:25 +0200118 unsigned int oldupdt;
119
120 /* note: we don't have a test-and-set yet in hathreads */
Willy Tarreau4d841862018-01-17 22:57:54 +0100121
Olivier Houchard6b96f722018-04-25 16:58:25 +0200122 if (HA_ATOMIC_BTS(&fdtab[fd].update_mask, tid))
123 return;
124
Olivier Houchardd3608792019-03-08 18:47:42 +0100125 oldupdt = _HA_ATOMIC_ADD(&fd_nbupdt, 1) - 1;
Olivier Houchard6b96f722018-04-25 16:58:25 +0200126 fd_updt[oldupdt] = fd;
127 } else {
128 unsigned long update_mask = fdtab[fd].update_mask;
129 do {
130 if (update_mask == fdtab[fd].thread_mask)
131 return;
Olivier Houchardd3608792019-03-08 18:47:42 +0100132 } while (!_HA_ATOMIC_CAS(&fdtab[fd].update_mask, &update_mask,
Olivier Houchard6b96f722018-04-25 16:58:25 +0200133 fdtab[fd].thread_mask));
134 fd_add_to_fd_list(&update_list, fd, offsetof(struct fdtab, update));
135 }
136
137}
Willy Tarreau4d841862018-01-17 22:57:54 +0100138
Olivier Houchard6b96f722018-04-25 16:58:25 +0200139/* Called from the poller to acknoledge we read an entry from the global
140 * update list, to remove our bit from the update_mask, and remove it from
141 * the list if we were the last one.
142 */
143static inline void done_update_polling(int fd)
144{
145 unsigned long update_mask;
146
Olivier Houchardd3608792019-03-08 18:47:42 +0100147 update_mask = _HA_ATOMIC_AND(&fdtab[fd].update_mask, ~tid_bit);
Olivier Houchard6b96f722018-04-25 16:58:25 +0200148 while ((update_mask & all_threads_mask)== 0) {
149 /* If we were the last one that had to update that entry, remove it from the list */
150 fd_rm_from_fd_list(&update_list, fd, offsetof(struct fdtab, update));
Olivier Houchard6b96f722018-04-25 16:58:25 +0200151 update_mask = (volatile unsigned long)fdtab[fd].update_mask;
152 if ((update_mask & all_threads_mask) != 0) {
153 /* Maybe it's been re-updated in the meanwhile, and we
154 * wrongly removed it from the list, if so, re-add it
155 */
156 fd_add_to_fd_list(&update_list, fd, offsetof(struct fdtab, update));
157 update_mask = (volatile unsigned long)(fdtab[fd].update_mask);
158 /* And then check again, just in case after all it
159 * should be removed, even if it's very unlikely, given
160 * the current thread wouldn't have been able to take
161 * care of it yet */
162 } else
163 break;
Willy Tarreau4d841862018-01-17 22:57:54 +0100164
Olivier Houchard6b96f722018-04-25 16:58:25 +0200165 }
Willy Tarreau7be79a42012-11-11 15:02:54 +0100166}
167
Willy Tarreau899d9572014-01-25 19:20:35 +0100168/* Allocates a cache entry for a file descriptor if it does not yet have one.
169 * This can be done at any time.
170 */
171static inline void fd_alloc_cache_entry(const int fd)
Willy Tarreau7be79a42012-11-11 15:02:54 +0100172{
Olivier Houchardd3608792019-03-08 18:47:42 +0100173 _HA_ATOMIC_OR(&fd_cache_mask, fdtab[fd].thread_mask);
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100174 if (!(fdtab[fd].thread_mask & (fdtab[fd].thread_mask - 1)))
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200175 fd_add_to_fd_list(&fd_cache_local[my_ffsl(fdtab[fd].thread_mask) - 1], fd, offsetof(struct fdtab, cache));
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100176 else
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200177 fd_add_to_fd_list(&fd_cache, fd, offsetof(struct fdtab, cache));
Willy Tarreau7be79a42012-11-11 15:02:54 +0100178}
179
Willy Tarreau899d9572014-01-25 19:20:35 +0100180/* Removes entry used by fd <fd> from the FD cache and replaces it with the
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100181 * last one.
Willy Tarreau7be79a42012-11-11 15:02:54 +0100182 * If the fd has no entry assigned, return immediately.
183 */
Willy Tarreau4cc67a22018-02-05 17:14:55 +0100184static inline void fd_release_cache_entry(const int fd)
Willy Tarreau7be79a42012-11-11 15:02:54 +0100185{
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100186 if (!(fdtab[fd].thread_mask & (fdtab[fd].thread_mask - 1)))
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200187 fd_rm_from_fd_list(&fd_cache_local[my_ffsl(fdtab[fd].thread_mask) - 1], fd, offsetof(struct fdtab, cache));
Olivier Houchard4815c8c2018-01-24 18:17:56 +0100188 else
Olivier Houchard6a2cf872018-04-25 15:10:30 +0200189 fd_rm_from_fd_list(&fd_cache, fd, offsetof(struct fdtab, cache));
Willy Tarreau7be79a42012-11-11 15:02:54 +0100190}
Willy Tarreau49b046d2012-08-09 12:11:58 +0200191
Willy Tarreau5be2f352014-11-19 19:43:05 +0100192/* This function automatically enables/disables caching for an entry depending
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100193 * on its state. It is only called on state changes.
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100194 */
Willy Tarreau5be2f352014-11-19 19:43:05 +0100195static inline void fd_update_cache(int fd)
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100196{
Willy Tarreau5be2f352014-11-19 19:43:05 +0100197 /* only READY and ACTIVE states (the two with both flags set) require a cache entry */
198 if (((fdtab[fd].state & (FD_EV_READY_R | FD_EV_ACTIVE_R)) == (FD_EV_READY_R | FD_EV_ACTIVE_R)) ||
199 ((fdtab[fd].state & (FD_EV_READY_W | FD_EV_ACTIVE_W)) == (FD_EV_READY_W | FD_EV_ACTIVE_W))) {
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100200 fd_alloc_cache_entry(fd);
201 }
202 else {
203 fd_release_cache_entry(fd);
204 }
205}
206
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100207/*
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100208 * returns the FD's recv state (FD_EV_*)
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100209 */
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100210static inline int fd_recv_state(const int fd)
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100211{
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100212 return ((unsigned)fdtab[fd].state >> (4 * DIR_RD)) & FD_EV_STATUS;
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100213}
214
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100215/*
216 * returns true if the FD is active for recv
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100217 */
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100218static inline int fd_recv_active(const int fd)
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100219{
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100220 return (unsigned)fdtab[fd].state & FD_EV_ACTIVE_R;
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100221}
222
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100223/*
224 * returns true if the FD is ready for recv
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100225 */
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100226static inline int fd_recv_ready(const int fd)
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100227{
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100228 return (unsigned)fdtab[fd].state & FD_EV_READY_R;
229}
230
231/*
232 * returns true if the FD is polled for recv
233 */
234static inline int fd_recv_polled(const int fd)
235{
236 return (unsigned)fdtab[fd].state & FD_EV_POLLED_R;
237}
238
239/*
240 * returns the FD's send state (FD_EV_*)
241 */
242static inline int fd_send_state(const int fd)
243{
244 return ((unsigned)fdtab[fd].state >> (4 * DIR_WR)) & FD_EV_STATUS;
245}
246
247/*
248 * returns true if the FD is active for send
249 */
250static inline int fd_send_active(const int fd)
251{
252 return (unsigned)fdtab[fd].state & FD_EV_ACTIVE_W;
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100253}
254
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100255/*
256 * returns true if the FD is ready for send
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100257 */
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100258static inline int fd_send_ready(const int fd)
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100259{
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100260 return (unsigned)fdtab[fd].state & FD_EV_READY_W;
261}
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100262
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100263/*
264 * returns true if the FD is polled for send
265 */
266static inline int fd_send_polled(const int fd)
267{
268 return (unsigned)fdtab[fd].state & FD_EV_POLLED_W;
269}
270
Christopher Faulet8db2fdf2017-08-30 09:59:38 +0200271/*
272 * returns true if the FD is active for recv or send
273 */
274static inline int fd_active(const int fd)
275{
276 return (unsigned)fdtab[fd].state & FD_EV_ACTIVE_RW;
277}
278
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100279/* Disable processing recv events on fd <fd> */
280static inline void fd_stop_recv(int fd)
281{
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100282 unsigned char old, new;
Richard Russobc9d9842019-02-20 12:43:45 -0800283 unsigned long locked;
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100284
285 old = fdtab[fd].state;
286 do {
287 if (!(old & FD_EV_ACTIVE_R))
288 return;
289 new = old & ~FD_EV_ACTIVE_R;
290 new &= ~FD_EV_POLLED_R;
Olivier Houchardd3608792019-03-08 18:47:42 +0100291 } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new)));
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100292
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100293 if ((old ^ new) & FD_EV_POLLED_R)
294 updt_fd_polling(fd);
295
Richard Russobc9d9842019-02-20 12:43:45 -0800296 locked = atleast2(fdtab[fd].thread_mask);
297 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200298 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100299 fd_update_cache(fd); /* need an update entry to change the state */
Richard Russobc9d9842019-02-20 12:43:45 -0800300 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200301 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100302}
303
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100304/* Disable processing send events on fd <fd> */
305static inline void fd_stop_send(int fd)
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100306{
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100307 unsigned char old, new;
Richard Russobc9d9842019-02-20 12:43:45 -0800308 unsigned long locked;
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100309
310 old = fdtab[fd].state;
311 do {
312 if (!(old & FD_EV_ACTIVE_W))
313 return;
314 new = old & ~FD_EV_ACTIVE_W;
315 new &= ~FD_EV_POLLED_W;
Olivier Houchardd3608792019-03-08 18:47:42 +0100316 } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new)));
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100317
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100318 if ((old ^ new) & FD_EV_POLLED_W)
319 updt_fd_polling(fd);
320
Richard Russobc9d9842019-02-20 12:43:45 -0800321 locked = atleast2(fdtab[fd].thread_mask);
322 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200323 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100324 fd_update_cache(fd); /* need an update entry to change the state */
Richard Russobc9d9842019-02-20 12:43:45 -0800325 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200326 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau6ea20b12012-11-11 16:05:19 +0100327}
328
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100329/* Disable processing of events on fd <fd> for both directions. */
330static inline void fd_stop_both(int fd)
Willy Tarreau49b046d2012-08-09 12:11:58 +0200331{
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100332 unsigned char old, new;
Richard Russobc9d9842019-02-20 12:43:45 -0800333 unsigned long locked;
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100334
335 old = fdtab[fd].state;
336 do {
337 if (!(old & FD_EV_ACTIVE_RW))
338 return;
339 new = old & ~FD_EV_ACTIVE_RW;
340 new &= ~FD_EV_POLLED_RW;
Olivier Houchardd3608792019-03-08 18:47:42 +0100341 } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new)));
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100342
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100343 if ((old ^ new) & FD_EV_POLLED_RW)
344 updt_fd_polling(fd);
345
Richard Russobc9d9842019-02-20 12:43:45 -0800346 locked = atleast2(fdtab[fd].thread_mask);
347 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200348 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100349 fd_update_cache(fd); /* need an update entry to change the state */
Richard Russobc9d9842019-02-20 12:43:45 -0800350 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200351 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau49b046d2012-08-09 12:11:58 +0200352}
353
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100354/* Report that FD <fd> cannot receive anymore without polling (EAGAIN detected). */
355static inline void fd_cant_recv(const int fd)
Willy Tarreau49b046d2012-08-09 12:11:58 +0200356{
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100357 unsigned char old, new;
Richard Russobc9d9842019-02-20 12:43:45 -0800358 unsigned long locked;
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100359
360 old = fdtab[fd].state;
361 do {
362 if (!(old & FD_EV_READY_R))
363 return;
364 new = old & ~FD_EV_READY_R;
365 if (new & FD_EV_ACTIVE_R)
366 new |= FD_EV_POLLED_R;
Olivier Houchardd3608792019-03-08 18:47:42 +0100367 } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new)));
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100368
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100369 if ((old ^ new) & FD_EV_POLLED_R)
370 updt_fd_polling(fd);
371
Richard Russobc9d9842019-02-20 12:43:45 -0800372 locked = atleast2(fdtab[fd].thread_mask);
373 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200374 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100375 fd_update_cache(fd); /* need an update entry to change the state */
Richard Russobc9d9842019-02-20 12:43:45 -0800376 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200377 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau49b046d2012-08-09 12:11:58 +0200378}
379
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100380/* Report that FD <fd> can receive anymore without polling. */
381static inline void fd_may_recv(const int fd)
Willy Tarreaubabd05a2012-08-09 12:14:03 +0200382{
Richard Russobc9d9842019-02-20 12:43:45 -0800383 unsigned long locked;
384
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100385 /* marking ready never changes polled status */
Olivier Houchardd3608792019-03-08 18:47:42 +0100386 _HA_ATOMIC_OR(&fdtab[fd].state, FD_EV_READY_R);
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100387
Richard Russobc9d9842019-02-20 12:43:45 -0800388 locked = atleast2(fdtab[fd].thread_mask);
389 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200390 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100391 fd_update_cache(fd); /* need an update entry to change the state */
Richard Russobc9d9842019-02-20 12:43:45 -0800392 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200393 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreaubabd05a2012-08-09 12:14:03 +0200394}
395
Willy Tarreau6c11bd22014-01-24 00:54:27 +0100396/* Disable readiness when polled. This is useful to interrupt reading when it
397 * is suspected that the end of data might have been reached (eg: short read).
398 * This can only be done using level-triggered pollers, so if any edge-triggered
399 * is ever implemented, a test will have to be added here.
400 */
401static inline void fd_done_recv(const int fd)
402{
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100403 unsigned char old, new;
Richard Russobc9d9842019-02-20 12:43:45 -0800404 unsigned long locked;
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100405
406 old = fdtab[fd].state;
407 do {
408 if ((old & (FD_EV_POLLED_R|FD_EV_READY_R)) != (FD_EV_POLLED_R|FD_EV_READY_R))
409 return;
410 new = old & ~FD_EV_READY_R;
411 if (new & FD_EV_ACTIVE_R)
412 new |= FD_EV_POLLED_R;
Olivier Houchardd3608792019-03-08 18:47:42 +0100413 } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new)));
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100414
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100415 if ((old ^ new) & FD_EV_POLLED_R)
416 updt_fd_polling(fd);
417
Richard Russobc9d9842019-02-20 12:43:45 -0800418 locked = atleast2(fdtab[fd].thread_mask);
419 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200420 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100421 fd_update_cache(fd); /* need an update entry to change the state */
Richard Russobc9d9842019-02-20 12:43:45 -0800422 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200423 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau6c11bd22014-01-24 00:54:27 +0100424}
425
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100426/* Report that FD <fd> cannot send anymore without polling (EAGAIN detected). */
427static inline void fd_cant_send(const int fd)
Willy Tarreau49b046d2012-08-09 12:11:58 +0200428{
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100429 unsigned char old, new;
Richard Russobc9d9842019-02-20 12:43:45 -0800430 unsigned long locked;
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100431
432 old = fdtab[fd].state;
433 do {
434 if (!(old & FD_EV_READY_W))
435 return;
436 new = old & ~FD_EV_READY_W;
437 if (new & FD_EV_ACTIVE_W)
438 new |= FD_EV_POLLED_W;
Olivier Houchardd3608792019-03-08 18:47:42 +0100439 } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new)));
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100440
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100441 if ((old ^ new) & FD_EV_POLLED_W)
442 updt_fd_polling(fd);
443
Richard Russobc9d9842019-02-20 12:43:45 -0800444 locked = atleast2(fdtab[fd].thread_mask);
445 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200446 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100447 fd_update_cache(fd); /* need an update entry to change the state */
Richard Russobc9d9842019-02-20 12:43:45 -0800448 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200449 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau49b046d2012-08-09 12:11:58 +0200450}
451
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100452/* Report that FD <fd> can send anymore without polling (EAGAIN detected). */
453static inline void fd_may_send(const int fd)
Willy Tarreau49b046d2012-08-09 12:11:58 +0200454{
Richard Russobc9d9842019-02-20 12:43:45 -0800455 unsigned long locked;
456
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100457 /* marking ready never changes polled status */
Olivier Houchardd3608792019-03-08 18:47:42 +0100458 _HA_ATOMIC_OR(&fdtab[fd].state, FD_EV_READY_W);
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100459
Richard Russobc9d9842019-02-20 12:43:45 -0800460 locked = atleast2(fdtab[fd].thread_mask);
461 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200462 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100463 fd_update_cache(fd); /* need an update entry to change the state */
Richard Russobc9d9842019-02-20 12:43:45 -0800464 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200465 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau49b046d2012-08-09 12:11:58 +0200466}
Willy Tarreau2a429502006-10-15 14:52:29 +0200467
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100468/* Prepare FD <fd> to try to receive */
469static inline void fd_want_recv(int fd)
Willy Tarreaubabd05a2012-08-09 12:14:03 +0200470{
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100471 unsigned char old, new;
Richard Russobc9d9842019-02-20 12:43:45 -0800472 unsigned long locked;
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100473
474 old = fdtab[fd].state;
475 do {
476 if (old & FD_EV_ACTIVE_R)
477 return;
478 new = old | FD_EV_ACTIVE_R;
479 if (!(new & FD_EV_READY_R))
480 new |= FD_EV_POLLED_R;
Olivier Houchardd3608792019-03-08 18:47:42 +0100481 } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new)));
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100482
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100483 if ((old ^ new) & FD_EV_POLLED_R)
484 updt_fd_polling(fd);
485
Richard Russobc9d9842019-02-20 12:43:45 -0800486 locked = atleast2(fdtab[fd].thread_mask);
487 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200488 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100489 fd_update_cache(fd); /* need an update entry to change the state */
Richard Russobc9d9842019-02-20 12:43:45 -0800490 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200491 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreaubabd05a2012-08-09 12:14:03 +0200492}
493
Willy Tarreauf817e9f2014-01-10 16:58:45 +0100494/* Prepare FD <fd> to try to send */
495static inline void fd_want_send(int fd)
Willy Tarreau49b046d2012-08-09 12:11:58 +0200496{
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100497 unsigned char old, new;
Richard Russobc9d9842019-02-20 12:43:45 -0800498 unsigned long locked;
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100499
500 old = fdtab[fd].state;
501 do {
502 if (old & FD_EV_ACTIVE_W)
503 return;
504 new = old | FD_EV_ACTIVE_W;
505 if (!(new & FD_EV_READY_W))
506 new |= FD_EV_POLLED_W;
Olivier Houchardd3608792019-03-08 18:47:42 +0100507 } while (unlikely(!_HA_ATOMIC_CAS(&fdtab[fd].state, &old, new)));
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100508
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100509 if ((old ^ new) & FD_EV_POLLED_W)
510 updt_fd_polling(fd);
511
Richard Russobc9d9842019-02-20 12:43:45 -0800512 locked = atleast2(fdtab[fd].thread_mask);
513 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200514 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau7ac0e352018-01-17 21:25:57 +0100515 fd_update_cache(fd); /* need an update entry to change the state */
Richard Russobc9d9842019-02-20 12:43:45 -0800516 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200517 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreau49b046d2012-08-09 12:11:58 +0200518}
Willy Tarreau2a429502006-10-15 14:52:29 +0200519
Christopher Faulet21e92672017-08-30 10:30:04 +0200520/* Update events seen for FD <fd> and its state if needed. This should be called
521 * by the poller to set FD_POLL_* flags. */
522static inline void fd_update_events(int fd, int evts)
523{
Richard Russobc9d9842019-02-20 12:43:45 -0800524 unsigned long locked = atleast2(fdtab[fd].thread_mask);
525
526 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200527 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Christopher Faulet21e92672017-08-30 10:30:04 +0200528 fdtab[fd].ev &= FD_POLL_STICKY;
529 fdtab[fd].ev |= evts;
Richard Russobc9d9842019-02-20 12:43:45 -0800530 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200531 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Christopher Faulet21e92672017-08-30 10:30:04 +0200532
533 if (fdtab[fd].ev & (FD_POLL_IN | FD_POLL_HUP | FD_POLL_ERR))
534 fd_may_recv(fd);
535
536 if (fdtab[fd].ev & (FD_POLL_OUT | FD_POLL_ERR))
537 fd_may_send(fd);
538}
539
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100540/* Prepares <fd> for being polled */
Willy Tarreaua9786b62018-01-25 07:22:13 +0100541static inline void fd_insert(int fd, void *owner, void (*iocb)(int fd), unsigned long thread_mask)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200542{
Richard Russobc9d9842019-02-20 12:43:45 -0800543 unsigned long locked = atleast2(thread_mask);
544
545 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200546 HA_SPIN_LOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreaua9786b62018-01-25 07:22:13 +0100547 fdtab[fd].owner = owner;
548 fdtab[fd].iocb = iocb;
Willy Tarreaud6f087e2008-01-18 17:20:13 +0100549 fdtab[fd].ev = 0;
Willy Tarreauad38ace2013-12-15 14:19:38 +0100550 fdtab[fd].linger_risk = 0;
Conrad Hoffmann041751c2014-05-20 14:28:24 +0200551 fdtab[fd].cloned = 0;
Willy Tarreauf65610a2017-10-31 16:06:06 +0100552 fdtab[fd].thread_mask = thread_mask;
Willy Tarreauc9c83782018-01-17 18:44:46 +0100553 /* note: do not reset polled_mask here as it indicates which poller
554 * still knows this FD from a possible previous round.
555 */
Richard Russobc9d9842019-02-20 12:43:45 -0800556 if (locked)
Willy Tarreau87d54a92018-10-15 09:44:46 +0200557 HA_SPIN_UNLOCK(FD_LOCK, &fdtab[fd].lock);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200558}
559
Willy Tarreauf37ba942018-10-17 11:25:54 +0200560/* Computes the bounded poll() timeout based on the next expiration timer <next>
561 * by bounding it to MAX_DELAY_MS. <next> may equal TICK_ETERNITY. The pollers
562 * just needs to call this function right before polling to get their timeout
563 * value. Timeouts that are already expired (possibly due to a pending event)
564 * are accounted for in activity.poll_exp.
565 */
566static inline int compute_poll_timeout(int next)
567{
568 int wait_time;
569
570 if (!tick_isset(next))
571 wait_time = MAX_DELAY_MS;
572 else if (tick_is_expired(next, now_ms)) {
573 activity[tid].poll_exp++;
574 wait_time = 0;
575 }
576 else {
577 wait_time = TICKS_TO_MS(tick_remain(now_ms, next)) + 1;
578 if (wait_time > MAX_DELAY_MS)
579 wait_time = MAX_DELAY_MS;
580 }
581 return wait_time;
582}
583
Willy Tarreau322e6c72018-01-25 16:37:04 +0100584/* These are replacements for FD_SET, FD_CLR, FD_ISSET, working on uints */
585static inline void hap_fd_set(int fd, unsigned int *evts)
586{
Olivier Houchardd3608792019-03-08 18:47:42 +0100587 _HA_ATOMIC_OR(&evts[fd / (8*sizeof(*evts))], 1U << (fd & (8*sizeof(*evts) - 1)));
Willy Tarreau322e6c72018-01-25 16:37:04 +0100588}
589
590static inline void hap_fd_clr(int fd, unsigned int *evts)
591{
Olivier Houchardd3608792019-03-08 18:47:42 +0100592 _HA_ATOMIC_AND(&evts[fd / (8*sizeof(*evts))], ~(1U << (fd & (8*sizeof(*evts) - 1))));
Willy Tarreau322e6c72018-01-25 16:37:04 +0100593}
594
595static inline unsigned int hap_fd_isset(int fd, unsigned int *evts)
596{
597 return evts[fd / (8*sizeof(*evts))] & (1U << (fd & (8*sizeof(*evts) - 1)));
598}
599
Olivier Houchard79321b92018-07-26 17:55:11 +0200600static inline void wake_thread(int tid)
601{
602 char c = 'c';
603
604 shut_your_big_mouth_gcc(write(poller_wr_pipe[tid], &c, 1));
605}
606
Willy Tarreaubaaee002006-06-26 02:48:02 +0200607
608#endif /* _PROTO_FD_H */
609
610/*
611 * Local variables:
612 * c-indent-level: 8
613 * c-basic-offset: 8
614 * End:
615 */