blob: 2cd596aa39f374c4a0e90d1d8b03da77f9bf4265 [file] [log] [blame]
Willy Tarreau4f60f162007-04-08 16:39:58 +02001/*
Willy Tarreaue9f49e72012-11-11 17:42:00 +01002 * FD polling functions for Linux epoll
Willy Tarreau4f60f162007-04-08 16:39:58 +02003 *
Willy Tarreaue9f49e72012-11-11 17:42:00 +01004 * Copyright 2000-2012 Willy Tarreau <w@1wt.eu>
Willy Tarreau4f60f162007-04-08 16:39:58 +02005 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
Willy Tarreau4f60f162007-04-08 16:39:58 +020010 */
11
12#include <unistd.h>
13#include <sys/time.h>
14#include <sys/types.h>
15
16#include <common/compat.h>
17#include <common/config.h>
Willy Tarreaue9f49e72012-11-11 17:42:00 +010018#include <common/debug.h>
Willy Tarreau43d8fb22011-08-22 17:12:02 +020019#include <common/epoll.h>
Willy Tarreau58094f22007-04-10 01:33:20 +020020#include <common/standard.h>
Willy Tarreau0c303ee2008-07-07 00:09:58 +020021#include <common/ticks.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020022#include <common/time.h>
Willy Tarreau1db37712007-06-03 17:16:49 +020023#include <common/tools.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020024
Willy Tarreau4f60f162007-04-08 16:39:58 +020025#include <types/global.h>
26
Willy Tarreaue9f49e72012-11-11 17:42:00 +010027#include <proto/fd.h>
Willy Tarreau332740d2009-05-10 09:57:21 +020028#include <proto/signal.h>
Willy Tarreau4f60f162007-04-08 16:39:58 +020029#include <proto/task.h>
30
Willy Tarreau58094f22007-04-10 01:33:20 +020031
Willy Tarreaue9f49e72012-11-11 17:42:00 +010032static int absmaxevents = 0; // absolute maximum amounts of polled events
Willy Tarreau4f60f162007-04-08 16:39:58 +020033
34/* private data */
35static struct epoll_event *epoll_events;
36static int epoll_fd;
37
Willy Tarreau58094f22007-04-10 01:33:20 +020038/* This structure may be used for any purpose. Warning! do not use it in
39 * recursive functions !
40 */
41static struct epoll_event ev;
42
Willy Tarreau4f60f162007-04-08 16:39:58 +020043/*
Willy Tarreaue9f49e72012-11-11 17:42:00 +010044 * speculative epoll() poller
Willy Tarreau4f60f162007-04-08 16:39:58 +020045 */
Willy Tarreaue9f49e72012-11-11 17:42:00 +010046REGPRM2 static void _do_poll(struct poller *p, int exp)
Willy Tarreau4f60f162007-04-08 16:39:58 +020047{
Willy Tarreaue9f49e72012-11-11 17:42:00 +010048 int status, eo, en;
49 int fd, opcode;
50 int count;
51 int updt_idx;
52 int wait_time;
Willy Tarreau58094f22007-04-10 01:33:20 +020053
Willy Tarreaue9f49e72012-11-11 17:42:00 +010054 /* first, scan the update list to find changes */
55 for (updt_idx = 0; updt_idx < fd_nbupdt; updt_idx++) {
56 fd = fd_updt[updt_idx];
57 en = fdtab[fd].spec_e & 15; /* new events */
58 eo = fdtab[fd].spec_e >> 4; /* previous events */
Willy Tarreau58094f22007-04-10 01:33:20 +020059
Willy Tarreaue9f49e72012-11-11 17:42:00 +010060 if (fdtab[fd].owner && (eo ^ en)) {
61 if ((eo ^ en) & FD_EV_POLLED_RW) {
62 /* poll status changed */
63 if ((en & FD_EV_POLLED_RW) == 0) {
64 /* fd removed from poll list */
65 opcode = EPOLL_CTL_DEL;
66 }
67 else if ((eo & FD_EV_POLLED_RW) == 0) {
68 /* new fd in the poll list */
69 opcode = EPOLL_CTL_ADD;
70 }
71 else {
72 /* fd status changed */
73 opcode = EPOLL_CTL_MOD;
74 }
Willy Tarreau58094f22007-04-10 01:33:20 +020075
Willy Tarreaue9f49e72012-11-11 17:42:00 +010076 /* construct the epoll events based on new state */
77 ev.events = 0;
78 if (en & FD_EV_POLLED_R)
79 ev.events |= EPOLLIN;
Willy Tarreau58094f22007-04-10 01:33:20 +020080
Willy Tarreaue9f49e72012-11-11 17:42:00 +010081 if (en & FD_EV_POLLED_W)
82 ev.events |= EPOLLOUT;
Willy Tarreau58094f22007-04-10 01:33:20 +020083
Willy Tarreaue9f49e72012-11-11 17:42:00 +010084 ev.data.fd = fd;
85 epoll_ctl(epoll_fd, opcode, fd, &ev);
Willy Tarreau58094f22007-04-10 01:33:20 +020086 }
Willy Tarreau58094f22007-04-10 01:33:20 +020087
Willy Tarreaue9f49e72012-11-11 17:42:00 +010088 fdtab[fd].spec_e = (en << 4) + en; /* save new events */
Willy Tarreauf8cfa442012-10-04 21:54:41 +020089
Willy Tarreaue9f49e72012-11-11 17:42:00 +010090 if (!(en & FD_EV_ACTIVE_RW)) {
91 /* This fd doesn't use any active entry anymore, we can
92 * kill its entry.
93 */
94 release_spec_entry(fd);
95 }
96 else if ((en & ~eo) & FD_EV_ACTIVE_RW) {
97 /* we need a new spec entry now */
98 alloc_spec_entry(fd);
99 }
Willy Tarreau4f60f162007-04-08 16:39:58 +0200100
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100101 }
102 fdtab[fd].updated = 0;
103 fdtab[fd].new = 0;
Willy Tarreau58094f22007-04-10 01:33:20 +0200104 }
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100105 fd_nbupdt = 0;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200106
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100107 /* compute the epoll_wait() timeout */
Willy Tarreau4f60f162007-04-08 16:39:58 +0200108
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100109 if (fd_nbspec || run_queue || signal_queue_len) {
110 /* Maybe we still have events in the spec list, or there are
111 * some tasks left pending in the run_queue, so we must not
112 * wait in epoll() otherwise we would delay their delivery by
113 * the next timeout.
114 */
Willy Tarreaubdefc512007-05-14 02:02:04 +0200115 wait_time = 0;
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100116 }
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200117 else {
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100118 if (!exp)
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200119 wait_time = MAX_DELAY_MS;
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100120 else if (tick_is_expired(exp, now_ms))
121 wait_time = 0;
122 else {
123 wait_time = TICKS_TO_MS(tick_remain(now_ms, exp)) + 1;
124 if (wait_time > MAX_DELAY_MS)
125 wait_time = MAX_DELAY_MS;
126 }
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200127 }
Willy Tarreaud825eef2007-05-12 22:35:00 +0200128
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100129 /* now let's wait for polled events */
130
Willy Tarreau1db37712007-06-03 17:16:49 +0200131 fd = MIN(maxfd, global.tune.maxpollevents);
Willy Tarreau45a12512011-09-10 16:56:42 +0200132 gettimeofday(&before_poll, NULL);
Willy Tarreau1db37712007-06-03 17:16:49 +0200133 status = epoll_wait(epoll_fd, epoll_events, fd, wait_time);
Willy Tarreaub0b37bc2008-06-23 14:00:57 +0200134 tv_update_date(wait_time, status);
Willy Tarreau45a12512011-09-10 16:56:42 +0200135 measure_idle();
Willy Tarreau4f60f162007-04-08 16:39:58 +0200136
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100137 /* process polled events */
138
Willy Tarreau4f60f162007-04-08 16:39:58 +0200139 for (count = 0; count < status; count++) {
Willy Tarreau491c4982012-07-06 11:16:01 +0200140 int e = epoll_events[count].events;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200141 fd = epoll_events[count].data.fd;
142
Willy Tarreau076be252012-07-06 16:02:29 +0200143 if (!fdtab[fd].owner)
144 continue;
145
Willy Tarreau491c4982012-07-06 11:16:01 +0200146 /* it looks complicated but gcc can optimize it away when constants
147 * have same values.
148 */
149 fdtab[fd].ev &= FD_POLL_STICKY;
150 fdtab[fd].ev |=
151 ((e & EPOLLIN ) ? FD_POLL_IN : 0) |
152 ((e & EPOLLPRI) ? FD_POLL_PRI : 0) |
153 ((e & EPOLLOUT) ? FD_POLL_OUT : 0) |
154 ((e & EPOLLERR) ? FD_POLL_ERR : 0) |
155 ((e & EPOLLHUP) ? FD_POLL_HUP : 0);
156
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100157 if (fdtab[fd].iocb && fdtab[fd].owner && fdtab[fd].ev) {
158 int new_updt, old_updt = fd_nbupdt; /* Save number of updates to detect creation of new FDs. */
159
160 /* Mark the events as speculative before processing
161 * them so that if nothing can be done we don't need
162 * to poll again.
163 */
Willy Tarreau26d7cfc2012-12-07 00:09:43 +0100164 if (fdtab[fd].ev & FD_POLL_IN)
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100165 fd_ev_set(fd, DIR_RD);
166
Willy Tarreau26d7cfc2012-12-07 00:09:43 +0100167 if (fdtab[fd].ev & FD_POLL_OUT)
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100168 fd_ev_set(fd, DIR_WR);
169
Willy Tarreau9845e752012-07-06 11:44:28 +0200170 fdtab[fd].iocb(fd);
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100171
172 /* One or more fd might have been created during the iocb().
173 * This mainly happens with new incoming connections that have
174 * just been accepted, so we'd like to process them immediately
175 * for better efficiency. Second benefit, if at the end the fds
176 * are disabled again, we can safely destroy their update entry
177 * to reduce the scope of later scans. This is the reason we
178 * scan the new entries backwards.
179 */
180
181 for (new_updt = fd_nbupdt; new_updt > old_updt; new_updt--) {
182 fd = fd_updt[new_updt - 1];
183 if (!fdtab[fd].new)
184 continue;
185
186 fdtab[fd].new = 0;
187 fdtab[fd].ev &= FD_POLL_STICKY;
188
189 if ((fdtab[fd].spec_e & FD_EV_STATUS_R) == FD_EV_ACTIVE_R)
190 fdtab[fd].ev |= FD_POLL_IN;
191
192 if ((fdtab[fd].spec_e & FD_EV_STATUS_W) == FD_EV_ACTIVE_W)
193 fdtab[fd].ev |= FD_POLL_OUT;
194
195 if (fdtab[fd].ev && fdtab[fd].iocb && fdtab[fd].owner)
196 fdtab[fd].iocb(fd);
197
198 /* we can remove this update entry if it's the last one and is
199 * unused, otherwise we don't touch anything.
200 */
201 if (new_updt == fd_nbupdt && fdtab[fd].spec_e == 0) {
202 fdtab[fd].updated = 0;
203 fd_nbupdt--;
204 }
205 }
206 }
Willy Tarreau4f60f162007-04-08 16:39:58 +0200207 }
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100208
209 /* the caller will take care of speculative events */
Willy Tarreau4f60f162007-04-08 16:39:58 +0200210}
211
212/*
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100213 * Initialization of the speculative epoll() poller.
Willy Tarreaue54e9172007-04-09 09:23:31 +0200214 * Returns 0 in case of failure, non-zero in case of success. If it fails, it
215 * disables the poller by setting its pref to 0.
216 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200217REGPRM1 static int _do_init(struct poller *p)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200218{
Willy Tarreaue54e9172007-04-09 09:23:31 +0200219 p->private = NULL;
Willy Tarreaue54e9172007-04-09 09:23:31 +0200220
221 epoll_fd = epoll_create(global.maxsock + 1);
222 if (epoll_fd < 0)
223 goto fail_fd;
224
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100225 /* See comments at the top of the file about this formula. */
226 absmaxevents = MAX(global.tune.maxpollevents, global.maxsock);
Willy Tarreaue54e9172007-04-09 09:23:31 +0200227 epoll_events = (struct epoll_event*)
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100228 calloc(1, sizeof(struct epoll_event) * absmaxevents);
Willy Tarreaue54e9172007-04-09 09:23:31 +0200229
230 if (epoll_events == NULL)
231 goto fail_ee;
232
Willy Tarreaue54e9172007-04-09 09:23:31 +0200233 return 1;
234
Willy Tarreaue54e9172007-04-09 09:23:31 +0200235 fail_ee:
236 close(epoll_fd);
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200237 epoll_fd = -1;
Willy Tarreaue54e9172007-04-09 09:23:31 +0200238 fail_fd:
239 p->pref = 0;
240 return 0;
241}
242
243/*
Willy Tarreaue9f49e72012-11-11 17:42:00 +0100244 * Termination of the speculative epoll() poller.
Willy Tarreaue54e9172007-04-09 09:23:31 +0200245 * Memory is released and the poller is marked as unselectable.
246 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200247REGPRM1 static void _do_term(struct poller *p)
Willy Tarreaue54e9172007-04-09 09:23:31 +0200248{
Willy Tarreaua534fea2008-08-03 12:19:50 +0200249 free(epoll_events);
Willy Tarreaue54e9172007-04-09 09:23:31 +0200250
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200251 if (epoll_fd >= 0) {
252 close(epoll_fd);
253 epoll_fd = -1;
254 }
Willy Tarreaue54e9172007-04-09 09:23:31 +0200255
Willy Tarreau58094f22007-04-10 01:33:20 +0200256 epoll_events = NULL;
Willy Tarreaue54e9172007-04-09 09:23:31 +0200257 p->private = NULL;
258 p->pref = 0;
259}
260
261/*
Willy Tarreau2ff76222007-04-09 19:29:56 +0200262 * Check that the poller works.
263 * Returns 1 if OK, otherwise 0.
264 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200265REGPRM1 static int _do_test(struct poller *p)
Willy Tarreau2ff76222007-04-09 19:29:56 +0200266{
267 int fd;
268
269 fd = epoll_create(global.maxsock + 1);
270 if (fd < 0)
271 return 0;
272 close(fd);
273 return 1;
274}
275
276/*
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200277 * Recreate the epoll file descriptor after a fork(). Returns 1 if OK,
278 * otherwise 0. It will ensure that all processes will not share their
279 * epoll_fd. Some side effects were encountered because of this, such
280 * as epoll_wait() returning an FD which was previously deleted.
281 */
282REGPRM1 static int _do_fork(struct poller *p)
283{
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200284 if (epoll_fd >= 0)
285 close(epoll_fd);
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200286 epoll_fd = epoll_create(global.maxsock + 1);
287 if (epoll_fd < 0)
288 return 0;
289 return 1;
290}
291
292/*
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200293 * It is a constructor, which means that it will automatically be called before
294 * main(). This is GCC-specific but it works at least since 2.95.
295 * Special care must be taken so that it does not need any uninitialized data.
Willy Tarreau4f60f162007-04-08 16:39:58 +0200296 */
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200297__attribute__((constructor))
298static void _do_register(void)
Willy Tarreau4f60f162007-04-08 16:39:58 +0200299{
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200300 struct poller *p;
301
302 if (nbpollers >= MAX_POLLERS)
303 return;
Willy Tarreaud79e79b2009-05-10 10:18:54 +0200304
305 epoll_fd = -1;
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200306 p = &pollers[nbpollers++];
307
Willy Tarreau4f60f162007-04-08 16:39:58 +0200308 p->name = "epoll";
309 p->pref = 300;
310 p->private = NULL;
311
Willy Tarreau70c6fd82012-11-11 21:02:34 +0100312 p->clo = NULL;
Willy Tarreauef1d1f82007-04-16 00:25:25 +0200313 p->test = _do_test;
314 p->init = _do_init;
315 p->term = _do_term;
316 p->poll = _do_poll;
Willy Tarreaufb8983f2007-06-03 16:40:44 +0200317 p->fork = _do_fork;
Willy Tarreau4f60f162007-04-08 16:39:58 +0200318}
319
320
321/*
322 * Local variables:
323 * c-indent-level: 8
324 * c-basic-offset: 8
325 * End:
326 */