| /* |
| * FD polling functions for linux epoll() |
| * |
| * Copyright 2000-2008 Willy Tarreau <w@1wt.eu> |
| * |
| * This program is free software; you can redistribute it and/or |
| * modify it under the terms of the GNU General Public License |
| * as published by the Free Software Foundation; either version |
| * 2 of the License, or (at your option) any later version. |
| * |
| */ |
| |
| #include <unistd.h> |
| #include <sys/time.h> |
| #include <sys/types.h> |
| |
| #include <common/compat.h> |
| #include <common/config.h> |
| #include <common/standard.h> |
| #include <common/ticks.h> |
| #include <common/time.h> |
| #include <common/tools.h> |
| |
| #include <types/fd.h> |
| #include <types/global.h> |
| |
| #include <proto/signal.h> |
| #include <proto/task.h> |
| |
| #if defined(USE_MY_EPOLL) |
| #include <common/epoll.h> |
| #include <errno.h> |
| #include <sys/syscall.h> |
| static _syscall1 (int, epoll_create, int, size); |
| static _syscall4 (int, epoll_ctl, int, epfd, int, op, int, fd, struct epoll_event *, event); |
| static _syscall4 (int, epoll_wait, int, epfd, struct epoll_event *, events, int, maxevents, int, timeout); |
| #else |
| #include <sys/epoll.h> |
| #endif |
| |
| /* This is what we store in a list. It consists in old values and fds to detect changes. */ |
| struct fd_chg { |
| unsigned int prev:2; // previous state mask. New one is in fd_evts. |
| unsigned int fd:30; // file descriptor |
| }; |
| |
| static int nbchanges = 0; // number of changes pending |
| static struct fd_chg *chg_list = NULL; // list of changes |
| static struct fd_chg **chg_ptr = NULL; // per-fd changes |
| |
| /* Each 32-bit word contains 2-bit descriptors of the latest state for 16 FDs : |
| * desc = (u32 >> (2*fd)) & 3 |
| * desc = 0 : FD not set |
| * 1 : WRITE not set, READ set |
| * 2 : WRITE set, READ not set |
| * 3 : WRITE set, READ set |
| */ |
| |
| static uint32_t *fd_evts; |
| |
| /* private data */ |
| static struct epoll_event *epoll_events; |
| static int epoll_fd; |
| |
| /* This structure may be used for any purpose. Warning! do not use it in |
| * recursive functions ! |
| */ |
| static struct epoll_event ev; |
| |
| /* converts a direction to a single bitmask. |
| * 0 => 1 |
| * 1 => 2 |
| */ |
| #define DIR2MSK(dir) ((dir) + 1) |
| |
| /* converts an FD to an fd_evts offset and to a bit shift */ |
| #define FD2OFS(fd) ((uint32_t)(fd) >> 4) |
| #define FD2BIT(fd) (((uint32_t)(fd) & 0xF) << 1) |
| #define FD2MSK(fd) (3 << FD2BIT(fd)) |
| |
| /* |
| * Returns non-zero if direction <dir> is already set for <fd>. |
| */ |
| REGPRM2 static int __fd_is_set(const int fd, int dir) |
| { |
| return (fd_evts[FD2OFS(fd)] >> FD2BIT(fd)) & DIR2MSK(dir); |
| } |
| |
| /* |
| * Adds, mods or deletes <fd> according to current status and to new desired |
| * mask <dmask> : |
| * |
| * 0 = nothing |
| * 1 = EPOLLIN |
| * 2 = EPOLLOUT |
| * 3 = EPOLLIN | EPOLLOUT |
| * |
| */ |
| static int dmsk2event[4] = { 0, EPOLLIN, EPOLLOUT, EPOLLIN | EPOLLOUT }; |
| |
| |
| REGPRM2 static void fd_flush_changes() |
| { |
| uint32_t ofs; |
| int opcode; |
| int prev, next; |
| int chg, fd; |
| |
| for (chg = 0; chg < nbchanges; chg++) { |
| prev = chg_list[chg].prev; |
| fd = chg_list[chg].fd; |
| chg_ptr[fd] = NULL; |
| |
| ofs = FD2OFS(fd); |
| next = (fd_evts[ofs] >> FD2BIT(fd)) & 3; |
| |
| if (prev == next) |
| /* if the value was unchanged, do nothing */ |
| continue; |
| |
| ev.events = dmsk2event[next]; |
| ev.data.fd = fd; |
| |
| if (prev) { |
| if (!next) { |
| /* we want to delete it now */ |
| opcode = EPOLL_CTL_DEL; |
| } else { |
| /* we want to switch it */ |
| opcode = EPOLL_CTL_MOD; |
| } |
| } else { |
| /* the FD did not exist, let's add it */ |
| opcode = EPOLL_CTL_ADD; |
| } |
| |
| epoll_ctl(epoll_fd, opcode, fd, &ev); |
| } |
| nbchanges = 0; |
| } |
| |
| REGPRM2 static void alloc_chg_list(const int fd, int old_evt) |
| { |
| struct fd_chg *ptr; |
| |
| if (unlikely(chg_ptr[fd] != NULL)) |
| return; |
| |
| #if LIMIT_NUMBER_OF_CHANGES |
| if (nbchanges > 2) |
| fd_flush_changes(); |
| #endif |
| |
| ptr = &chg_list[nbchanges++]; |
| chg_ptr[fd] = ptr; |
| ptr->fd = fd; |
| ptr->prev = old_evt; |
| } |
| |
| REGPRM2 static int __fd_set(const int fd, int dir) |
| { |
| uint32_t ofs = FD2OFS(fd); |
| uint32_t dmsk = DIR2MSK(dir); |
| uint32_t old_evt; |
| |
| old_evt = fd_evts[ofs] >> FD2BIT(fd); |
| old_evt &= 3; |
| if (unlikely(old_evt & dmsk)) |
| return 0; |
| |
| alloc_chg_list(fd, old_evt); |
| dmsk <<= FD2BIT(fd); |
| fd_evts[ofs] |= dmsk; |
| return 1; |
| } |
| |
| REGPRM2 static int __fd_clr(const int fd, int dir) |
| { |
| uint32_t ofs = FD2OFS(fd); |
| uint32_t dmsk = DIR2MSK(dir); |
| uint32_t old_evt; |
| |
| old_evt = fd_evts[ofs] >> FD2BIT(fd); |
| old_evt &= 3; |
| if (unlikely(!(old_evt & dmsk))) |
| return 0; |
| |
| alloc_chg_list(fd, old_evt); |
| dmsk <<= FD2BIT(fd); |
| fd_evts[ofs] &= ~dmsk; |
| return 1; |
| } |
| |
| REGPRM1 static void __fd_rem(int fd) |
| { |
| uint32_t ofs = FD2OFS(fd); |
| |
| if (unlikely(!((fd_evts[ofs] >> FD2BIT(fd)) & 3))) |
| return; |
| |
| alloc_chg_list(fd, 0); |
| fd_evts[ofs] &= ~FD2MSK(fd); |
| return; |
| } |
| |
| /* |
| * On valid epoll() implementations, a call to close() automatically removes |
| * the fds. This means that the FD will appear as previously unset. |
| */ |
| REGPRM1 static void __fd_clo(int fd) |
| { |
| struct fd_chg *ptr; |
| fd_evts[FD2OFS(fd)] &= ~FD2MSK(fd); |
| ptr = chg_ptr[fd]; |
| if (ptr) { |
| ptr->prev = 0; |
| chg_ptr[fd] = NULL; |
| } |
| return; |
| } |
| |
| /* |
| * epoll() poller |
| */ |
| REGPRM2 static void _do_poll(struct poller *p, int exp) |
| { |
| int status; |
| int fd; |
| int count; |
| int wait_time; |
| |
| if (likely(nbchanges)) |
| fd_flush_changes(); |
| |
| /* now let's wait for events */ |
| if (run_queue || signal_queue_len) |
| wait_time = 0; |
| else if (!exp) |
| wait_time = MAX_DELAY_MS; |
| else if (tick_is_expired(exp, now_ms)) |
| wait_time = 0; |
| else { |
| wait_time = TICKS_TO_MS(tick_remain(now_ms, exp)) + 1; |
| if (wait_time > MAX_DELAY_MS) |
| wait_time = MAX_DELAY_MS; |
| } |
| |
| fd = MIN(maxfd, global.tune.maxpollevents); |
| status = epoll_wait(epoll_fd, epoll_events, fd, wait_time); |
| tv_update_date(wait_time, status); |
| |
| for (count = 0; count < status; count++) { |
| fd = epoll_events[count].data.fd; |
| |
| if ((fd_evts[FD2OFS(fd)] >> FD2BIT(fd)) & DIR2MSK(DIR_RD)) { |
| if (fdtab[fd].state == FD_STCLOSE) |
| continue; |
| if (epoll_events[count].events & ( EPOLLIN | EPOLLERR | EPOLLHUP )) |
| fdtab[fd].cb[DIR_RD].f(fd); |
| } |
| |
| if ((fd_evts[FD2OFS(fd)] >> FD2BIT(fd)) & DIR2MSK(DIR_WR)) { |
| if (fdtab[fd].state == FD_STCLOSE) |
| continue; |
| if (epoll_events[count].events & ( EPOLLOUT | EPOLLERR | EPOLLHUP )) |
| fdtab[fd].cb[DIR_WR].f(fd); |
| } |
| } |
| } |
| |
| /* |
| * Initialization of the epoll() poller. |
| * Returns 0 in case of failure, non-zero in case of success. If it fails, it |
| * disables the poller by setting its pref to 0. |
| */ |
| REGPRM1 static int _do_init(struct poller *p) |
| { |
| __label__ fail_chg_ptr, fail_chg_list, fail_fdevt, fail_ee, fail_fd; |
| int fd_set_bytes; |
| |
| p->private = NULL; |
| fd_set_bytes = 4 * (global.maxsock + 15) / 16; |
| |
| epoll_fd = epoll_create(global.maxsock + 1); |
| if (epoll_fd < 0) |
| goto fail_fd; |
| |
| epoll_events = (struct epoll_event*) |
| calloc(1, sizeof(struct epoll_event) * global.tune.maxpollevents); |
| |
| if (epoll_events == NULL) |
| goto fail_ee; |
| |
| if ((fd_evts = (uint32_t *)calloc(1, fd_set_bytes)) == NULL) |
| goto fail_fdevt; |
| |
| chg_list = (struct fd_chg *)calloc(1, sizeof(struct fd_chg) * global.maxsock); |
| if (chg_list == NULL) |
| goto fail_chg_list; |
| |
| chg_ptr = (struct fd_chg **)calloc(1, sizeof(struct fd_chg*) * global.maxsock); |
| if (chg_ptr == NULL) |
| goto fail_chg_ptr; |
| |
| return 1; |
| |
| fail_chg_ptr: |
| free(chg_list); |
| fail_chg_list: |
| free(fd_evts); |
| fail_fdevt: |
| free(epoll_events); |
| fail_ee: |
| close(epoll_fd); |
| epoll_fd = -1; |
| fail_fd: |
| p->pref = 0; |
| return 0; |
| } |
| |
| /* |
| * Termination of the epoll() poller. |
| * Memory is released and the poller is marked as unselectable. |
| */ |
| REGPRM1 static void _do_term(struct poller *p) |
| { |
| fd_flush_changes(); |
| |
| free(chg_ptr); |
| free(chg_list); |
| free(fd_evts); |
| free(epoll_events); |
| |
| if (epoll_fd >= 0) { |
| close(epoll_fd); |
| epoll_fd = -1; |
| } |
| |
| chg_ptr = NULL; |
| chg_list = NULL; |
| fd_evts = NULL; |
| epoll_events = NULL; |
| |
| p->private = NULL; |
| p->pref = 0; |
| } |
| |
| /* |
| * Check that the poller works. |
| * Returns 1 if OK, otherwise 0. |
| */ |
| REGPRM1 static int _do_test(struct poller *p) |
| { |
| int fd; |
| |
| fd = epoll_create(global.maxsock + 1); |
| if (fd < 0) |
| return 0; |
| close(fd); |
| return 1; |
| } |
| |
| /* |
| * Recreate the epoll file descriptor after a fork(). Returns 1 if OK, |
| * otherwise 0. It will ensure that all processes will not share their |
| * epoll_fd. Some side effects were encountered because of this, such |
| * as epoll_wait() returning an FD which was previously deleted. |
| */ |
| REGPRM1 static int _do_fork(struct poller *p) |
| { |
| if (epoll_fd >= 0) |
| close(epoll_fd); |
| epoll_fd = epoll_create(global.maxsock + 1); |
| if (epoll_fd < 0) |
| return 0; |
| return 1; |
| } |
| |
| /* |
| * It is a constructor, which means that it will automatically be called before |
| * main(). This is GCC-specific but it works at least since 2.95. |
| * Special care must be taken so that it does not need any uninitialized data. |
| */ |
| __attribute__((constructor)) |
| static void _do_register(void) |
| { |
| struct poller *p; |
| |
| if (nbpollers >= MAX_POLLERS) |
| return; |
| |
| epoll_fd = -1; |
| p = &pollers[nbpollers++]; |
| |
| p->name = "epoll"; |
| p->pref = 300; |
| p->private = NULL; |
| |
| p->test = _do_test; |
| p->init = _do_init; |
| p->term = _do_term; |
| p->poll = _do_poll; |
| p->fork = _do_fork; |
| |
| p->is_set = __fd_is_set; |
| p->cond_s = p->set = __fd_set; |
| p->cond_c = p->clr = __fd_clr; |
| p->rem = __fd_rem; |
| p->clo = __fd_clo; |
| } |
| |
| |
| /* |
| * Local variables: |
| * c-indent-level: 8 |
| * c-basic-offset: 8 |
| * End: |
| */ |