William Lallemand | 48dfbbd | 2019-04-01 11:29:53 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Master Worker |
| 3 | * |
| 4 | * Copyright HAProxy Technologies 2019 - William Lallemand <wlallemand@haproxy.com> |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or |
| 7 | * modify it under the terms of the GNU General Public License |
| 8 | * as published by the Free Software Foundation; either version |
| 9 | * 2 of the License, or (at your option) any later version. |
| 10 | * |
| 11 | */ |
| 12 | |
William Lallemand | 3fa724d | 2019-04-01 11:29:55 +0200 | [diff] [blame] | 13 | #include <errno.h> |
| 14 | #include <fcntl.h> |
| 15 | #include <signal.h> |
William Lallemand | 48dfbbd | 2019-04-01 11:29:53 +0200 | [diff] [blame] | 16 | #include <stdlib.h> |
| 17 | #include <string.h> |
William Lallemand | e25473c | 2019-04-01 11:29:56 +0200 | [diff] [blame^] | 18 | #include <sys/wait.h> |
William Lallemand | 48dfbbd | 2019-04-01 11:29:53 +0200 | [diff] [blame] | 19 | |
| 20 | #include <common/mini-clist.h> |
| 21 | |
William Lallemand | 3fa724d | 2019-04-01 11:29:55 +0200 | [diff] [blame] | 22 | #include <proto/fd.h> |
| 23 | #include <proto/listener.h> |
William Lallemand | e25473c | 2019-04-01 11:29:56 +0200 | [diff] [blame^] | 24 | #include <proto/log.h> |
William Lallemand | 48dfbbd | 2019-04-01 11:29:53 +0200 | [diff] [blame] | 25 | #include <proto/mworker.h> |
William Lallemand | 3cd95d2 | 2019-04-01 11:29:54 +0200 | [diff] [blame] | 26 | #include <proto/signal.h> |
William Lallemand | 48dfbbd | 2019-04-01 11:29:53 +0200 | [diff] [blame] | 27 | |
| 28 | #include <types/global.h> |
William Lallemand | e25473c | 2019-04-01 11:29:56 +0200 | [diff] [blame^] | 29 | #include <types/signal.h> |
William Lallemand | 48dfbbd | 2019-04-01 11:29:53 +0200 | [diff] [blame] | 30 | |
William Lallemand | e25473c | 2019-04-01 11:29:56 +0200 | [diff] [blame^] | 31 | #if defined(USE_SYSTEMD) |
| 32 | #include <systemd/sd-daemon.h> |
| 33 | #endif |
| 34 | |
| 35 | static int exitcode = -1; |
| 36 | |
| 37 | int *children = NULL; /* store PIDs of children in master workers mode */ |
| 38 | |
| 39 | /* ----- children processes handling ----- */ |
William Lallemand | 48dfbbd | 2019-04-01 11:29:53 +0200 | [diff] [blame] | 40 | |
William Lallemand | 48dfbbd | 2019-04-01 11:29:53 +0200 | [diff] [blame] | 41 | /* |
William Lallemand | e25473c | 2019-04-01 11:29:56 +0200 | [diff] [blame^] | 42 | * Send signal to every known children. |
| 43 | */ |
| 44 | |
| 45 | static void mworker_kill(int sig) |
| 46 | { |
| 47 | int i; |
| 48 | |
| 49 | /* TODO: merge mworker_kill and tell_old_pids for mworker mode */ |
| 50 | tell_old_pids(sig); |
| 51 | if (children) { |
| 52 | for (i = 0; i < global.nbproc; i++) |
| 53 | kill(children[i], sig); |
| 54 | } |
| 55 | } |
| 56 | |
| 57 | |
| 58 | /* return 1 if a pid is a current child otherwise 0 */ |
| 59 | int current_child(int pid) |
| 60 | { |
| 61 | int i; |
| 62 | |
| 63 | for (i = 0; i < global.nbproc; i++) { |
| 64 | if (children[i] == pid) |
| 65 | return 1; |
| 66 | } |
| 67 | return 0; |
| 68 | } |
| 69 | |
| 70 | /* |
William Lallemand | 48dfbbd | 2019-04-01 11:29:53 +0200 | [diff] [blame] | 71 | * serialize the proc list and put it in the environment |
| 72 | */ |
| 73 | void mworker_proc_list_to_env() |
| 74 | { |
| 75 | char *msg = NULL; |
| 76 | struct mworker_proc *child; |
| 77 | |
| 78 | list_for_each_entry(child, &proc_list, list) { |
| 79 | if (child->pid > -1) |
| 80 | memprintf(&msg, "%s|type=%c;fd=%d;pid=%d;rpid=%d;reloads=%d;timestamp=%d", msg ? msg : "", child->type, child->ipc_fd[0], child->pid, child->relative_pid, child->reloads, child->timestamp); |
| 81 | } |
| 82 | if (msg) |
| 83 | setenv("HAPROXY_PROCESSES", msg, 1); |
| 84 | } |
| 85 | |
| 86 | /* |
| 87 | * unserialize the proc list from the environment |
| 88 | */ |
| 89 | void mworker_env_to_proc_list() |
| 90 | { |
| 91 | char *msg, *token = NULL, *s1; |
| 92 | |
| 93 | msg = getenv("HAPROXY_PROCESSES"); |
| 94 | if (!msg) |
| 95 | return; |
| 96 | |
| 97 | while ((token = strtok_r(msg, "|", &s1))) { |
| 98 | struct mworker_proc *child; |
| 99 | char *subtoken = NULL; |
| 100 | char *s2; |
| 101 | |
| 102 | msg = NULL; |
| 103 | |
| 104 | child = calloc(1, sizeof(*child)); |
| 105 | |
| 106 | while ((subtoken = strtok_r(token, ";", &s2))) { |
| 107 | |
| 108 | token = NULL; |
| 109 | |
| 110 | if (strncmp(subtoken, "type=", 5) == 0) { |
| 111 | child->type = *(subtoken+5); |
| 112 | if (child->type == 'm') /* we are in the master, assign it */ |
| 113 | proc_self = child; |
| 114 | } else if (strncmp(subtoken, "fd=", 3) == 0) { |
| 115 | child->ipc_fd[0] = atoi(subtoken+3); |
| 116 | } else if (strncmp(subtoken, "pid=", 4) == 0) { |
| 117 | child->pid = atoi(subtoken+4); |
| 118 | } else if (strncmp(subtoken, "rpid=", 5) == 0) { |
| 119 | child->relative_pid = atoi(subtoken+5); |
| 120 | } else if (strncmp(subtoken, "reloads=", 8) == 0) { |
| 121 | /* we reloaded this process once more */ |
| 122 | child->reloads = atoi(subtoken+8) + 1; |
| 123 | } else if (strncmp(subtoken, "timestamp=", 10) == 0) { |
| 124 | child->timestamp = atoi(subtoken+10); |
| 125 | } |
| 126 | } |
| 127 | if (child->pid) |
| 128 | LIST_ADDQ(&proc_list, &child->list); |
| 129 | else |
| 130 | free(child); |
| 131 | } |
| 132 | |
| 133 | unsetenv("HAPROXY_PROCESSES"); |
| 134 | } |
William Lallemand | 3cd95d2 | 2019-04-01 11:29:54 +0200 | [diff] [blame] | 135 | |
| 136 | /* Signal blocking and unblocking */ |
| 137 | |
| 138 | void mworker_block_signals() |
| 139 | { |
| 140 | sigset_t set; |
| 141 | |
| 142 | sigemptyset(&set); |
| 143 | sigaddset(&set, SIGUSR1); |
| 144 | sigaddset(&set, SIGUSR2); |
| 145 | sigaddset(&set, SIGHUP); |
| 146 | sigaddset(&set, SIGCHLD); |
| 147 | ha_sigmask(SIG_SETMASK, &set, NULL); |
| 148 | } |
| 149 | |
| 150 | void mworker_unblock_signals() |
| 151 | { |
| 152 | haproxy_unblock_signals(); |
| 153 | } |
William Lallemand | 3fa724d | 2019-04-01 11:29:55 +0200 | [diff] [blame] | 154 | |
William Lallemand | e25473c | 2019-04-01 11:29:56 +0200 | [diff] [blame^] | 155 | /* ----- mworker signal handlers ----- */ |
| 156 | |
| 157 | /* |
| 158 | * When called, this function reexec haproxy with -sf followed by current |
| 159 | * children PIDs and possibly old children PIDs if they didn't leave yet. |
| 160 | */ |
| 161 | void mworker_catch_sighup(struct sig_handler *sh) |
| 162 | { |
| 163 | mworker_reload(); |
| 164 | } |
| 165 | |
| 166 | void mworker_catch_sigterm(struct sig_handler *sh) |
| 167 | { |
| 168 | int sig = sh->arg; |
| 169 | |
| 170 | #if defined(USE_SYSTEMD) |
| 171 | if (global.tune.options & GTUNE_USE_SYSTEMD) { |
| 172 | sd_notify(0, "STOPPING=1"); |
| 173 | } |
| 174 | #endif |
| 175 | ha_warning("Exiting Master process...\n"); |
| 176 | mworker_kill(sig); |
| 177 | } |
| 178 | |
| 179 | /* |
| 180 | * Wait for every children to exit |
| 181 | */ |
| 182 | |
| 183 | void mworker_catch_sigchld(struct sig_handler *sh) |
| 184 | { |
| 185 | int exitpid = -1; |
| 186 | int status = 0; |
| 187 | struct mworker_proc *child, *it; |
| 188 | int childfound; |
| 189 | |
| 190 | restart_wait: |
| 191 | |
| 192 | childfound = 0; |
| 193 | |
| 194 | exitpid = waitpid(-1, &status, WNOHANG); |
| 195 | if (exitpid > 0) { |
| 196 | if (WIFEXITED(status)) |
| 197 | status = WEXITSTATUS(status); |
| 198 | else if (WIFSIGNALED(status)) |
| 199 | status = 128 + WTERMSIG(status); |
| 200 | else if (WIFSTOPPED(status)) |
| 201 | status = 128 + WSTOPSIG(status); |
| 202 | else |
| 203 | status = 255; |
| 204 | |
| 205 | list_for_each_entry_safe(child, it, &proc_list, list) { |
| 206 | if (child->pid != exitpid) |
| 207 | continue; |
| 208 | |
| 209 | LIST_DEL(&child->list); |
| 210 | close(child->ipc_fd[0]); |
| 211 | childfound = 1; |
| 212 | break; |
| 213 | } |
| 214 | |
| 215 | if (!children || !childfound) { |
| 216 | ha_warning("Worker %d exited with code %d (%s)\n", exitpid, status, (status >= 128) ? strsignal(status - 128) : "Exit"); |
| 217 | } else { |
| 218 | /* check if exited child was in the current children list */ |
| 219 | if (current_child(exitpid)) { |
| 220 | ha_alert("Current worker #%d (%d) exited with code %d (%s)\n", child->relative_pid, exitpid, status, (status >= 128) ? strsignal(status - 128) : "Exit"); |
| 221 | if (status != 0 && status != 130 && status != 143 |
| 222 | && !(global.tune.options & GTUNE_NOEXIT_ONFAILURE)) { |
| 223 | ha_alert("exit-on-failure: killing every workers with SIGTERM\n"); |
| 224 | if (exitcode < 0) |
| 225 | exitcode = status; |
| 226 | mworker_kill(SIGTERM); |
| 227 | } |
| 228 | } else { |
| 229 | ha_warning("Former worker #%d (%d) exited with code %d (%s)\n", child->relative_pid, exitpid, status, (status >= 128) ? strsignal(status - 128) : "Exit"); |
| 230 | /* TODO: merge children and oldpids list in mworker mode */ |
| 231 | delete_oldpid(exitpid); |
| 232 | } |
| 233 | free(child); |
| 234 | } |
| 235 | |
| 236 | /* do it again to check if it was the last worker */ |
| 237 | goto restart_wait; |
| 238 | } |
| 239 | /* Better rely on the system than on a list of process to check if it was the last one */ |
| 240 | else if (exitpid == -1 && errno == ECHILD) { |
| 241 | ha_warning("All workers exited. Exiting... (%d)\n", (exitcode > 0) ? exitcode : status); |
| 242 | atexit_flag = 0; |
| 243 | if (exitcode > 0) |
| 244 | exit(exitcode); |
| 245 | exit(status); /* parent must leave using the latest status code known */ |
| 246 | } |
| 247 | |
| 248 | } |
| 249 | |
William Lallemand | 3fa724d | 2019-04-01 11:29:55 +0200 | [diff] [blame] | 250 | /* ----- IPC FD (sockpair) related ----- */ |
| 251 | |
| 252 | /* This wrapper is called from the workers. It is registered instead of the |
| 253 | * normal listener_accept() so the worker can exit() when it detects that the |
| 254 | * master closed the IPC FD. If it's not a close, we just call the regular |
| 255 | * listener_accept() function */ |
| 256 | void mworker_accept_wrapper(int fd) |
| 257 | { |
| 258 | char c; |
| 259 | int ret; |
| 260 | |
| 261 | while (1) { |
| 262 | ret = recv(fd, &c, 1, MSG_PEEK); |
| 263 | if (ret == -1) { |
| 264 | if (errno == EINTR) |
| 265 | continue; |
| 266 | if (errno == EAGAIN) { |
| 267 | fd_cant_recv(fd); |
| 268 | return; |
| 269 | } |
| 270 | break; |
| 271 | } else if (ret > 0) { |
| 272 | listener_accept(fd); |
| 273 | return; |
| 274 | } else if (ret == 0) { |
| 275 | /* At this step the master is down before |
| 276 | * this worker perform a 'normal' exit. |
| 277 | * So we want to exit with an error but |
| 278 | * other threads could currently process |
| 279 | * some stuff so we can't perform a clean |
| 280 | * deinit(). |
| 281 | */ |
| 282 | exit(EXIT_FAILURE); |
| 283 | } |
| 284 | } |
| 285 | return; |
| 286 | } |
| 287 | |
| 288 | /* |
| 289 | * This function register the accept wrapper for the sockpair of the master worker |
| 290 | */ |
| 291 | void mworker_pipe_register() |
| 292 | { |
| 293 | /* The iocb should be already initialized with listener_accept */ |
| 294 | if (fdtab[proc_self->ipc_fd[1]].iocb == mworker_accept_wrapper) |
| 295 | return; |
| 296 | |
| 297 | fcntl(proc_self->ipc_fd[1], F_SETFL, O_NONBLOCK); |
| 298 | /* In multi-tread, we need only one thread to process |
| 299 | * events on the pipe with master |
| 300 | */ |
| 301 | fd_insert(proc_self->ipc_fd[1], fdtab[proc_self->ipc_fd[1]].owner, mworker_accept_wrapper, 1); |
| 302 | fd_want_recv(proc_self->ipc_fd[1]); |
| 303 | } |