Willy Tarreau | 18b7df7 | 2020-08-28 12:07:22 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Generic code for native (BSD-compatible) sockets |
| 3 | * |
| 4 | * Copyright 2000-2020 Willy Tarreau <w@1wt.eu> |
| 5 | * |
| 6 | * This program is free software; you can redistribute it and/or |
| 7 | * modify it under the terms of the GNU General Public License |
| 8 | * as published by the Free Software Foundation; either version |
| 9 | * 2 of the License, or (at your option) any later version. |
| 10 | * |
| 11 | */ |
| 12 | |
| 13 | #include <ctype.h> |
| 14 | #include <errno.h> |
| 15 | #include <fcntl.h> |
| 16 | #include <stdio.h> |
| 17 | #include <stdlib.h> |
| 18 | #include <string.h> |
| 19 | #include <time.h> |
| 20 | |
| 21 | #include <sys/param.h> |
| 22 | #include <sys/socket.h> |
| 23 | #include <sys/types.h> |
| 24 | |
Willy Tarreau | 4296174 | 2020-08-28 18:42:45 +0200 | [diff] [blame] | 25 | #include <net/if.h> |
| 26 | |
Willy Tarreau | 18b7df7 | 2020-08-28 12:07:22 +0200 | [diff] [blame] | 27 | #include <haproxy/api.h> |
| 28 | #include <haproxy/connection.h> |
Willy Tarreau | 063d47d | 2020-08-28 16:29:53 +0200 | [diff] [blame] | 29 | #include <haproxy/listener-t.h> |
Willy Tarreau | 18b7df7 | 2020-08-28 12:07:22 +0200 | [diff] [blame] | 30 | #include <haproxy/namespace.h> |
| 31 | #include <haproxy/sock.h> |
Willy Tarreau | 2d34a71 | 2020-08-28 16:49:41 +0200 | [diff] [blame] | 32 | #include <haproxy/sock_inet.h> |
Willy Tarreau | 18b7df7 | 2020-08-28 12:07:22 +0200 | [diff] [blame] | 33 | #include <haproxy/tools.h> |
| 34 | |
Willy Tarreau | 063d47d | 2020-08-28 16:29:53 +0200 | [diff] [blame] | 35 | /* the list of remaining sockets transferred from an older process */ |
| 36 | struct xfer_sock_list *xfer_sock_list = NULL; |
Willy Tarreau | 18b7df7 | 2020-08-28 12:07:22 +0200 | [diff] [blame] | 37 | |
| 38 | /* Create a socket to connect to the server in conn->dst (which MUST be valid), |
| 39 | * using the configured namespace if needed, or the one passed by the proxy |
| 40 | * protocol if required to do so. It ultimately calls socket() or socketat() |
| 41 | * and returns the FD or error code. |
| 42 | */ |
| 43 | int sock_create_server_socket(struct connection *conn) |
| 44 | { |
| 45 | const struct netns_entry *ns = NULL; |
| 46 | |
| 47 | #ifdef USE_NS |
| 48 | if (objt_server(conn->target)) { |
| 49 | if (__objt_server(conn->target)->flags & SRV_F_USE_NS_FROM_PP) |
| 50 | ns = conn->proxy_netns; |
| 51 | else |
| 52 | ns = __objt_server(conn->target)->netns; |
| 53 | } |
| 54 | #endif |
| 55 | return my_socketat(ns, conn->dst->ss_family, SOCK_STREAM, 0); |
| 56 | } |
| 57 | |
| 58 | /* |
| 59 | * Retrieves the source address for the socket <fd>, with <dir> indicating |
| 60 | * if we're a listener (=0) or an initiator (!=0). It returns 0 in case of |
| 61 | * success, -1 in case of error. The socket's source address is stored in |
| 62 | * <sa> for <salen> bytes. |
| 63 | */ |
| 64 | int sock_get_src(int fd, struct sockaddr *sa, socklen_t salen, int dir) |
| 65 | { |
| 66 | if (dir) |
| 67 | return getsockname(fd, sa, &salen); |
| 68 | else |
| 69 | return getpeername(fd, sa, &salen); |
| 70 | } |
| 71 | |
| 72 | /* |
| 73 | * Retrieves the original destination address for the socket <fd>, with <dir> |
| 74 | * indicating if we're a listener (=0) or an initiator (!=0). It returns 0 in |
| 75 | * case of success, -1 in case of error. The socket's source address is stored |
| 76 | * in <sa> for <salen> bytes. |
| 77 | */ |
| 78 | int sock_get_dst(int fd, struct sockaddr *sa, socklen_t salen, int dir) |
| 79 | { |
| 80 | if (dir) |
| 81 | return getpeername(fd, sa, &salen); |
| 82 | else |
| 83 | return getsockname(fd, sa, &salen); |
| 84 | } |
| 85 | |
Willy Tarreau | 4296174 | 2020-08-28 18:42:45 +0200 | [diff] [blame] | 86 | /* Try to retrieve exported sockets from worker at CLI <unixsocket>. These |
| 87 | * ones will be placed into the xfer_sock_list for later use by function |
| 88 | * sock_find_compatible_fd(). Returns 0 on success, -1 on failure. |
| 89 | */ |
| 90 | int sock_get_old_sockets(const char *unixsocket) |
| 91 | { |
| 92 | char *cmsgbuf = NULL, *tmpbuf = NULL; |
| 93 | int *tmpfd = NULL; |
| 94 | struct sockaddr_un addr; |
| 95 | struct cmsghdr *cmsg; |
| 96 | struct msghdr msghdr; |
| 97 | struct iovec iov; |
| 98 | struct xfer_sock_list *xfer_sock = NULL; |
| 99 | struct timeval tv = { .tv_sec = 1, .tv_usec = 0 }; |
| 100 | int sock = -1; |
| 101 | int ret = -1; |
| 102 | int ret2 = -1; |
| 103 | int fd_nb; |
| 104 | int got_fd = 0; |
| 105 | int cur_fd = 0; |
| 106 | size_t maxoff = 0, curoff = 0; |
| 107 | |
| 108 | memset(&msghdr, 0, sizeof(msghdr)); |
| 109 | cmsgbuf = malloc(CMSG_SPACE(sizeof(int)) * MAX_SEND_FD); |
| 110 | if (!cmsgbuf) { |
| 111 | ha_warning("Failed to allocate memory to send sockets\n"); |
| 112 | goto out; |
| 113 | } |
| 114 | |
| 115 | sock = socket(PF_UNIX, SOCK_STREAM, 0); |
| 116 | if (sock < 0) { |
| 117 | ha_warning("Failed to connect to the old process socket '%s'\n", unixsocket); |
| 118 | goto out; |
| 119 | } |
| 120 | |
| 121 | strncpy(addr.sun_path, unixsocket, sizeof(addr.sun_path) - 1); |
| 122 | addr.sun_path[sizeof(addr.sun_path) - 1] = 0; |
| 123 | addr.sun_family = PF_UNIX; |
| 124 | |
| 125 | ret = connect(sock, (struct sockaddr *)&addr, sizeof(addr)); |
| 126 | if (ret < 0) { |
| 127 | ha_warning("Failed to connect to the old process socket '%s'\n", unixsocket); |
| 128 | goto out; |
| 129 | } |
| 130 | |
| 131 | setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (void *)&tv, sizeof(tv)); |
| 132 | iov.iov_base = &fd_nb; |
| 133 | iov.iov_len = sizeof(fd_nb); |
| 134 | msghdr.msg_iov = &iov; |
| 135 | msghdr.msg_iovlen = 1; |
| 136 | |
| 137 | if (send(sock, "_getsocks\n", strlen("_getsocks\n"), 0) != strlen("_getsocks\n")) { |
| 138 | ha_warning("Failed to get the number of sockets to be transferred !\n"); |
| 139 | goto out; |
| 140 | } |
| 141 | |
| 142 | /* First, get the number of file descriptors to be received */ |
| 143 | if (recvmsg(sock, &msghdr, MSG_WAITALL) != sizeof(fd_nb)) { |
| 144 | ha_warning("Failed to get the number of sockets to be transferred !\n"); |
| 145 | goto out; |
| 146 | } |
| 147 | |
| 148 | if (fd_nb == 0) { |
| 149 | ret2 = 0; |
| 150 | goto out; |
| 151 | } |
| 152 | |
| 153 | tmpbuf = malloc(fd_nb * (1 + MAXPATHLEN + 1 + IFNAMSIZ + sizeof(int))); |
| 154 | if (tmpbuf == NULL) { |
| 155 | ha_warning("Failed to allocate memory while receiving sockets\n"); |
| 156 | goto out; |
| 157 | } |
| 158 | |
| 159 | tmpfd = malloc(fd_nb * sizeof(int)); |
| 160 | if (tmpfd == NULL) { |
| 161 | ha_warning("Failed to allocate memory while receiving sockets\n"); |
| 162 | goto out; |
| 163 | } |
| 164 | |
| 165 | msghdr.msg_control = cmsgbuf; |
| 166 | msghdr.msg_controllen = CMSG_SPACE(sizeof(int)) * MAX_SEND_FD; |
| 167 | iov.iov_len = MAX_SEND_FD * (1 + MAXPATHLEN + 1 + IFNAMSIZ + sizeof(int)); |
| 168 | |
| 169 | do { |
| 170 | int ret3; |
| 171 | |
| 172 | iov.iov_base = tmpbuf + curoff; |
| 173 | |
| 174 | ret = recvmsg(sock, &msghdr, 0); |
| 175 | |
| 176 | if (ret == -1 && errno == EINTR) |
| 177 | continue; |
| 178 | |
| 179 | if (ret <= 0) |
| 180 | break; |
| 181 | |
| 182 | /* Send an ack to let the sender know we got the sockets |
| 183 | * and it can send some more |
| 184 | */ |
| 185 | do { |
| 186 | ret3 = send(sock, &got_fd, sizeof(got_fd), 0); |
| 187 | } while (ret3 == -1 && errno == EINTR); |
| 188 | |
| 189 | for (cmsg = CMSG_FIRSTHDR(&msghdr); cmsg != NULL; cmsg = CMSG_NXTHDR(&msghdr, cmsg)) { |
| 190 | if (cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS) { |
| 191 | size_t totlen = cmsg->cmsg_len - CMSG_LEN(0); |
| 192 | |
| 193 | if (totlen / sizeof(int) + got_fd > fd_nb) { |
| 194 | ha_warning("Got to many sockets !\n"); |
| 195 | goto out; |
| 196 | } |
| 197 | |
| 198 | /* |
| 199 | * Be paranoid and use memcpy() to avoid any |
| 200 | * potential alignment issue. |
| 201 | */ |
| 202 | memcpy(&tmpfd[got_fd], CMSG_DATA(cmsg), totlen); |
| 203 | got_fd += totlen / sizeof(int); |
| 204 | } |
| 205 | } |
| 206 | curoff += ret; |
| 207 | } while (got_fd < fd_nb); |
| 208 | |
| 209 | if (got_fd != fd_nb) { |
| 210 | ha_warning("We didn't get the expected number of sockets (expecting %d got %d)\n", |
| 211 | fd_nb, got_fd); |
| 212 | goto out; |
| 213 | } |
| 214 | |
| 215 | maxoff = curoff; |
| 216 | curoff = 0; |
| 217 | |
| 218 | for (cur_fd = 0; cur_fd < got_fd; cur_fd++) { |
| 219 | int fd = tmpfd[cur_fd]; |
| 220 | socklen_t socklen; |
| 221 | int val; |
| 222 | int len; |
| 223 | |
| 224 | xfer_sock = calloc(1, sizeof(*xfer_sock)); |
| 225 | if (!xfer_sock) { |
| 226 | ha_warning("Failed to allocate memory in get_old_sockets() !\n"); |
| 227 | break; |
| 228 | } |
| 229 | xfer_sock->fd = -1; |
| 230 | |
| 231 | socklen = sizeof(xfer_sock->addr); |
| 232 | if (getsockname(fd, (struct sockaddr *)&xfer_sock->addr, &socklen) != 0) { |
| 233 | ha_warning("Failed to get socket address\n"); |
| 234 | free(xfer_sock); |
| 235 | xfer_sock = NULL; |
| 236 | continue; |
| 237 | } |
| 238 | |
| 239 | if (curoff >= maxoff) { |
| 240 | ha_warning("Inconsistency while transferring sockets\n"); |
| 241 | goto out; |
| 242 | } |
| 243 | |
| 244 | len = tmpbuf[curoff++]; |
| 245 | if (len > 0) { |
| 246 | /* We have a namespace */ |
| 247 | if (curoff + len > maxoff) { |
| 248 | ha_warning("Inconsistency while transferring sockets\n"); |
| 249 | goto out; |
| 250 | } |
| 251 | xfer_sock->namespace = malloc(len + 1); |
| 252 | if (!xfer_sock->namespace) { |
| 253 | ha_warning("Failed to allocate memory while transferring sockets\n"); |
| 254 | goto out; |
| 255 | } |
| 256 | memcpy(xfer_sock->namespace, &tmpbuf[curoff], len); |
| 257 | xfer_sock->namespace[len] = 0; |
| 258 | xfer_sock->ns_namelen = len; |
| 259 | curoff += len; |
| 260 | } |
| 261 | |
| 262 | if (curoff >= maxoff) { |
| 263 | ha_warning("Inconsistency while transferring sockets\n"); |
| 264 | goto out; |
| 265 | } |
| 266 | |
| 267 | len = tmpbuf[curoff++]; |
| 268 | if (len > 0) { |
| 269 | /* We have an interface */ |
| 270 | if (curoff + len > maxoff) { |
| 271 | ha_warning("Inconsistency while transferring sockets\n"); |
| 272 | goto out; |
| 273 | } |
| 274 | xfer_sock->iface = malloc(len + 1); |
| 275 | if (!xfer_sock->iface) { |
| 276 | ha_warning("Failed to allocate memory while transferring sockets\n"); |
| 277 | goto out; |
| 278 | } |
| 279 | memcpy(xfer_sock->iface, &tmpbuf[curoff], len); |
| 280 | xfer_sock->iface[len] = 0; |
| 281 | xfer_sock->if_namelen = len; |
| 282 | curoff += len; |
| 283 | } |
| 284 | |
| 285 | if (curoff + sizeof(int) > maxoff) { |
| 286 | ha_warning("Inconsistency while transferring sockets\n"); |
| 287 | goto out; |
| 288 | } |
| 289 | |
| 290 | /* we used to have 32 bits of listener options here but we don't |
| 291 | * use them anymore. |
| 292 | */ |
| 293 | curoff += sizeof(int); |
| 294 | |
| 295 | /* determine the foreign status directly from the socket itself */ |
| 296 | if (sock_inet_is_foreign(fd, xfer_sock->addr.ss_family)) |
Willy Tarreau | a2c1787 | 2020-08-28 19:09:19 +0200 | [diff] [blame] | 297 | xfer_sock->options |= SOCK_XFER_OPT_FOREIGN; |
Willy Tarreau | 4296174 | 2020-08-28 18:42:45 +0200 | [diff] [blame] | 298 | |
Willy Tarreau | 9dbb6c4 | 2020-08-28 19:20:23 +0200 | [diff] [blame] | 299 | socklen = sizeof(val); |
| 300 | if (getsockopt(fd, SOL_SOCKET, SO_TYPE, &val, &socklen) == 0 && val == SOCK_DGRAM) |
| 301 | xfer_sock->options |= SOCK_XFER_OPT_DGRAM; |
| 302 | |
Willy Tarreau | 4296174 | 2020-08-28 18:42:45 +0200 | [diff] [blame] | 303 | #if defined(IPV6_V6ONLY) |
| 304 | /* keep only the v6only flag depending on what's currently |
| 305 | * active on the socket, and always drop the v4v6 one. |
| 306 | */ |
| 307 | socklen = sizeof(val); |
| 308 | if (xfer_sock->addr.ss_family == AF_INET6 && |
| 309 | getsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &val, &socklen) == 0 && val > 0) |
Willy Tarreau | a2c1787 | 2020-08-28 19:09:19 +0200 | [diff] [blame] | 310 | xfer_sock->options |= SOCK_XFER_OPT_V6ONLY; |
Willy Tarreau | 4296174 | 2020-08-28 18:42:45 +0200 | [diff] [blame] | 311 | #endif |
| 312 | |
| 313 | xfer_sock->fd = fd; |
| 314 | if (xfer_sock_list) |
| 315 | xfer_sock_list->prev = xfer_sock; |
| 316 | xfer_sock->next = xfer_sock_list; |
| 317 | xfer_sock->prev = NULL; |
| 318 | xfer_sock_list = xfer_sock; |
| 319 | xfer_sock = NULL; |
| 320 | } |
| 321 | |
| 322 | ret2 = 0; |
| 323 | out: |
| 324 | /* If we failed midway make sure to close the remaining |
| 325 | * file descriptors |
| 326 | */ |
| 327 | if (tmpfd != NULL && cur_fd < got_fd) { |
| 328 | for (; cur_fd < got_fd; cur_fd++) { |
| 329 | close(tmpfd[cur_fd]); |
| 330 | } |
| 331 | } |
| 332 | |
| 333 | free(tmpbuf); |
| 334 | free(tmpfd); |
| 335 | free(cmsgbuf); |
| 336 | |
| 337 | if (sock != -1) |
| 338 | close(sock); |
| 339 | |
| 340 | if (xfer_sock) { |
| 341 | free(xfer_sock->namespace); |
| 342 | free(xfer_sock->iface); |
| 343 | if (xfer_sock->fd != -1) |
| 344 | close(xfer_sock->fd); |
| 345 | free(xfer_sock); |
| 346 | } |
| 347 | return (ret2); |
| 348 | } |
| 349 | |
Willy Tarreau | c049c0d | 2020-09-01 15:20:52 +0200 | [diff] [blame] | 350 | /* When binding the receivers, check if a socket has been sent to us by the |
Willy Tarreau | 2d34a71 | 2020-08-28 16:49:41 +0200 | [diff] [blame] | 351 | * previous process that we could reuse, instead of creating a new one. Note |
| 352 | * that some address family-specific options are checked on the listener and |
| 353 | * on the socket. Typically for AF_INET and AF_INET6, we check for transparent |
| 354 | * mode, and for AF_INET6 we also check for "v4v6" or "v6only". The reused |
| 355 | * socket is automatically removed from the list so that it's not proposed |
| 356 | * anymore. |
| 357 | */ |
Willy Tarreau | c049c0d | 2020-09-01 15:20:52 +0200 | [diff] [blame] | 358 | int sock_find_compatible_fd(const struct receiver *rx) |
Willy Tarreau | 2d34a71 | 2020-08-28 16:49:41 +0200 | [diff] [blame] | 359 | { |
| 360 | struct xfer_sock_list *xfer_sock = xfer_sock_list; |
Willy Tarreau | a2c1787 | 2020-08-28 19:09:19 +0200 | [diff] [blame] | 361 | int options = 0; |
Willy Tarreau | 2d34a71 | 2020-08-28 16:49:41 +0200 | [diff] [blame] | 362 | int if_namelen = 0; |
| 363 | int ns_namelen = 0; |
| 364 | int ret = -1; |
| 365 | |
Willy Tarreau | f1f6609 | 2020-09-04 08:15:31 +0200 | [diff] [blame] | 366 | if (!rx->proto->fam->addrcmp) |
Willy Tarreau | 2d34a71 | 2020-08-28 16:49:41 +0200 | [diff] [blame] | 367 | return -1; |
| 368 | |
Willy Tarreau | c049c0d | 2020-09-01 15:20:52 +0200 | [diff] [blame] | 369 | if (rx->proto->sock_type == SOCK_DGRAM) |
Willy Tarreau | 9dbb6c4 | 2020-08-28 19:20:23 +0200 | [diff] [blame] | 370 | options |= SOCK_XFER_OPT_DGRAM; |
| 371 | |
Willy Tarreau | c049c0d | 2020-09-01 15:20:52 +0200 | [diff] [blame] | 372 | if (rx->settings->options & RX_O_FOREIGN) |
Willy Tarreau | a2c1787 | 2020-08-28 19:09:19 +0200 | [diff] [blame] | 373 | options |= SOCK_XFER_OPT_FOREIGN; |
| 374 | |
Willy Tarreau | c049c0d | 2020-09-01 15:20:52 +0200 | [diff] [blame] | 375 | if (rx->addr.ss_family == AF_INET6) { |
Willy Tarreau | 2d34a71 | 2020-08-28 16:49:41 +0200 | [diff] [blame] | 376 | /* Prepare to match the v6only option against what we really want. Note |
| 377 | * that sadly the two options are not exclusive to each other and that |
| 378 | * v6only is stronger than v4v6. |
| 379 | */ |
Willy Tarreau | c049c0d | 2020-09-01 15:20:52 +0200 | [diff] [blame] | 380 | if ((rx->settings->options & RX_O_V6ONLY) || |
| 381 | (sock_inet6_v6only_default && !(rx->settings->options & RX_O_V4V6))) |
Willy Tarreau | a2c1787 | 2020-08-28 19:09:19 +0200 | [diff] [blame] | 382 | options |= SOCK_XFER_OPT_V6ONLY; |
Willy Tarreau | 2d34a71 | 2020-08-28 16:49:41 +0200 | [diff] [blame] | 383 | } |
Willy Tarreau | 2d34a71 | 2020-08-28 16:49:41 +0200 | [diff] [blame] | 384 | |
Willy Tarreau | c049c0d | 2020-09-01 15:20:52 +0200 | [diff] [blame] | 385 | if (rx->settings->interface) |
| 386 | if_namelen = strlen(rx->settings->interface); |
Willy Tarreau | 2d34a71 | 2020-08-28 16:49:41 +0200 | [diff] [blame] | 387 | #ifdef USE_NS |
Willy Tarreau | c049c0d | 2020-09-01 15:20:52 +0200 | [diff] [blame] | 388 | if (rx->settings->netns) |
| 389 | ns_namelen = rx->settings->netns->name_len; |
Willy Tarreau | 2d34a71 | 2020-08-28 16:49:41 +0200 | [diff] [blame] | 390 | #endif |
| 391 | |
| 392 | while (xfer_sock) { |
Willy Tarreau | a2c1787 | 2020-08-28 19:09:19 +0200 | [diff] [blame] | 393 | if ((options == xfer_sock->options) && |
Willy Tarreau | 2d34a71 | 2020-08-28 16:49:41 +0200 | [diff] [blame] | 394 | (if_namelen == xfer_sock->if_namelen) && |
| 395 | (ns_namelen == xfer_sock->ns_namelen) && |
Willy Tarreau | c049c0d | 2020-09-01 15:20:52 +0200 | [diff] [blame] | 396 | (!if_namelen || strcmp(rx->settings->interface, xfer_sock->iface) == 0) && |
Willy Tarreau | 2d34a71 | 2020-08-28 16:49:41 +0200 | [diff] [blame] | 397 | #ifdef USE_NS |
Willy Tarreau | c049c0d | 2020-09-01 15:20:52 +0200 | [diff] [blame] | 398 | (!ns_namelen || strcmp(rx->settings->netns->node.key, xfer_sock->namespace) == 0) && |
Willy Tarreau | 2d34a71 | 2020-08-28 16:49:41 +0200 | [diff] [blame] | 399 | #endif |
Willy Tarreau | f1f6609 | 2020-09-04 08:15:31 +0200 | [diff] [blame] | 400 | rx->proto->fam->addrcmp(&xfer_sock->addr, &rx->addr) == 0) |
Willy Tarreau | 2d34a71 | 2020-08-28 16:49:41 +0200 | [diff] [blame] | 401 | break; |
| 402 | xfer_sock = xfer_sock->next; |
| 403 | } |
| 404 | |
| 405 | if (xfer_sock != NULL) { |
| 406 | ret = xfer_sock->fd; |
| 407 | if (xfer_sock == xfer_sock_list) |
| 408 | xfer_sock_list = xfer_sock->next; |
| 409 | if (xfer_sock->prev) |
| 410 | xfer_sock->prev->next = xfer_sock->next; |
| 411 | if (xfer_sock->next) |
| 412 | xfer_sock->next->prev = xfer_sock->prev; |
| 413 | free(xfer_sock->iface); |
| 414 | free(xfer_sock->namespace); |
| 415 | free(xfer_sock); |
| 416 | } |
| 417 | return ret; |
| 418 | } |
| 419 | |
Willy Tarreau | 18b7df7 | 2020-08-28 12:07:22 +0200 | [diff] [blame] | 420 | /* |
| 421 | * Local variables: |
| 422 | * c-indent-level: 8 |
| 423 | * c-basic-offset: 8 |
| 424 | * End: |
| 425 | */ |