Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 1 | /* |
Willy Tarreau | f7bc57c | 2012-10-03 00:19:48 +0200 | [diff] [blame] | 2 | * RAW transport layer over SOCK_STREAM sockets. |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 3 | * |
Willy Tarreau | b277d6e | 2012-05-11 16:59:14 +0200 | [diff] [blame] | 4 | * Copyright 2000-2012 Willy Tarreau <w@1wt.eu> |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 5 | * |
| 6 | * This program is free software; you can redistribute it and/or |
| 7 | * modify it under the terms of the GNU General Public License |
| 8 | * as published by the Free Software Foundation; either version |
| 9 | * 2 of the License, or (at your option) any later version. |
| 10 | * |
| 11 | */ |
| 12 | |
Willy Tarreau | 6b4aad4 | 2009-01-18 21:59:13 +0100 | [diff] [blame] | 13 | #define _GNU_SOURCE |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 14 | #include <errno.h> |
| 15 | #include <fcntl.h> |
| 16 | #include <stdio.h> |
| 17 | #include <stdlib.h> |
| 18 | |
| 19 | #include <sys/socket.h> |
| 20 | #include <sys/stat.h> |
| 21 | #include <sys/types.h> |
| 22 | |
Dmitry Sivachenko | caf5898 | 2009-08-24 15:11:06 +0400 | [diff] [blame] | 23 | #include <netinet/tcp.h> |
| 24 | |
Willy Tarreau | 4c7e4b7 | 2020-05-27 12:58:42 +0200 | [diff] [blame] | 25 | #include <haproxy/api.h> |
Willy Tarreau | 2741c8c | 2020-06-02 11:28:02 +0200 | [diff] [blame] | 26 | #include <haproxy/buf.h> |
Willy Tarreau | 7ea393d | 2020-06-04 18:02:10 +0200 | [diff] [blame] | 27 | #include <haproxy/connection.h> |
Willy Tarreau | f268ee8 | 2020-06-04 17:05:57 +0200 | [diff] [blame] | 28 | #include <haproxy/global.h> |
Willy Tarreau | aeed4a8 | 2020-06-04 22:01:04 +0200 | [diff] [blame^] | 29 | #include <haproxy/log.h> |
Willy Tarreau | 5e539c9 | 2020-06-04 20:45:39 +0200 | [diff] [blame] | 30 | #include <haproxy/stream_interface.h> |
Willy Tarreau | 48fbcae | 2020-06-03 18:09:46 +0200 | [diff] [blame] | 31 | #include <haproxy/tools.h> |
Willy Tarreau | c2f7c58 | 2020-06-02 18:15:32 +0200 | [diff] [blame] | 32 | #include <haproxy/ticks.h> |
Willy Tarreau | 92b4f13 | 2020-06-01 11:05:15 +0200 | [diff] [blame] | 33 | #include <haproxy/time.h> |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 34 | |
Willy Tarreau | 0f6ffd6 | 2020-06-03 19:33:00 +0200 | [diff] [blame] | 35 | #include <haproxy/fd.h> |
Willy Tarreau | 6634794 | 2020-06-01 12:18:08 +0200 | [diff] [blame] | 36 | #include <haproxy/freq_ctr.h> |
Willy Tarreau | 551271d | 2020-06-04 08:32:23 +0200 | [diff] [blame] | 37 | #include <haproxy/pipe.h> |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 38 | |
Willy Tarreau | b277d6e | 2012-05-11 16:59:14 +0200 | [diff] [blame] | 39 | |
Willy Tarreau | e573323 | 2019-05-22 19:24:06 +0200 | [diff] [blame] | 40 | #if defined(USE_LINUX_SPLICE) |
Willy Tarreau | 5bd8c37 | 2009-01-19 00:32:22 +0100 | [diff] [blame] | 41 | |
| 42 | /* A pipe contains 16 segments max, and it's common to see segments of 1448 bytes |
| 43 | * because of timestamps. Use this as a hint for not looping on splice(). |
| 44 | */ |
| 45 | #define SPLICE_FULL_HINT 16*1448 |
| 46 | |
Willy Tarreau | a9de333 | 2009-11-28 07:47:10 +0100 | [diff] [blame] | 47 | /* how many data we attempt to splice at once when the buffer is configured for |
| 48 | * infinite forwarding */ |
| 49 | #define MAX_SPLICE_AT_ONCE (1<<30) |
| 50 | |
Willy Tarreau | 5bd8c37 | 2009-01-19 00:32:22 +0100 | [diff] [blame] | 51 | /* Returns : |
Willy Tarreau | 96199b1 | 2012-08-24 00:46:52 +0200 | [diff] [blame] | 52 | * -1 if splice() is not supported |
| 53 | * >= 0 to report the amount of spliced bytes. |
| 54 | * connection flags are updated (error, read0, wait_room, wait_data). |
| 55 | * The caller must have previously allocated the pipe. |
Willy Tarreau | 5bd8c37 | 2009-01-19 00:32:22 +0100 | [diff] [blame] | 56 | */ |
Olivier Houchard | e179d0e | 2019-03-21 18:27:17 +0100 | [diff] [blame] | 57 | int raw_sock_to_pipe(struct connection *conn, void *xprt_ctx, struct pipe *pipe, unsigned int count) |
Willy Tarreau | 5bd8c37 | 2009-01-19 00:32:22 +0100 | [diff] [blame] | 58 | { |
Willy Tarreau | 31971e5 | 2009-09-20 12:07:52 +0200 | [diff] [blame] | 59 | int ret; |
Willy Tarreau | afad0e0 | 2012-08-09 14:45:22 +0200 | [diff] [blame] | 60 | int retval = 0; |
Willy Tarreau | 5bd8c37 | 2009-01-19 00:32:22 +0100 | [diff] [blame] | 61 | |
Willy Tarreau | f79c817 | 2013-10-21 16:30:56 +0200 | [diff] [blame] | 62 | |
Willy Tarreau | fd803bb | 2014-01-20 15:13:07 +0100 | [diff] [blame] | 63 | if (!conn_ctrl_ready(conn)) |
Willy Tarreau | f79c817 | 2013-10-21 16:30:56 +0200 | [diff] [blame] | 64 | return 0; |
| 65 | |
Willy Tarreau | 585744b | 2017-08-24 14:31:19 +0200 | [diff] [blame] | 66 | if (!fd_recv_ready(conn->handle.fd)) |
Willy Tarreau | fd803bb | 2014-01-20 15:13:07 +0100 | [diff] [blame] | 67 | return 0; |
| 68 | |
Willy Tarreau | e2a0eec | 2020-01-17 09:59:40 +0100 | [diff] [blame] | 69 | conn->flags &= ~CO_FL_WAIT_ROOM; |
Willy Tarreau | ce3eda7 | 2013-12-05 00:49:40 +0100 | [diff] [blame] | 70 | errno = 0; |
| 71 | |
Willy Tarreau | 96199b1 | 2012-08-24 00:46:52 +0200 | [diff] [blame] | 72 | /* Under Linux, if FD_POLL_HUP is set, we have reached the end. |
| 73 | * Since older splice() implementations were buggy and returned |
| 74 | * EAGAIN on end of read, let's bypass the call to splice() now. |
| 75 | */ |
Willy Tarreau | 585744b | 2017-08-24 14:31:19 +0200 | [diff] [blame] | 76 | if (unlikely(!(fdtab[conn->handle.fd].ev & FD_POLL_IN))) { |
Willy Tarreau | 6f5d141 | 2012-10-04 20:38:49 +0200 | [diff] [blame] | 77 | /* stop here if we reached the end of data */ |
Willy Tarreau | 585744b | 2017-08-24 14:31:19 +0200 | [diff] [blame] | 78 | if ((fdtab[conn->handle.fd].ev & (FD_POLL_ERR|FD_POLL_HUP)) == FD_POLL_HUP) |
Willy Tarreau | 6f5d141 | 2012-10-04 20:38:49 +0200 | [diff] [blame] | 79 | goto out_read0; |
| 80 | |
| 81 | /* report error on POLL_ERR before connection establishment */ |
Willy Tarreau | 585744b | 2017-08-24 14:31:19 +0200 | [diff] [blame] | 82 | if ((fdtab[conn->handle.fd].ev & FD_POLL_ERR) && (conn->flags & CO_FL_WAIT_L4_CONN)) { |
Willy Tarreau | 26f4a04 | 2013-12-04 23:44:10 +0100 | [diff] [blame] | 83 | conn->flags |= CO_FL_ERROR | CO_FL_SOCK_RD_SH | CO_FL_SOCK_WR_SH; |
Willy Tarreau | ce3eda7 | 2013-12-05 00:49:40 +0100 | [diff] [blame] | 84 | errno = 0; /* let the caller do a getsockopt() if it wants it */ |
Willy Tarreau | 256b9c5 | 2017-10-25 09:30:13 +0200 | [diff] [blame] | 85 | goto leave; |
Willy Tarreau | 6f5d141 | 2012-10-04 20:38:49 +0200 | [diff] [blame] | 86 | } |
| 87 | } |
Willy Tarreau | a9de333 | 2009-11-28 07:47:10 +0100 | [diff] [blame] | 88 | |
Willy Tarreau | 96199b1 | 2012-08-24 00:46:52 +0200 | [diff] [blame] | 89 | while (count) { |
| 90 | if (count > MAX_SPLICE_AT_ONCE) |
| 91 | count = MAX_SPLICE_AT_ONCE; |
Willy Tarreau | 5bd8c37 | 2009-01-19 00:32:22 +0100 | [diff] [blame] | 92 | |
Willy Tarreau | 585744b | 2017-08-24 14:31:19 +0200 | [diff] [blame] | 93 | ret = splice(conn->handle.fd, NULL, pipe->prod, NULL, count, |
Willy Tarreau | 5bd8c37 | 2009-01-19 00:32:22 +0100 | [diff] [blame] | 94 | SPLICE_F_MOVE|SPLICE_F_NONBLOCK); |
| 95 | |
| 96 | if (ret <= 0) { |
Willy Tarreau | 3844747 | 2019-05-22 19:55:24 +0200 | [diff] [blame] | 97 | if (ret == 0) |
Willy Tarreau | 96199b1 | 2012-08-24 00:46:52 +0200 | [diff] [blame] | 98 | goto out_read0; |
Willy Tarreau | 5bd8c37 | 2009-01-19 00:32:22 +0100 | [diff] [blame] | 99 | |
| 100 | if (errno == EAGAIN) { |
| 101 | /* there are two reasons for EAGAIN : |
| 102 | * - nothing in the socket buffer (standard) |
| 103 | * - pipe is full |
Willy Tarreau | 3844747 | 2019-05-22 19:55:24 +0200 | [diff] [blame] | 104 | * The difference between these two situations |
| 105 | * is problematic. Since we don't know if the |
| 106 | * pipe is full, we'll stop if the pipe is not |
| 107 | * empty. Anyway, we will almost always fill or |
| 108 | * empty the pipe. |
Willy Tarreau | 5bd8c37 | 2009-01-19 00:32:22 +0100 | [diff] [blame] | 109 | */ |
Willy Tarreau | 96199b1 | 2012-08-24 00:46:52 +0200 | [diff] [blame] | 110 | if (pipe->data) { |
Ilya Shipitsin | ce7b00f | 2020-03-23 22:28:40 +0500 | [diff] [blame] | 111 | /* always stop reading until the pipe is flushed */ |
Willy Tarreau | 96199b1 | 2012-08-24 00:46:52 +0200 | [diff] [blame] | 112 | conn->flags |= CO_FL_WAIT_ROOM; |
Willy Tarreau | 5bd8c37 | 2009-01-19 00:32:22 +0100 | [diff] [blame] | 113 | break; |
| 114 | } |
Willy Tarreau | 8dd348c | 2020-02-28 14:09:12 +0100 | [diff] [blame] | 115 | /* socket buffer exhausted */ |
| 116 | fd_cant_recv(conn->handle.fd); |
Willy Tarreau | 5bd8c37 | 2009-01-19 00:32:22 +0100 | [diff] [blame] | 117 | break; |
| 118 | } |
Willy Tarreau | 45b8893 | 2012-11-12 12:00:09 +0100 | [diff] [blame] | 119 | else if (errno == ENOSYS || errno == EINVAL || errno == EBADF) { |
Willy Tarreau | 96199b1 | 2012-08-24 00:46:52 +0200 | [diff] [blame] | 120 | /* splice not supported on this end, disable it. |
| 121 | * We can safely return -1 since there is no |
| 122 | * chance that any data has been piped yet. |
| 123 | */ |
Willy Tarreau | 256b9c5 | 2017-10-25 09:30:13 +0200 | [diff] [blame] | 124 | retval = -1; |
| 125 | goto leave; |
Willy Tarreau | dc340a9 | 2009-06-28 23:10:19 +0200 | [diff] [blame] | 126 | } |
Willy Tarreau | 96199b1 | 2012-08-24 00:46:52 +0200 | [diff] [blame] | 127 | else if (errno == EINTR) { |
| 128 | /* try again */ |
| 129 | continue; |
| 130 | } |
Willy Tarreau | 5bd8c37 | 2009-01-19 00:32:22 +0100 | [diff] [blame] | 131 | /* here we have another error */ |
Willy Tarreau | 96199b1 | 2012-08-24 00:46:52 +0200 | [diff] [blame] | 132 | conn->flags |= CO_FL_ERROR; |
Willy Tarreau | 5bd8c37 | 2009-01-19 00:32:22 +0100 | [diff] [blame] | 133 | break; |
| 134 | } /* ret <= 0 */ |
| 135 | |
Willy Tarreau | 96199b1 | 2012-08-24 00:46:52 +0200 | [diff] [blame] | 136 | retval += ret; |
| 137 | pipe->data += ret; |
Willy Tarreau | 4fc90ef | 2013-04-06 11:29:39 +0200 | [diff] [blame] | 138 | count -= ret; |
Willy Tarreau | baf2a50 | 2013-01-07 16:38:26 +0100 | [diff] [blame] | 139 | |
Willy Tarreau | 96199b1 | 2012-08-24 00:46:52 +0200 | [diff] [blame] | 140 | if (pipe->data >= SPLICE_FULL_HINT || ret >= global.tune.recv_enough) { |
| 141 | /* We've read enough of it for this time, let's stop before |
| 142 | * being asked to poll. |
| 143 | */ |
Willy Tarreau | 61d39a0 | 2013-07-18 21:49:32 +0200 | [diff] [blame] | 144 | conn->flags |= CO_FL_WAIT_ROOM; |
Willy Tarreau | 5bd8c37 | 2009-01-19 00:32:22 +0100 | [diff] [blame] | 145 | break; |
| 146 | } |
| 147 | } /* while */ |
| 148 | |
Willy Tarreau | 665e6ee | 2012-10-04 20:20:46 +0200 | [diff] [blame] | 149 | if (unlikely(conn->flags & CO_FL_WAIT_L4_CONN) && retval) |
| 150 | conn->flags &= ~CO_FL_WAIT_L4_CONN; |
Willy Tarreau | 256b9c5 | 2017-10-25 09:30:13 +0200 | [diff] [blame] | 151 | |
| 152 | leave: |
Willy Tarreau | 7cf0e45 | 2019-05-23 11:39:14 +0200 | [diff] [blame] | 153 | if (retval > 0) { |
| 154 | /* we count the total bytes sent, and the send rate for 32-byte |
| 155 | * blocks. The reason for the latter is that freq_ctr are |
| 156 | * limited to 4GB and that it's not enough per second. |
| 157 | */ |
| 158 | _HA_ATOMIC_ADD(&global.out_bytes, retval); |
| 159 | update_freq_ctr(&global.out_32bps, (retval + 16) / 32); |
| 160 | } |
Willy Tarreau | 96199b1 | 2012-08-24 00:46:52 +0200 | [diff] [blame] | 161 | return retval; |
Willy Tarreau | 3eba98a | 2009-01-25 13:56:13 +0100 | [diff] [blame] | 162 | |
Willy Tarreau | 96199b1 | 2012-08-24 00:46:52 +0200 | [diff] [blame] | 163 | out_read0: |
| 164 | conn_sock_read0(conn); |
Willy Tarreau | 665e6ee | 2012-10-04 20:20:46 +0200 | [diff] [blame] | 165 | conn->flags &= ~CO_FL_WAIT_L4_CONN; |
Willy Tarreau | 256b9c5 | 2017-10-25 09:30:13 +0200 | [diff] [blame] | 166 | goto leave; |
Willy Tarreau | 5bd8c37 | 2009-01-19 00:32:22 +0100 | [diff] [blame] | 167 | } |
| 168 | |
Willy Tarreau | 96199b1 | 2012-08-24 00:46:52 +0200 | [diff] [blame] | 169 | /* Send as many bytes as possible from the pipe to the connection's socket. |
| 170 | */ |
Olivier Houchard | e179d0e | 2019-03-21 18:27:17 +0100 | [diff] [blame] | 171 | int raw_sock_from_pipe(struct connection *conn, void *xprt_ctx, struct pipe *pipe) |
Willy Tarreau | 96199b1 | 2012-08-24 00:46:52 +0200 | [diff] [blame] | 172 | { |
| 173 | int ret, done; |
| 174 | |
Willy Tarreau | fd803bb | 2014-01-20 15:13:07 +0100 | [diff] [blame] | 175 | if (!conn_ctrl_ready(conn)) |
| 176 | return 0; |
| 177 | |
Willy Tarreau | 585744b | 2017-08-24 14:31:19 +0200 | [diff] [blame] | 178 | if (!fd_send_ready(conn->handle.fd)) |
Willy Tarreau | f79c817 | 2013-10-21 16:30:56 +0200 | [diff] [blame] | 179 | return 0; |
| 180 | |
Willy Tarreau | a8c7e8e | 2020-01-23 18:17:55 +0100 | [diff] [blame] | 181 | if (conn->flags & CO_FL_SOCK_WR_SH) { |
| 182 | /* it's already closed */ |
| 183 | conn->flags |= CO_FL_ERROR | CO_FL_SOCK_RD_SH; |
| 184 | errno = EPIPE; |
| 185 | return 0; |
| 186 | } |
| 187 | |
Willy Tarreau | 96199b1 | 2012-08-24 00:46:52 +0200 | [diff] [blame] | 188 | done = 0; |
| 189 | while (pipe->data) { |
Willy Tarreau | 585744b | 2017-08-24 14:31:19 +0200 | [diff] [blame] | 190 | ret = splice(pipe->cons, NULL, conn->handle.fd, NULL, pipe->data, |
Willy Tarreau | 96199b1 | 2012-08-24 00:46:52 +0200 | [diff] [blame] | 191 | SPLICE_F_MOVE|SPLICE_F_NONBLOCK); |
| 192 | |
| 193 | if (ret <= 0) { |
| 194 | if (ret == 0 || errno == EAGAIN) { |
Willy Tarreau | 8dd348c | 2020-02-28 14:09:12 +0100 | [diff] [blame] | 195 | fd_cant_send(conn->handle.fd); |
Willy Tarreau | 96199b1 | 2012-08-24 00:46:52 +0200 | [diff] [blame] | 196 | break; |
| 197 | } |
| 198 | else if (errno == EINTR) |
| 199 | continue; |
| 200 | |
| 201 | /* here we have another error */ |
| 202 | conn->flags |= CO_FL_ERROR; |
| 203 | break; |
| 204 | } |
| 205 | |
| 206 | done += ret; |
| 207 | pipe->data -= ret; |
| 208 | } |
Willy Tarreau | ccf3f6d | 2019-09-05 17:05:05 +0200 | [diff] [blame] | 209 | if (unlikely(conn->flags & CO_FL_WAIT_L4_CONN) && done) { |
Willy Tarreau | 665e6ee | 2012-10-04 20:20:46 +0200 | [diff] [blame] | 210 | conn->flags &= ~CO_FL_WAIT_L4_CONN; |
Willy Tarreau | ccf3f6d | 2019-09-05 17:05:05 +0200 | [diff] [blame] | 211 | } |
Willy Tarreau | 256b9c5 | 2017-10-25 09:30:13 +0200 | [diff] [blame] | 212 | |
Willy Tarreau | 96199b1 | 2012-08-24 00:46:52 +0200 | [diff] [blame] | 213 | return done; |
| 214 | } |
| 215 | |
Willy Tarreau | e573323 | 2019-05-22 19:24:06 +0200 | [diff] [blame] | 216 | #endif /* USE_LINUX_SPLICE */ |
Willy Tarreau | 6b4aad4 | 2009-01-18 21:59:13 +0100 | [diff] [blame] | 217 | |
| 218 | |
Willy Tarreau | 2ba4465 | 2012-08-20 17:30:32 +0200 | [diff] [blame] | 219 | /* Receive up to <count> bytes from connection <conn>'s socket and store them |
Willy Tarreau | abf08d9 | 2014-01-14 11:31:27 +0100 | [diff] [blame] | 220 | * into buffer <buf>. Only one call to recv() is performed, unless the |
Willy Tarreau | 2ba4465 | 2012-08-20 17:30:32 +0200 | [diff] [blame] | 221 | * buffer wraps, in which case a second call may be performed. The connection's |
| 222 | * flags are updated with whatever special event is detected (error, read0, |
| 223 | * empty). The caller is responsible for taking care of those events and |
| 224 | * avoiding the call if inappropriate. The function does not call the |
| 225 | * connection's polling update function, so the caller is responsible for this. |
Willy Tarreau | ce3eda7 | 2013-12-05 00:49:40 +0100 | [diff] [blame] | 226 | * errno is cleared before starting so that the caller knows that if it spots an |
| 227 | * error without errno, it's pending and can be retrieved via getsockopt(SO_ERROR). |
Willy Tarreau | 2ba4465 | 2012-08-20 17:30:32 +0200 | [diff] [blame] | 228 | */ |
Olivier Houchard | e179d0e | 2019-03-21 18:27:17 +0100 | [diff] [blame] | 229 | static size_t raw_sock_to_buf(struct connection *conn, void *xprt_ctx, struct buffer *buf, size_t count, int flags) |
Willy Tarreau | 2ba4465 | 2012-08-20 17:30:32 +0200 | [diff] [blame] | 230 | { |
Willy Tarreau | bfc4d77 | 2018-07-18 11:22:03 +0200 | [diff] [blame] | 231 | ssize_t ret; |
| 232 | size_t try, done = 0; |
Willy Tarreau | 2ba4465 | 2012-08-20 17:30:32 +0200 | [diff] [blame] | 233 | |
Willy Tarreau | fd803bb | 2014-01-20 15:13:07 +0100 | [diff] [blame] | 234 | if (!conn_ctrl_ready(conn)) |
Willy Tarreau | f79c817 | 2013-10-21 16:30:56 +0200 | [diff] [blame] | 235 | return 0; |
| 236 | |
Willy Tarreau | 585744b | 2017-08-24 14:31:19 +0200 | [diff] [blame] | 237 | if (!fd_recv_ready(conn->handle.fd)) |
Willy Tarreau | fd803bb | 2014-01-20 15:13:07 +0100 | [diff] [blame] | 238 | return 0; |
| 239 | |
Willy Tarreau | e2a0eec | 2020-01-17 09:59:40 +0100 | [diff] [blame] | 240 | conn->flags &= ~CO_FL_WAIT_ROOM; |
Willy Tarreau | ce3eda7 | 2013-12-05 00:49:40 +0100 | [diff] [blame] | 241 | errno = 0; |
| 242 | |
Willy Tarreau | 585744b | 2017-08-24 14:31:19 +0200 | [diff] [blame] | 243 | if (unlikely(!(fdtab[conn->handle.fd].ev & FD_POLL_IN))) { |
Willy Tarreau | 6f5d141 | 2012-10-04 20:38:49 +0200 | [diff] [blame] | 244 | /* stop here if we reached the end of data */ |
Willy Tarreau | 585744b | 2017-08-24 14:31:19 +0200 | [diff] [blame] | 245 | if ((fdtab[conn->handle.fd].ev & (FD_POLL_ERR|FD_POLL_HUP)) == FD_POLL_HUP) |
Willy Tarreau | 6f5d141 | 2012-10-04 20:38:49 +0200 | [diff] [blame] | 246 | goto read0; |
| 247 | |
| 248 | /* report error on POLL_ERR before connection establishment */ |
Willy Tarreau | 585744b | 2017-08-24 14:31:19 +0200 | [diff] [blame] | 249 | if ((fdtab[conn->handle.fd].ev & FD_POLL_ERR) && (conn->flags & CO_FL_WAIT_L4_CONN)) { |
Willy Tarreau | 26f4a04 | 2013-12-04 23:44:10 +0100 | [diff] [blame] | 250 | conn->flags |= CO_FL_ERROR | CO_FL_SOCK_RD_SH | CO_FL_SOCK_WR_SH; |
Willy Tarreau | 256b9c5 | 2017-10-25 09:30:13 +0200 | [diff] [blame] | 251 | goto leave; |
Willy Tarreau | 6f5d141 | 2012-10-04 20:38:49 +0200 | [diff] [blame] | 252 | } |
| 253 | } |
Willy Tarreau | 2ba4465 | 2012-08-20 17:30:32 +0200 | [diff] [blame] | 254 | |
Willy Tarreau | 2ba4465 | 2012-08-20 17:30:32 +0200 | [diff] [blame] | 255 | /* read the largest possible block. For this, we perform only one call |
| 256 | * to recv() unless the buffer wraps and we exactly fill the first hunk, |
| 257 | * in which case we accept to do it once again. A new attempt is made on |
| 258 | * EINTR too. |
| 259 | */ |
Willy Tarreau | abf08d9 | 2014-01-14 11:31:27 +0100 | [diff] [blame] | 260 | while (count > 0) { |
Willy Tarreau | 591d445 | 2018-06-15 17:21:00 +0200 | [diff] [blame] | 261 | try = b_contig_space(buf); |
| 262 | if (!try) |
| 263 | break; |
| 264 | |
Willy Tarreau | abf08d9 | 2014-01-14 11:31:27 +0100 | [diff] [blame] | 265 | if (try > count) |
| 266 | try = count; |
| 267 | |
Willy Tarreau | 8f9c72d | 2018-06-07 18:46:28 +0200 | [diff] [blame] | 268 | ret = recv(conn->handle.fd, b_tail(buf), try, 0); |
Willy Tarreau | 2ba4465 | 2012-08-20 17:30:32 +0200 | [diff] [blame] | 269 | |
| 270 | if (ret > 0) { |
Olivier Houchard | acd1403 | 2018-06-28 18:17:23 +0200 | [diff] [blame] | 271 | b_add(buf, ret); |
Willy Tarreau | 2ba4465 | 2012-08-20 17:30:32 +0200 | [diff] [blame] | 272 | done += ret; |
| 273 | if (ret < try) { |
Willy Tarreau | 8dd348c | 2020-02-28 14:09:12 +0100 | [diff] [blame] | 274 | /* socket buffer exhausted */ |
| 275 | fd_cant_recv(conn->handle.fd); |
| 276 | |
Willy Tarreau | 2ba4465 | 2012-08-20 17:30:32 +0200 | [diff] [blame] | 277 | /* unfortunately, on level-triggered events, POLL_HUP |
| 278 | * is generally delivered AFTER the system buffer is |
Willy Tarreau | 6812871 | 2017-03-13 12:04:34 +0100 | [diff] [blame] | 279 | * empty, unless the poller supports POLL_RDHUP. If |
| 280 | * we know this is the case, we don't try to read more |
| 281 | * as we know there's no more available. Similarly, if |
| 282 | * there's no problem with lingering we don't even try |
| 283 | * to read an unlikely close from the client since we'll |
| 284 | * close first anyway. |
Willy Tarreau | 2ba4465 | 2012-08-20 17:30:32 +0200 | [diff] [blame] | 285 | */ |
Willy Tarreau | 585744b | 2017-08-24 14:31:19 +0200 | [diff] [blame] | 286 | if (fdtab[conn->handle.fd].ev & FD_POLL_HUP) |
Willy Tarreau | 2ba4465 | 2012-08-20 17:30:32 +0200 | [diff] [blame] | 287 | goto read0; |
Willy Tarreau | 6c11bd2 | 2014-01-24 00:54:27 +0100 | [diff] [blame] | 288 | |
Willy Tarreau | 585744b | 2017-08-24 14:31:19 +0200 | [diff] [blame] | 289 | if ((!fdtab[conn->handle.fd].linger_risk) || |
Willy Tarreau | 6812871 | 2017-03-13 12:04:34 +0100 | [diff] [blame] | 290 | (cur_poller.flags & HAP_POLL_F_RDHUP)) { |
Willy Tarreau | 6812871 | 2017-03-13 12:04:34 +0100 | [diff] [blame] | 291 | break; |
| 292 | } |
Willy Tarreau | 2ba4465 | 2012-08-20 17:30:32 +0200 | [diff] [blame] | 293 | } |
| 294 | count -= ret; |
Willy Tarreau | 716bec2 | 2020-02-20 11:04:40 +0100 | [diff] [blame] | 295 | |
| 296 | if (flags & CO_RFL_READ_ONCE) |
| 297 | break; |
Willy Tarreau | 2ba4465 | 2012-08-20 17:30:32 +0200 | [diff] [blame] | 298 | } |
| 299 | else if (ret == 0) { |
| 300 | goto read0; |
| 301 | } |
Joshua M. Clulow | 0724903 | 2014-03-03 13:48:42 -0800 | [diff] [blame] | 302 | else if (errno == EAGAIN || errno == ENOTCONN) { |
Willy Tarreau | 8dd348c | 2020-02-28 14:09:12 +0100 | [diff] [blame] | 303 | /* socket buffer exhausted */ |
| 304 | fd_cant_recv(conn->handle.fd); |
Willy Tarreau | 2ba4465 | 2012-08-20 17:30:32 +0200 | [diff] [blame] | 305 | break; |
| 306 | } |
| 307 | else if (errno != EINTR) { |
Willy Tarreau | 26f4a04 | 2013-12-04 23:44:10 +0100 | [diff] [blame] | 308 | conn->flags |= CO_FL_ERROR | CO_FL_SOCK_RD_SH | CO_FL_SOCK_WR_SH; |
Willy Tarreau | 2ba4465 | 2012-08-20 17:30:32 +0200 | [diff] [blame] | 309 | break; |
| 310 | } |
| 311 | } |
Willy Tarreau | 665e6ee | 2012-10-04 20:20:46 +0200 | [diff] [blame] | 312 | |
| 313 | if (unlikely(conn->flags & CO_FL_WAIT_L4_CONN) && done) |
| 314 | conn->flags &= ~CO_FL_WAIT_L4_CONN; |
Willy Tarreau | 256b9c5 | 2017-10-25 09:30:13 +0200 | [diff] [blame] | 315 | |
| 316 | leave: |
Willy Tarreau | 2ba4465 | 2012-08-20 17:30:32 +0200 | [diff] [blame] | 317 | return done; |
| 318 | |
| 319 | read0: |
| 320 | conn_sock_read0(conn); |
Willy Tarreau | 665e6ee | 2012-10-04 20:20:46 +0200 | [diff] [blame] | 321 | conn->flags &= ~CO_FL_WAIT_L4_CONN; |
Willy Tarreau | debdc4b | 2012-12-07 00:01:33 +0100 | [diff] [blame] | 322 | |
| 323 | /* Now a final check for a possible asynchronous low-level error |
| 324 | * report. This can happen when a connection receives a reset |
| 325 | * after a shutdown, both POLL_HUP and POLL_ERR are queued, and |
| 326 | * we might have come from there by just checking POLL_HUP instead |
| 327 | * of recv()'s return value 0, so we have no way to tell there was |
| 328 | * an error without checking. |
| 329 | */ |
Willy Tarreau | 585744b | 2017-08-24 14:31:19 +0200 | [diff] [blame] | 330 | if (unlikely(fdtab[conn->handle.fd].ev & FD_POLL_ERR)) |
Willy Tarreau | 26f4a04 | 2013-12-04 23:44:10 +0100 | [diff] [blame] | 331 | conn->flags |= CO_FL_ERROR | CO_FL_SOCK_RD_SH | CO_FL_SOCK_WR_SH; |
Willy Tarreau | 256b9c5 | 2017-10-25 09:30:13 +0200 | [diff] [blame] | 332 | goto leave; |
Willy Tarreau | 2ba4465 | 2012-08-20 17:30:32 +0200 | [diff] [blame] | 333 | } |
| 334 | |
| 335 | |
Willy Tarreau | 787db9a | 2018-06-14 18:31:46 +0200 | [diff] [blame] | 336 | /* Send up to <count> pending bytes from buffer <buf> to connection <conn>'s |
| 337 | * socket. <flags> may contain some CO_SFL_* flags to hint the system about |
| 338 | * other pending data for example, but this flag is ignored at the moment. |
Willy Tarreau | 5368d80 | 2012-08-21 18:22:06 +0200 | [diff] [blame] | 339 | * Only one call to send() is performed, unless the buffer wraps, in which case |
| 340 | * a second call may be performed. The connection's flags are updated with |
| 341 | * whatever special event is detected (error, empty). The caller is responsible |
| 342 | * for taking care of those events and avoiding the call if inappropriate. The |
| 343 | * function does not call the connection's polling update function, so the caller |
Willy Tarreau | 787db9a | 2018-06-14 18:31:46 +0200 | [diff] [blame] | 344 | * is responsible for this. It's up to the caller to update the buffer's contents |
| 345 | * based on the return value. |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 346 | */ |
Olivier Houchard | e179d0e | 2019-03-21 18:27:17 +0100 | [diff] [blame] | 347 | static size_t raw_sock_from_buf(struct connection *conn, void *xprt_ctx, const struct buffer *buf, size_t count, int flags) |
Willy Tarreau | 0c2fc1f | 2009-01-18 15:30:37 +0100 | [diff] [blame] | 348 | { |
Willy Tarreau | 787db9a | 2018-06-14 18:31:46 +0200 | [diff] [blame] | 349 | ssize_t ret; |
| 350 | size_t try, done; |
| 351 | int send_flag; |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 352 | |
Willy Tarreau | fd803bb | 2014-01-20 15:13:07 +0100 | [diff] [blame] | 353 | if (!conn_ctrl_ready(conn)) |
| 354 | return 0; |
| 355 | |
Willy Tarreau | 585744b | 2017-08-24 14:31:19 +0200 | [diff] [blame] | 356 | if (!fd_send_ready(conn->handle.fd)) |
Willy Tarreau | f79c817 | 2013-10-21 16:30:56 +0200 | [diff] [blame] | 357 | return 0; |
| 358 | |
Willy Tarreau | a8c7e8e | 2020-01-23 18:17:55 +0100 | [diff] [blame] | 359 | if (conn->flags & CO_FL_SOCK_WR_SH) { |
| 360 | /* it's already closed */ |
| 361 | conn->flags |= CO_FL_ERROR | CO_FL_SOCK_RD_SH; |
| 362 | errno = EPIPE; |
| 363 | return 0; |
| 364 | } |
| 365 | |
Willy Tarreau | 5368d80 | 2012-08-21 18:22:06 +0200 | [diff] [blame] | 366 | done = 0; |
| 367 | /* send the largest possible block. For this we perform only one call |
| 368 | * to send() unless the buffer wraps and we exactly fill the first hunk, |
| 369 | * in which case we accept to do it once again. |
Willy Tarreau | 5bd8c37 | 2009-01-19 00:32:22 +0100 | [diff] [blame] | 370 | */ |
Willy Tarreau | 787db9a | 2018-06-14 18:31:46 +0200 | [diff] [blame] | 371 | while (count) { |
| 372 | try = b_contig_data(buf, done); |
| 373 | if (try > count) |
| 374 | try = count; |
Willy Tarreau | f890dc9 | 2008-12-13 21:12:26 +0100 | [diff] [blame] | 375 | |
Willy Tarreau | 5368d80 | 2012-08-21 18:22:06 +0200 | [diff] [blame] | 376 | send_flag = MSG_DONTWAIT | MSG_NOSIGNAL; |
Willy Tarreau | 787db9a | 2018-06-14 18:31:46 +0200 | [diff] [blame] | 377 | if (try < count || flags & CO_SFL_MSG_MORE) |
Willy Tarreau | 7e4086d | 2014-02-02 01:44:13 +0100 | [diff] [blame] | 378 | send_flag |= MSG_MORE; |
Willy Tarreau | fb14edc | 2009-06-14 15:24:37 +0200 | [diff] [blame] | 379 | |
Willy Tarreau | 787db9a | 2018-06-14 18:31:46 +0200 | [diff] [blame] | 380 | ret = send(conn->handle.fd, b_peek(buf, done), try, send_flag); |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 381 | |
| 382 | if (ret > 0) { |
Willy Tarreau | 787db9a | 2018-06-14 18:31:46 +0200 | [diff] [blame] | 383 | count -= ret; |
Willy Tarreau | 5368d80 | 2012-08-21 18:22:06 +0200 | [diff] [blame] | 384 | done += ret; |
Willy Tarreau | b38903c | 2008-11-23 21:33:29 +0100 | [diff] [blame] | 385 | |
Willy Tarreau | ab3e1d3 | 2007-06-03 14:10:36 +0200 | [diff] [blame] | 386 | /* if the system buffer is full, don't insist */ |
Willy Tarreau | 8dd348c | 2020-02-28 14:09:12 +0100 | [diff] [blame] | 387 | if (ret < try) { |
| 388 | fd_cant_send(conn->handle.fd); |
Willy Tarreau | 6996e15 | 2007-04-30 14:37:43 +0200 | [diff] [blame] | 389 | break; |
Willy Tarreau | 8dd348c | 2020-02-28 14:09:12 +0100 | [diff] [blame] | 390 | } |
Willy Tarreau | 3381bf8 | 2020-01-17 17:39:35 +0100 | [diff] [blame] | 391 | if (!count) |
Willy Tarreau | 3110eb7 | 2020-02-21 10:21:46 +0100 | [diff] [blame] | 392 | fd_stop_send(conn->handle.fd); |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 393 | } |
Willy Tarreau | 034c88c | 2017-01-23 23:36:45 +0100 | [diff] [blame] | 394 | else if (ret == 0 || errno == EAGAIN || errno == ENOTCONN || errno == EINPROGRESS) { |
Willy Tarreau | 0c2fc1f | 2009-01-18 15:30:37 +0100 | [diff] [blame] | 395 | /* nothing written, we need to poll for write first */ |
Willy Tarreau | 8dd348c | 2020-02-28 14:09:12 +0100 | [diff] [blame] | 396 | fd_cant_send(conn->handle.fd); |
Willy Tarreau | 5368d80 | 2012-08-21 18:22:06 +0200 | [diff] [blame] | 397 | break; |
Willy Tarreau | 8374918 | 2007-04-15 20:56:27 +0200 | [diff] [blame] | 398 | } |
Willy Tarreau | 5368d80 | 2012-08-21 18:22:06 +0200 | [diff] [blame] | 399 | else if (errno != EINTR) { |
Willy Tarreau | 26f4a04 | 2013-12-04 23:44:10 +0100 | [diff] [blame] | 400 | conn->flags |= CO_FL_ERROR | CO_FL_SOCK_RD_SH | CO_FL_SOCK_WR_SH; |
Willy Tarreau | 5368d80 | 2012-08-21 18:22:06 +0200 | [diff] [blame] | 401 | break; |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 402 | } |
Willy Tarreau | 5368d80 | 2012-08-21 18:22:06 +0200 | [diff] [blame] | 403 | } |
Willy Tarreau | ccf3f6d | 2019-09-05 17:05:05 +0200 | [diff] [blame] | 404 | if (unlikely(conn->flags & CO_FL_WAIT_L4_CONN) && done) { |
Willy Tarreau | 665e6ee | 2012-10-04 20:20:46 +0200 | [diff] [blame] | 405 | conn->flags &= ~CO_FL_WAIT_L4_CONN; |
Willy Tarreau | ccf3f6d | 2019-09-05 17:05:05 +0200 | [diff] [blame] | 406 | } |
Willy Tarreau | 256b9c5 | 2017-10-25 09:30:13 +0200 | [diff] [blame] | 407 | |
Willy Tarreau | 7cf0e45 | 2019-05-23 11:39:14 +0200 | [diff] [blame] | 408 | if (done > 0) { |
| 409 | /* we count the total bytes sent, and the send rate for 32-byte |
| 410 | * blocks. The reason for the latter is that freq_ctr are |
| 411 | * limited to 4GB and that it's not enough per second. |
| 412 | */ |
| 413 | _HA_ATOMIC_ADD(&global.out_bytes, done); |
| 414 | update_freq_ctr(&global.out_32bps, (done + 16) / 32); |
| 415 | } |
Willy Tarreau | 5368d80 | 2012-08-21 18:22:06 +0200 | [diff] [blame] | 416 | return done; |
Willy Tarreau | 0c2fc1f | 2009-01-18 15:30:37 +0100 | [diff] [blame] | 417 | } |
Willy Tarreau | 6996e15 | 2007-04-30 14:37:43 +0200 | [diff] [blame] | 418 | |
Willy Tarreau | ee1a6fc | 2020-01-17 07:52:13 +0100 | [diff] [blame] | 419 | /* Called from the upper layer, to subscribe <es> to events <event_type>. The |
| 420 | * event subscriber <es> is not allowed to change from a previous call as long |
| 421 | * as at least one event is still subscribed. The <event_type> must only be a |
| 422 | * combination of SUB_RETRY_RECV and SUB_RETRY_SEND. It always returns 0. |
| 423 | */ |
| 424 | static int raw_sock_subscribe(struct connection *conn, void *xprt_ctx, int event_type, struct wait_event *es) |
Olivier Houchard | e179d0e | 2019-03-21 18:27:17 +0100 | [diff] [blame] | 425 | { |
Willy Tarreau | ee1a6fc | 2020-01-17 07:52:13 +0100 | [diff] [blame] | 426 | return conn_subscribe(conn, xprt_ctx, event_type, es); |
Olivier Houchard | e179d0e | 2019-03-21 18:27:17 +0100 | [diff] [blame] | 427 | } |
| 428 | |
Willy Tarreau | ee1a6fc | 2020-01-17 07:52:13 +0100 | [diff] [blame] | 429 | /* Called from the upper layer, to unsubscribe <es> from events <event_type>. |
| 430 | * The <es> pointer is not allowed to differ from the one passed to the |
| 431 | * subscribe() call. It always returns zero. |
| 432 | */ |
| 433 | static int raw_sock_unsubscribe(struct connection *conn, void *xprt_ctx, int event_type, struct wait_event *es) |
Olivier Houchard | e179d0e | 2019-03-21 18:27:17 +0100 | [diff] [blame] | 434 | { |
Willy Tarreau | ee1a6fc | 2020-01-17 07:52:13 +0100 | [diff] [blame] | 435 | return conn_unsubscribe(conn, xprt_ctx, event_type, es); |
Olivier Houchard | e179d0e | 2019-03-21 18:27:17 +0100 | [diff] [blame] | 436 | } |
Willy Tarreau | 0c2fc1f | 2009-01-18 15:30:37 +0100 | [diff] [blame] | 437 | |
Olivier Houchard | 5149b59 | 2019-05-23 17:47:36 +0200 | [diff] [blame] | 438 | /* We can't have an underlying XPRT, so just return -1 to signify failure */ |
| 439 | static int raw_sock_remove_xprt(struct connection *conn, void *xprt_ctx, void *toremove_ctx, const struct xprt_ops *newops, void *newctx) |
| 440 | { |
| 441 | /* This is the lowest xprt we can have, so if we get there we didn't |
| 442 | * find the xprt we wanted to remove, that's a bug |
| 443 | */ |
| 444 | BUG_ON(1); |
| 445 | return -1; |
| 446 | } |
| 447 | |
Willy Tarreau | f7bc57c | 2012-10-03 00:19:48 +0200 | [diff] [blame] | 448 | /* transport-layer operations for RAW sockets */ |
Willy Tarreau | d9f5cca | 2016-12-22 21:08:52 +0100 | [diff] [blame] | 449 | static struct xprt_ops raw_sock = { |
Willy Tarreau | c578891 | 2012-08-24 18:12:41 +0200 | [diff] [blame] | 450 | .snd_buf = raw_sock_from_buf, |
| 451 | .rcv_buf = raw_sock_to_buf, |
Olivier Houchard | e179d0e | 2019-03-21 18:27:17 +0100 | [diff] [blame] | 452 | .subscribe = raw_sock_subscribe, |
| 453 | .unsubscribe = raw_sock_unsubscribe, |
Olivier Houchard | 5149b59 | 2019-05-23 17:47:36 +0200 | [diff] [blame] | 454 | .remove_xprt = raw_sock_remove_xprt, |
Willy Tarreau | e573323 | 2019-05-22 19:24:06 +0200 | [diff] [blame] | 455 | #if defined(USE_LINUX_SPLICE) |
Willy Tarreau | 96199b1 | 2012-08-24 00:46:52 +0200 | [diff] [blame] | 456 | .rcv_pipe = raw_sock_to_pipe, |
| 457 | .snd_pipe = raw_sock_from_pipe, |
| 458 | #endif |
Willy Tarreau | c578891 | 2012-08-24 18:12:41 +0200 | [diff] [blame] | 459 | .shutr = NULL, |
| 460 | .shutw = NULL, |
| 461 | .close = NULL, |
Willy Tarreau | 8e0bb0a | 2016-11-24 16:58:12 +0100 | [diff] [blame] | 462 | .name = "RAW", |
Willy Tarreau | 5c979a9 | 2012-05-07 17:15:39 +0200 | [diff] [blame] | 463 | }; |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 464 | |
Willy Tarreau | 13e1410 | 2016-12-22 20:25:26 +0100 | [diff] [blame] | 465 | |
| 466 | __attribute__((constructor)) |
Olivier Houchard | 0d00593 | 2017-08-14 15:59:44 +0200 | [diff] [blame] | 467 | static void __raw_sock_init(void) |
Willy Tarreau | 13e1410 | 2016-12-22 20:25:26 +0100 | [diff] [blame] | 468 | { |
| 469 | xprt_register(XPRT_RAW, &raw_sock); |
| 470 | } |
| 471 | |
Willy Tarreau | baaee00 | 2006-06-26 02:48:02 +0200 | [diff] [blame] | 472 | /* |
| 473 | * Local variables: |
| 474 | * c-indent-level: 8 |
| 475 | * c-basic-offset: 8 |
| 476 | * End: |
| 477 | */ |