blob: 167b75ae7dbd62e56e1f57deda2179cc5981338c [file] [log] [blame]
Willy Tarreauc7e42382012-08-24 19:22:53 +02001/*
2 * Buffer management functions.
3 *
4 * Copyright 2000-2012 Willy Tarreau <w@1wt.eu>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
12
William Lallemandbe0efd82012-11-22 18:01:40 +010013#include <ctype.h>
Willy Tarreauc7e42382012-08-24 19:22:53 +020014#include <stdio.h>
15#include <string.h>
16
17#include <common/config.h>
18#include <common/buffer.h>
Willy Tarreau9b28e032012-10-12 23:49:43 +020019#include <common/memory.h>
Willy Tarreauc7e42382012-08-24 19:22:53 +020020
21#include <types/global.h>
22
Willy Tarreaubafbe012017-11-24 17:34:44 +010023struct pool_head *pool_head_buffer;
Willy Tarreau9b28e032012-10-12 23:49:43 +020024
Willy Tarreauf2f7d6b2014-11-24 11:55:08 +010025/* These buffers are used to always have a valid pointer to an empty buffer in
26 * channels. The first buffer is set once a buffer is empty. The second one is
27 * set when a buffer is desired but no more are available. It helps knowing
28 * what channel wants a buffer. They can reliably be exchanged, the split
29 * between the two is only an optimization.
Willy Tarreau2a4b5432014-11-24 11:39:34 +010030 */
31struct buffer buf_empty = { .p = buf_empty.data };
Willy Tarreauf2f7d6b2014-11-24 11:55:08 +010032struct buffer buf_wanted = { .p = buf_wanted.data };
Willy Tarreau9b28e032012-10-12 23:49:43 +020033
Christopher Fauleta73e59b2016-12-09 17:30:18 +010034/* list of objects waiting for at least one buffer */
35struct list buffer_wq = LIST_HEAD_INIT(buffer_wq);
Willy Tarreaub6a2f582017-11-26 11:08:14 +010036__decl_hathreads(HA_SPINLOCK_T __attribute__((aligned(64))) buffer_wq_lock);
Christopher Fauleta73e59b2016-12-09 17:30:18 +010037
Christopher Fauletad405f12017-08-29 15:30:11 +020038/* this buffer is always the same size as standard buffers and is used for
39 * swapping data inside a buffer.
40 */
Christopher Fauletba39f232017-08-29 14:43:04 +020041static THREAD_LOCAL char *swap_buffer = NULL;
42
43static int init_buffer_per_thread()
44{
45 swap_buffer = calloc(1, global.tune.bufsize);
46 if (swap_buffer == NULL)
47 return 0;
48 return 1;
49}
50
51static void deinit_buffer_per_thread()
52{
53 free(swap_buffer); swap_buffer = NULL;
54}
Christopher Fauletad405f12017-08-29 15:30:11 +020055
Willy Tarreau9b28e032012-10-12 23:49:43 +020056/* perform minimal intializations, report 0 in case of error, 1 if OK. */
57int init_buffer()
58{
Willy Tarreaua24adf02014-11-27 01:11:56 +010059 void *buffer;
60
Willy Tarreaubafbe012017-11-24 17:34:44 +010061 pool_head_buffer = create_pool("buffer", sizeof (struct buffer) + global.tune.bufsize, MEM_F_SHARED|MEM_F_EXACT);
62 if (!pool_head_buffer)
Willy Tarreaua24adf02014-11-27 01:11:56 +010063 return 0;
64
65 /* The reserved buffer is what we leave behind us. Thus we always need
66 * at least one extra buffer in minavail otherwise we'll end up waking
67 * up tasks with no memory available, causing a lot of useless wakeups.
68 * That means that we always want to have at least 3 buffers available
69 * (2 for current session, one for next session that might be needed to
70 * release a server connection).
71 */
Willy Tarreaubafbe012017-11-24 17:34:44 +010072 pool_head_buffer->minavail = MAX(global.tune.reserved_bufs, 3);
Willy Tarreau33cb0652014-12-23 22:52:37 +010073 if (global.tune.buf_limit)
Willy Tarreaubafbe012017-11-24 17:34:44 +010074 pool_head_buffer->limit = global.tune.buf_limit;
Willy Tarreaua24adf02014-11-27 01:11:56 +010075
Christopher Faulet2a944ee2017-11-07 10:42:54 +010076 HA_SPIN_INIT(&buffer_wq_lock);
Emeric Bruna1dd2432017-06-21 15:42:52 +020077
Willy Tarreaubafbe012017-11-24 17:34:44 +010078 buffer = pool_refill_alloc(pool_head_buffer, pool_head_buffer->minavail - 1);
Willy Tarreaua24adf02014-11-27 01:11:56 +010079 if (!buffer)
80 return 0;
81
Willy Tarreaubafbe012017-11-24 17:34:44 +010082 pool_free(pool_head_buffer, buffer);
Christopher Fauletad405f12017-08-29 15:30:11 +020083
Christopher Fauletcd7879a2017-10-27 13:53:47 +020084 hap_register_per_thread_init(init_buffer_per_thread);
85 hap_register_per_thread_deinit(deinit_buffer_per_thread);
Willy Tarreaua24adf02014-11-27 01:11:56 +010086 return 1;
Willy Tarreau9b28e032012-10-12 23:49:43 +020087}
88
Christopher Fauletad405f12017-08-29 15:30:11 +020089void deinit_buffer()
90{
Willy Tarreaubafbe012017-11-24 17:34:44 +010091 pool_destroy(pool_head_buffer);
Christopher Fauletad405f12017-08-29 15:30:11 +020092}
93
Willy Tarreauaf819352012-08-27 22:08:00 +020094/* This function writes the string <str> at position <pos> which must be in
95 * buffer <b>, and moves <end> just after the end of <str>. <b>'s parameters
96 * <l> and <r> are updated to be valid after the shift. The shift value
97 * (positive or negative) is returned. If there's no space left, the move is
98 * not done. The function does not adjust ->o because it does not make sense to
99 * use it on data scheduled to be sent. For the same reason, it does not make
100 * sense to call this function on unparsed data, so <orig> is not updated. The
101 * string length is taken from parameter <len>. If <len> is null, the <str>
102 * pointer is allowed to be null.
103 */
104int buffer_replace2(struct buffer *b, char *pos, char *end, const char *str, int len)
105{
106 int delta;
107
108 delta = len - (end - pos);
109
Thierry FOURNIERfdda6772015-03-10 01:55:01 +0100110 if (bi_end(b) + delta > b->data + b->size)
Willy Tarreauaf819352012-08-27 22:08:00 +0200111 return 0; /* no space left */
112
113 if (buffer_not_empty(b) &&
114 bi_end(b) + delta > bo_ptr(b) &&
115 bo_ptr(b) >= bi_end(b))
116 return 0; /* no space left before wrapping data */
117
118 /* first, protect the end of the buffer */
119 memmove(end + delta, end, bi_end(b) - end);
120
121 /* now, copy str over pos */
122 if (len)
123 memcpy(pos, str, len);
124
125 b->i += delta;
126
Willy Tarreau5fb38032012-12-16 19:39:09 +0100127 if (buffer_empty(b))
Willy Tarreauaf819352012-08-27 22:08:00 +0200128 b->p = b->data;
129
130 return delta;
131}
132
133/*
134 * Inserts <str> followed by "\r\n" at position <pos> in buffer <b>. The <len>
135 * argument informs about the length of string <str> so that we don't have to
136 * measure it. It does not include the "\r\n". If <str> is NULL, then the buffer
137 * is only opened for len+2 bytes but nothing is copied in. It may be useful in
138 * some circumstances. The send limit is *not* adjusted. Same comments as above
139 * for the valid use cases.
140 *
141 * The number of bytes added is returned on success. 0 is returned on failure.
142 */
143int buffer_insert_line2(struct buffer *b, char *pos, const char *str, int len)
144{
145 int delta;
146
147 delta = len + 2;
148
149 if (bi_end(b) + delta >= b->data + b->size)
150 return 0; /* no space left */
151
Godbacha6547c12014-10-31 13:16:37 +0800152 if (buffer_not_empty(b) &&
153 bi_end(b) + delta > bo_ptr(b) &&
154 bo_ptr(b) >= bi_end(b))
155 return 0; /* no space left before wrapping data */
156
Willy Tarreauaf819352012-08-27 22:08:00 +0200157 /* first, protect the end of the buffer */
158 memmove(pos + delta, pos, bi_end(b) - pos);
159
160 /* now, copy str over pos */
161 if (len && str) {
162 memcpy(pos, str, len);
163 pos[len] = '\r';
164 pos[len + 1] = '\n';
165 }
166
167 b->i += delta;
168 return delta;
169}
170
Willy Tarreau27187ab2015-07-02 12:50:23 +0200171/* This function realigns a possibly wrapping buffer so that the input part is
172 * contiguous and starts at the beginning of the buffer and the output part
173 * ends at the end of the buffer. This provides the best conditions since it
174 * allows the largest inputs to be processed at once and ensures that once the
175 * output data leaves, the whole buffer is available at once.
Willy Tarreauc7e42382012-08-24 19:22:53 +0200176 */
177void buffer_slow_realign(struct buffer *buf)
178{
Willy Tarreau27187ab2015-07-02 12:50:23 +0200179 int block1 = buf->o;
180 int block2 = 0;
181
182 /* process output data in two steps to cover wrapping */
183 if (block1 > buf->p - buf->data) {
184 block2 = buf->p - buf->data;
185 block1 -= block2;
186 }
187 memcpy(swap_buffer + buf->size - buf->o, bo_ptr(buf), block1);
188 memcpy(swap_buffer + buf->size - block2, buf->data, block2);
189
190 /* process input data in two steps to cover wrapping */
191 block1 = buf->i;
192 block2 = 0;
193
194 if (block1 > buf->data + buf->size - buf->p) {
195 block1 = buf->data + buf->size - buf->p;
196 block2 = buf->i - block1;
Willy Tarreauc7e42382012-08-24 19:22:53 +0200197 }
Willy Tarreau27187ab2015-07-02 12:50:23 +0200198 memcpy(swap_buffer, bi_ptr(buf), block1);
199 memcpy(swap_buffer + block1, buf->data, block2);
200
201 /* reinject changes into the buffer */
202 memcpy(buf->data, swap_buffer, buf->i);
203 memcpy(buf->data + buf->size - buf->o, swap_buffer + buf->size - buf->o, buf->o);
Willy Tarreauc7e42382012-08-24 19:22:53 +0200204
205 buf->p = buf->data;
206}
207
Willy Tarreauc7e42382012-08-24 19:22:53 +0200208/*
209 * Dumps part or all of a buffer.
210 */
211void buffer_dump(FILE *o, struct buffer *b, int from, int to)
212{
213 fprintf(o, "Dumping buffer %p\n", b);
William Lallemandbe0efd82012-11-22 18:01:40 +0100214 fprintf(o, " data=%p o=%d i=%d p=%p\n"
215 " relative: p=0x%04x\n",
216 b->data, b->o, b->i, b->p, (unsigned int)(b->p - b->data));
Willy Tarreauc7e42382012-08-24 19:22:53 +0200217
218 fprintf(o, "Dumping contents from byte %d to byte %d\n", from, to);
William Lallemandbe0efd82012-11-22 18:01:40 +0100219 fprintf(o, " 0 1 2 3 4 5 6 7 8 9 a b c d e f\n");
220 /* dump hexa */
221 while (from < to) {
222 int i;
223
224 fprintf(o, " %04x: ", from);
225 for (i = 0; ((from + i) < to) && (i < 16) ; i++) {
226 fprintf(o, "%02x ", (unsigned char)b->data[from + i]);
227 if (((from + i) & 15) == 7)
228 fprintf(o, "- ");
229 }
Godbachc08057c2013-11-14 10:15:20 +0800230 if (to - from < 16) {
Godbachc3916a72013-11-21 10:21:22 +0800231 int j = 0;
232
Godbachc08057c2013-11-14 10:15:20 +0800233 for (j = 0; j < from + 16 - to; j++)
234 fprintf(o, " ");
Godbachc3916a72013-11-21 10:21:22 +0800235 if (j > 8)
236 fprintf(o, " ");
Godbachc08057c2013-11-14 10:15:20 +0800237 }
William Lallemandbe0efd82012-11-22 18:01:40 +0100238 fprintf(o, " ");
239 for (i = 0; (from + i < to) && (i < 16) ; i++) {
Willy Tarreau95898ac2012-11-26 00:57:40 +0100240 fprintf(o, "%c", isprint((int)b->data[from + i]) ? b->data[from + i] : '.') ;
William Lallemandbe0efd82012-11-22 18:01:40 +0100241 if ((((from + i) & 15) == 15) && ((from + i) != to-1))
242 fprintf(o, "\n");
243 }
244 from += i;
Willy Tarreauc7e42382012-08-24 19:22:53 +0200245 }
246 fprintf(o, "\n--\n");
William Lallemandbe0efd82012-11-22 18:01:40 +0100247 fflush(o);
Willy Tarreauc7e42382012-08-24 19:22:53 +0200248}
249
Christopher Fauleta73e59b2016-12-09 17:30:18 +0100250void __offer_buffer(void *from, unsigned int threshold)
251{
252 struct buffer_wait *wait, *bak;
253 int avail;
254
255 /* For now, we consider that all objects need 1 buffer, so we can stop
256 * waking up them once we have enough of them to eat all the available
257 * buffers. Note that we don't really know if they are streams or just
258 * other tasks, but that's a rough estimate. Similarly, for each cached
259 * event we'll need 1 buffer. If no buffer is currently used, always
260 * wake up the number of tasks we can offer a buffer based on what is
261 * allocated, and in any case at least one task per two reserved
262 * buffers.
263 */
Willy Tarreaubafbe012017-11-24 17:34:44 +0100264 avail = pool_head_buffer->allocated - pool_head_buffer->used - global.tune.reserved_bufs / 2;
Christopher Fauleta73e59b2016-12-09 17:30:18 +0100265
266 list_for_each_entry_safe(wait, bak, &buffer_wq, list) {
267 if (avail <= threshold)
268 break;
269
270 if (wait->target == from || !wait->wakeup_cb(wait->target))
271 continue;
272
273 LIST_DEL(&wait->list);
274 LIST_INIT(&wait->list);
275
276 avail--;
277 }
278}
Willy Tarreauc7e42382012-08-24 19:22:53 +0200279
280/*
281 * Local variables:
282 * c-indent-level: 8
283 * c-basic-offset: 8
284 * End:
285 */