blob: ce6d38c63e3d01f239ae1f4035b62aabfa706353 [file] [log] [blame]
Willy Tarreauc7e42382012-08-24 19:22:53 +02001/*
2 * include/common/buffer.h
3 * Buffer management definitions, macros and inline functions.
4 *
5 * Copyright (C) 2000-2012 Willy Tarreau - w@1wt.eu
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation, version 2.1
10 * exclusively.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
21
22#ifndef _COMMON_BUFFER_H
23#define _COMMON_BUFFER_H
24
25#include <stdio.h>
26#include <stdlib.h>
27#include <string.h>
28
Willy Tarreau41806d12018-07-11 09:39:05 +020029#include <common/buf.h>
Willy Tarreau8c89c202012-09-28 16:02:48 +020030#include <common/chunk.h>
Willy Tarreauc7e42382012-08-24 19:22:53 +020031#include <common/config.h>
Willy Tarreau6634b632017-09-22 15:02:54 +020032#include <common/ist.h>
Willy Tarreauea1b06d2018-07-12 09:02:47 +020033#include <common/istbuf.h>
Willy Tarreau9b28e032012-10-12 23:49:43 +020034#include <common/memory.h>
Willy Tarreauc7e42382012-08-24 19:22:53 +020035
36
Christopher Fauleta73e59b2016-12-09 17:30:18 +010037/* an element of the <buffer_wq> list. It represents an object that need to
38 * acquire a buffer to continue its process. */
39struct buffer_wait {
40 void *target; /* The waiting object that should be woken up */
41 int (*wakeup_cb)(void *); /* The function used to wake up the <target>, passed as argument */
42 struct list list; /* Next element in the <buffer_wq> list */
43};
44
Willy Tarreaubafbe012017-11-24 17:34:44 +010045extern struct pool_head *pool_head_buffer;
Christopher Fauleta73e59b2016-12-09 17:30:18 +010046extern struct list buffer_wq;
Willy Tarreau53bae852017-11-26 11:00:37 +010047__decl_hathreads(extern HA_SPINLOCK_T buffer_wq_lock);
Willy Tarreauc7e42382012-08-24 19:22:53 +020048
Willy Tarreau9b28e032012-10-12 23:49:43 +020049int init_buffer();
Willy Tarreauc7e42382012-08-24 19:22:53 +020050void buffer_dump(FILE *o, struct buffer *b, int from, int to);
Willy Tarreauc7e42382012-08-24 19:22:53 +020051
52/*****************************************************************/
53/* These functions are used to compute various buffer area sizes */
54/*****************************************************************/
55
Willy Tarreauc7e42382012-08-24 19:22:53 +020056/* Return 1 if the buffer has less than 1/4 of its capacity free, otherwise 0 */
57static inline int buffer_almost_full(const struct buffer *buf)
58{
Willy Tarreauc9fa0482018-07-10 17:43:27 +020059 if (b_is_null(buf))
Willy Tarreau4428a292014-11-28 20:54:13 +010060 return 0;
61
Willy Tarreaubbc68df2018-06-06 14:30:50 +020062 return b_almost_full(buf);
Willy Tarreauc7e42382012-08-24 19:22:53 +020063}
64
Willy Tarreau7b04cc42018-07-10 10:35:02 +020065/**************************************************/
66/* Functions below are used for buffer allocation */
67/**************************************************/
Willy Tarreauaf819352012-08-27 22:08:00 +020068
Willy Tarreauc9fa0482018-07-10 17:43:27 +020069/* Allocates a buffer and assigns it to *buf. If no memory is available,
70 * ((char *)1) is assigned instead with a zero size. No control is made to
71 * check if *buf already pointed to another buffer. The allocated buffer is
72 * returned, or NULL in case no memory is available.
Willy Tarreaue583ea52014-11-24 11:30:16 +010073 */
Willy Tarreauc9fa0482018-07-10 17:43:27 +020074static inline struct buffer *b_alloc(struct buffer *buf)
Willy Tarreaue583ea52014-11-24 11:30:16 +010075{
Willy Tarreauc9fa0482018-07-10 17:43:27 +020076 char *area;
Willy Tarreauf2f7d6b2014-11-24 11:55:08 +010077
Willy Tarreauc9fa0482018-07-10 17:43:27 +020078 *buf = BUF_WANTED;
79 area = pool_alloc_dirty(pool_head_buffer);
80 if (unlikely(!area))
81 return NULL;
82
83 buf->area = area;
84 buf->size = pool_head_buffer->size;
85 return buf;
Willy Tarreaue583ea52014-11-24 11:30:16 +010086}
87
Willy Tarreauc9fa0482018-07-10 17:43:27 +020088/* Allocates a buffer and assigns it to *buf. If no memory is available,
89 * ((char *)1) is assigned instead with a zero size. No control is made to
90 * check if *buf already pointed to another buffer. The allocated buffer is
91 * returned, or NULL in case no memory is available. The difference with
92 * b_alloc() is that this function only picks from the pool and never calls
93 * malloc(), so it can fail even if some memory is available.
Willy Tarreau620bd6c2014-12-08 16:37:26 +010094 */
Willy Tarreauc9fa0482018-07-10 17:43:27 +020095static inline struct buffer *b_alloc_fast(struct buffer *buf)
Willy Tarreau620bd6c2014-12-08 16:37:26 +010096{
Willy Tarreauc9fa0482018-07-10 17:43:27 +020097 char *area;
Willy Tarreau620bd6c2014-12-08 16:37:26 +010098
Willy Tarreauc9fa0482018-07-10 17:43:27 +020099 *buf = BUF_WANTED;
100 area = pool_get_first(pool_head_buffer);
101 if (unlikely(!area))
102 return NULL;
103
104 buf->area = area;
105 buf->size = pool_head_buffer->size;
106 return buf;
Willy Tarreau620bd6c2014-12-08 16:37:26 +0100107}
108
Willy Tarreauc9fa0482018-07-10 17:43:27 +0200109/* Releases buffer <buf> (no check of emptiness) */
110static inline void __b_drop(struct buffer *buf)
Willy Tarreau7dfca9d2014-11-25 19:45:11 +0100111{
Willy Tarreauc9fa0482018-07-10 17:43:27 +0200112 pool_free(pool_head_buffer, buf->area);
Willy Tarreau7dfca9d2014-11-25 19:45:11 +0100113}
114
Willy Tarreauc9fa0482018-07-10 17:43:27 +0200115/* Releases buffer <buf> if allocated. */
116static inline void b_drop(struct buffer *buf)
Willy Tarreau2a4b5432014-11-24 11:39:34 +0100117{
Willy Tarreauc9fa0482018-07-10 17:43:27 +0200118 if (buf->size)
119 __b_drop(buf);
Willy Tarreau2a4b5432014-11-24 11:39:34 +0100120}
121
Willy Tarreauc9fa0482018-07-10 17:43:27 +0200122/* Releases buffer <buf> if allocated, and marks it empty. */
123static inline void b_free(struct buffer *buf)
Willy Tarreau2a4b5432014-11-24 11:39:34 +0100124{
125 b_drop(buf);
Willy Tarreauc9fa0482018-07-10 17:43:27 +0200126 *buf = BUF_NULL;
Willy Tarreau2a4b5432014-11-24 11:39:34 +0100127}
128
Willy Tarreauf4718e82014-12-02 13:54:01 +0100129/* Ensures that <buf> is allocated. If an allocation is needed, it ensures that
130 * there are still at least <margin> buffers available in the pool after this
131 * allocation so that we don't leave the pool in a condition where a session or
132 * a response buffer could not be allocated anymore, resulting in a deadlock.
133 * This means that we sometimes need to try to allocate extra entries even if
134 * only one buffer is needed.
Christopher Fauletfa5c8122017-11-10 10:39:16 +0100135 *
136 * We need to lock the pool here to be sure to have <margin> buffers available
137 * after the allocation, regardless how many threads that doing it in the same
138 * time. So, we use internal and lockless memory functions (prefixed with '__').
Willy Tarreauf4718e82014-12-02 13:54:01 +0100139 */
Willy Tarreauc9fa0482018-07-10 17:43:27 +0200140static inline struct buffer *b_alloc_margin(struct buffer *buf, int margin)
Willy Tarreauf4718e82014-12-02 13:54:01 +0100141{
Willy Tarreauc9fa0482018-07-10 17:43:27 +0200142 char *area;
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200143 ssize_t idx;
144 unsigned int cached;
Willy Tarreauf4718e82014-12-02 13:54:01 +0100145
Willy Tarreauc9fa0482018-07-10 17:43:27 +0200146 if (buf->size)
147 return buf;
Willy Tarreauf4718e82014-12-02 13:54:01 +0100148
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200149 cached = 0;
150 idx = pool_get_index(pool_head_buffer);
151 if (idx >= 0)
Willy Tarreau7f0165e2018-11-26 17:09:46 +0100152 cached = pool_cache[tid][idx].count;
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200153
Willy Tarreauc9fa0482018-07-10 17:43:27 +0200154 *buf = BUF_WANTED;
155
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100156#ifndef CONFIG_HAP_LOCKLESS_POOLS
Willy Tarreaubafbe012017-11-24 17:34:44 +0100157 HA_SPIN_LOCK(POOL_LOCK, &pool_head_buffer->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100158#endif
Christopher Fauletfa5c8122017-11-10 10:39:16 +0100159
Willy Tarreauf4718e82014-12-02 13:54:01 +0100160 /* fast path */
Willy Tarreaue18db9e2018-10-16 10:28:54 +0200161 if ((pool_head_buffer->allocated - pool_head_buffer->used + cached) > margin) {
Willy Tarreauc9fa0482018-07-10 17:43:27 +0200162 area = __pool_get_first(pool_head_buffer);
163 if (likely(area)) {
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100164#ifndef CONFIG_HAP_LOCKLESS_POOLS
Willy Tarreaubafbe012017-11-24 17:34:44 +0100165 HA_SPIN_UNLOCK(POOL_LOCK, &pool_head_buffer->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100166#endif
Willy Tarreauc9fa0482018-07-10 17:43:27 +0200167 goto done;
Christopher Fauletfa5c8122017-11-10 10:39:16 +0100168 }
169 }
Willy Tarreauf4718e82014-12-02 13:54:01 +0100170
Christopher Fauletfa5c8122017-11-10 10:39:16 +0100171 /* slow path, uses malloc() */
Willy Tarreauc9fa0482018-07-10 17:43:27 +0200172 area = __pool_refill_alloc(pool_head_buffer, margin);
Christopher Fauletfa5c8122017-11-10 10:39:16 +0100173
Willy Tarreauf161d0f2018-02-22 14:05:55 +0100174#ifndef CONFIG_HAP_LOCKLESS_POOLS
Willy Tarreaubafbe012017-11-24 17:34:44 +0100175 HA_SPIN_UNLOCK(POOL_LOCK, &pool_head_buffer->lock);
Olivier Houchardcf975d42018-01-24 18:38:31 +0100176#endif
Willy Tarreauf4718e82014-12-02 13:54:01 +0100177
Willy Tarreauc9fa0482018-07-10 17:43:27 +0200178 if (unlikely(!area))
179 return NULL;
180
181 done:
182 buf->area = area;
183 buf->size = pool_head_buffer->size;
184 return buf;
Willy Tarreauf4718e82014-12-02 13:54:01 +0100185}
186
Christopher Fauleta73e59b2016-12-09 17:30:18 +0100187
Willy Tarreauc41b3e82018-03-02 10:27:12 +0100188/* Offer a buffer currently belonging to target <from> to whoever needs one.
189 * Any pointer is valid for <from>, including NULL. Its purpose is to avoid
190 * passing a buffer to oneself in case of failed allocations (e.g. need two
191 * buffers, get one, fail, release it and wake up self again). In case of
192 * normal buffer release where it is expected that the caller is not waiting
193 * for a buffer, NULL is fine.
194 */
Christopher Fauleta73e59b2016-12-09 17:30:18 +0100195void __offer_buffer(void *from, unsigned int threshold);
196
197static inline void offer_buffers(void *from, unsigned int threshold)
198{
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100199 HA_SPIN_LOCK(BUF_WQ_LOCK, &buffer_wq_lock);
Emeric Bruna1dd2432017-06-21 15:42:52 +0200200 if (LIST_ISEMPTY(&buffer_wq)) {
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100201 HA_SPIN_UNLOCK(BUF_WQ_LOCK, &buffer_wq_lock);
Christopher Fauleta73e59b2016-12-09 17:30:18 +0100202 return;
Emeric Bruna1dd2432017-06-21 15:42:52 +0200203 }
Christopher Fauleta73e59b2016-12-09 17:30:18 +0100204 __offer_buffer(from, threshold);
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100205 HA_SPIN_UNLOCK(BUF_WQ_LOCK, &buffer_wq_lock);
Christopher Fauleta73e59b2016-12-09 17:30:18 +0100206}
207
Willy Tarreaue5676e72017-09-22 15:47:51 +0200208
Willy Tarreauc7e42382012-08-24 19:22:53 +0200209#endif /* _COMMON_BUFFER_H */
210
211/*
212 * Local variables:
213 * c-indent-level: 8
214 * c-basic-offset: 8
215 * End:
216 */