blob: 25bf377f23c50631c791adbec681029ab32fbc7e [file] [log] [blame]
Emeric Brun3e541d12012-09-03 11:14:36 +02001/*
2 * shctx.h - shared context management functions for SSL
3 *
4 * Copyright (C) 2011-2012 EXCELIANCE
5 *
6 * Author: Emeric Brun - emeric@exceliance.fr
7 *
8 * This program is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU General Public License
10 * as published by the Free Software Foundation; either version
11 * 2 of the License, or (at your option) any later version.
12 */
13
14#ifndef SHCTX_H
15#define SHCTX_H
William Lallemand24a7a752017-10-09 14:17:39 +020016
William Lallemand4f45bb92017-10-30 20:08:51 +010017#include <common/mini-clist.h>
William Lallemand24a7a752017-10-09 14:17:39 +020018#include <types/shctx.h>
19
Willy Tarreaua1bd1fa2019-03-29 17:26:33 +010020#include <inttypes.h>
Emeric Brun3e541d12012-09-03 11:14:36 +020021
William Lallemand24a7a752017-10-09 14:17:39 +020022#ifndef USE_PRIVATE_CACHE
23#ifdef USE_PTHREAD_PSHARED
24#include <pthread.h>
25#else
26#ifdef USE_SYSCALL_FUTEX
27#include <unistd.h>
28#include <linux/futex.h>
29#include <sys/syscall.h>
Emeric Brun3e541d12012-09-03 11:14:36 +020030#endif
Emeric Brun3e541d12012-09-03 11:14:36 +020031#endif
Emeric Brun786991e2012-11-26 18:37:12 +010032#endif
33
Frédéric Lécailleb7838af2018-10-22 16:21:39 +020034int shctx_init(struct shared_context **orig_shctx,
Frédéric Lécailleb80bc272018-10-25 20:31:40 +020035 int maxblocks, int blocksize, unsigned int maxobjsz,
36 int extra, int shared);
Frédéric Lécaille0bec8072018-10-22 17:55:57 +020037struct shared_block *shctx_row_reserve_hot(struct shared_context *shctx,
38 struct shared_block *last, int data_len);
William Lallemand4f45bb92017-10-30 20:08:51 +010039void shctx_row_inc_hot(struct shared_context *shctx, struct shared_block *first);
40void shctx_row_dec_hot(struct shared_context *shctx, struct shared_block *first);
41int shctx_row_data_append(struct shared_context *shctx,
Frédéric Lécaille0bec8072018-10-22 17:55:57 +020042 struct shared_block *first, struct shared_block *from,
43 unsigned char *data, int len);
William Lallemand4f45bb92017-10-30 20:08:51 +010044int shctx_row_data_get(struct shared_context *shctx, struct shared_block *first,
45 unsigned char *dst, int offset, int len);
Emeric Bruncaa19cc2014-05-07 16:10:18 +020046
William Lallemanded0b5ad2017-10-30 19:36:36 +010047
William Lallemand24a7a752017-10-09 14:17:39 +020048/* Lock functions */
49
50#if defined (USE_PRIVATE_CACHE)
51
William Lallemanda3c77cf2017-10-30 23:44:40 +010052#define shctx_lock(shctx)
53#define shctx_unlock(shctx)
William Lallemand24a7a752017-10-09 14:17:39 +020054
55#elif defined (USE_PTHREAD_PSHARED)
56extern int use_shared_mem;
57
William Lallemanda3c77cf2017-10-30 23:44:40 +010058#define shctx_lock(shctx) if (use_shared_mem) pthread_mutex_lock(&shctx->mutex)
59#define shctx_unlock(shctx) if (use_shared_mem) pthread_mutex_unlock(&shctx->mutex)
William Lallemand24a7a752017-10-09 14:17:39 +020060
61#else
62extern int use_shared_mem;
63
64#ifdef USE_SYSCALL_FUTEX
William Lallemanda3c77cf2017-10-30 23:44:40 +010065static inline void _shctx_wait4lock(unsigned int *count, unsigned int *uaddr, int value)
William Lallemand24a7a752017-10-09 14:17:39 +020066{
67 syscall(SYS_futex, uaddr, FUTEX_WAIT, value, NULL, 0, 0);
68}
69
William Lallemanda3c77cf2017-10-30 23:44:40 +010070static inline void _shctx_awakelocker(unsigned int *uaddr)
William Lallemand24a7a752017-10-09 14:17:39 +020071{
72 syscall(SYS_futex, uaddr, FUTEX_WAKE, 1, NULL, 0, 0);
73}
74
75#else /* internal spin lock */
76
77#if defined (__i486__) || defined (__i586__) || defined (__i686__) || defined (__x86_64__)
78static inline void relax()
79{
80 __asm volatile("rep;nop\n" ::: "memory");
81}
82#else /* if no x86_64 or i586 arch: use less optimized but generic asm */
83static inline void relax()
84{
85 __asm volatile("" ::: "memory");
86}
87#endif
88
William Lallemanda3c77cf2017-10-30 23:44:40 +010089static inline void _shctx_wait4lock(unsigned int *count, unsigned int *uaddr, int value)
William Lallemand24a7a752017-10-09 14:17:39 +020090{
91 int i;
92
93 for (i = 0; i < *count; i++) {
94 relax();
95 relax();
Willy Tarreau7f767912020-05-01 13:05:29 +020096 if (*uaddr != value)
97 return;
William Lallemand24a7a752017-10-09 14:17:39 +020098 }
Willy Tarreau62d5e9c2020-05-01 13:15:32 +020099 *count = (unsigned char)((*count << 1) + 1);
William Lallemand24a7a752017-10-09 14:17:39 +0200100}
101
William Lallemanda3c77cf2017-10-30 23:44:40 +0100102#define _shctx_awakelocker(a)
William Lallemand24a7a752017-10-09 14:17:39 +0200103
104#endif
105
106#if defined (__i486__) || defined (__i586__) || defined (__i686__) || defined (__x86_64__)
107static inline unsigned int xchg(unsigned int *ptr, unsigned int x)
108{
109 __asm volatile("lock xchgl %0,%1"
110 : "=r" (x), "+m" (*ptr)
111 : "0" (x)
112 : "memory");
113 return x;
114}
115
116static inline unsigned int cmpxchg(unsigned int *ptr, unsigned int old, unsigned int new)
117{
118 unsigned int ret;
119
120 __asm volatile("lock cmpxchgl %2,%1"
121 : "=a" (ret), "+m" (*ptr)
122 : "r" (new), "0" (old)
123 : "memory");
124 return ret;
125}
126
127static inline unsigned char atomic_dec(unsigned int *ptr)
128{
129 unsigned char ret;
130 __asm volatile("lock decl %0\n"
131 "setne %1\n"
132 : "+m" (*ptr), "=qm" (ret)
133 :
134 : "memory");
135 return ret;
136}
137
138#else /* if no x86_64 or i586 arch: use less optimized gcc >= 4.1 built-ins */
139static inline unsigned int xchg(unsigned int *ptr, unsigned int x)
140{
141 return __sync_lock_test_and_set(ptr, x);
142}
143
144static inline unsigned int cmpxchg(unsigned int *ptr, unsigned int old, unsigned int new)
145{
146 return __sync_val_compare_and_swap(ptr, old, new);
147}
148
149static inline unsigned char atomic_dec(unsigned int *ptr)
150{
151 return __sync_sub_and_fetch(ptr, 1) ? 1 : 0;
152}
153
154#endif
155
William Lallemanda3c77cf2017-10-30 23:44:40 +0100156static inline void _shctx_lock(struct shared_context *shctx)
William Lallemand24a7a752017-10-09 14:17:39 +0200157{
158 unsigned int x;
Willy Tarreau62d5e9c2020-05-01 13:15:32 +0200159 unsigned int count = 3;
William Lallemand24a7a752017-10-09 14:17:39 +0200160
161 x = cmpxchg(&shctx->waiters, 0, 1);
162 if (x) {
163 if (x != 2)
164 x = xchg(&shctx->waiters, 2);
165
166 while (x) {
William Lallemanda3c77cf2017-10-30 23:44:40 +0100167 _shctx_wait4lock(&count, &shctx->waiters, 2);
William Lallemand24a7a752017-10-09 14:17:39 +0200168 x = xchg(&shctx->waiters, 2);
169 }
170 }
171}
172
William Lallemanda3c77cf2017-10-30 23:44:40 +0100173static inline void _shctx_unlock(struct shared_context *shctx)
William Lallemand24a7a752017-10-09 14:17:39 +0200174{
175 if (atomic_dec(&shctx->waiters)) {
176 shctx->waiters = 0;
William Lallemanda3c77cf2017-10-30 23:44:40 +0100177 _shctx_awakelocker(&shctx->waiters);
William Lallemand24a7a752017-10-09 14:17:39 +0200178 }
179}
180
William Lallemanda3c77cf2017-10-30 23:44:40 +0100181#define shctx_lock(shctx) if (use_shared_mem) _shctx_lock(shctx)
William Lallemand24a7a752017-10-09 14:17:39 +0200182
William Lallemanda3c77cf2017-10-30 23:44:40 +0100183#define shctx_unlock(shctx) if (use_shared_mem) _shctx_unlock(shctx)
William Lallemand24a7a752017-10-09 14:17:39 +0200184
185#endif
186
William Lallemanded0b5ad2017-10-30 19:36:36 +0100187/* List Macros */
188
Frédéric Lécaille0bec8072018-10-22 17:55:57 +0200189/*
190 * Insert <s> block after <head> which is not necessarily the head of a list,
191 * so between <head> and the next element after <head>.
192 */
193static inline void shctx_block_append_hot(struct shared_context *shctx,
194 struct list *head,
195 struct shared_block *s)
196{
197 shctx->nbav--;
198 LIST_DEL(&s->list);
199 LIST_ADD(head, &s->list);
200}
201
William Lallemand4f45bb92017-10-30 20:08:51 +0100202static inline void shctx_block_set_hot(struct shared_context *shctx,
William Lallemanded0b5ad2017-10-30 19:36:36 +0100203 struct shared_block *s)
204{
William Lallemand4f45bb92017-10-30 20:08:51 +0100205 shctx->nbav--;
206 LIST_DEL(&s->list);
207 LIST_ADDQ(&shctx->hot, &s->list);
William Lallemanded0b5ad2017-10-30 19:36:36 +0100208}
209
William Lallemand4f45bb92017-10-30 20:08:51 +0100210static inline void shctx_block_set_avail(struct shared_context *shctx,
William Lallemanded0b5ad2017-10-30 19:36:36 +0100211 struct shared_block *s)
212{
William Lallemand4f45bb92017-10-30 20:08:51 +0100213 shctx->nbav++;
214 LIST_DEL(&s->list);
215 LIST_ADDQ(&shctx->avail, &s->list);
William Lallemanded0b5ad2017-10-30 19:36:36 +0100216}
William Lallemand24a7a752017-10-09 14:17:39 +0200217
Emeric Brun3e541d12012-09-03 11:14:36 +0200218#endif /* SHCTX_H */
219