blob: 4e1b7307f90d1fc49235db89bf89848d6b79a213 [file] [log] [blame]
Willy Tarreauf89c1872009-10-01 11:19:37 +02001/*
2 * Fast Weighted Least Connection load balancing algorithm.
3 *
4 * Copyright 2000-2009 Willy Tarreau <w@1wt.eu>
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 *
11 */
12
Willy Tarreau1e56f922020-06-04 23:20:13 +020013#include <import/eb32tree.h>
Willy Tarreau4c7e4b72020-05-27 12:58:42 +020014#include <haproxy/api.h>
Willy Tarreaub2551052020-06-09 09:07:15 +020015#include <haproxy/backend.h>
Willy Tarreaua55c4542020-06-04 22:59:39 +020016#include <haproxy/queue.h>
Willy Tarreau1e56f922020-06-04 23:20:13 +020017#include <haproxy/server-t.h>
Willy Tarreauf89c1872009-10-01 11:19:37 +020018
19
20/* Remove a server from a tree. It must have previously been dequeued. This
21 * function is meant to be called when a server is going down or has its
22 * weight disabled.
Willy Tarreau1b877482018-08-21 19:44:53 +020023 *
24 * The server's lock and the lbprm's lock must be held.
Willy Tarreauf89c1872009-10-01 11:19:37 +020025 */
26static inline void fwlc_remove_from_tree(struct server *s)
27{
28 s->lb_tree = NULL;
29}
30
Willy Tarreau1b877482018-08-21 19:44:53 +020031/* simply removes a server from a tree.
32 *
Willy Tarreau17807ed2021-02-17 16:14:00 +010033 * The lbprm's lock must be held.
Willy Tarreau1b877482018-08-21 19:44:53 +020034 */
Willy Tarreauf89c1872009-10-01 11:19:37 +020035static inline void fwlc_dequeue_srv(struct server *s)
36{
37 eb32_delete(&s->lb_node);
38}
39
Christopher Faulet08818422020-12-11 15:36:01 +010040/* Queue a server in its associated tree, assuming the <eweight> is >0.
Willy Tarreau1eb6c552018-12-14 08:33:28 +010041 * Servers are sorted by (#conns+1)/weight. To ensure maximum accuracy,
42 * we use (#conns+1)*SRV_EWGHT_MAX/eweight as the sorting key. The reason
43 * for using #conns+1 is to sort by weights in case the server is picked
44 * and not before it is picked. This provides a better load accuracy for
45 * low connection counts when weights differ and makes sure the round-robin
Willy Tarreaued5ac9c2019-09-06 17:04:04 +020046 * applies between servers of highest weight first. However servers with no
47 * connection are always picked first so that under low loads, it's not
48 * always the single server with the highest weight that gets picked.
Willy Tarreau1b877482018-08-21 19:44:53 +020049 *
Christopher Faulet08818422020-12-11 15:36:01 +010050 * NOTE: Depending on the calling context, we use s->next_eweight or
51 * s->cur_eweight. The next value is used when the server state is updated
52 * (because the weight changed for instance). During this step, the server
53 * state is not yet committed. The current value is used to reposition the
54 * server in the tree. This happens when the server is used.
55 *
Willy Tarreau17807ed2021-02-17 16:14:00 +010056 * The lbprm's lock must be held.
Willy Tarreauf89c1872009-10-01 11:19:37 +020057 */
Christopher Faulet08818422020-12-11 15:36:01 +010058static inline void fwlc_queue_srv(struct server *s, unsigned int eweight)
Willy Tarreauf89c1872009-10-01 11:19:37 +020059{
Willy Tarreau17807ed2021-02-17 16:14:00 +010060 unsigned int inflight = _HA_ATOMIC_LOAD(&s->served) + _HA_ATOMIC_LOAD(&s->nbpend);
Willy Tarreau8c855f62020-10-22 17:41:45 +020061
Christopher Faulet08818422020-12-11 15:36:01 +010062 s->lb_node.key = inflight ? (inflight + 1) * SRV_EWGHT_MAX / eweight : 0;
Willy Tarreauf89c1872009-10-01 11:19:37 +020063 eb32_insert(s->lb_tree, &s->lb_node);
64}
65
66/* Re-position the server in the FWLC tree after it has been assigned one
67 * connection or after it has released one. Note that it is possible that
68 * the server has been moved out of the tree due to failed health-checks.
Willy Tarreau1b877482018-08-21 19:44:53 +020069 *
Willy Tarreau2b437002021-02-17 16:01:37 +010070 * <locked> must reflect the server's lock ownership. The lbprm's lock will
71 * be used.
Willy Tarreauf89c1872009-10-01 11:19:37 +020072 */
Willy Tarreau2b437002021-02-17 16:01:37 +010073static void fwlc_srv_reposition(struct server *s, int locked)
Willy Tarreauf89c1872009-10-01 11:19:37 +020074{
Willy Tarreau55d78a72021-02-17 16:26:55 +010075 unsigned int inflight = _HA_ATOMIC_LOAD(&s->served) + _HA_ATOMIC_LOAD(&s->nbpend);
76 unsigned int new_key = inflight ? (inflight + 1) * SRV_EWGHT_MAX / s->cur_eweight : 0;
77
78 /* some calls will be made for no change (e.g connect_server() after
79 * assign_server(). Let's check that first.
80 */
81 if (s->lb_node.node.leaf_p && s->lb_node.key == new_key)
82 return;
83
Willy Tarreaucd10def2020-10-17 18:48:47 +020084 HA_RWLOCK_WRLOCK(LBPRM_LOCK, &s->proxy->lbprm.lock);
Christopher Faulet1ae2a882019-06-19 10:50:38 +020085 if (s->lb_tree) {
Willy Tarreau55d78a72021-02-17 16:26:55 +010086 /* we might have been waiting for a while on the lock above
87 * so it's worth testing again because other threads are very
88 * likely to have released a connection or taken one leading
89 * to our target value (50% of the case in measurements).
90 */
91 inflight = _HA_ATOMIC_LOAD(&s->served) + _HA_ATOMIC_LOAD(&s->nbpend);
92 new_key = inflight ? (inflight + 1) * SRV_EWGHT_MAX / s->cur_eweight : 0;
93 if (!s->lb_node.node.leaf_p || s->lb_node.key != new_key) {
94 eb32_delete(&s->lb_node);
95 s->lb_node.key = new_key;
96 eb32_insert(s->lb_tree, &s->lb_node);
97 }
Christopher Faulet1ae2a882019-06-19 10:50:38 +020098 }
Willy Tarreaucd10def2020-10-17 18:48:47 +020099 HA_RWLOCK_WRUNLOCK(LBPRM_LOCK, &s->proxy->lbprm.lock);
Willy Tarreauf89c1872009-10-01 11:19:37 +0200100}
101
102/* This function updates the server trees according to server <srv>'s new
103 * state. It should be called when server <srv>'s status changes to down.
104 * It is not important whether the server was already down or not. It is not
105 * important either that the new state is completely down (the caller may not
106 * know all the variables of a server's state).
Willy Tarreau1b877482018-08-21 19:44:53 +0200107 *
108 * The server's lock must be held. The lbprm's lock will be used.
Willy Tarreauf89c1872009-10-01 11:19:37 +0200109 */
110static void fwlc_set_server_status_down(struct server *srv)
111{
112 struct proxy *p = srv->proxy;
113
Willy Tarreauc5150da2014-05-13 19:27:31 +0200114 if (!srv_lb_status_changed(srv))
Willy Tarreauf89c1872009-10-01 11:19:37 +0200115 return;
116
Emeric Brun52a91d32017-08-31 14:41:55 +0200117 if (srv_willbe_usable(srv))
Willy Tarreauf89c1872009-10-01 11:19:37 +0200118 goto out_update_state;
Willy Tarreaucd10def2020-10-17 18:48:47 +0200119 HA_RWLOCK_WRLOCK(LBPRM_LOCK, &p->lbprm.lock);
Willy Tarreau1b877482018-08-21 19:44:53 +0200120
Willy Tarreauf89c1872009-10-01 11:19:37 +0200121
Emeric Brun52a91d32017-08-31 14:41:55 +0200122 if (!srv_currently_usable(srv))
Willy Tarreauf89c1872009-10-01 11:19:37 +0200123 /* server was already down */
124 goto out_update_backend;
125
Willy Tarreauc93cd162014-05-13 15:54:22 +0200126 if (srv->flags & SRV_F_BACKUP) {
Emeric Brun52a91d32017-08-31 14:41:55 +0200127 p->lbprm.tot_wbck -= srv->cur_eweight;
Willy Tarreauf89c1872009-10-01 11:19:37 +0200128 p->srv_bck--;
129
130 if (srv == p->lbprm.fbck) {
131 /* we lost the first backup server in a single-backup
132 * configuration, we must search another one.
133 */
134 struct server *srv2 = p->lbprm.fbck;
135 do {
136 srv2 = srv2->next;
137 } while (srv2 &&
Willy Tarreauc93cd162014-05-13 15:54:22 +0200138 !((srv2->flags & SRV_F_BACKUP) &&
Emeric Brun52a91d32017-08-31 14:41:55 +0200139 srv_willbe_usable(srv2)));
Willy Tarreauf89c1872009-10-01 11:19:37 +0200140 p->lbprm.fbck = srv2;
141 }
142 } else {
Emeric Brun52a91d32017-08-31 14:41:55 +0200143 p->lbprm.tot_wact -= srv->cur_eweight;
Willy Tarreauf89c1872009-10-01 11:19:37 +0200144 p->srv_act--;
145 }
146
147 fwlc_dequeue_srv(srv);
148 fwlc_remove_from_tree(srv);
149
150out_update_backend:
151 /* check/update tot_used, tot_weight */
152 update_backend_weight(p);
Willy Tarreaucd10def2020-10-17 18:48:47 +0200153 HA_RWLOCK_WRUNLOCK(LBPRM_LOCK, &p->lbprm.lock);
Willy Tarreau1b877482018-08-21 19:44:53 +0200154
Willy Tarreauf89c1872009-10-01 11:19:37 +0200155 out_update_state:
Willy Tarreauc5150da2014-05-13 19:27:31 +0200156 srv_lb_commit_status(srv);
Willy Tarreauf89c1872009-10-01 11:19:37 +0200157}
158
159/* This function updates the server trees according to server <srv>'s new
160 * state. It should be called when server <srv>'s status changes to up.
161 * It is not important whether the server was already down or not. It is not
162 * important either that the new state is completely UP (the caller may not
163 * know all the variables of a server's state). This function will not change
164 * the weight of a server which was already up.
Willy Tarreau1b877482018-08-21 19:44:53 +0200165 *
166 * The server's lock must be held. The lbprm's lock will be used.
Willy Tarreauf89c1872009-10-01 11:19:37 +0200167 */
168static void fwlc_set_server_status_up(struct server *srv)
169{
170 struct proxy *p = srv->proxy;
171
Willy Tarreauc5150da2014-05-13 19:27:31 +0200172 if (!srv_lb_status_changed(srv))
Willy Tarreauf89c1872009-10-01 11:19:37 +0200173 return;
174
Emeric Brun52a91d32017-08-31 14:41:55 +0200175 if (!srv_willbe_usable(srv))
Willy Tarreauf89c1872009-10-01 11:19:37 +0200176 goto out_update_state;
177
Willy Tarreaucd10def2020-10-17 18:48:47 +0200178 HA_RWLOCK_WRLOCK(LBPRM_LOCK, &p->lbprm.lock);
Willy Tarreau1b877482018-08-21 19:44:53 +0200179
Emeric Brun52a91d32017-08-31 14:41:55 +0200180 if (srv_currently_usable(srv))
Willy Tarreauf89c1872009-10-01 11:19:37 +0200181 /* server was already up */
182 goto out_update_backend;
183
Willy Tarreauc93cd162014-05-13 15:54:22 +0200184 if (srv->flags & SRV_F_BACKUP) {
Willy Tarreauf89c1872009-10-01 11:19:37 +0200185 srv->lb_tree = &p->lbprm.fwlc.bck;
Emeric Brun52a91d32017-08-31 14:41:55 +0200186 p->lbprm.tot_wbck += srv->next_eweight;
Willy Tarreauf89c1872009-10-01 11:19:37 +0200187 p->srv_bck++;
188
189 if (!(p->options & PR_O_USE_ALL_BK)) {
190 if (!p->lbprm.fbck) {
191 /* there was no backup server anymore */
192 p->lbprm.fbck = srv;
193 } else {
194 /* we may have restored a backup server prior to fbck,
195 * in which case it should replace it.
196 */
197 struct server *srv2 = srv;
198 do {
199 srv2 = srv2->next;
200 } while (srv2 && (srv2 != p->lbprm.fbck));
201 if (srv2)
202 p->lbprm.fbck = srv;
203 }
204 }
205 } else {
206 srv->lb_tree = &p->lbprm.fwlc.act;
Emeric Brun52a91d32017-08-31 14:41:55 +0200207 p->lbprm.tot_wact += srv->next_eweight;
Willy Tarreauf89c1872009-10-01 11:19:37 +0200208 p->srv_act++;
209 }
210
211 /* note that eweight cannot be 0 here */
Christopher Faulet08818422020-12-11 15:36:01 +0100212 fwlc_queue_srv(srv, srv->next_eweight);
Willy Tarreauf89c1872009-10-01 11:19:37 +0200213
214 out_update_backend:
215 /* check/update tot_used, tot_weight */
216 update_backend_weight(p);
Willy Tarreaucd10def2020-10-17 18:48:47 +0200217 HA_RWLOCK_WRUNLOCK(LBPRM_LOCK, &p->lbprm.lock);
Willy Tarreau1b877482018-08-21 19:44:53 +0200218
Willy Tarreauf89c1872009-10-01 11:19:37 +0200219 out_update_state:
Willy Tarreauc5150da2014-05-13 19:27:31 +0200220 srv_lb_commit_status(srv);
Willy Tarreauf89c1872009-10-01 11:19:37 +0200221}
222
223/* This function must be called after an update to server <srv>'s effective
224 * weight. It may be called after a state change too.
Willy Tarreau1b877482018-08-21 19:44:53 +0200225 *
226 * The server's lock must be held. The lbprm's lock will be used.
Willy Tarreauf89c1872009-10-01 11:19:37 +0200227 */
228static void fwlc_update_server_weight(struct server *srv)
229{
230 int old_state, new_state;
231 struct proxy *p = srv->proxy;
232
Willy Tarreauc5150da2014-05-13 19:27:31 +0200233 if (!srv_lb_status_changed(srv))
Willy Tarreauf89c1872009-10-01 11:19:37 +0200234 return;
235
236 /* If changing the server's weight changes its state, we simply apply
237 * the procedures we already have for status change. If the state
238 * remains down, the server is not in any tree, so it's as easy as
239 * updating its values. If the state remains up with different weights,
240 * there are some computations to perform to find a new place and
241 * possibly a new tree for this server.
242 */
243
Emeric Brun52a91d32017-08-31 14:41:55 +0200244 old_state = srv_currently_usable(srv);
245 new_state = srv_willbe_usable(srv);
Willy Tarreauf89c1872009-10-01 11:19:37 +0200246
247 if (!old_state && !new_state) {
Willy Tarreauc5150da2014-05-13 19:27:31 +0200248 srv_lb_commit_status(srv);
Willy Tarreauf89c1872009-10-01 11:19:37 +0200249 return;
250 }
251 else if (!old_state && new_state) {
252 fwlc_set_server_status_up(srv);
253 return;
254 }
255 else if (old_state && !new_state) {
256 fwlc_set_server_status_down(srv);
257 return;
258 }
259
Willy Tarreaucd10def2020-10-17 18:48:47 +0200260 HA_RWLOCK_WRLOCK(LBPRM_LOCK, &p->lbprm.lock);
Willy Tarreau1b877482018-08-21 19:44:53 +0200261
Willy Tarreauf89c1872009-10-01 11:19:37 +0200262 if (srv->lb_tree)
263 fwlc_dequeue_srv(srv);
264
Willy Tarreauc93cd162014-05-13 15:54:22 +0200265 if (srv->flags & SRV_F_BACKUP) {
Emeric Brun52a91d32017-08-31 14:41:55 +0200266 p->lbprm.tot_wbck += srv->next_eweight - srv->cur_eweight;
Willy Tarreauf89c1872009-10-01 11:19:37 +0200267 srv->lb_tree = &p->lbprm.fwlc.bck;
268 } else {
Emeric Brun52a91d32017-08-31 14:41:55 +0200269 p->lbprm.tot_wact += srv->next_eweight - srv->cur_eweight;
Willy Tarreauf89c1872009-10-01 11:19:37 +0200270 srv->lb_tree = &p->lbprm.fwlc.act;
271 }
272
Christopher Faulet08818422020-12-11 15:36:01 +0100273 fwlc_queue_srv(srv, srv->next_eweight);
Willy Tarreauf89c1872009-10-01 11:19:37 +0200274
275 update_backend_weight(p);
Willy Tarreaucd10def2020-10-17 18:48:47 +0200276 HA_RWLOCK_WRUNLOCK(LBPRM_LOCK, &p->lbprm.lock);
Willy Tarreau1b877482018-08-21 19:44:53 +0200277
Willy Tarreauc5150da2014-05-13 19:27:31 +0200278 srv_lb_commit_status(srv);
Willy Tarreauf89c1872009-10-01 11:19:37 +0200279}
280
281/* This function is responsible for building the trees in case of fast
282 * weighted least-conns. It also sets p->lbprm.wdiv to the eweight to
283 * uweight ratio. Both active and backup groups are initialized.
284 */
285void fwlc_init_server_tree(struct proxy *p)
286{
287 struct server *srv;
288 struct eb_root init_head = EB_ROOT;
289
290 p->lbprm.set_server_status_up = fwlc_set_server_status_up;
291 p->lbprm.set_server_status_down = fwlc_set_server_status_down;
292 p->lbprm.update_server_eweight = fwlc_update_server_weight;
293 p->lbprm.server_take_conn = fwlc_srv_reposition;
294 p->lbprm.server_drop_conn = fwlc_srv_reposition;
295
296 p->lbprm.wdiv = BE_WEIGHT_SCALE;
297 for (srv = p->srv; srv; srv = srv->next) {
Emeric Brun52a91d32017-08-31 14:41:55 +0200298 srv->next_eweight = (srv->uweight * p->lbprm.wdiv + p->lbprm.wmult - 1) / p->lbprm.wmult;
Willy Tarreauc5150da2014-05-13 19:27:31 +0200299 srv_lb_commit_status(srv);
Willy Tarreauf89c1872009-10-01 11:19:37 +0200300 }
301
302 recount_servers(p);
303 update_backend_weight(p);
304
305 p->lbprm.fwlc.act = init_head;
306 p->lbprm.fwlc.bck = init_head;
307
308 /* queue active and backup servers in two distinct groups */
309 for (srv = p->srv; srv; srv = srv->next) {
Emeric Brun52a91d32017-08-31 14:41:55 +0200310 if (!srv_currently_usable(srv))
Willy Tarreauf89c1872009-10-01 11:19:37 +0200311 continue;
Willy Tarreauc93cd162014-05-13 15:54:22 +0200312 srv->lb_tree = (srv->flags & SRV_F_BACKUP) ? &p->lbprm.fwlc.bck : &p->lbprm.fwlc.act;
Christopher Faulet08818422020-12-11 15:36:01 +0100313 fwlc_queue_srv(srv, srv->next_eweight);
Willy Tarreauf89c1872009-10-01 11:19:37 +0200314 }
315}
316
317/* Return next server from the FWLC tree in backend <p>. If the tree is empty,
318 * return NULL. Saturated servers are skipped.
Willy Tarreau1b877482018-08-21 19:44:53 +0200319 *
320 * The server's lock must be held. The lbprm's lock will be used.
Willy Tarreauf89c1872009-10-01 11:19:37 +0200321 */
322struct server *fwlc_get_next_server(struct proxy *p, struct server *srvtoavoid)
323{
324 struct server *srv, *avoided;
325 struct eb32_node *node;
326
327 srv = avoided = NULL;
328
Willy Tarreau58bc9c12020-10-17 19:32:09 +0200329 HA_RWLOCK_RDLOCK(LBPRM_LOCK, &p->lbprm.lock);
Willy Tarreauf89c1872009-10-01 11:19:37 +0200330 if (p->srv_act)
331 node = eb32_first(&p->lbprm.fwlc.act);
Christopher Faulet5b517552017-06-09 14:17:53 +0200332 else if (p->lbprm.fbck) {
333 srv = p->lbprm.fbck;
334 goto out;
335 }
Willy Tarreauf89c1872009-10-01 11:19:37 +0200336 else if (p->srv_bck)
337 node = eb32_first(&p->lbprm.fwlc.bck);
Christopher Faulet5b517552017-06-09 14:17:53 +0200338 else {
339 srv = NULL;
340 goto out;
341 }
Willy Tarreauf89c1872009-10-01 11:19:37 +0200342
343 while (node) {
344 /* OK, we have a server. However, it may be saturated, in which
345 * case we don't want to reconsider it for now, so we'll simply
346 * skip it. Same if it's the server we try to avoid, in which
347 * case we simply remember it for later use if needed.
348 */
349 struct server *s;
350
351 s = eb32_entry(node, struct server, lb_node);
Willy Tarreau8ae8c482020-10-22 17:19:07 +0200352 if (!s->maxconn || s->served + s->nbpend < srv_dynamic_maxconn(s) + s->maxqueue) {
Willy Tarreauf89c1872009-10-01 11:19:37 +0200353 if (s != srvtoavoid) {
354 srv = s;
355 break;
356 }
357 avoided = s;
358 }
359 node = eb32_next(node);
360 }
361
362 if (!srv)
363 srv = avoided;
Christopher Faulet5b517552017-06-09 14:17:53 +0200364 out:
Willy Tarreau58bc9c12020-10-17 19:32:09 +0200365 HA_RWLOCK_RDUNLOCK(LBPRM_LOCK, &p->lbprm.lock);
Willy Tarreauf89c1872009-10-01 11:19:37 +0200366 return srv;
367}
368
369
370/*
371 * Local variables:
372 * c-indent-level: 8
373 * c-basic-offset: 8
374 * End:
375 */