MEDIUM: stick-tables: Add srvkey option to stick-table
This allows using the address of the server rather than the name of the
server for keeping track of servers in a backend for stickiness.
The peers code was also extended to support feeding the dictionary using
this key instead of the name.
Fixes #814
diff --git a/doc/configuration.txt b/doc/configuration.txt
index be8cb9e..e5e8546 100644
--- a/doc/configuration.txt
+++ b/doc/configuration.txt
@@ -10647,7 +10647,7 @@
stick-table type {ip | integer | string [len <length>] | binary [len <length>]}
- size <size> [expire <expire>] [nopurge] [peers <peersect>]
+ size <size> [expire <expire>] [nopurge] [peers <peersect>] [srvkey <srvkey>]
[store <data_type>]*
Configure the stickiness table for the current section
May be used in sections : defaults | frontend | listen | backend
@@ -10724,6 +10724,16 @@
be removed once full. Be sure not to use the "nopurge" parameter
if not expiration delay is specified.
+ <srvkey> specifies how each server is identified for the purposes of the
+ stick table. The valid values are "name" and "addr". If "name" is
+ given, then <name> argument for the server (may be generated by
+ a template). If "addr" is given, then the server is identified
+ by its current network address, including the port. "addr" is
+ especially useful if you are using service discovery to generate
+ the addresses for servers with peered stick-tables and want
+ to consistently use the same host across peers for a stickiness
+ token.
+
<data_type> is used to store additional information in the stick-table. This
may be used by ACLs in order to control various criteria related
to the activity of the client matching the stick-table. For each
diff --git a/include/haproxy/dict.h b/include/haproxy/dict.h
index 59e8135..c55834c 100644
--- a/include/haproxy/dict.h
+++ b/include/haproxy/dict.h
@@ -31,5 +31,6 @@
struct dict *new_dict(const char *name);
struct dict_entry *dict_insert(struct dict *d, char *str);
+void dict_entry_unref(struct dict *d, struct dict_entry *de);
#endif /* _HAPROXY_DICT_H */
diff --git a/include/haproxy/proxy-t.h b/include/haproxy/proxy-t.h
index 998e210..e62b797 100644
--- a/include/haproxy/proxy-t.h
+++ b/include/haproxy/proxy-t.h
@@ -424,6 +424,7 @@
char *lfsd_file; /* file name where the structured-data logformat string for RFC5424 appears (strdup) */
int lfsd_line; /* file name where the structured-data logformat string for RFC5424 appears */
} conf; /* config information */
+ struct eb_root used_server_addr; /* list of server addresses in use */
void *parent; /* parent of the proxy when applicable */
struct comp *comp; /* http compression */
diff --git a/include/haproxy/server-t.h b/include/haproxy/server-t.h
index 512489a..3107e04 100644
--- a/include/haproxy/server-t.h
+++ b/include/haproxy/server-t.h
@@ -342,6 +342,7 @@
struct ebpt_node name; /* place in the tree of used names */
int line; /* line where the section appears */
} conf; /* config information */
+ struct ebpt_node addr_node; /* Node for string representation of address for the server (including port number) */
/* Template information used only for server objects which
* serve as template filled at parsing time and used during
* server allocations from server templates.
diff --git a/include/haproxy/server.h b/include/haproxy/server.h
index d63eb01..00036b1 100644
--- a/include/haproxy/server.h
+++ b/include/haproxy/server.h
@@ -38,7 +38,7 @@
__decl_thread(extern HA_SPINLOCK_T idle_conn_srv_lock);
extern struct eb_root idle_conn_srv;
extern struct task *idle_conn_task;
-extern struct dict server_name_dict;
+extern struct dict server_key_dict;
int srv_downtime(const struct server *s);
int srv_lastsession(const struct server *s);
diff --git a/include/haproxy/stick_table-t.h b/include/haproxy/stick_table-t.h
index 59aadea..2cb4a1b 100644
--- a/include/haproxy/stick_table-t.h
+++ b/include/haproxy/stick_table-t.h
@@ -56,7 +56,7 @@
STKTABLE_DT_BYTES_OUT_RATE,/* bytes rate from servers to client */
STKTABLE_DT_GPC1, /* General Purpose Counter 1 (unsigned 32-bit integer) */
STKTABLE_DT_GPC1_RATE, /* General Purpose Counter 1's event rate */
- STKTABLE_DT_SERVER_NAME, /* The server name */
+ STKTABLE_DT_SERVER_KEY, /* The server key */
STKTABLE_STATIC_DATA_TYPES,/* number of types above */
/* up to STKTABLE_EXTRA_DATA_TYPES types may be registered here, always
* followed by the number of data types, must always be last.
@@ -80,6 +80,12 @@
ARG_T_DELAY, /* a delay which supports time units */
};
+/* They types of keys that servers can be identified by */
+enum {
+ STKTABLE_SRV_NAME = 0,
+ STKTABLE_SRV_ADDR,
+};
+
/* stick table key type flags */
#define STK_F_CUSTOM_KEYSIZE 0x00000001 /* this table's key size is configurable */
@@ -112,7 +118,7 @@
/* types of each storable data */
int server_id;
- struct dict_entry *server_name;
+ struct dict_entry *server_key;
unsigned int gpt0;
unsigned int gpc0;
struct freq_ctr_period gpc0_rate;
@@ -188,6 +194,7 @@
} peers;
unsigned long type; /* type of table (determines key format) */
+ unsigned int server_key_type; /* What type of key is used to identify servers */
size_t key_size; /* size of a key, maximum size in case of string */
unsigned int size; /* maximum number of sticky sessions in table */
unsigned int current; /* number of sticky sessions currently in table */
diff --git a/include/haproxy/tools.h b/include/haproxy/tools.h
index 4080d7a..651c070 100644
--- a/include/haproxy/tools.h
+++ b/include/haproxy/tools.h
@@ -245,6 +245,19 @@
struct protocol **proto, char **err,
const char *pfx, char **fqdn, unsigned int opts);
+
+/* converts <addr> and <port> into a string representation of the address and port. This is sort
+ * of an inverse of str2sa_range, with some restrictions. The supported families are AF_INET,
+ * AF_INET6, AF_UNIX, and AF_CUST_SOCKPAIR. If the family is unsopported NULL is returned.
+ * If map_ports is true, then the sign of the port is included in the output, to indicate it is
+ * relative to the incoming port. AF_INET and AF_INET6 will be in the form "<addr>:<port>".
+ * AF_UNIX will either be just the path (if using a pathname) or "abns@<path>" if it is abstract.
+ * AF_CUST_SOCKPAIR will be of the form "sockpair@<fd>".
+ *
+ * The returned char* is allocated, and it is the responsibility of the caller to free it.
+ */
+char *sa2str(const struct sockaddr_storage *addr, int port, int map_ports);
+
/* converts <str> to a struct in_addr containing a network mask. It can be
* passed in dotted form (255.255.255.0) or in CIDR form (24). It returns 1
* if the conversion succeeds otherwise zero.
diff --git a/src/cfgparse-listen.c b/src/cfgparse-listen.c
index 97a97e7..a493e74 100644
--- a/src/cfgparse-listen.c
+++ b/src/cfgparse-listen.c
@@ -457,6 +457,7 @@
curproxy->grace = defproxy.grace;
curproxy->conf.used_listener_id = EB_ROOT;
curproxy->conf.used_server_id = EB_ROOT;
+ curproxy->used_server_addr = EB_ROOT_UNIQUE;
if (defproxy.check_path)
curproxy->check_path = strdup(defproxy.check_path);
diff --git a/src/cfgparse.c b/src/cfgparse.c
index 6ac6872..a485a46 100644
--- a/src/cfgparse.c
+++ b/src/cfgparse.c
@@ -2682,7 +2682,7 @@
free((void *)mrule->table.name);
mrule->table.t = target;
stktable_alloc_data_type(target, STKTABLE_DT_SERVER_ID, NULL);
- stktable_alloc_data_type(target, STKTABLE_DT_SERVER_NAME, NULL);
+ stktable_alloc_data_type(target, STKTABLE_DT_SERVER_KEY, NULL);
if (!in_proxies_list(target->proxies_list, curproxy)) {
curproxy->next_stkt_ref = target->proxies_list;
target->proxies_list = curproxy;
@@ -2720,7 +2720,7 @@
free((void *)mrule->table.name);
mrule->table.t = target;
stktable_alloc_data_type(target, STKTABLE_DT_SERVER_ID, NULL);
- stktable_alloc_data_type(target, STKTABLE_DT_SERVER_NAME, NULL);
+ stktable_alloc_data_type(target, STKTABLE_DT_SERVER_KEY, NULL);
if (!in_proxies_list(target->proxies_list, curproxy)) {
curproxy->next_stkt_ref = target->proxies_list;
target->proxies_list = curproxy;
diff --git a/src/dict.c b/src/dict.c
index 903f073..9b3536d 100644
--- a/src/dict.c
+++ b/src/dict.c
@@ -87,8 +87,10 @@
HA_RWLOCK_RDLOCK(DICT_LOCK, &d->rwlock);
de = __dict_lookup(d, s);
HA_RWLOCK_RDUNLOCK(DICT_LOCK, &d->rwlock);
- if (de)
+ if (de) {
+ HA_ATOMIC_ADD(&de->refcount, 1);
return de;
+ }
de = new_dict_entry(s);
if (!de)
@@ -105,3 +107,23 @@
return de;
}
+
+/*
+ * Unreference a dict entry previously acquired with <dict_insert>.
+ * If this is the last live reference to the entry, it is
+ * removed from the dictionary.
+ */
+void dict_entry_unref(struct dict *d, struct dict_entry *de)
+{
+ if (!de)
+ return;
+
+ if (HA_ATOMIC_SUB(&de->refcount, 1) != 0)
+ return;
+
+ HA_RWLOCK_WRLOCK(DICT_LOCK, &d->rwlock);
+ ebpt_delete(&de->value);
+ HA_RWLOCK_WRUNLOCK(DICT_LOCK, &d->rwlock);
+
+ free_dict_entry(de);
+}
diff --git a/src/peers.c b/src/peers.c
index b5c1d42..3fa1a28 100644
--- a/src/peers.c
+++ b/src/peers.c
@@ -1633,13 +1633,16 @@
chunk->area[chunk->data] = '\0';
*msg_cur += value_len;
- de = dict_insert(&server_name_dict, chunk->area);
+ de = dict_insert(&server_key_dict, chunk->area);
+ dict_entry_unref(&server_key_dict, dc->rx[id - 1].de);
dc->rx[id - 1].de = de;
}
if (de) {
data_ptr = stktable_data_ptr(st->table, ts, data_type);
- if (data_ptr)
+ if (data_ptr) {
+ HA_ATOMIC_ADD(&de->refcount, 1);
stktable_data_cast(data_ptr, std_t_dict) = de;
+ }
}
break;
}
@@ -3059,6 +3062,8 @@
for (i = 0; i < dc->max_entries; i++) {
ebpt_delete(&dc->tx->entries[i]);
dc->tx->entries[i].key = NULL;
+ dict_entry_unref(&server_key_dict, dc->rx[i].de);
+ dc->rx[i].de = NULL;
}
dc->tx->prev_lookup = NULL;
dc->tx->lru_key = 0;
diff --git a/src/server.c b/src/server.c
index 8df763b..8906f0c 100644
--- a/src/server.c
+++ b/src/server.c
@@ -65,8 +65,8 @@
struct task *idle_conn_task = NULL;
/* The server names dictionary */
-struct dict server_name_dict = {
- .name = "server names",
+struct dict server_key_dict = {
+ .name = "server keys",
.values = EB_ROOT_UNIQUE,
};
@@ -194,6 +194,36 @@
}
/*
+ * Must be called with the server lock held, and will write-lock the proxy.
+ */
+static void srv_set_addr_desc(struct server *s)
+{
+ struct proxy *p = s->proxy;
+ char *key;
+
+ key = sa2str(&s->addr, s->svc_port, s->flags & SRV_F_MAPPORTS);
+
+ if (s->addr_node.key) {
+ if (strcmp(key, s->addr_node.key) == 0) {
+ free(key);
+ return;
+ }
+
+ HA_RWLOCK_WRLOCK(PROXY_LOCK, &p->lock);
+ ebpt_delete(&s->addr_node);
+ HA_RWLOCK_WRUNLOCK(PROXY_LOCK, &p->lock);
+
+ free(s->addr_node.key);
+ }
+
+ s->addr_node.key = key;
+
+ HA_RWLOCK_WRLOCK(PROXY_LOCK, &p->lock);
+ ebis_insert(&p->used_server_addr, &s->addr_node);
+ HA_RWLOCK_WRUNLOCK(PROXY_LOCK, &p->lock);
+}
+
+/*
* Registers the server keyword list <kwl> as a list of valid keywords for next
* parsing sessions.
*/
@@ -2055,6 +2085,9 @@
newsrv->addr = *sk;
newsrv->svc_port = port;
+ // we don't need to lock the server here, because
+ // we are in the process of initializing
+ srv_set_addr_desc(newsrv);
if (!newsrv->srvrq && !newsrv->hostname && !protocol_by_family(newsrv->addr.ss_family)) {
ha_alert("parsing [%s:%d] : Unknown protocol family %d '%s'\n",
@@ -3522,6 +3555,7 @@
break;
};
srv_set_dyncookie(s);
+ srv_set_addr_desc(s);
return 0;
}
@@ -3694,6 +3728,7 @@
/* force connection cleanup on the given server */
srv_cleanup_connections(s);
srv_set_dyncookie(s);
+ srv_set_addr_desc(s);
}
if (updater)
chunk_appendf(msg, " by '%s'", updater);
@@ -4174,6 +4209,7 @@
return return_code;
out:
srv_set_dyncookie(srv);
+ srv_set_addr_desc(srv);
return return_code;
}
diff --git a/src/stick_table.c b/src/stick_table.c
index 5059164..825005c 100644
--- a/src/stick_table.c
+++ b/src/stick_table.c
@@ -22,6 +22,7 @@
#include <haproxy/arg.h>
#include <haproxy/cfgparse.h>
#include <haproxy/cli.h>
+#include <haproxy/dict.h>
#include <haproxy/errors.h>
#include <haproxy/global.h>
#include <haproxy/http_rules.h>
@@ -94,6 +95,12 @@
*/
void stksess_free(struct stktable *t, struct stksess *ts)
{
+ void *data;
+ data = stktable_data_ptr(t, ts, STKTABLE_DT_SERVER_KEY);
+ if (data) {
+ dict_entry_unref(&server_key_dict, stktable_data_cast(data, server_key));
+ stktable_data_cast(data, server_key) = NULL;
+ }
HA_SPIN_LOCK(STK_TABLE_LOCK, &t->lock);
__stksess_free(t, ts);
HA_SPIN_UNLOCK(STK_TABLE_LOCK, &t->lock);
@@ -877,6 +884,25 @@
}
idx++;
}
+ else if (strcmp(args[idx], "srvkey") == 0) {
+ char *keytype;
+ idx++;
+ keytype = args[idx];
+ if (strcmp(keytype, "name") == 0) {
+ t->server_key_type = STKTABLE_SRV_NAME;
+ }
+ else if (strcmp(keytype, "addr") == 0) {
+ t->server_key_type = STKTABLE_SRV_ADDR;
+ }
+ else {
+ ha_alert("parsing [%s:%d] : %s : unknown server key type '%s'.\n",
+ file, linenum, args[0], keytype);
+ err_code |= ERR_ALERT | ERR_FATAL;
+ goto out;
+
+ }
+ idx++;
+ }
else {
ha_alert("parsing [%s:%d] : %s: unknown argument '%s'.\n",
file, linenum, args[0], args[idx]);
@@ -1048,7 +1074,7 @@
[STKTABLE_DT_BYTES_OUT_RATE]= { .name = "bytes_out_rate", .std_type = STD_T_FRQP, .arg_type = ARG_T_DELAY },
[STKTABLE_DT_GPC1] = { .name = "gpc1", .std_type = STD_T_UINT },
[STKTABLE_DT_GPC1_RATE] = { .name = "gpc1_rate", .std_type = STD_T_FRQP, .arg_type = ARG_T_DELAY },
- [STKTABLE_DT_SERVER_NAME] = { .name = "server_name", .std_type = STD_T_DICT },
+ [STKTABLE_DT_SERVER_KEY] = { .name = "server_key", .std_type = STD_T_DICT },
};
/* Registers stick-table extra data type with index <idx>, name <name>, type
@@ -1095,6 +1121,9 @@
if (strcmp(name, stktable_data_types[type].name) == 0)
return type;
}
+ /* For backwards compatibility */
+ if (strcmp(name, "server_name") == 0)
+ return STKTABLE_DT_SERVER_KEY;
return -1;
}
diff --git a/src/stream.c b/src/stream.c
index 6ca1538..e24ea49 100644
--- a/src/stream.c
+++ b/src/stream.c
@@ -1202,17 +1202,27 @@
/* Look for the server name previously stored in <t> stick-table */
HA_RWLOCK_RDLOCK(STK_SESS_LOCK, &ts->lock);
- ptr = __stktable_data_ptr(t, ts, STKTABLE_DT_SERVER_NAME);
- de = stktable_data_cast(ptr, server_name);
+ ptr = __stktable_data_ptr(t, ts, STKTABLE_DT_SERVER_KEY);
+ de = stktable_data_cast(ptr, server_key);
HA_RWLOCK_RDUNLOCK(STK_SESS_LOCK, &ts->lock);
if (de) {
- struct ebpt_node *name;
+ struct ebpt_node *node;
- name = ebis_lookup(&px->conf.used_server_name, de->value.key);
- if (name) {
- srv = container_of(name, struct server, conf.name);
- goto found;
+ if (t->server_key_type == STKTABLE_SRV_NAME) {
+ node = ebis_lookup(&px->conf.used_server_name, de->value.key);
+ if (node) {
+ srv = container_of(node, struct server, conf.name);
+ goto found;
+ }
+ } else if (t->server_key_type == STKTABLE_SRV_ADDR) {
+ HA_RWLOCK_RDLOCK(PROXY_LOCK, &px->lock);
+ node = ebis_lookup(&px->used_server_addr, de->value.key);
+ HA_RWLOCK_RDUNLOCK(PROXY_LOCK, &px->lock);
+ if (node) {
+ srv = container_of(node, struct server, addr_node);
+ goto found;
+ }
}
}
@@ -1378,7 +1388,9 @@
for (i = 0; i < s->store_count; i++) {
struct stksess *ts;
void *ptr;
+ char *key;
struct dict_entry *de;
+ struct stktable *t = s->store[i].table;
if (objt_server(s->target) && objt_server(s->target)->flags & SRV_F_NON_STICK) {
stksess_free(s->store[i].table, s->store[i].ts);
@@ -1386,27 +1398,34 @@
continue;
}
- ts = stktable_set_entry(s->store[i].table, s->store[i].ts);
+ ts = stktable_set_entry(t, s->store[i].ts);
if (ts != s->store[i].ts) {
/* the entry already existed, we can free ours */
- stksess_free(s->store[i].table, s->store[i].ts);
+ stksess_free(t, s->store[i].ts);
}
s->store[i].ts = NULL;
HA_RWLOCK_WRLOCK(STK_SESS_LOCK, &ts->lock);
- ptr = __stktable_data_ptr(s->store[i].table, ts, STKTABLE_DT_SERVER_ID);
+ ptr = __stktable_data_ptr(t, ts, STKTABLE_DT_SERVER_ID);
stktable_data_cast(ptr, server_id) = __objt_server(s->target)->puid;
HA_RWLOCK_WRUNLOCK(STK_SESS_LOCK, &ts->lock);
+ if (t->server_key_type == STKTABLE_SRV_NAME)
+ key = __objt_server(s->target)->id;
+ else if (t->server_key_type == STKTABLE_SRV_ADDR)
+ key = __objt_server(s->target)->addr_node.key;
+ else
+ continue;
+
HA_RWLOCK_WRLOCK(STK_SESS_LOCK, &ts->lock);
- de = dict_insert(&server_name_dict, __objt_server(s->target)->id);
+ de = dict_insert(&server_key_dict, key);
if (de) {
- ptr = __stktable_data_ptr(s->store[i].table, ts, STKTABLE_DT_SERVER_NAME);
- stktable_data_cast(ptr, server_name) = de;
+ ptr = __stktable_data_ptr(t, ts, STKTABLE_DT_SERVER_KEY);
+ stktable_data_cast(ptr, server_key) = de;
}
HA_RWLOCK_WRUNLOCK(STK_SESS_LOCK, &ts->lock);
- stktable_touch_local(s->store[i].table, ts, 1);
+ stktable_touch_local(t, ts, 1);
}
s->store_count = 0; /* everything is stored */
diff --git a/src/tools.c b/src/tools.c
index 6e84fb2..5895559 100644
--- a/src/tools.c
+++ b/src/tools.c
@@ -1214,6 +1214,51 @@
return ret;
}
+/* converts <addr> and <port> into a string representation of the address and port. This is sort
+ * of an inverse of str2sa_range, with some restrictions. The supported families are AF_INET,
+ * AF_INET6, AF_UNIX, and AF_CUST_SOCKPAIR. If the family is unsopported NULL is returned.
+ * If map_ports is true, then the sign of the port is included in the output, to indicate it is
+ * relative to the incoming port. AF_INET and AF_INET6 will be in the form "<addr>:<port>".
+ * AF_UNIX will either be just the path (if using a pathname) or "abns@<path>" if it is abstract.
+ * AF_CUST_SOCKPAIR will be of the form "sockpair@<fd>".
+ *
+ * The returned char* is allocated, and it is the responsibility of the caller to free it.
+ */
+char * sa2str(const struct sockaddr_storage *addr, int port, int map_ports)
+{
+ char buffer[INET6_ADDRSTRLEN];
+ char *out = NULL;
+ const void *ptr;
+ const char *path;
+
+ switch (addr->ss_family) {
+ case AF_INET:
+ ptr = &((struct sockaddr_in *)addr)->sin_addr;
+ break;
+ case AF_INET6:
+ ptr = &((struct sockaddr_in6 *)addr)->sin6_addr;
+ break;
+ case AF_UNIX:
+ path = ((struct sockaddr_un *)addr)->sun_path;
+ if (path[0] == '\0') {
+ const int max_length = sizeof(struct sockaddr_un) - offsetof(struct sockaddr_un, sun_path) - 1;
+ return memprintf(&out, "abns@%.*s", max_length, path+1);
+ } else {
+ return strdup(path);
+ }
+ case AF_CUST_SOCKPAIR:
+ return memprintf(&out, "sockpair@%d", ((struct sockaddr_in *)addr)->sin_addr.s_addr);
+ default:
+ return NULL;
+ }
+ inet_ntop(addr->ss_family, ptr, buffer, get_addr_len(addr));
+ if (map_ports)
+ return memprintf(&out, "%s:%+d", buffer, port);
+ else
+ return memprintf(&out, "%s:%d", buffer, port);
+}
+
+
/* converts <str> to a struct in_addr containing a network mask. It can be
* passed in dotted form (255.255.255.0) or in CIDR form (24). It returns 1
* if the conversion succeeds otherwise zero.