MINOR: stats: report the number of times the global maxconn was reached
As discussed a few times over the years, it's quite difficult to know
how often we stop accepting connections because the global maxconn was
reached. This is not easy to know because when we reach the limit we
stop accepting but we don't know if incoming connections are pending,
so it's not possible to know how many were delayed just because of this.
However, an interesting equivalent metric consist in counting the number
of times an accepted incoming connection resulted in the limit being
reached. I.e. "we've accepted the last one for now". That doesn't imply
any other one got delayed but it's a factual indicator that something
might have been delayed. And by counting the number of such events, it
becomes easier to know whether some limits need to be adjusted because
they're reached often, or if it's exceptionally rare.
The metric is reported as a counter in show info and on the stats page
in the info section right next to "maxconn".
diff --git a/include/haproxy/listener.h b/include/haproxy/listener.h
index 46d9ee2..5b3dc18 100644
--- a/include/haproxy/listener.h
+++ b/include/haproxy/listener.h
@@ -220,6 +220,9 @@
extern const char* li_status_st[LI_STATE_COUNT];
enum li_status get_li_status(struct listener *l);
+/* number of times an accepted connection resulted in maxconn being reached */
+extern ullong maxconn_reached;
+
static inline uint accept_queue_ring_len(const struct accept_queue_ring *ring)
{
uint idx, head, tail, len;
diff --git a/include/haproxy/stats-t.h b/include/haproxy/stats-t.h
index b15ca86..d185d85 100644
--- a/include/haproxy/stats-t.h
+++ b/include/haproxy/stats-t.h
@@ -344,6 +344,7 @@
INF_START_TIME_SEC,
INF_TAINTED,
INF_WARNINGS,
+ INF_MAXCONN_REACHED,
/* must always be the last one */
INF_TOTAL_FIELDS
diff --git a/src/listener.c b/src/listener.c
index 3da01df..a1160da 100644
--- a/src/listener.c
+++ b/src/listener.c
@@ -47,6 +47,8 @@
/* list of the temporarily limited listeners because of lack of resource */
static struct mt_list global_listener_queue = MT_LIST_HEAD_INIT(global_listener_queue);
static struct task *global_listener_queue_task;
+/* number of times an accepted connection resulted in maxconn being reached */
+ullong maxconn_reached = 0;
__decl_thread(static HA_RWLOCK_T global_listener_rwlock);
/* listener status for stats */
@@ -1164,6 +1166,12 @@
_HA_ATOMIC_INC(&activity[tid].accepted);
+ /* count the number of times an accepted connection resulted in
+ * maxconn being reached.
+ */
+ if (unlikely(_HA_ATOMIC_LOAD(&actconn) + 1 >= global.maxconn))
+ _HA_ATOMIC_INC(&maxconn_reached);
+
/* past this point, l->bind_conf->accept() will automatically decrement
* l->nbconn, feconn and actconn once done. Setting next_*conn=0
* allows the error path not to rollback on nbconn. It's more
diff --git a/src/stats.c b/src/stats.c
index eab75ee..cc3668f 100644
--- a/src/stats.c
+++ b/src/stats.c
@@ -157,6 +157,7 @@
[INF_BUILD_INFO] = { .name = "Build info", .desc = "Build info" },
[INF_TAINTED] = { .name = "Tainted", .desc = "Experimental features used" },
[INF_WARNINGS] = { .name = "TotalWarnings", .desc = "Total warnings issued" },
+ [INF_MAXCONN_REACHED] = { .name = "MaxconnReached", .desc = "Number of times an accepted connection resulted in Maxconn being reached" },
};
const struct name_desc stat_fields[ST_F_TOTAL_FIELDS] = {
@@ -3603,7 +3604,7 @@
"<p><b>pid = </b> %d (process #%d, nbproc = %d, nbthread = %d)<br>\n"
"<b>uptime = </b> %dd %dh%02dm%02ds; warnings = %u<br>\n"
"<b>system limits:</b> memmax = %s%s; ulimit-n = %d<br>\n"
- "<b>maxsock = </b> %d; <b>maxconn = </b> %d; <b>maxpipes = </b> %d<br>\n"
+ "<b>maxsock = </b> %d; <b>maxconn = </b> %d; <b>reached = </b> %llu; <b>maxpipes = </b> %d<br>\n"
"current conns = %d; current pipes = %d/%d; conn rate = %d/sec; bit rate = %.3f %cbps<br>\n"
"Running tasks: %d/%d; idle = %d %%<br>\n"
"</td><td align=\"center\" nowrap>\n"
@@ -3641,7 +3642,7 @@
global.rlimit_memmax ? ultoa(global.rlimit_memmax) : "unlimited",
global.rlimit_memmax ? " MB" : "",
global.rlimit_nofile,
- global.maxsock, global.maxconn, global.maxpipes,
+ global.maxsock, global.maxconn, HA_ATOMIC_LOAD(&maxconn_reached), global.maxpipes,
actconn, pipes_used, pipes_used+pipes_free, read_freq_ctr(&global.conn_per_sec),
bps >= 1000000000UL ? (bps / 1000000000.0) : bps >= 1000000UL ? (bps / 1000000.0) : (bps / 1000.0),
bps >= 1000000000UL ? 'G' : bps >= 1000000UL ? 'M' : 'k',
@@ -4740,6 +4741,7 @@
info[INF_TAINTED] = mkf_str(FO_STATUS, chunk_newstr(out));
info[INF_WARNINGS] = mkf_u32(FN_COUNTER, HA_ATOMIC_LOAD(&tot_warnings));
+ info[INF_MAXCONN_REACHED] = mkf_u32(FN_COUNTER, HA_ATOMIC_LOAD(&maxconn_reached));
chunk_appendf(out, "%#x", get_tainted());
return 1;