blob: 68a0a18d1b5a16e5b0100a235c4f8ff725d0fe61 [file] [log] [blame]
Willy Tarreaubaaee002006-06-26 02:48:02 +02001/*
Willy Tarreau24f4efa2010-08-27 17:56:48 +02002 * include/proto/task.h
3 * Functions for task management.
4 *
5 * Copyright (C) 2000-2010 Willy Tarreau - w@1wt.eu
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation, version 2.1
10 * exclusively.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 *
17 * You should have received a copy of the GNU Lesser General Public
18 * License along with this library; if not, write to the Free Software
19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20 */
Willy Tarreaubaaee002006-06-26 02:48:02 +020021
22#ifndef _PROTO_TASK_H
23#define _PROTO_TASK_H
24
25
26#include <sys/time.h>
Willy Tarreaue3ba5f02006-06-29 18:54:54 +020027
28#include <common/config.h>
Willy Tarreau2dd0d472006-06-29 17:53:05 +020029#include <common/memory.h>
Willy Tarreau96bcfd72007-04-29 10:41:56 +020030#include <common/mini-clist.h>
31#include <common/standard.h>
Willy Tarreaud0a201b2009-03-08 15:53:06 +010032#include <common/ticks.h>
Emeric Brunc60def82017-09-27 14:59:38 +020033#include <common/hathreads.h>
34
Willy Tarreau8d388052017-11-05 13:34:20 +010035#include <eb32sctree.h>
Willy Tarreau45cb4fb2009-10-26 21:10:04 +010036#include <eb32tree.h>
Willy Tarreau96bcfd72007-04-29 10:41:56 +020037
Willy Tarreaueb118892014-11-13 16:57:19 +010038#include <types/global.h>
Willy Tarreaue3ba5f02006-06-29 18:54:54 +020039#include <types/task.h>
Willy Tarreaubaaee002006-06-26 02:48:02 +020040
Olivier Houchardbba1a262019-09-24 14:55:28 +020041#include <proto/fd.h>
42
Willy Tarreaud0a201b2009-03-08 15:53:06 +010043/* Principle of the wait queue.
44 *
45 * We want to be able to tell whether an expiration date is before of after the
46 * current time <now>. We KNOW that expiration dates are never too far apart,
47 * because they are measured in ticks (milliseconds). We also know that almost
48 * all dates will be in the future, and that a very small part of them will be
49 * in the past, they are the ones which have expired since last time we checked
50 * them. Using ticks, we know if a date is in the future or in the past, but we
51 * cannot use that to store sorted information because that reference changes
52 * all the time.
53 *
Willy Tarreaue35c94a2009-03-21 10:01:42 +010054 * We'll use the fact that the time wraps to sort timers. Timers above <now>
55 * are in the future, timers below <now> are in the past. Here, "above" and
56 * "below" are to be considered modulo 2^31.
Willy Tarreaud0a201b2009-03-08 15:53:06 +010057 *
Willy Tarreaue35c94a2009-03-21 10:01:42 +010058 * Timers are stored sorted in an ebtree. We use the new ability for ebtrees to
59 * lookup values starting from X to only expire tasks between <now> - 2^31 and
60 * <now>. If the end of the tree is reached while walking over it, we simply
61 * loop back to the beginning. That way, we have no problem keeping sorted
62 * wrapping timers in a tree, between (now - 24 days) and (now + 24 days). The
63 * keys in the tree always reflect their real position, none can be infinite.
64 * This reduces the number of checks to be performed.
Willy Tarreaud0a201b2009-03-08 15:53:06 +010065 *
66 * Another nice optimisation is to allow a timer to stay at an old place in the
67 * queue as long as it's not further than the real expiration date. That way,
68 * we use the tree as a place holder for a minorant of the real expiration
69 * date. Since we have a very low chance of hitting a timeout anyway, we can
70 * bounce the nodes to their right place when we scan the tree if we encounter
71 * a misplaced node once in a while. This even allows us not to remove the
72 * infinite timers from the wait queue.
73 *
74 * So, to summarize, we have :
75 * - node->key always defines current position in the wait queue
76 * - timer is the real expiration date (possibly infinite)
Willy Tarreaue35c94a2009-03-21 10:01:42 +010077 * - node->key is always before or equal to timer
Willy Tarreaud0a201b2009-03-08 15:53:06 +010078 *
79 * The run queue works similarly to the wait queue except that the current date
80 * is replaced by an insertion counter which can also wrap without any problem.
81 */
82
Willy Tarreaue35c94a2009-03-21 10:01:42 +010083/* The farthest we can look back in a timer tree */
84#define TIMER_LOOK_BACK (1U << 31)
Willy Tarreaud0a201b2009-03-08 15:53:06 +010085
86/* a few exported variables */
Willy Tarreaua4613182009-03-21 18:13:21 +010087extern unsigned int nb_tasks; /* total number of tasks */
Willy Tarreauaa1e1be2019-05-16 17:37:27 +020088extern volatile unsigned long global_tasks_mask; /* Mask of threads with tasks in the global runqueue */
Christopher Faulet34c5cc92016-12-06 09:15:30 +010089extern unsigned int tasks_run_queue; /* run queue size */
90extern unsigned int tasks_run_queue_cur;
Willy Tarreauc7bdf092009-03-21 18:33:52 +010091extern unsigned int nb_tasks_cur;
Willy Tarreau91e99932008-06-30 07:51:00 +020092extern unsigned int niced_tasks; /* number of niced tasks in the run queue */
Willy Tarreaubafbe012017-11-24 17:34:44 +010093extern struct pool_head *pool_head_task;
Olivier Houchardb0bdae72018-05-18 18:45:28 +020094extern struct pool_head *pool_head_tasklet;
Willy Tarreaubafbe012017-11-24 17:34:44 +010095extern struct pool_head *pool_head_notification;
Willy Tarreaud022e9c2019-09-24 08:25:15 +020096extern THREAD_LOCAL struct task_per_thread *sched; /* current's thread scheduler context */
Olivier Houchardb1ca58b2018-06-06 14:22:03 +020097#ifdef USE_THREAD
Willy Tarreaub20aa9e2018-10-15 14:52:21 +020098extern struct eb_root timers; /* sorted timers tree, global */
Olivier Houchardf6e6dc12018-05-18 18:38:23 +020099extern struct eb_root rqueue; /* tree constituting the run queue */
Olivier Houchard77551ee2018-07-26 15:59:38 +0200100extern int global_rqueue_size; /* Number of element sin the global runqueue */
Olivier Houchardb1ca58b2018-06-06 14:22:03 +0200101#endif
Olivier Houchard77551ee2018-07-26 15:59:38 +0200102
Willy Tarreau8d8747a2018-10-15 16:12:48 +0200103extern struct task_per_thread task_per_thread[MAX_THREADS];
Christopher Faulet9dcf9b62017-11-13 10:34:01 +0100104
105__decl_hathreads(extern HA_SPINLOCK_T rq_lock); /* spin lock related to run queue */
Willy Tarreauef28dc12019-05-28 18:48:07 +0200106__decl_hathreads(extern HA_RWLOCK_T wq_lock); /* RW lock related to the wait queue */
Willy Tarreauc6ca1a02007-05-13 19:43:47 +0200107
Olivier Houchard0742c312019-12-05 15:11:19 +0100108static inline struct task *task_unlink_wq(struct task *t);
109static inline void task_queue(struct task *task);
Olivier Houchard5d187182018-08-01 15:58:44 +0200110
Willy Tarreau4726f532009-03-07 17:25:21 +0100111/* return 0 if task is in run queue, otherwise non-zero */
112static inline int task_in_rq(struct task *t)
113{
Olivier Houchardb0bdae72018-05-18 18:45:28 +0200114 /* Check if leaf_p is NULL, in case he's not in the runqueue, and if
115 * it's not 0x1, which would mean it's in the tasklet list.
116 */
Olivier Houchard4a1be0c2019-04-17 19:13:07 +0200117 return t->rq.node.leaf_p != NULL;
Willy Tarreau4726f532009-03-07 17:25:21 +0100118}
119
120/* return 0 if task is in wait queue, otherwise non-zero */
121static inline int task_in_wq(struct task *t)
122{
123 return t->wq.node.leaf_p != NULL;
124}
125
Willy Tarreaufdccded2008-08-29 18:19:04 +0200126/* puts the task <t> in run queue with reason flags <f>, and returns <t> */
Olivier Houchardf6e6dc12018-05-18 18:38:23 +0200127/* This will put the task in the local runqueue if the task is only runnable
128 * by the current thread, in the global runqueue otherwies.
129 */
130void __task_wakeup(struct task *t, struct eb_root *);
131static inline void task_wakeup(struct task *t, unsigned int f)
Willy Tarreau4df82062008-08-29 15:26:14 +0200132{
Olivier Houchardf6e6dc12018-05-18 18:38:23 +0200133 unsigned short state;
Emeric Brunc60def82017-09-27 14:59:38 +0200134
Olivier Houchardf6e6dc12018-05-18 18:38:23 +0200135#ifdef USE_THREAD
136 struct eb_root *root;
Emeric Brunc60def82017-09-27 14:59:38 +0200137
Olivier Houchardb0bdae72018-05-18 18:45:28 +0200138 if (t->thread_mask == tid_bit || global.nbthread == 1)
Willy Tarreaud022e9c2019-09-24 08:25:15 +0200139 root = &sched->rqueue;
Olivier Houchardf6e6dc12018-05-18 18:38:23 +0200140 else
141 root = &rqueue;
142#else
Willy Tarreaud022e9c2019-09-24 08:25:15 +0200143 struct eb_root *root = &sched->rqueue;
Olivier Houchardf6e6dc12018-05-18 18:38:23 +0200144#endif
145
Willy Tarreaub0380072019-04-17 11:47:18 +0200146 state = _HA_ATOMIC_OR(&t->state, f);
147 while (!(state & (TASK_RUNNING | TASK_QUEUED))) {
Willy Tarreau8c12e2f2019-04-17 20:52:51 +0200148 if (_HA_ATOMIC_CAS(&t->state, &state, state | TASK_QUEUED)) {
149 __task_wakeup(t, root);
Willy Tarreaub0380072019-04-17 11:47:18 +0200150 break;
Willy Tarreau8c12e2f2019-04-17 20:52:51 +0200151 }
Willy Tarreaub0380072019-04-17 11:47:18 +0200152 }
Willy Tarreau4df82062008-08-29 15:26:14 +0200153}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200154
Willy Tarreaudd0e89a2019-12-19 07:39:06 +0100155/* change the thread affinity of a task to <thread_mask>.
156 * This may only be done from within the running task itself or during its
157 * initialization. It will unqueue and requeue the task from the wait queue
158 * if it was in it. This is safe against a concurrent task_queue() call because
159 * task_queue() itself will unlink again if needed after taking into account
160 * the new thread_mask.
161 */
Emeric Brunc60def82017-09-27 14:59:38 +0200162static inline void task_set_affinity(struct task *t, unsigned long thread_mask)
163{
Willy Tarreau440d09b2019-12-11 09:11:58 +0100164 if (unlikely(task_in_wq(t))) {
Olivier Houchard0742c312019-12-05 15:11:19 +0100165 task_unlink_wq(t);
Willy Tarreau440d09b2019-12-11 09:11:58 +0100166 t->thread_mask = thread_mask;
Olivier Houchard0742c312019-12-05 15:11:19 +0100167 task_queue(t);
Willy Tarreau440d09b2019-12-11 09:11:58 +0100168 }
169 else
170 t->thread_mask = thread_mask;
Emeric Brunc60def82017-09-27 14:59:38 +0200171}
Willy Tarreauf65610a2017-10-31 16:06:06 +0100172
Willy Tarreau4726f532009-03-07 17:25:21 +0100173/*
174 * Unlink the task from the wait queue, and possibly update the last_timer
175 * pointer. A pointer to the task itself is returned. The task *must* already
176 * be in the wait queue before calling this function. If unsure, use the safer
177 * task_unlink_wq() function.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200178 */
Willy Tarreau4726f532009-03-07 17:25:21 +0100179static inline struct task *__task_unlink_wq(struct task *t)
180{
181 eb32_delete(&t->wq);
Willy Tarreau4726f532009-03-07 17:25:21 +0100182 return t;
183}
184
Willy Tarreaub20aa9e2018-10-15 14:52:21 +0200185/* remove a task from its wait queue. It may either be the local wait queue if
Willy Tarreaudd0e89a2019-12-19 07:39:06 +0100186 * the task is bound to a single thread or the global queue. If the task uses a
187 * shared wait queue, the global wait queue lock is used.
Willy Tarreaub20aa9e2018-10-15 14:52:21 +0200188 */
Willy Tarreau4726f532009-03-07 17:25:21 +0100189static inline struct task *task_unlink_wq(struct task *t)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200190{
Richard Russobc9d9842019-02-20 12:43:45 -0800191 unsigned long locked;
192
Willy Tarreaub20aa9e2018-10-15 14:52:21 +0200193 if (likely(task_in_wq(t))) {
Willy Tarreaudd0e89a2019-12-19 07:39:06 +0100194 locked = t->state & TASK_SHARED_WQ;
Richard Russobc9d9842019-02-20 12:43:45 -0800195 if (locked)
Willy Tarreauef28dc12019-05-28 18:48:07 +0200196 HA_RWLOCK_WRLOCK(TASK_WQ_LOCK, &wq_lock);
Willy Tarreau4726f532009-03-07 17:25:21 +0100197 __task_unlink_wq(t);
Richard Russobc9d9842019-02-20 12:43:45 -0800198 if (locked)
Willy Tarreauef28dc12019-05-28 18:48:07 +0200199 HA_RWLOCK_WRUNLOCK(TASK_WQ_LOCK, &wq_lock);
Willy Tarreaub20aa9e2018-10-15 14:52:21 +0200200 }
Willy Tarreau96bcfd72007-04-29 10:41:56 +0200201 return t;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200202}
203
204/*
Christopher Faulet34c5cc92016-12-06 09:15:30 +0100205 * Unlink the task from the run queue. The tasks_run_queue size and number of
206 * niced tasks are updated too. A pointer to the task itself is returned. The
207 * task *must* already be in the run queue before calling this function. If
208 * unsure, use the safer task_unlink_rq() function. Note that the pointer to the
209 * next run queue entry is neither checked nor updated.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200210 */
Willy Tarreau4726f532009-03-07 17:25:21 +0100211static inline struct task *__task_unlink_rq(struct task *t)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200212{
Olivier Houchard4c2832852019-03-08 18:48:47 +0100213 _HA_ATOMIC_SUB(&tasks_run_queue, 1);
Olivier Houchard77551ee2018-07-26 15:59:38 +0200214#ifdef USE_THREAD
215 if (t->state & TASK_GLOBAL) {
Olivier Houchard4c2832852019-03-08 18:48:47 +0100216 _HA_ATOMIC_AND(&t->state, ~TASK_GLOBAL);
Olivier Houchard77551ee2018-07-26 15:59:38 +0200217 global_rqueue_size--;
218 } else
219#endif
Willy Tarreaud022e9c2019-09-24 08:25:15 +0200220 sched->rqueue_size--;
Olivier Houchard77551ee2018-07-26 15:59:38 +0200221 eb32sc_delete(&t->rq);
Willy Tarreau4726f532009-03-07 17:25:21 +0100222 if (likely(t->nice))
Olivier Houchard4c2832852019-03-08 18:48:47 +0100223 _HA_ATOMIC_SUB(&niced_tasks, 1);
Willy Tarreauce44f122008-07-05 18:16:19 +0200224 return t;
225}
Willy Tarreau9789f7b2008-06-24 08:17:16 +0200226
Willy Tarreau501260b2015-02-23 16:07:01 +0100227/* This function unlinks task <t> from the run queue if it is in it. It also
228 * takes care of updating the next run queue task if it was this task.
229 */
Willy Tarreau4726f532009-03-07 17:25:21 +0100230static inline struct task *task_unlink_rq(struct task *t)
231{
Olivier Houchard1d7f37a2019-03-14 16:14:04 +0100232 int is_global = t->state & TASK_GLOBAL;
233
234 if (is_global)
Olivier Houchardf6e6dc12018-05-18 18:38:23 +0200235 HA_SPIN_LOCK(TASK_RQ_LOCK, &rq_lock);
Willy Tarreau24f382f2019-04-12 16:10:55 +0200236 if (likely(task_in_rq(t)))
Willy Tarreau4726f532009-03-07 17:25:21 +0100237 __task_unlink_rq(t);
Olivier Houchard1d7f37a2019-03-14 16:14:04 +0100238 if (is_global)
Olivier Houchardf6e6dc12018-05-18 18:38:23 +0200239 HA_SPIN_UNLOCK(TASK_RQ_LOCK, &rq_lock);
Willy Tarreau4726f532009-03-07 17:25:21 +0100240 return t;
241}
242
Olivier Houchardb0bdae72018-05-18 18:45:28 +0200243static inline void tasklet_wakeup(struct tasklet *tl)
244{
Willy Tarreau8cdc1672019-10-18 06:43:53 +0200245 if (likely(tl->tid < 0)) {
246 /* this tasklet runs on the caller thread */
Olivier Houchard06910462019-10-11 16:35:01 +0200247 if (LIST_ISEMPTY(&tl->list)) {
Willy Tarreaubb238832020-01-31 10:48:10 +0100248 if (tl->state & TASK_SELF_WAKING) {
Willy Tarreaua17664d2020-01-30 18:59:43 +0100249 LIST_ADDQ(&task_per_thread[tid].tasklets[TL_BULK], &tl->list);
Willy Tarreaubb238832020-01-31 10:48:10 +0100250 }
Willy Tarreaub30a1532020-01-31 16:37:34 +0100251 else if ((struct task *)tl == sched->current) {
Willy Tarreaubb238832020-01-31 10:48:10 +0100252 _HA_ATOMIC_OR(&tl->state, TASK_SELF_WAKING);
253 LIST_ADDQ(&task_per_thread[tid].tasklets[TL_BULK], &tl->list);
254 }
255 else {
Willy Tarreaua17664d2020-01-30 18:59:43 +0100256 LIST_ADDQ(&task_per_thread[tid].tasklets[TL_URGENT], &tl->list);
Willy Tarreaubb238832020-01-31 10:48:10 +0100257 }
Olivier Houchard06910462019-10-11 16:35:01 +0200258 _HA_ATOMIC_ADD(&tasks_run_queue, 1);
259 }
260 } else {
Willy Tarreau8cdc1672019-10-18 06:43:53 +0200261 /* this tasklet runs on a specific thread */
Olivier Houchard06910462019-10-11 16:35:01 +0200262 if (MT_LIST_ADDQ(&task_per_thread[tl->tid].shared_tasklet_list, (struct mt_list *)&tl->list) == 1) {
263 _HA_ATOMIC_ADD(&tasks_run_queue, 1);
Willy Tarreau891b5ef2019-10-18 08:45:41 +0200264 if (sleeping_thread_mask & (1UL << tl->tid)) {
265 _HA_ATOMIC_AND(&sleeping_thread_mask, ~(1UL << tl->tid));
Olivier Houchard06910462019-10-11 16:35:01 +0200266 wake_thread(tl->tid);
267 }
Olivier Houchardbba1a262019-09-24 14:55:28 +0200268 }
269 }
Olivier Houchardb0bdae72018-05-18 18:45:28 +0200270
271}
272
Willy Tarreaubd20a9d2019-06-14 18:05:54 +0200273/* Insert a tasklet into the tasklet list. If used with a plain task instead,
274 * the caller must update the task_list_size.
275 */
Willy Tarreaua62917b2020-01-30 18:37:28 +0100276static inline void tasklet_insert_into_tasklet_list(struct list *list, struct tasklet *tl)
Olivier Houchardb0bdae72018-05-18 18:45:28 +0200277{
Olivier Houchard06910462019-10-11 16:35:01 +0200278 _HA_ATOMIC_ADD(&tasks_run_queue, 1);
Willy Tarreaua62917b2020-01-30 18:37:28 +0100279 LIST_ADDQ(list, &tl->list);
Olivier Houchardb0bdae72018-05-18 18:45:28 +0200280}
281
Willy Tarreaubd20a9d2019-06-14 18:05:54 +0200282/* Remove the tasklet from the tasklet list. The tasklet MUST already be there.
283 * If unsure, use tasklet_remove_from_tasklet_list() instead. If used with a
284 * plain task, the caller must update the task_list_size.
Olivier Houchard06910462019-10-11 16:35:01 +0200285 * This should only be used by the thread that owns the tasklet, any other
286 * thread should use tasklet_cancel().
Willy Tarreaue73256f2019-03-25 18:02:54 +0100287 */
Willy Tarreau86eded62019-06-14 14:47:49 +0200288static inline void __tasklet_remove_from_tasklet_list(struct tasklet *t)
Olivier Houchardb0bdae72018-05-18 18:45:28 +0200289{
Olivier Houchard06910462019-10-11 16:35:01 +0200290 LIST_DEL_INIT(&t->list);
291 _HA_ATOMIC_SUB(&tasks_run_queue, 1);
Olivier Houchardb0bdae72018-05-18 18:45:28 +0200292}
293
Willy Tarreau86eded62019-06-14 14:47:49 +0200294static inline void tasklet_remove_from_tasklet_list(struct tasklet *t)
Willy Tarreaue73256f2019-03-25 18:02:54 +0100295{
Olivier Houchard7031e3d2019-11-08 15:41:55 +0100296 if (MT_LIST_DEL((struct mt_list *)&t->list))
297 _HA_ATOMIC_SUB(&tasks_run_queue, 1);
Willy Tarreaue73256f2019-03-25 18:02:54 +0100298}
299
Willy Tarreauce44f122008-07-05 18:16:19 +0200300/*
Willy Tarreaua4613182009-03-21 18:13:21 +0100301 * Initialize a new task. The bare minimum is performed (queue pointers and
302 * state). The task is returned. This function should not be used outside of
Willy Tarreaudd0e89a2019-12-19 07:39:06 +0100303 * task_new(). If the thread mask contains more than one thread, TASK_SHARED_WQ
304 * is set.
Willy Tarreau9789f7b2008-06-24 08:17:16 +0200305 */
Emeric Brunc60def82017-09-27 14:59:38 +0200306static inline struct task *task_init(struct task *t, unsigned long thread_mask)
Willy Tarreau9789f7b2008-06-24 08:17:16 +0200307{
Willy Tarreau4726f532009-03-07 17:25:21 +0100308 t->wq.node.leaf_p = NULL;
309 t->rq.node.leaf_p = NULL;
Olivier Houchardf6e6dc12018-05-18 18:38:23 +0200310 t->state = TASK_SLEEPING;
Willy Tarreauf65610a2017-10-31 16:06:06 +0100311 t->thread_mask = thread_mask;
Willy Tarreaudd0e89a2019-12-19 07:39:06 +0100312 if (atleast2(thread_mask))
313 t->state |= TASK_SHARED_WQ;
Willy Tarreau91e99932008-06-30 07:51:00 +0200314 t->nice = 0;
Willy Tarreau3884cba2009-03-28 17:54:35 +0100315 t->calls = 0;
Willy Tarreau9efd7452018-05-31 14:48:54 +0200316 t->call_date = 0;
317 t->cpu_time = 0;
318 t->lat_time = 0;
Willy Tarreauf4219992017-07-24 17:52:58 +0200319 t->expire = TICK_ETERNITY;
Willy Tarreaubaaee002006-06-26 02:48:02 +0200320 return t;
321}
322
Willy Tarreau8cdc1672019-10-18 06:43:53 +0200323/* Initialize a new tasklet. It's identified as a tasklet by ->nice=-32768. It
324 * is expected to run on the calling thread by default, it's up to the caller
325 * to change ->tid if it wants to own it.
326 */
Olivier Houchardb0bdae72018-05-18 18:45:28 +0200327static inline void tasklet_init(struct tasklet *t)
328{
329 t->nice = -32768;
330 t->calls = 0;
331 t->state = 0;
Olivier Houchard9ddaf792018-07-19 16:02:16 +0200332 t->process = NULL;
Willy Tarreau8cdc1672019-10-18 06:43:53 +0200333 t->tid = -1;
Olivier Houchard06910462019-10-11 16:35:01 +0200334 LIST_INIT(&t->list);
Olivier Houchardb0bdae72018-05-18 18:45:28 +0200335}
336
Willy Tarreau8cdc1672019-10-18 06:43:53 +0200337/* Allocate and initialize a new tasklet, local to the thread by default. The
338 * caller may assing its tid if it wants to own the tasklet.
339 */
Olivier Houchardb0bdae72018-05-18 18:45:28 +0200340static inline struct tasklet *tasklet_new(void)
341{
342 struct tasklet *t = pool_alloc(pool_head_tasklet);
343
344 if (t) {
345 tasklet_init(t);
346 }
347 return t;
348}
349
Willy Tarreaubaaee002006-06-26 02:48:02 +0200350/*
Willy Tarreaua4613182009-03-21 18:13:21 +0100351 * Allocate and initialise a new task. The new task is returned, or NULL in
352 * case of lack of memory. The task count is incremented. Tasks should only
353 * be allocated this way, and must be freed using task_free().
354 */
Emeric Brunc60def82017-09-27 14:59:38 +0200355static inline struct task *task_new(unsigned long thread_mask)
Willy Tarreaua4613182009-03-21 18:13:21 +0100356{
Willy Tarreaubafbe012017-11-24 17:34:44 +0100357 struct task *t = pool_alloc(pool_head_task);
Willy Tarreaua4613182009-03-21 18:13:21 +0100358 if (t) {
Olivier Houchard4c2832852019-03-08 18:48:47 +0100359 _HA_ATOMIC_ADD(&nb_tasks, 1);
Emeric Brunc60def82017-09-27 14:59:38 +0200360 task_init(t, thread_mask);
Willy Tarreaua4613182009-03-21 18:13:21 +0100361 }
362 return t;
363}
364
365/*
Willy Tarreau29bf96d2019-05-17 14:16:51 +0200366 * Free a task. Its context must have been freed since it will be lost. The
367 * task count is decremented. It it is the current task, this one is reset.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200368 */
Olivier Houchard9b36cb42018-05-04 15:46:16 +0200369static inline void __task_free(struct task *t)
Willy Tarreaubaaee002006-06-26 02:48:02 +0200370{
Willy Tarreaud022e9c2019-09-24 08:25:15 +0200371 if (t == sched->current) {
372 sched->current = NULL;
Willy Tarreau29bf96d2019-05-17 14:16:51 +0200373 __ha_barrier_store();
374 }
Willy Tarreaubafbe012017-11-24 17:34:44 +0100375 pool_free(pool_head_task, t);
Willy Tarreaueb118892014-11-13 16:57:19 +0100376 if (unlikely(stopping))
Willy Tarreaubafbe012017-11-24 17:34:44 +0100377 pool_flush(pool_head_task);
Olivier Houchard4c2832852019-03-08 18:48:47 +0100378 _HA_ATOMIC_SUB(&nb_tasks, 1);
Willy Tarreaubaaee002006-06-26 02:48:02 +0200379}
380
Willy Tarreauf6562792019-05-07 19:05:35 +0200381/* Destroys a task : it's unlinked from the wait queues and is freed if it's
382 * the current task or not queued otherwise it's marked to be freed by the
383 * scheduler. It does nothing if <t> is NULL.
384 */
Olivier Houchard3f795f72019-04-17 22:51:06 +0200385static inline void task_destroy(struct task *t)
Olivier Houchard9b36cb42018-05-04 15:46:16 +0200386{
Dragan Dosen75bc6d32019-05-07 15:25:25 +0200387 if (!t)
388 return;
389
Olivier Houchard3f795f72019-04-17 22:51:06 +0200390 task_unlink_wq(t);
391 /* We don't have to explicitely remove from the run queue.
392 * If we are in the runqueue, the test below will set t->process
393 * to NULL, and the task will be free'd when it'll be its turn
394 * to run.
395 */
396
Olivier Houchard9b36cb42018-05-04 15:46:16 +0200397 /* There's no need to protect t->state with a lock, as the task
398 * has to run on the current thread.
399 */
Willy Tarreaud022e9c2019-09-24 08:25:15 +0200400 if (t == sched->current || !(t->state & (TASK_QUEUED | TASK_RUNNING)))
Olivier Houchard9b36cb42018-05-04 15:46:16 +0200401 __task_free(t);
402 else
403 t->process = NULL;
404}
405
Olivier Houchard06910462019-10-11 16:35:01 +0200406/* Should only be called by the thread responsible for the tasklet */
Olivier Houchardb0bdae72018-05-18 18:45:28 +0200407static inline void tasklet_free(struct tasklet *tl)
408{
Olivier Houchard3c4f40a2020-01-10 16:46:48 +0100409 if (MT_LIST_DEL((struct mt_list *)&tl->list))
Olivier Houchard06910462019-10-11 16:35:01 +0200410 _HA_ATOMIC_SUB(&tasks_run_queue, 1);
Olivier Houcharddcd6f3a2018-06-08 17:08:19 +0200411
Olivier Houchardb0bdae72018-05-18 18:45:28 +0200412 pool_free(pool_head_tasklet, tl);
413 if (unlikely(stopping))
414 pool_flush(pool_head_tasklet);
415}
416
Olivier Houchardff1e9f32019-09-20 17:18:35 +0200417static inline void tasklet_set_tid(struct tasklet *tl, int tid)
418{
419 tl->tid = tid;
420}
421
Willy Tarreaub20aa9e2018-10-15 14:52:21 +0200422void __task_queue(struct task *task, struct eb_root *wq);
423
Willy Tarreau4726f532009-03-07 17:25:21 +0100424/* Place <task> into the wait queue, where it may already be. If the expiration
Willy Tarreau531cf0c2009-03-08 16:35:27 +0100425 * timer is infinite, do nothing and rely on wake_expired_task to clean up.
Willy Tarreaudd0e89a2019-12-19 07:39:06 +0100426 * If the task uses a shared wait queue, it's queued into the global wait queue,
427 * protected by the global wq_lock, otherwise by it necessarily belongs to the
428 * current thread'sand is queued without locking.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200429 */
Willy Tarreau531cf0c2009-03-08 16:35:27 +0100430static inline void task_queue(struct task *task)
431{
432 /* If we already have a place in the wait queue no later than the
433 * timeout we're trying to set, we'll stay there, because it is very
434 * unlikely that we will reach the timeout anyway. If the timeout
435 * has been disabled, it's useless to leave the queue as well. We'll
436 * rely on wake_expired_tasks() to catch the node and move it to the
437 * proper place should it ever happen. Finally we only add the task
438 * to the queue if it was not there or if it was further than what
439 * we want.
440 */
441 if (!tick_isset(task->expire))
442 return;
443
Willy Tarreaub20aa9e2018-10-15 14:52:21 +0200444#ifdef USE_THREAD
Willy Tarreaudd0e89a2019-12-19 07:39:06 +0100445 if (task->state & TASK_SHARED_WQ) {
Willy Tarreauef28dc12019-05-28 18:48:07 +0200446 HA_RWLOCK_WRLOCK(TASK_WQ_LOCK, &wq_lock);
Willy Tarreaub20aa9e2018-10-15 14:52:21 +0200447 if (!task_in_wq(task) || tick_is_lt(task->expire, task->wq.key))
448 __task_queue(task, &timers);
Willy Tarreauef28dc12019-05-28 18:48:07 +0200449 HA_RWLOCK_WRUNLOCK(TASK_WQ_LOCK, &wq_lock);
Willy Tarreaub20aa9e2018-10-15 14:52:21 +0200450 } else
451#endif
452 {
Willy Tarreaudd0e89a2019-12-19 07:39:06 +0100453 BUG_ON((task->thread_mask & tid_bit) == 0); // should have TASK_SHARED_WQ
Willy Tarreaub20aa9e2018-10-15 14:52:21 +0200454 if (!task_in_wq(task) || tick_is_lt(task->expire, task->wq.key))
Willy Tarreaud022e9c2019-09-24 08:25:15 +0200455 __task_queue(task, &sched->timers);
Willy Tarreaub20aa9e2018-10-15 14:52:21 +0200456 }
Willy Tarreau531cf0c2009-03-08 16:35:27 +0100457}
Willy Tarreaubaaee002006-06-26 02:48:02 +0200458
Willy Tarreau26e48812011-07-25 14:30:42 +0200459/* Ensure <task> will be woken up at most at <when>. If the task is already in
460 * the run queue (but not running), nothing is done. It may be used that way
461 * with a delay : task_schedule(task, tick_add(now_ms, delay));
462 */
463static inline void task_schedule(struct task *task, int when)
464{
Emeric Brunc60def82017-09-27 14:59:38 +0200465 /* TODO: mthread, check if there is no tisk with this test */
Willy Tarreau26e48812011-07-25 14:30:42 +0200466 if (task_in_rq(task))
467 return;
468
Willy Tarreaub20aa9e2018-10-15 14:52:21 +0200469#ifdef USE_THREAD
Willy Tarreaudd0e89a2019-12-19 07:39:06 +0100470 if (task->state & TASK_SHARED_WQ) {
Willy Tarreauef28dc12019-05-28 18:48:07 +0200471 /* FIXME: is it really needed to lock the WQ during the check ? */
472 HA_RWLOCK_WRLOCK(TASK_WQ_LOCK, &wq_lock);
Willy Tarreaub20aa9e2018-10-15 14:52:21 +0200473 if (task_in_wq(task))
474 when = tick_first(when, task->expire);
475
476 task->expire = when;
477 if (!task_in_wq(task) || tick_is_lt(task->expire, task->wq.key))
478 __task_queue(task, &timers);
Willy Tarreauef28dc12019-05-28 18:48:07 +0200479 HA_RWLOCK_WRUNLOCK(TASK_WQ_LOCK, &wq_lock);
Willy Tarreaub20aa9e2018-10-15 14:52:21 +0200480 } else
481#endif
482 {
Willy Tarreaudd0e89a2019-12-19 07:39:06 +0100483 BUG_ON((task->thread_mask & tid_bit) == 0); // should have TASK_SHARED_WQ
Willy Tarreaub20aa9e2018-10-15 14:52:21 +0200484 if (task_in_wq(task))
485 when = tick_first(when, task->expire);
Willy Tarreau26e48812011-07-25 14:30:42 +0200486
Willy Tarreaub20aa9e2018-10-15 14:52:21 +0200487 task->expire = when;
488 if (!task_in_wq(task) || tick_is_lt(task->expire, task->wq.key))
Willy Tarreaud022e9c2019-09-24 08:25:15 +0200489 __task_queue(task, &sched->timers);
Willy Tarreaub20aa9e2018-10-15 14:52:21 +0200490 }
Willy Tarreau26e48812011-07-25 14:30:42 +0200491}
492
Thierry FOURNIERd6975962017-07-12 14:31:10 +0200493/* This function register a new signal. "lua" is the current lua
494 * execution context. It contains a pointer to the associated task.
495 * "link" is a list head attached to an other task that must be wake
496 * the lua task if an event occurs. This is useful with external
497 * events like TCP I/O or sleep functions. This funcion allocate
498 * memory for the signal.
499 */
500static inline struct notification *notification_new(struct list *purge, struct list *event, struct task *wakeup)
501{
Willy Tarreaubafbe012017-11-24 17:34:44 +0100502 struct notification *com = pool_alloc(pool_head_notification);
Thierry FOURNIERd6975962017-07-12 14:31:10 +0200503 if (!com)
504 return NULL;
505 LIST_ADDQ(purge, &com->purge_me);
506 LIST_ADDQ(event, &com->wake_me);
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100507 HA_SPIN_INIT(&com->lock);
Thierry FOURNIERd6975962017-07-12 14:31:10 +0200508 com->task = wakeup;
509 return com;
510}
511
512/* This function purge all the pending signals when the LUA execution
513 * is finished. This prevent than a coprocess try to wake a deleted
514 * task. This function remove the memory associated to the signal.
Thierry FOURNIERd5b79832017-12-10 17:14:07 +0100515 * The purge list is not locked because it is owned by only one
516 * process. before browsing this list, the caller must ensure to be
517 * the only one browser.
Thierry FOURNIERd6975962017-07-12 14:31:10 +0200518 */
519static inline void notification_purge(struct list *purge)
520{
521 struct notification *com, *back;
522
523 /* Delete all pending communication signals. */
524 list_for_each_entry_safe(com, back, purge, purge_me) {
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100525 HA_SPIN_LOCK(NOTIF_LOCK, &com->lock);
Thierry FOURNIERd6975962017-07-12 14:31:10 +0200526 LIST_DEL(&com->purge_me);
Thierry FOURNIER738a6d72017-07-17 00:14:07 +0200527 if (!com->task) {
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100528 HA_SPIN_UNLOCK(NOTIF_LOCK, &com->lock);
Willy Tarreaubafbe012017-11-24 17:34:44 +0100529 pool_free(pool_head_notification, com);
Thierry FOURNIER738a6d72017-07-17 00:14:07 +0200530 continue;
531 }
532 com->task = NULL;
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100533 HA_SPIN_UNLOCK(NOTIF_LOCK, &com->lock);
Thierry FOURNIERd6975962017-07-12 14:31:10 +0200534 }
535}
536
Thierry FOURNIERcb146882017-12-10 17:10:57 +0100537/* In some cases, the disconnected notifications must be cleared.
538 * This function just release memory blocs. The purge list is not
539 * locked because it is owned by only one process. Before browsing
540 * this list, the caller must ensure to be the only one browser.
541 * The "com" is not locked because when com->task is NULL, the
542 * notification is no longer used.
543 */
544static inline void notification_gc(struct list *purge)
545{
546 struct notification *com, *back;
547
548 /* Delete all pending communication signals. */
549 list_for_each_entry_safe (com, back, purge, purge_me) {
550 if (com->task)
551 continue;
552 LIST_DEL(&com->purge_me);
553 pool_free(pool_head_notification, com);
554 }
555}
556
Thierry FOURNIERd6975962017-07-12 14:31:10 +0200557/* This function sends signals. It wakes all the tasks attached
558 * to a list head, and remove the signal, and free the used
Thierry FOURNIERd5b79832017-12-10 17:14:07 +0100559 * memory. The wake list is not locked because it is owned by
560 * only one process. before browsing this list, the caller must
561 * ensure to be the only one browser.
Thierry FOURNIERd6975962017-07-12 14:31:10 +0200562 */
563static inline void notification_wake(struct list *wake)
564{
565 struct notification *com, *back;
566
567 /* Wake task and delete all pending communication signals. */
568 list_for_each_entry_safe(com, back, wake, wake_me) {
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100569 HA_SPIN_LOCK(NOTIF_LOCK, &com->lock);
Thierry FOURNIERd6975962017-07-12 14:31:10 +0200570 LIST_DEL(&com->wake_me);
Thierry FOURNIER738a6d72017-07-17 00:14:07 +0200571 if (!com->task) {
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100572 HA_SPIN_UNLOCK(NOTIF_LOCK, &com->lock);
Willy Tarreaubafbe012017-11-24 17:34:44 +0100573 pool_free(pool_head_notification, com);
Thierry FOURNIER738a6d72017-07-17 00:14:07 +0200574 continue;
575 }
Thierry FOURNIERd6975962017-07-12 14:31:10 +0200576 task_wakeup(com->task, TASK_WOKEN_MSG);
Thierry FOURNIER738a6d72017-07-17 00:14:07 +0200577 com->task = NULL;
Christopher Faulet2a944ee2017-11-07 10:42:54 +0100578 HA_SPIN_UNLOCK(NOTIF_LOCK, &com->lock);
Thierry FOURNIERd6975962017-07-12 14:31:10 +0200579 }
580}
581
Thierry FOURNIER9d5422a2018-05-30 11:40:08 +0200582/* This function returns true is some notification are pending
583 */
584static inline int notification_registered(struct list *wake)
585{
586 return !LIST_ISEMPTY(wake);
587}
588
Olivier Houchardcfbb3e62019-05-29 19:22:43 +0200589static inline int thread_has_tasks(void)
590{
591 return (!!(global_tasks_mask & tid_bit) |
Willy Tarreaud022e9c2019-09-24 08:25:15 +0200592 (sched->rqueue_size > 0) |
Willy Tarreaua62917b2020-01-30 18:37:28 +0100593 !LIST_ISEMPTY(&sched->tasklets[TL_URGENT]) |
594 !LIST_ISEMPTY(&sched->tasklets[TL_NORMAL]) |
595 !LIST_ISEMPTY(&sched->tasklets[TL_BULK]) |
596 !MT_LIST_ISEMPTY(&sched->shared_tasklet_list));
Olivier Houchardcfbb3e62019-05-29 19:22:43 +0200597}
598
Willy Tarreau64e60122019-07-12 08:31:17 +0200599/* adds list item <item> to work list <work> and wake up the associated task */
Olivier Houchard859dc802019-08-08 15:47:21 +0200600static inline void work_list_add(struct work_list *work, struct mt_list *item)
Willy Tarreau64e60122019-07-12 08:31:17 +0200601{
Olivier Houchard859dc802019-08-08 15:47:21 +0200602 MT_LIST_ADDQ(&work->head, item);
Willy Tarreau64e60122019-07-12 08:31:17 +0200603 task_wakeup(work->task, TASK_WOKEN_OTHER);
604}
605
606struct work_list *work_list_create(int nbthread,
607 struct task *(*fct)(struct task *, void *, unsigned short),
608 void *arg);
609
610void work_list_destroy(struct work_list *work, int nbthread);
611
Willy Tarreaubaaee002006-06-26 02:48:02 +0200612/*
Willy Tarreau918ff602011-07-25 16:33:49 +0200613 * This does 3 things :
Willy Tarreaubaaee002006-06-26 02:48:02 +0200614 * - wake up all expired tasks
615 * - call all runnable tasks
Willy Tarreaud825eef2007-05-12 22:35:00 +0200616 * - return the date of next event in <next> or eternity.
Willy Tarreaubaaee002006-06-26 02:48:02 +0200617 */
618
Thierry FOURNIER9cf7c4b2014-12-15 13:26:01 +0100619void process_runnable_tasks();
Willy Tarreaubaaee002006-06-26 02:48:02 +0200620
Willy Tarreau58b458d2008-06-29 22:40:23 +0200621/*
622 * Extract all expired timers from the timer queue, and wakes up all
Willy Tarreauc49ba522019-12-11 08:12:23 +0100623 * associated tasks.
624 */
625void wake_expired_tasks();
626
627/* Checks the next timer for the current thread by looking into its own timer
628 * list and the global one. It may return TICK_ETERNITY if no timer is present.
629 * Note that the next timer might very well be slighly in the past.
Willy Tarreau58b458d2008-06-29 22:40:23 +0200630 */
Willy Tarreauc49ba522019-12-11 08:12:23 +0100631int next_timer_expiry();
Willy Tarreau58b458d2008-06-29 22:40:23 +0200632
William Lallemand27f3fa52018-12-06 14:05:20 +0100633/*
634 * Delete every tasks before running the master polling loop
635 */
636void mworker_cleantasks();
637
Willy Tarreaubaaee002006-06-26 02:48:02 +0200638#endif /* _PROTO_TASK_H */
639
640/*
641 * Local variables:
642 * c-indent-level: 8
643 * c-basic-offset: 8
644 * End:
645 */