CLEANUP: task: cache the task_per_thread pointer
In process_runnable_tasks() we perform a lot of dereferences to
task_per_thread[tid] but tid is thread_local and the compiler cannot
know that it doesn't change so this results in making lots of thread
local accesses and array dereferences. By just keeping a copy pointer
of this, we let the compiler optimize the code. Just doing this has
reduced process_runnable_tasks() by 124 bytes in the fast path. Doing
the same in wake_expired_tasks() results in 16 extra bytes saved.
diff --git a/src/task.c b/src/task.c
index 33ce05f..1ad6b01 100644
--- a/src/task.c
+++ b/src/task.c
@@ -159,6 +159,7 @@
*/
int wake_expired_tasks()
{
+ struct task_per_thread * const tt = &task_per_thread[tid]; // thread's tasks
struct task *task;
struct eb32_node *eb;
int ret = TICK_ETERNITY;
@@ -166,13 +167,13 @@
while (1) {
lookup_next_local:
- eb = eb32_lookup_ge(&task_per_thread[tid].timers, now_ms - TIMER_LOOK_BACK);
+ eb = eb32_lookup_ge(&tt->timers, now_ms - TIMER_LOOK_BACK);
if (!eb) {
/* we might have reached the end of the tree, typically because
* <now_ms> is in the first half and we're first scanning the last
* half. Let's loop back to the beginning of the tree now.
*/
- eb = eb32_first(&task_per_thread[tid].timers);
+ eb = eb32_first(&tt->timers);
if (likely(!eb))
break;
}
@@ -202,7 +203,7 @@
*/
if (!tick_is_expired(task->expire, now_ms)) {
if (tick_isset(task->expire))
- __task_queue(task, &task_per_thread[tid].timers);
+ __task_queue(task, &tt->timers);
goto lookup_next_local;
}
task_wakeup(task, TASK_WOKEN_TIMER);
@@ -299,6 +300,7 @@
*/
void process_runnable_tasks()
{
+ struct task_per_thread * const tt = &task_per_thread[tid]; // thread's tasks
struct eb32sc_node *lrq = NULL; // next local run queue entry
struct eb32sc_node *grq = NULL; // next global run queue entry
struct task *t;
@@ -320,7 +322,7 @@
/* Note: the grq lock is always held when grq is not null */
- while (task_per_thread[tid].task_list_size < max_processed) {
+ while (tt->task_list_size < max_processed) {
if ((global_tasks_mask & tid_bit) && !grq) {
#ifdef USE_THREAD
HA_SPIN_LOCK(TASK_RQ_LOCK, &rq_lock);
@@ -340,9 +342,9 @@
*/
if (!lrq) {
- lrq = eb32sc_lookup_ge(&task_per_thread[tid].rqueue, rqueue_ticks - TIMER_LOOK_BACK, tid_bit);
+ lrq = eb32sc_lookup_ge(&tt->rqueue, rqueue_ticks - TIMER_LOOK_BACK, tid_bit);
if (unlikely(!lrq))
- lrq = eb32sc_first(&task_per_thread[tid].rqueue, tid_bit);
+ lrq = eb32sc_first(&tt->rqueue, tid_bit);
}
if (!lrq && !grq)
@@ -372,7 +374,7 @@
MT_LIST_INIT(&((struct tasklet *)t)->list);
/* And add it to the local task list */
tasklet_insert_into_tasklet_list((struct tasklet *)t);
- HA_ATOMIC_ADD(&task_per_thread[tid].task_list_size, 1);
+ HA_ATOMIC_ADD(&tt->task_list_size, 1);
activity[tid].tasksw++;
}
@@ -382,19 +384,19 @@
grq = NULL;
}
- while (max_processed > 0 && !MT_LIST_ISEMPTY(&task_per_thread[tid].task_list)) {
+ while (max_processed > 0 && !MT_LIST_ISEMPTY(&tt->task_list)) {
struct task *t;
unsigned short state;
void *ctx;
struct task *(*process)(struct task *t, void *ctx, unsigned short state);
- t = (struct task *)MT_LIST_POP(&task_per_thread[tid].task_list, struct tasklet *, list);
+ t = (struct task *)MT_LIST_POP(&tt->task_list, struct tasklet *, list);
if (!t)
break;
state = _HA_ATOMIC_XCHG(&t->state, TASK_RUNNING);
__ha_barrier_atomic_store();
if (!TASK_IS_TASKLET(t))
- _HA_ATOMIC_SUB(&task_per_thread[tid].task_list_size, 1);
+ _HA_ATOMIC_SUB(&tt->task_list_size, 1);
ti->flags &= ~TI_FL_STUCK; // this thread is still running
activity[tid].ctxsw++;
@@ -446,7 +448,7 @@
max_processed--;
}
- if (!MT_LIST_ISEMPTY(&task_per_thread[tid].task_list))
+ if (!MT_LIST_ISEMPTY(&tt->task_list))
activity[tid].long_rq++;
}