From 7eb03ba629b3205696efaefaa51dd37b519ac128 Mon Sep 17 00:00:00 2001
From: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Date: Thu, 9 Jul 2026 15:48:33 -0400
Subject: [PATCH] call-rcu worker: set CPU affinity on first non-empty dequeue

commit 28f282af1905 ("call_rcu: pin per-CPU worker at thread startup")
attempts to prevent the per-cpu call-rcu worker from running on the
wrong CPU at the beginning of the application lifetime (first ~2.5s).

The goal is to ensure good performance in that time-frame.

However, pinning each worker threads at thread startup for each CPU for
per-cpu call-rcu worker configurations will trigger a lot of CPU wakeups
on short-lived processes using few CPUs when they are deployed on large
SMP machines: this wakes up every CPU just to set the affinity. This is
detrimental for energy consumption of the CPUs.

Refactor the call-rcu worker thread code to call set_thread_cpu_affinity()
only when call-rcu work is dequeued, and introduce an "affinity_init"
boolean which allows the first call to enforce affinity initialization,
bypassing the period mask check and the current CPU vs affinity CPU
check.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Change-Id: I24ee28b5133cbebd33af076321ba18aaab4d0b8e
---
 src/urcu-call-rcu-impl.h | 34 ++++++++++++++++------------------
 1 file changed, 16 insertions(+), 18 deletions(-)

diff --git a/src/urcu-call-rcu-impl.h b/src/urcu-call-rcu-impl.h
index 0dfa53fe..dc253d7c 100644
--- a/src/urcu-call-rcu-impl.h
+++ b/src/urcu-call-rcu-impl.h
@@ -53,6 +53,7 @@ struct call_rcu_data {
 	pthread_t tid;
 	int cpu_affinity;
 	unsigned long gp_count;
+	bool affinity_init;
 	struct cds_list_head list;
 } __attribute__((__aligned__(CAA_CACHE_LINE_SIZE)));
 
@@ -190,12 +191,6 @@ static void call_rcu_unlock(pthread_mutex_t *pmp)
  * cpuset(7).
  */
 #ifdef HAVE_SCHED_SETAFFINITY
-/*
- * Actually pin the calling thread to crdp->cpu_affinity.  Unconditional
- * (no rate-limit), so it is used for the initial placement at call_rcu
- * thread startup; the work loop reaches it through the rate-limited
- * set_thread_cpu_affinity() wrapper below.
- */
 static
 int do_set_thread_cpu_affinity(struct call_rcu_data *crdp)
 {
@@ -226,11 +221,18 @@ int set_thread_cpu_affinity(struct call_rcu_data *crdp)
 {
 	if (crdp->cpu_affinity < 0)
 		return 0;
-	/* Rate-limit the migration-recovery re-pin in the work loop. */
-	if (++crdp->gp_count & SET_AFFINITY_CHECK_PERIOD_MASK)
-		return 0;
-	if (urcu_sched_getcpu() == crdp->cpu_affinity)
-		return 0;
+	crdp->gp_count++;
+	/* Force setting the affinity if it was not already initialized. */
+	if (!crdp->affinity_init) {
+		crdp->affinity_init = true;
+	} else {
+		/* Rate-limit the migration-recovery re-pin in the work loop. */
+		if (crdp->gp_count & SET_AFFINITY_CHECK_PERIOD_MASK)
+			return 0;
+		/* Only set affinity if we happen to be running on the wrong CPU. */
+		if (urcu_sched_getcpu() == crdp->cpu_affinity)
+			return 0;
+	}
 	return do_set_thread_cpu_affinity(crdp);
 }
 #else
@@ -338,10 +340,6 @@ static void *call_rcu_thread(void *arg)
 	struct call_rcu_data *crdp = (struct call_rcu_data *) arg;
 	int rt = !!(uatomic_load(&crdp->flags) & URCU_CALL_RCU_RT);
 
-	/* Initial placement: pin immediately, before processing any callback. */
-	if (do_set_thread_cpu_affinity(crdp))
-		urcu_die(errno);
-
 	/*
 	 * If callbacks take a read-side lock, we need to be registered.
 	 */
@@ -359,9 +357,6 @@ static void *call_rcu_thread(void *arg)
 		struct cds_wfcq_node *cbs, *cbs_tmp_n;
 		enum cds_wfcq_ret splice_ret;
 
-		if (set_thread_cpu_affinity(crdp))
-			urcu_die(errno);
-
 		if (uatomic_load(&crdp->flags) & URCU_CALL_RCU_PAUSE) {
 			/*
 			 * Pause requested. Become quiescent: remove
@@ -385,6 +380,8 @@ static void *call_rcu_thread(void *arg)
 		urcu_posix_assert(splice_ret != CDS_WFCQ_RET_WOULDBLOCK);
 		urcu_posix_assert(splice_ret != CDS_WFCQ_RET_DEST_NON_EMPTY);
 		if (splice_ret != CDS_WFCQ_RET_SRC_EMPTY) {
+			if (set_thread_cpu_affinity(crdp))
+				urcu_die(errno);
 			synchronize_rcu();
 			cbcount = 0;
 			__cds_wfcq_for_each_blocking_safe(&cbs_tmp_head,
@@ -457,6 +454,7 @@ static void call_rcu_data_init(struct call_rcu_data **crdpp,
 	cds_list_add(&crdp->list, &call_rcu_data_list);
 	crdp->cpu_affinity = cpu_affinity;
 	crdp->gp_count = 0;
+	crdp->affinity_init = false;
 	rcu_set_pointer(crdpp, crdp);
 
 	ret = sigfillset(&newmask);
