From 5c33a80edb016c2241cebabc07fc42698ee5d513 Mon Sep 17 00:00:00 2001
From: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Date: Wed, 1 Jul 2026 12:49:43 -0400
Subject: [PATCH] call_rcu: pin per-CPU worker at thread startup

create_all_cpu_call_rcu_data() gives each reclaim worker a target CPU
(crdp->cpu_affinity), but the worker only pins to it lazily.  Both the
startup call and the in-loop re-check go through set_thread_cpu_affinity(),
which rate-limits itself with

	if (++crdp->gp_count & SET_AFFINITY_CHECK_PERIOD_MASK)
		return 0;

gp_count starts at 0, so the first call returns without pinning; the pin
only fires once gp_count reaches a multiple of SET_AFFINITY_CHECK_PERIOD
(256).  set_thread_cpu_affinity() is called once per work-loop iteration
and each iteration sleeps at least 10 ms (poll), so the first pin does not
happen until roughly 2.5 s into the thread's life.  Until then the worker
keeps its inherited (wide) affinity mask and the scheduler may place it on
a different last-level-cache domain than the CPU whose callbacks it
services: every freed object's cache line then bounces across the LLC
boundary and the wake-up futex crosses it too.

On a 192-core / 24-CCX machine this shows up as bimodal single-writer
reclaim throughput (worker same-CCX vs cross-CCX) and depressed,
high-variance throughput at scale.

Split the actual pinning into do_set_thread_cpu_affinity() and call it
unconditionally once at call_rcu thread startup, before processing any
callback.  set_thread_cpu_affinity() keeps the rate-limited
migration-recovery re-check in the work loop.  This only makes the
already-intended placement happen promptly instead of ~2.5 s late.

Measured (churn micro-benchmark, per-CPU call_rcu, EPYC 9654):
single-updater reclaim goes from a bimodal ~2-6 Mops/s to a stable ~7.8;
a multi-writer coherent-list workload improves +29% at 64 writers and
+53% at 191 writers, with the run-to-run variance removed.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Change-Id: I76230a964126d14ad6ebd7fcd6783f4238b5ae85
---
 src/urcu-call-rcu-impl.h | 33 +++++++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/src/urcu-call-rcu-impl.h b/src/urcu-call-rcu-impl.h
index e473f4a93..6db4ec015 100644
--- a/src/urcu-call-rcu-impl.h
+++ b/src/urcu-call-rcu-impl.h
@@ -190,18 +190,20 @@ static void call_rcu_unlock(pthread_mutex_t *pmp)
  * cpuset(7).
  */
 #ifdef HAVE_SCHED_SETAFFINITY
+/*
+ * Actually pin the calling thread to crdp->cpu_affinity.  Unconditional
+ * (no rate-limit), so it is used for the initial placement at call_rcu
+ * thread startup; the work loop reaches it through the rate-limited
+ * set_thread_cpu_affinity() wrapper below.
+ */
 static
-int set_thread_cpu_affinity(struct call_rcu_data *crdp)
+int do_set_thread_cpu_affinity(struct call_rcu_data *crdp)
 {
 	cpu_set_t mask;
 	int ret;
 
 	if (crdp->cpu_affinity < 0)
 		return 0;
-	if (++crdp->gp_count & SET_AFFINITY_CHECK_PERIOD_MASK)
-		return 0;
-	if (urcu_sched_getcpu() == crdp->cpu_affinity)
-		return 0;
 
 	CPU_ZERO(&mask);
 	CPU_SET(crdp->cpu_affinity, &mask);
@@ -218,8 +220,26 @@ int set_thread_cpu_affinity(struct call_rcu_data *crdp)
 	}
 	return ret;
 }
+
+static
+int set_thread_cpu_affinity(struct call_rcu_data *crdp)
+{
+	if (crdp->cpu_affinity < 0)
+		return 0;
+	/* Rate-limit the migration-recovery re-pin in the work loop. */
+	if (++crdp->gp_count & SET_AFFINITY_CHECK_PERIOD_MASK)
+		return 0;
+	if (urcu_sched_getcpu() == crdp->cpu_affinity)
+		return 0;
+	return do_set_thread_cpu_affinity(crdp);
+}
 #else
 static
+int do_set_thread_cpu_affinity(struct call_rcu_data *crdp __attribute__((__unused__)))
+{
+	return 0;
+}
+static
 int set_thread_cpu_affinity(struct call_rcu_data *crdp __attribute__((__unused__)))
 {
 	return 0;
@@ -318,7 +338,8 @@ static void *call_rcu_thread(void *arg)
 	struct call_rcu_data *crdp = (struct call_rcu_data *) arg;
 	int rt = !!(uatomic_load(&crdp->flags) & URCU_CALL_RCU_RT);
 
-	if (set_thread_cpu_affinity(crdp))
+	/* Initial placement: pin immediately, before processing any callback. */
+	if (do_set_thread_cpu_affinity(crdp))
 		urcu_die(errno);
 
 	/*
