scx_lavd: restrict direct dispatch to usable, uncongested CPUs

scheds/rust/scx_lavd/src/bpf/lavd.bpf.h

@@ -121,6 +121,7 @@ enum consts_internal {
 	LAVD_CPDOM_MIG_SHIFT		= 3, /* when mildly loaded: 1/2**3 = [-12.5%, +12.5%] */
 	LAVD_CPDOM_MIG_SHIFT_OL		= 4, /* when over-loaded:   1/2**4 = [-6.25%, +6.25%] */
 	LAVD_CPDOM_MIG_PROB_FT		= (LAVD_SYS_STAT_INTERVAL_NS / LAVD_SLICE_MAX_NS_DFL), /* roughly twice per interval */
+	LAVD_CPU_CONGESTED_THRES	= 1, /* the CPU is congested when one or more tasks are waiting across its DSQs */
 
 	LAVD_FUTEX_OP_INVALID		= -1,
 };
@@ -697,6 +698,7 @@ static __always_inline bool use_cpdom_dsq(void)
 }
 
 bool queued_on_cpu(struct cpu_ctx *cpuc);
+bool is_cpu_congested(struct cpu_ctx *cpuc);
 u64 get_target_dsq_id(struct task_struct *p, struct cpu_ctx *cpuc, task_ctx *taskc);
 u16 normalize_lat_cri(u16 lat_cri);
 
@@ -800,7 +802,7 @@ void preempt_at_tick(struct task_struct *p, struct cpu_ctx *cpuc);
 void try_find_and_kick_victim_cpu(struct task_struct *p,
 					 task_ctx *taskc,
 					 s32 preferred_cpu,
-					 u64 dsq_id);
+					 u64 cpdom_id);
 
 extern volatile bool is_monitored;
 

scheds/rust/scx_lavd/src/bpf/main.bpf.c

@@ -675,9 +675,27 @@ static void update_stat_for_refill(struct task_struct *p,
 
 static bool can_direct_dispatch(struct cpu_ctx *cpuc, bool is_cpu_idle)
 {
-	return (is_cpu_idle && !queued_on_cpu(cpuc)) ||
-	       (lb_local_dsq_util_wall > 0 &&
-		cpuc->avg_util_wall < lb_local_dsq_util_wall);
+	/*
+	 * An idle CPU with nothing queued cannot be congested --
+	 * queued_on_cpu() covers every DSQ that is_cpu_congested()
+	 * counts -- so no congestion check is needed on this path.
+	 */
+	if (is_cpu_idle && !queued_on_cpu(cpuc))
+		return true;
+
+	/*
+	 * Bypass deadline ordering under low utilization, but never
+	 * direct-dispatch into a congested CPU (tasks are already waiting
+	 * across its DSQs, and inserting into the local DSQ would let the
+	 * new task jump ahead of them) nor into a CPU an RT/DL task has
+	 * taken (the task would be stranded in a non-stealable local DSQ
+	 * until the higher class yields). Both walk/peek remote state, so
+	 * evaluate them last, only after the cheap utilization checks pass.
+	 */
+	return lb_local_dsq_util_wall > 0 &&
+	       cpuc->avg_util_wall < lb_local_dsq_util_wall &&
+	       !is_cpu_congested(cpuc) &&
+	       !is_rt_or_dl_task_running(cpuc->cpu_id);
 }
 
 /*
@@ -1093,16 +1111,11 @@ void BPF_STRUCT_OPS(lavd_enqueue, struct task_struct *p, u64 enq_flags)
 	}
 
 	/*
-	 * If there is no idle CPU for an eligible task, try to preempt a task.
-	 * Try to find and kick a victim CPU, which runs a less urgent task,
-	 * from dsq_id. The kick will be done asynchronously.
-	 *
-	 * In the case of the forced enqueue mode, we don't try preemption
-	 * since it is a batch of bulk enqueues.
+	 * If there is no idle CPU, try to preempt a task. Find and kick a
+	 * victim CPU, which runs a less urgent task.
 	 */
 	if (!no_preemption) {
-		try_find_and_kick_victim_cpu(p, taskc, cpu,
-					     cpdom_to_dsq(cpuc->cpdom_id));
+		try_find_and_kick_victim_cpu(p, taskc, cpu, cpuc->cpdom_id);
 	}
 }
 

scheds/rust/scx_lavd/src/bpf/preempt.bpf.c

@@ -347,14 +347,14 @@ __hidden
 void try_find_and_kick_victim_cpu(struct task_struct *p,
 					 task_ctx *taskc,
 					 s32 preferred_cpu,
-					 u64 dsq_id)
+					 u64 cpdom_id)
 {
 	struct preemption_info prm_t, prm_c;
 	struct bpf_cpumask *cd_cpumask, *cpumask;
 	struct cpdom_ctx *cpdomc;
 	struct cpu_ctx *cpuc_victim;
 	struct cpu_ctx *cpuc_cur = NULL;
-	u64 now, duration_wall, cpdom_id, new_slice_wall = 0;
+	u64 now, duration_wall, new_slice_wall = 0;
 
 	/*
 	 * Don't even try to perform expensive preemption for greedy tasks.
@@ -414,7 +414,6 @@ void try_find_and_kick_victim_cpu(struct task_struct *p,
 		return;
 
 	cpumask = cpuc_cur->temp_mask;
-	cpdom_id = dsq_to_cpdom(dsq_id);
 	cpdomc = MEMBER_VPTR(cpdom_ctxs, [cpdom_id]);
 	cd_cpumask = MEMBER_VPTR(cpdom_cpumask, [cpdom_id]);
 	if (!cpdomc || !cd_cpumask || !cpumask)

scheds/rust/scx_lavd/src/bpf/util.bpf.c

@@ -411,6 +411,34 @@ bool queued_on_cpu(struct cpu_ctx *cpuc)
 	return false;
 }
 
+__hidden
+bool is_cpu_congested(struct cpu_ctx *cpuc)
+{
+	int nr;
+
+	nr = scx_bpf_dsq_nr_queued(SCX_DSQ_LOCAL_ON | cpuc->cpu_id);
+	if (nr >= LAVD_CPU_CONGESTED_THRES)
+		return true;
+
+	if (use_cpdom_dsq()) {
+		nr += scx_bpf_dsq_nr_queued(cpdom_to_dsq(cpuc->cpdom_id));
+		if (nr >= LAVD_CPU_CONGESTED_THRES)
+			return true;
+
+		nr += scx_bpf_dsq_nr_queued(cpdom_to_turb_dsq(cpuc->cpdom_id));
+		if (nr >= LAVD_CPU_CONGESTED_THRES)
+			return true;
+	}
+
+	if (use_per_cpu_dsq()) {
+		nr += scx_bpf_dsq_nr_queued(cpu_to_dsq(cpuc->cpu_id));
+		if (nr >= LAVD_CPU_CONGESTED_THRES)
+			return true;
+	}
+
+	return false;
+}
+
 __hidden
 u64 peek_dsq_vtime(u64 dsq_id)
 {