diff --git a/scheds/rust/scx_lavd/src/bpf/lavd.bpf.h b/scheds/rust/scx_lavd/src/bpf/lavd.bpf.h index f9834b227e..de706eee6e 100644 --- a/scheds/rust/scx_lavd/src/bpf/lavd.bpf.h +++ b/scheds/rust/scx_lavd/src/bpf/lavd.bpf.h @@ -121,6 +121,7 @@ enum consts_internal { LAVD_CPDOM_MIG_SHIFT = 3, /* when mildly loaded: 1/2**3 = [-12.5%, +12.5%] */ LAVD_CPDOM_MIG_SHIFT_OL = 4, /* when over-loaded: 1/2**4 = [-6.25%, +6.25%] */ LAVD_CPDOM_MIG_PROB_FT = (LAVD_SYS_STAT_INTERVAL_NS / LAVD_SLICE_MAX_NS_DFL), /* roughly twice per interval */ + LAVD_CPU_CONGESTED_THRES = 1, /* the CPU is congested when one or more tasks are waiting across its DSQs */ LAVD_FUTEX_OP_INVALID = -1, }; @@ -697,6 +698,7 @@ static __always_inline bool use_cpdom_dsq(void) } bool queued_on_cpu(struct cpu_ctx *cpuc); +bool is_cpu_congested(struct cpu_ctx *cpuc); u64 get_target_dsq_id(struct task_struct *p, struct cpu_ctx *cpuc, task_ctx *taskc); u16 normalize_lat_cri(u16 lat_cri); @@ -800,7 +802,7 @@ void preempt_at_tick(struct task_struct *p, struct cpu_ctx *cpuc); void try_find_and_kick_victim_cpu(struct task_struct *p, task_ctx *taskc, s32 preferred_cpu, - u64 dsq_id); + u64 cpdom_id); extern volatile bool is_monitored; diff --git a/scheds/rust/scx_lavd/src/bpf/main.bpf.c b/scheds/rust/scx_lavd/src/bpf/main.bpf.c index 30f495b422..eaef4a7df4 100644 --- a/scheds/rust/scx_lavd/src/bpf/main.bpf.c +++ b/scheds/rust/scx_lavd/src/bpf/main.bpf.c @@ -675,9 +675,27 @@ static void update_stat_for_refill(struct task_struct *p, static bool can_direct_dispatch(struct cpu_ctx *cpuc, bool is_cpu_idle) { - return (is_cpu_idle && !queued_on_cpu(cpuc)) || - (lb_local_dsq_util_wall > 0 && - cpuc->avg_util_wall < lb_local_dsq_util_wall); + /* + * An idle CPU with nothing queued cannot be congested -- + * queued_on_cpu() covers every DSQ that is_cpu_congested() + * counts -- so no congestion check is needed on this path. + */ + if (is_cpu_idle && !queued_on_cpu(cpuc)) + return true; + + /* + * Bypass deadline ordering under low utilization, but never + * direct-dispatch into a congested CPU (tasks are already waiting + * across its DSQs, and inserting into the local DSQ would let the + * new task jump ahead of them) nor into a CPU an RT/DL task has + * taken (the task would be stranded in a non-stealable local DSQ + * until the higher class yields). Both walk/peek remote state, so + * evaluate them last, only after the cheap utilization checks pass. + */ + return lb_local_dsq_util_wall > 0 && + cpuc->avg_util_wall < lb_local_dsq_util_wall && + !is_cpu_congested(cpuc) && + !is_rt_or_dl_task_running(cpuc->cpu_id); } /* @@ -1093,16 +1111,11 @@ void BPF_STRUCT_OPS(lavd_enqueue, struct task_struct *p, u64 enq_flags) } /* - * If there is no idle CPU for an eligible task, try to preempt a task. - * Try to find and kick a victim CPU, which runs a less urgent task, - * from dsq_id. The kick will be done asynchronously. - * - * In the case of the forced enqueue mode, we don't try preemption - * since it is a batch of bulk enqueues. + * If there is no idle CPU, try to preempt a task. Find and kick a + * victim CPU, which runs a less urgent task. */ if (!no_preemption) { - try_find_and_kick_victim_cpu(p, taskc, cpu, - cpdom_to_dsq(cpuc->cpdom_id)); + try_find_and_kick_victim_cpu(p, taskc, cpu, cpuc->cpdom_id); } } diff --git a/scheds/rust/scx_lavd/src/bpf/preempt.bpf.c b/scheds/rust/scx_lavd/src/bpf/preempt.bpf.c index 5f43cdfaef..49121e66de 100644 --- a/scheds/rust/scx_lavd/src/bpf/preempt.bpf.c +++ b/scheds/rust/scx_lavd/src/bpf/preempt.bpf.c @@ -347,14 +347,14 @@ __hidden void try_find_and_kick_victim_cpu(struct task_struct *p, task_ctx *taskc, s32 preferred_cpu, - u64 dsq_id) + u64 cpdom_id) { struct preemption_info prm_t, prm_c; struct bpf_cpumask *cd_cpumask, *cpumask; struct cpdom_ctx *cpdomc; struct cpu_ctx *cpuc_victim; struct cpu_ctx *cpuc_cur = NULL; - u64 now, duration_wall, cpdom_id, new_slice_wall = 0; + u64 now, duration_wall, new_slice_wall = 0; /* * Don't even try to perform expensive preemption for greedy tasks. @@ -414,7 +414,6 @@ void try_find_and_kick_victim_cpu(struct task_struct *p, return; cpumask = cpuc_cur->temp_mask; - cpdom_id = dsq_to_cpdom(dsq_id); cpdomc = MEMBER_VPTR(cpdom_ctxs, [cpdom_id]); cd_cpumask = MEMBER_VPTR(cpdom_cpumask, [cpdom_id]); if (!cpdomc || !cd_cpumask || !cpumask) diff --git a/scheds/rust/scx_lavd/src/bpf/util.bpf.c b/scheds/rust/scx_lavd/src/bpf/util.bpf.c index b264a2eabc..031c0ce75a 100644 --- a/scheds/rust/scx_lavd/src/bpf/util.bpf.c +++ b/scheds/rust/scx_lavd/src/bpf/util.bpf.c @@ -411,6 +411,34 @@ bool queued_on_cpu(struct cpu_ctx *cpuc) return false; } +__hidden +bool is_cpu_congested(struct cpu_ctx *cpuc) +{ + int nr; + + nr = scx_bpf_dsq_nr_queued(SCX_DSQ_LOCAL_ON | cpuc->cpu_id); + if (nr >= LAVD_CPU_CONGESTED_THRES) + return true; + + if (use_cpdom_dsq()) { + nr += scx_bpf_dsq_nr_queued(cpdom_to_dsq(cpuc->cpdom_id)); + if (nr >= LAVD_CPU_CONGESTED_THRES) + return true; + + nr += scx_bpf_dsq_nr_queued(cpdom_to_turb_dsq(cpuc->cpdom_id)); + if (nr >= LAVD_CPU_CONGESTED_THRES) + return true; + } + + if (use_per_cpu_dsq()) { + nr += scx_bpf_dsq_nr_queued(cpu_to_dsq(cpuc->cpu_id)); + if (nr >= LAVD_CPU_CONGESTED_THRES) + return true; + } + + return false; +} + __hidden u64 peek_dsq_vtime(u64 dsq_id) {