Skip to content

scx_lavd: decouple per-cpdom cpumasks from build-time NR_CPUS #3633

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jkkm wants to merge 3 commits into
base: main
Choose a base branch
from
Open

scx_lavd: decouple per-cpdom cpumasks from build-time NR_CPUS #3633

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0b1996d
scx_lavd: allocate per-cpdom cpumasks dynamically to decouple from NR…
Jun 5, 2026
9474cb2
scx_lavd: inline lookup_cpdom_cpumask() to avoid a hot-path call
Jun 5, 2026
208a5fc
Merge branch 'main' into lavd-cpdom-cpumask-kptr
htejun Jul 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions scheds/rust/scx_lavd/src/bpf/idle.bpf.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ static s32 pick_idle_cpu_at_cpdom(struct pick_ctx *ctx, s64 cpdom, u64 scope,
struct cpdom_ctx *cpdc;
s32 cpu;

cpd_mask = MEMBER_VPTR(cpdom_cpumask, [cpdom]);
cpd_mask = lookup_cpdom_cpumask(cpdom);
cpdc = MEMBER_VPTR(cpdom_ctxs, [cpdom]);
if (!ctx || !cpdc || !cpd_mask || !cpdc->is_valid)
return -ENOENT;
Expand Down Expand Up @@ -313,7 +313,7 @@ s32 find_sticky_cpu_at_cpdom(struct pick_ctx *ctx, s32 sticky_cpu, s64 sticky_cp
if (sticky_cpdom < 0)
return -ENOENT;

cpd_mask = MEMBER_VPTR(cpdom_cpumask, [sticky_cpdom]);
cpd_mask = lookup_cpdom_cpumask(sticky_cpdom);
if (cpd_mask) {
if (ctx->a_mask) {
cpu = bpf_cpumask_any_and_distribute(
Expand Down Expand Up @@ -368,7 +368,7 @@ bool can_run_on_domain(struct pick_ctx *ctx, s64 cpdom)
if (!test_task_flag(ctx->taskc, LAVD_FLAG_IS_AFFINITIZED))
return true;

cpd_mask = MEMBER_VPTR(cpdom_cpumask, [cpdom]);
cpd_mask = lookup_cpdom_cpumask(cpdom);
cpdc = MEMBER_VPTR(cpdom_ctxs, [cpdom]);
if (!cpd_mask || !cpdc)
return false;
Expand Down
35 changes: 34 additions & 1 deletion scheds/rust/scx_lavd/src/bpf/lavd.bpf.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -329,9 +329,42 @@ static __always_inline void decrement_stealer_budget(struct cpdom_ctx *cpdomc,
}

extern struct cpdom_ctx cpdom_ctxs[LAVD_CPDOM_MAX_NR];
extern struct bpf_cpumask cpdom_cpumask[LAVD_CPDOM_MAX_NR];
extern int nr_cpdoms;

/*
* Per-compute-domain online CPU mask.
*
* Stored as a kptr cpumask (allocated via bpf_cpumask_create() at init) rather
* than an embedded "struct bpf_cpumask" so its storage is sized by the kernel
* for the running nr_cpu_ids, independent of the NR_CPUS the scheduler was
* built against. The wrapper lives in a BPF_MAP_TYPE_ARRAY because kptrs may
* not be placed in mmap-able global (.bss/.data) storage.
*/
struct cpdom_cpumask_wrapper {
struct bpf_cpumask __kptr *mask;
};

struct {
__uint(type, BPF_MAP_TYPE_ARRAY);
__type(key, u32);
__type(value, struct cpdom_cpumask_wrapper);
__uint(max_entries, LAVD_CPDOM_MAX_NR);
} cpdom_cpumasks __weak SEC(".maps");

static __always_inline
struct bpf_cpumask *lookup_cpdom_cpumask(u32 cpdom_id)
{
struct cpdom_cpumask_wrapper *w;

w = bpf_map_lookup_elem(&cpdom_cpumasks, &cpdom_id);
if (!w)
return NULL;

return w->mask;
}

int init_cpdom_cpumasks(void);

typedef struct task_ctx __arena task_ctx;

struct cpu_ctx *get_cpu_ctx(void);
Expand Down
38 changes: 35 additions & 3 deletions scheds/rust/scx_lavd/src/bpf/main.bpf.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1784,12 +1784,36 @@ void BPF_STRUCT_OPS(lavd_quiescent, struct task_struct *p, u64 deq_flags)
}
}

__hidden
int init_cpdom_cpumasks(void)
{
struct cpdom_cpumask_wrapper *w;
struct bpf_cpumask *cpumask;
u32 cpdom_id;

bpf_for(cpdom_id, 0, LAVD_CPDOM_MAX_NR) {
w = bpf_map_lookup_elem(&cpdom_cpumasks, &cpdom_id);
if (!w)
return -ESRCH;

cpumask = bpf_cpumask_create();
if (!cpumask)
return -ENOMEM;

cpumask = bpf_kptr_xchg(&w->mask, cpumask);
if (cpumask)
bpf_cpumask_release(cpumask);
}

return 0;
}

static void cpu_ctx_init_online(struct cpu_ctx *cpuc, u32 cpu_id)
{
struct bpf_cpumask *cd_cpumask;

bpf_rcu_read_lock();
cd_cpumask = MEMBER_VPTR(cpdom_cpumask, [cpuc->cpdom_id]);
cd_cpumask = lookup_cpdom_cpumask(cpuc->cpdom_id);
if (!cd_cpumask)
goto unlock_out;
bpf_cpumask_set_cpu(cpu_id, cd_cpumask);
Expand All @@ -1814,7 +1838,7 @@ static void cpu_ctx_init_offline(struct cpu_ctx *cpuc, u32 cpu_id)
struct bpf_cpumask *cd_cpumask;

bpf_rcu_read_lock();
cd_cpumask = MEMBER_VPTR(cpdom_cpumask, [cpuc->cpdom_id]);
cd_cpumask = lookup_cpdom_cpumask(cpuc->cpdom_id);
if (!cd_cpumask)
goto unlock_out;
bpf_cpumask_clear_cpu(cpu_id, cd_cpumask);
Expand Down Expand Up @@ -2455,7 +2479,7 @@ static s32 init_per_cpu_ctx(u64 now)
break;

cpdomc = MEMBER_VPTR(cpdom_ctxs, [cpdom_id]);
cd_cpumask = MEMBER_VPTR(cpdom_cpumask, [cpdom_id]);
cd_cpumask = lookup_cpdom_cpumask(cpdom_id);
if (!cpdomc || !cd_cpumask) {
scx_bpf_error("Failed to lookup cpdom_ctx for %llu", cpdom_id);
err = -ESRCH;
Expand Down Expand Up @@ -2647,6 +2671,14 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(lavd_init)
if (err)
return err;

/*
* Allocate the per-compute-domain online cpumasks before the per-CPU
* context init populates them.
*/
err = init_cpdom_cpumasks();
if (err)
return err;

/*
* Initialize per-CPU context.
*/
Expand Down
4 changes: 0 additions & 4 deletions scheds/rust/scx_lavd/src/bpf/power.bpf.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,6 @@ int nr_cpdoms;
/* contexts for compute domains */
struct cpdom_ctx cpdom_ctxs[LAVD_CPDOM_MAX_NR];

/* online CPU mask for each compute domain */
private(LAVD) struct bpf_cpumask cpdom_cpumask[LAVD_CPDOM_MAX_NR];


/*
* Performance vs. CPU order (PCO) table
*/
Expand Down
2 changes: 1 addition & 1 deletion scheds/rust/scx_lavd/src/bpf/preempt.bpf.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -409,7 +409,7 @@ void try_find_and_kick_victim_cpu(struct task_struct *p,
cpumask = cpuc_cur->temp_mask;
cpdom_id = dsq_to_cpdom(dsq_id);
cpdomc = MEMBER_VPTR(cpdom_ctxs, [cpdom_id]);
cd_cpumask = MEMBER_VPTR(cpdom_cpumask, [cpdom_id]);
cd_cpumask = lookup_cpdom_cpumask(cpdom_id);
if (!cpdomc || !cd_cpumask || !cpumask)
return;

Expand Down
Loading