diff --git a/scheds/rust/scx_lavd/src/bpf/idle.bpf.c b/scheds/rust/scx_lavd/src/bpf/idle.bpf.c index 401f7e7bc5..a8a081f041 100644 --- a/scheds/rust/scx_lavd/src/bpf/idle.bpf.c +++ b/scheds/rust/scx_lavd/src/bpf/idle.bpf.c @@ -224,7 +224,7 @@ static s32 pick_idle_cpu_at_cpdom(struct pick_ctx *ctx, s64 cpdom, u64 scope, struct cpdom_ctx *cpdc; s32 cpu; - cpd_mask = MEMBER_VPTR(cpdom_cpumask, [cpdom]); + cpd_mask = lookup_cpdom_cpumask(cpdom); cpdc = MEMBER_VPTR(cpdom_ctxs, [cpdom]); if (!ctx || !cpdc || !cpd_mask || !cpdc->is_valid) return -ENOENT; @@ -313,7 +313,7 @@ s32 find_sticky_cpu_at_cpdom(struct pick_ctx *ctx, s32 sticky_cpu, s64 sticky_cp if (sticky_cpdom < 0) return -ENOENT; - cpd_mask = MEMBER_VPTR(cpdom_cpumask, [sticky_cpdom]); + cpd_mask = lookup_cpdom_cpumask(sticky_cpdom); if (cpd_mask) { if (ctx->a_mask) { cpu = bpf_cpumask_any_and_distribute( @@ -368,7 +368,7 @@ bool can_run_on_domain(struct pick_ctx *ctx, s64 cpdom) if (!test_task_flag(ctx->taskc, LAVD_FLAG_IS_AFFINITIZED)) return true; - cpd_mask = MEMBER_VPTR(cpdom_cpumask, [cpdom]); + cpd_mask = lookup_cpdom_cpumask(cpdom); cpdc = MEMBER_VPTR(cpdom_ctxs, [cpdom]); if (!cpd_mask || !cpdc) return false; diff --git a/scheds/rust/scx_lavd/src/bpf/lavd.bpf.h b/scheds/rust/scx_lavd/src/bpf/lavd.bpf.h index a9f6214a1d..80bb9db5f9 100644 --- a/scheds/rust/scx_lavd/src/bpf/lavd.bpf.h +++ b/scheds/rust/scx_lavd/src/bpf/lavd.bpf.h @@ -329,9 +329,42 @@ static __always_inline void decrement_stealer_budget(struct cpdom_ctx *cpdomc, } extern struct cpdom_ctx cpdom_ctxs[LAVD_CPDOM_MAX_NR]; -extern struct bpf_cpumask cpdom_cpumask[LAVD_CPDOM_MAX_NR]; extern int nr_cpdoms; +/* + * Per-compute-domain online CPU mask. + * + * Stored as a kptr cpumask (allocated via bpf_cpumask_create() at init) rather + * than an embedded "struct bpf_cpumask" so its storage is sized by the kernel + * for the running nr_cpu_ids, independent of the NR_CPUS the scheduler was + * built against. The wrapper lives in a BPF_MAP_TYPE_ARRAY because kptrs may + * not be placed in mmap-able global (.bss/.data) storage. + */ +struct cpdom_cpumask_wrapper { + struct bpf_cpumask __kptr *mask; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, u32); + __type(value, struct cpdom_cpumask_wrapper); + __uint(max_entries, LAVD_CPDOM_MAX_NR); +} cpdom_cpumasks __weak SEC(".maps"); + +static __always_inline +struct bpf_cpumask *lookup_cpdom_cpumask(u32 cpdom_id) +{ + struct cpdom_cpumask_wrapper *w; + + w = bpf_map_lookup_elem(&cpdom_cpumasks, &cpdom_id); + if (!w) + return NULL; + + return w->mask; +} + +int init_cpdom_cpumasks(void); + typedef struct task_ctx __arena task_ctx; struct cpu_ctx *get_cpu_ctx(void); diff --git a/scheds/rust/scx_lavd/src/bpf/main.bpf.c b/scheds/rust/scx_lavd/src/bpf/main.bpf.c index ff19684cf6..2be094a4de 100644 --- a/scheds/rust/scx_lavd/src/bpf/main.bpf.c +++ b/scheds/rust/scx_lavd/src/bpf/main.bpf.c @@ -1784,12 +1784,36 @@ void BPF_STRUCT_OPS(lavd_quiescent, struct task_struct *p, u64 deq_flags) } } +__hidden +int init_cpdom_cpumasks(void) +{ + struct cpdom_cpumask_wrapper *w; + struct bpf_cpumask *cpumask; + u32 cpdom_id; + + bpf_for(cpdom_id, 0, LAVD_CPDOM_MAX_NR) { + w = bpf_map_lookup_elem(&cpdom_cpumasks, &cpdom_id); + if (!w) + return -ESRCH; + + cpumask = bpf_cpumask_create(); + if (!cpumask) + return -ENOMEM; + + cpumask = bpf_kptr_xchg(&w->mask, cpumask); + if (cpumask) + bpf_cpumask_release(cpumask); + } + + return 0; +} + static void cpu_ctx_init_online(struct cpu_ctx *cpuc, u32 cpu_id) { struct bpf_cpumask *cd_cpumask; bpf_rcu_read_lock(); - cd_cpumask = MEMBER_VPTR(cpdom_cpumask, [cpuc->cpdom_id]); + cd_cpumask = lookup_cpdom_cpumask(cpuc->cpdom_id); if (!cd_cpumask) goto unlock_out; bpf_cpumask_set_cpu(cpu_id, cd_cpumask); @@ -1814,7 +1838,7 @@ static void cpu_ctx_init_offline(struct cpu_ctx *cpuc, u32 cpu_id) struct bpf_cpumask *cd_cpumask; bpf_rcu_read_lock(); - cd_cpumask = MEMBER_VPTR(cpdom_cpumask, [cpuc->cpdom_id]); + cd_cpumask = lookup_cpdom_cpumask(cpuc->cpdom_id); if (!cd_cpumask) goto unlock_out; bpf_cpumask_clear_cpu(cpu_id, cd_cpumask); @@ -2455,7 +2479,7 @@ static s32 init_per_cpu_ctx(u64 now) break; cpdomc = MEMBER_VPTR(cpdom_ctxs, [cpdom_id]); - cd_cpumask = MEMBER_VPTR(cpdom_cpumask, [cpdom_id]); + cd_cpumask = lookup_cpdom_cpumask(cpdom_id); if (!cpdomc || !cd_cpumask) { scx_bpf_error("Failed to lookup cpdom_ctx for %llu", cpdom_id); err = -ESRCH; @@ -2647,6 +2671,14 @@ s32 BPF_STRUCT_OPS_SLEEPABLE(lavd_init) if (err) return err; + /* + * Allocate the per-compute-domain online cpumasks before the per-CPU + * context init populates them. + */ + err = init_cpdom_cpumasks(); + if (err) + return err; + /* * Initialize per-CPU context. */ diff --git a/scheds/rust/scx_lavd/src/bpf/power.bpf.c b/scheds/rust/scx_lavd/src/bpf/power.bpf.c index d7aeb7f038..9c97bd2cc7 100644 --- a/scheds/rust/scx_lavd/src/bpf/power.bpf.c +++ b/scheds/rust/scx_lavd/src/bpf/power.bpf.c @@ -43,10 +43,6 @@ int nr_cpdoms; /* contexts for compute domains */ struct cpdom_ctx cpdom_ctxs[LAVD_CPDOM_MAX_NR]; -/* online CPU mask for each compute domain */ -private(LAVD) struct bpf_cpumask cpdom_cpumask[LAVD_CPDOM_MAX_NR]; - - /* * Performance vs. CPU order (PCO) table */ diff --git a/scheds/rust/scx_lavd/src/bpf/preempt.bpf.c b/scheds/rust/scx_lavd/src/bpf/preempt.bpf.c index 9321feaa91..164889ff6e 100644 --- a/scheds/rust/scx_lavd/src/bpf/preempt.bpf.c +++ b/scheds/rust/scx_lavd/src/bpf/preempt.bpf.c @@ -409,7 +409,7 @@ void try_find_and_kick_victim_cpu(struct task_struct *p, cpumask = cpuc_cur->temp_mask; cpdom_id = dsq_to_cpdom(dsq_id); cpdomc = MEMBER_VPTR(cpdom_ctxs, [cpdom_id]); - cd_cpumask = MEMBER_VPTR(cpdom_cpumask, [cpdom_id]); + cd_cpumask = lookup_cpdom_cpumask(cpdom_id); if (!cpdomc || !cd_cpumask || !cpumask) return;