From dd0a4f62d3732daf5828393d6b1e62d83516cc3a Mon Sep 17 00:00:00 2001 From: Ryan Newton Date: Tue, 10 Mar 2026 09:01:20 -0700 Subject: [PATCH] Add struct_ops program support and kfunc call resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support for BPF_PROG_TYPE_STRUCT_OPS programs and BPF_MAP_TYPE_STRUCT_OPS maps, enabling use cases like sched_ext custom schedulers that implement kernel struct interfaces from Rust eBPF. aya-obj changes: - Add EbpfSectionKind::StructOps/StructOpsLink and ProgramSection::StructOps for .struct_ops ELF sections - Add Map::StructOps variant with StructOpsMap type - Parse .struct_ops/.struct_ops.link sections, resolving struct type names from BTF VAR types - Make Btf::type_by_id, type_name, string_at public for kernel BTF introspection - Make BtfMember and Struct.{size,members} public - Add Btf::fixup_func_linkage() to patch GLOBAL→STATIC for struct_ops (Rust compiler emits all as GLOBAL) - Sanitize EXTERN FUNCs and .struct_ops DATASECs in to_bytes() to avoid kernel rejection during BPF_BTF_LOAD - Recompute BTF header in to_bytes() to match serialized sizes - Detect extern symbol (kfunc) call relocations and patch src_reg from BPF_PSEUDO_CALL to BPF_PSEUDO_KFUNC_CALL - Add Object::fixup_kfunc_calls() to resolve kfunc imm fields against vmlinux BTF aya changes: - Add StructOps program type with StructOpsLink - Wire StructOps into Program enum and all impl_*! macros - Defer struct_ops map creation until attach time - Add Ebpf::attach_struct_ops() that loads programs, creates the struct_ops map with program FDs, and attaches via BPF_LINK_CREATE - Cache kernel BTF in Ebpf for reuse during attachment - Store btf_fd for struct_ops map creation ~215 of the ~855 added lines are unit tests covering section parsing, BTF fixups, and serialization sanitization. Tested with a pure-Rust sched_ext FIFO scheduler (scx_purerust) running for 5+ minutes under normal workload. --- aya-obj/src/btf/btf.rs | 219 +++++++++++++++++++++++++++- aya-obj/src/btf/types.rs | 18 ++- aya-obj/src/lib.rs | 2 +- aya-obj/src/maps.rs | 38 ++++- aya-obj/src/obj.rs | 254 +++++++++++++++++++++++++++++++- aya-obj/src/relocation.rs | 32 +++- aya/src/bpf.rs | 259 +++++++++++++++++++++++++++++++-- aya/src/programs/mod.rs | 16 ++ aya/src/programs/struct_ops.rs | 52 +++++++ aya/src/sys/bpf.rs | 2 +- 10 files changed, 855 insertions(+), 37 deletions(-) create mode 100644 aya/src/programs/struct_ops.rs diff --git a/aya-obj/src/btf/btf.rs b/aya-obj/src/btf/btf.rs index 4a2be735b..25e16cbff 100644 --- a/aya-obj/src/btf/btf.rs +++ b/aya-obj/src/btf/btf.rs @@ -393,7 +393,8 @@ impl Btf { Ok(types) } - pub(crate) fn string_at(&self, offset: u32) -> Result, BtfError> { + /// Returns the string at the given offset in the BTF string table. + pub fn string_at(&self, offset: u32) -> Result, BtfError> { let btf_header { hdr_len, mut str_off, @@ -415,7 +416,8 @@ impl Btf { Ok(s.to_string_lossy()) } - pub(crate) fn type_by_id(&self, type_id: u32) -> Result<&BtfType, BtfError> { + /// Returns the BTF type at the given type id. + pub fn type_by_id(&self, type_id: u32) -> Result<&BtfType, BtfError> { self.types.type_by_id(type_id) } @@ -423,7 +425,8 @@ impl Btf { self.types.resolve_type(root_type_id) } - pub(crate) fn type_name(&self, ty: &BtfType) -> Result, BtfError> { + /// Returns the name of a BTF type. + pub fn type_name(&self, ty: &BtfType) -> Result, BtfError> { self.string_at(ty.name_offset()) } @@ -477,11 +480,62 @@ impl Btf { }) } - /// Encodes the metadata as BTF format + /// Patches all GLOBAL functions to STATIC linkage. + /// + /// This is needed for `struct_ops` programs where the Rust compiler emits + /// all functions as `BTF_FUNC_GLOBAL`, but the BPF verifier rejects + /// global functions that return void. + pub fn fixup_func_linkage(&mut self) { + for t in &mut self.types.types { + if let BtfType::Func(func) = t { + if func.linkage() == FuncLinkage::Global { + func.set_linkage(FuncLinkage::Static); + } + } + } + } + + /// Encodes the metadata as BTF format. + /// + /// Sanitizes types that the kernel rejects during `BPF_BTF_LOAD`: + /// - EXTERN FUNCs are replaced with INT placeholders + /// - Unsupported DATASECs are replaced with INT placeholders pub fn to_bytes(&self) -> Vec { + // Serialize types, sanitizing problematic entries + let mut type_buf = Vec::new(); + for ty in self.types() { + match ty { + BtfType::Func(func) if func.linkage() == FuncLinkage::Extern => { + // Replace EXTERN FUNC with INT (same size: 12 bytes) + let placeholder = + BtfType::Int(Int::new(func.name_offset, 1, IntEncoding::None, 0)); + type_buf.extend(placeholder.to_bytes()); + } + BtfType::DataSec(d) => { + let name = self.string_at(d.name_offset).unwrap_or_default(); + if name == ".ksyms" + || name.starts_with(".struct_ops") + { + // Replace unsupported DATASEC with INT placeholder + let placeholder = + BtfType::Int(Int::new(d.name_offset, 1, IntEncoding::None, 0)); + type_buf.extend(placeholder.to_bytes()); + } else { + type_buf.extend(ty.to_bytes()); + } + } + _ => { + type_buf.extend(ty.to_bytes()); + } + } + } + let mut header = self.header; + header.type_len = type_buf.len() as u32; + header.str_off = header.type_len; + header.str_len = self.strings.len() as u32; // Safety: btf_header is POD - let mut buf = unsafe { bytes_of::(&self.header).to_vec() }; - buf.extend(self.types.to_bytes()); + let mut buf = unsafe { bytes_of::(&header).to_vec() }; + buf.extend(type_buf); buf.put(self.strings.as_slice()); buf } @@ -568,9 +622,13 @@ impl Btf { d.name_offset = self.add_string(&fixed_name); } - // There are some cases when the compiler does indeed populate the size. if d.size > 0 { debug!("{kind} {name}: size fixup not required"); + } else if name == ".ksyms" + || name.starts_with(".struct_ops") + { + // Virtual sections without ELF backing; sanitized in to_bytes(). + debug!("{kind} {name}: skipping fixup (sanitized in to_bytes)"); } else { // We need to get the size of the section from the ELF file. // Fortunately, we cached these when parsing it initially @@ -693,6 +751,9 @@ impl Btf { if !features.btf_func { debug!("{kind}: not supported. replacing with TYPEDEF"); *t = BtfType::Typedef(Typedef::new(ty.name_offset, ty.btf_type)); + } else if ty.linkage() == FuncLinkage::Extern { + // Keep EXTERN FUNC in-memory for kfunc call resolution. + // The to_bytes() method sanitizes them before kernel load. } else if !features.btf_func_global || name == "memset" || name == "memcpy" @@ -716,7 +777,6 @@ impl Btf { } } } - // Sanitize FLOAT. BtfType::Float(ty) if !features.btf_float => { debug!("{kind}: not supported. replacing with STRUCT"); *t = BtfType::Struct(Struct::new(0, vec![], ty.size)); @@ -822,6 +882,19 @@ impl Object { if obj_btf.is_empty() { return Ok(None); } + + // For struct_ops objects, patch all GLOBAL functions to STATIC. + // The Rust compiler emits all functions as BTF_FUNC_GLOBAL, but the + // BPF verifier rejects global functions that return void. Since + // struct_ops functions are logically static (single compilation + // unit), this is always safe. + let has_struct_ops = self.programs.values().any(|p| { + matches!(p.section, crate::ProgramSection::StructOps { .. }) + }); + if has_struct_ops { + obj_btf.fixup_func_linkage(); + } + // fixup btf obj_btf.fixup_and_sanitize( &self.section_infos, @@ -1108,6 +1181,7 @@ impl Default for BtfTypes { } impl BtfTypes { + #[expect(dead_code, reason = "retained for potential future use")] pub(crate) fn to_bytes(&self) -> Vec { let mut buf = vec![]; for t in self.types.iter().skip(1) { @@ -1998,4 +2072,133 @@ mod tests { let raw = btf.to_bytes(); Btf::parse(&raw, Endianness::default()).unwrap(); } + + #[test] + fn test_fixup_func_linkage_global_to_static() { + let mut btf = Btf::new(); + let name = btf.add_string("my_func"); + let _proto_name = btf.add_string(""); + let int_name = btf.add_string("int"); + let int_id = btf.add_type(BtfType::Int(Int::new(int_name, 4, IntEncoding::Signed, 0))); + let proto_id = btf.add_type(BtfType::FuncProto(FuncProto::new(vec![], int_id))); + let func_id = btf.add_type(BtfType::Func(Func::new(name, proto_id, FuncLinkage::Global))); + + btf.fixup_func_linkage(); + + assert_matches!(btf.type_by_id(func_id).unwrap(), BtfType::Func(f) => { + assert_eq!(f.linkage(), FuncLinkage::Static); + }); + } + + #[test] + fn test_fixup_func_linkage_leaves_static_alone() { + let mut btf = Btf::new(); + let name = btf.add_string("my_func"); + let int_name = btf.add_string("int"); + let int_id = btf.add_type(BtfType::Int(Int::new(int_name, 4, IntEncoding::Signed, 0))); + let proto_id = btf.add_type(BtfType::FuncProto(FuncProto::new(vec![], int_id))); + let func_id = btf.add_type(BtfType::Func(Func::new(name, proto_id, FuncLinkage::Static))); + + btf.fixup_func_linkage(); + + assert_matches!(btf.type_by_id(func_id).unwrap(), BtfType::Func(f) => { + assert_eq!(f.linkage(), FuncLinkage::Static); + }); + } + + #[test] + fn test_fixup_func_linkage_leaves_extern_alone() { + let mut btf = Btf::new(); + let name = btf.add_string("kfunc"); + let int_name = btf.add_string("int"); + let int_id = btf.add_type(BtfType::Int(Int::new(int_name, 4, IntEncoding::Signed, 0))); + let proto_id = btf.add_type(BtfType::FuncProto(FuncProto::new(vec![], int_id))); + let func_id = btf.add_type(BtfType::Func(Func::new(name, proto_id, FuncLinkage::Extern))); + + btf.fixup_func_linkage(); + + assert_matches!(btf.type_by_id(func_id).unwrap(), BtfType::Func(f) => { + assert_eq!(f.linkage(), FuncLinkage::Extern); + }); + } + + #[test] + fn test_to_bytes_sanitizes_extern_func() { + let mut btf = Btf::new(); + let name = btf.add_string("my_kfunc"); + let int_name = btf.add_string("int"); + let int_id = btf.add_type(BtfType::Int(Int::new(int_name, 4, IntEncoding::Signed, 0))); + let proto_id = btf.add_type(BtfType::FuncProto(FuncProto::new(vec![], int_id))); + let func_id = btf.add_type(BtfType::Func(Func::new(name, proto_id, FuncLinkage::Extern))); + + // In-memory: still EXTERN + assert_matches!(btf.type_by_id(func_id).unwrap(), BtfType::Func(f) => { + assert_eq!(f.linkage(), FuncLinkage::Extern); + }); + + // Serialized: should be parseable (EXTERN replaced with INT) + let raw = btf.to_bytes(); + let parsed = Btf::parse(&raw, Endianness::default()).unwrap(); + + // The EXTERN FUNC should have been replaced with INT in serialized form + assert_matches!(parsed.type_by_id(func_id).unwrap(), BtfType::Int(_)); + } + + #[test] + fn test_to_bytes_sanitizes_struct_ops_datasec() { + let mut btf = Btf::new(); + let sec_name = btf.add_string(".struct_ops.link"); + let int_name = btf.add_string("int"); + let int_id = btf.add_type(BtfType::Int(Int::new(int_name, 4, IntEncoding::Signed, 0))); + let var_name = btf.add_string("ops"); + let var_id = btf.add_type(BtfType::Var(Var::new(var_name, int_id, VarLinkage::Global))); + let datasec_id = btf.add_type(BtfType::DataSec(DataSec::new(sec_name, vec![ + DataSecEntry { btf_type: var_id, offset: 0, size: 4 }, + ], 4))); + + // Serialized: should be parseable (DATASEC replaced with INT) + let raw = btf.to_bytes(); + let parsed = Btf::parse(&raw, Endianness::default()).unwrap(); + assert_matches!(parsed.type_by_id(datasec_id).unwrap(), BtfType::Int(_)); + } + + #[test] + fn test_to_bytes_preserves_normal_datasec() { + let mut btf = Btf::new(); + let sec_name = btf.add_string(".data"); + let int_name = btf.add_string("int"); + let int_id = btf.add_type(BtfType::Int(Int::new(int_name, 4, IntEncoding::Signed, 0))); + let var_name = btf.add_string("my_var"); + let var_id = btf.add_type(BtfType::Var(Var::new(var_name, int_id, VarLinkage::Global))); + let datasec_id = btf.add_type(BtfType::DataSec(DataSec::new(sec_name, vec![ + DataSecEntry { btf_type: var_id, offset: 0, size: 4 }, + ], 4))); + + // Normal .data DATASEC should NOT be replaced + let raw = btf.to_bytes(); + let parsed = Btf::parse(&raw, Endianness::default()).unwrap(); + assert_matches!(parsed.type_by_id(datasec_id).unwrap(), BtfType::DataSec(_)); + } + + #[test] + fn test_to_bytes_header_consistency() { + // Verify that to_bytes produces a BTF blob where header sizes match actual content + let mut btf = Btf::new(); + let sec_name = btf.add_string(".struct_ops.link"); + let int_name = btf.add_string("int"); + let int_id = btf.add_type(BtfType::Int(Int::new(int_name, 4, IntEncoding::Signed, 0))); + let var_name = btf.add_string("ops"); + let var_id = btf.add_type(BtfType::Var(Var::new(var_name, int_id, VarLinkage::Global))); + btf.add_type(BtfType::DataSec(DataSec::new(sec_name, vec![ + DataSecEntry { btf_type: var_id, offset: 0, size: 4 }, + ], 4))); + + let raw = btf.to_bytes(); + + // Parse the header and verify consistency + let header: btf_header = unsafe { core::ptr::read_unaligned(raw.as_ptr().cast()) }; + let expected_len = header.hdr_len as usize + header.type_len as usize + header.str_len as usize; + assert_eq!(raw.len(), expected_len, "BTF blob size must match header"); + assert_eq!(header.str_off, header.type_len, "str_off must equal type_len"); + } } diff --git a/aya-obj/src/btf/types.rs b/aya-obj/src/btf/types.rs index 4c11fa2b6..6ea3977cf 100644 --- a/aya-obj/src/btf/types.rs +++ b/aya-obj/src/btf/types.rs @@ -567,10 +567,14 @@ impl Enum64 { #[repr(C)] #[derive(Clone, Debug)] -pub(crate) struct BtfMember { - pub(crate) name_offset: u32, - pub(crate) btf_type: u32, - pub(crate) offset: u32, +/// A BTF struct member +pub struct BtfMember { + /// Name offset in the string table + pub name_offset: u32, + /// Type ID of the member + pub btf_type: u32, + /// Bit offset of the member + pub offset: u32, } #[repr(C)] @@ -578,8 +582,10 @@ pub(crate) struct BtfMember { pub struct Struct { pub(crate) name_offset: u32, info: u32, - pub(crate) size: u32, - pub(crate) members: Vec, + /// Size of the struct in bytes + pub size: u32, + /// Members of the struct + pub members: Vec, } impl Struct { diff --git a/aya-obj/src/lib.rs b/aya-obj/src/lib.rs index 73c0c1ac6..f4e9a3583 100644 --- a/aya-obj/src/lib.rs +++ b/aya-obj/src/lib.rs @@ -107,7 +107,7 @@ pub mod programs; pub mod relocation; mod util; -pub use maps::Map; +pub use maps::{Map, StructOpsMap}; pub use obj::*; /// An error returned from the verifier. diff --git a/aya-obj/src/maps.rs b/aya-obj/src/maps.rs index e9dbc4561..f9850174b 100644 --- a/aya-obj/src/maps.rs +++ b/aya-obj/src/maps.rs @@ -1,9 +1,24 @@ //! Map struct and type bindings. -use alloc::vec::Vec; +use alloc::{string::String, vec::Vec}; use crate::{EbpfSectionKind, InvalidTypeBinding, generated::bpf_map_type}; +/// Struct ops map definition +#[derive(Debug, Clone)] +pub struct StructOpsMap { + /// The section index + pub section_index: usize, + /// The symbol index + pub symbol_index: usize, + /// The struct type name (e.g., `sched_ext_ops`) + pub struct_type_name: String, + /// The initial section data (scalar field values) + pub data: Vec, + /// Whether auto-attach is requested (`.struct_ops.link` suffix) + pub auto_attach: bool, +} + impl TryFrom for bpf_map_type { type Error = InvalidTypeBinding; @@ -142,6 +157,8 @@ pub enum Map { Legacy(LegacyMap), /// A map defined in the `.maps` section Btf(BtfMap), + /// A `struct_ops` map defined in `.struct_ops` or `.struct_ops.link` sections + StructOps(StructOpsMap), } impl Map { @@ -150,6 +167,7 @@ impl Map { match self { Self::Legacy(m) => m.def.map_type, Self::Btf(m) => m.def.map_type, + Self::StructOps(_) => bpf_map_type::BPF_MAP_TYPE_STRUCT_OPS as u32, } } @@ -158,6 +176,7 @@ impl Map { match self { Self::Legacy(m) => m.def.key_size, Self::Btf(m) => m.def.key_size, + Self::StructOps(_) => size_of::() as u32, } } @@ -166,6 +185,7 @@ impl Map { match self { Self::Legacy(m) => m.def.value_size, Self::Btf(m) => m.def.value_size, + Self::StructOps(m) => m.data.len() as u32, } } @@ -174,6 +194,7 @@ impl Map { match self { Self::Legacy(m) => m.def.value_size = size, Self::Btf(m) => m.def.value_size = size, + Self::StructOps(_) => {} // struct_ops value size is determined by kernel BTF } } @@ -182,6 +203,7 @@ impl Map { match self { Self::Legacy(m) => m.def.max_entries, Self::Btf(m) => m.def.max_entries, + Self::StructOps(_) => 1, } } @@ -190,6 +212,7 @@ impl Map { match self { Self::Legacy(m) => m.def.max_entries = v, Self::Btf(m) => m.def.max_entries = v, + Self::StructOps(_) => {} // struct_ops always has max_entries=1 } } @@ -198,6 +221,7 @@ impl Map { match self { Self::Legacy(m) => m.def.map_flags, Self::Btf(m) => m.def.map_flags, + Self::StructOps(_) => 0, } } @@ -206,6 +230,7 @@ impl Map { match self { Self::Legacy(m) => m.def.pinning, Self::Btf(m) => m.def.pinning, + Self::StructOps(_) => PinningType::None, } } @@ -214,6 +239,7 @@ impl Map { match self { Self::Legacy(m) => &m.data, Self::Btf(m) => &m.data, + Self::StructOps(m) => &m.data, } } @@ -222,6 +248,7 @@ impl Map { match self { Self::Legacy(m) => m.data.as_mut(), Self::Btf(m) => m.data.as_mut(), + Self::StructOps(m) => m.data.as_mut(), } } @@ -230,6 +257,7 @@ impl Map { match self { Self::Legacy(m) => m.section_index, Self::Btf(m) => m.section_index, + Self::StructOps(m) => m.section_index, } } @@ -238,6 +266,13 @@ impl Map { match self { Self::Legacy(m) => m.section_kind, Self::Btf(_) => EbpfSectionKind::BtfMaps, + Self::StructOps(m) => { + if m.auto_attach { + EbpfSectionKind::StructOpsLink + } else { + EbpfSectionKind::StructOps + } + } } } @@ -249,6 +284,7 @@ impl Map { match self { Self::Legacy(m) => m.symbol_index, Self::Btf(m) => Some(m.symbol_index), + Self::StructOps(m) => Some(m.symbol_index), } } } diff --git a/aya-obj/src/obj.rs b/aya-obj/src/obj.rs index ccb37527e..5574713ca 100644 --- a/aya-obj/src/obj.rs +++ b/aya-obj/src/obj.rs @@ -30,7 +30,9 @@ use crate::{ BPF_CALL, BPF_F_RDONLY_PROG, BPF_JMP, BPF_K, bpf_func_id, bpf_insn, bpf_map_info, bpf_map_type::BPF_MAP_TYPE_ARRAY, }, - maps::{BtfMap, BtfMapDef, LegacyMap, MINIMUM_MAP_SIZE, Map, PinningType, bpf_map_def}, + maps::{ + self, BtfMap, BtfMapDef, LegacyMap, MINIMUM_MAP_SIZE, Map, PinningType, bpf_map_def, + }, programs::{ CgroupSockAddrAttachType, CgroupSockAttachType, CgroupSockoptAttachType, XdpAttachType, }, @@ -226,7 +228,6 @@ pub struct Function { /// - `action` /// - `sk_reuseport/migrate`, `sk_reuseport` /// - `syscall` -/// - `struct_ops+` /// - `fmod_ret+`, `fmod_ret.s+` /// - `iter+`, `iter.s+` #[derive(Debug, Clone)] @@ -285,6 +286,9 @@ pub enum ProgramSection { Iter { sleepable: bool, }, + StructOps { + sleepable: bool, + }, } impl FromStr for ProgramSection { @@ -438,6 +442,8 @@ impl FromStr for ProgramSection { "sk_lookup" => Self::SkLookup, "iter" => Self::Iter { sleepable: false }, "iter.s" => Self::Iter { sleepable: true }, + "struct_ops" => Self::StructOps { sleepable: false }, + "struct_ops.s" => Self::StructOps { sleepable: true }, _ => { return Err(ParseError::InvalidProgramSection { section: section.to_owned(), @@ -839,6 +845,79 @@ impl Object { Ok(()) } + /// Parses a `.struct_ops` or `.struct_ops.link` section into a [`Map::StructOps`] entry. + /// + /// These sections contain the initializer data for a kernel struct (e.g., + /// `sched_ext_ops`). The struct type name is resolved from BTF by following + /// the section's VAR type to its underlying STRUCT type. + fn parse_struct_ops_section(&mut self, section: &Section<'_>) -> Result<(), ParseError> { + let auto_attach = section.kind == EbpfSectionKind::StructOpsLink; + + // Find the symbol for this section to get the struct name + let syms = self.symbols_by_section.get(§ion.index).ok_or_else(|| { + ParseError::NoSymbolsForSection { + section_name: section.name.to_string(), + } + })?; + + for symbol_index in syms { + let symbol = self + .symbol_table + .get(symbol_index) + .expect("all symbols in symbols_by_section are also in symbol_table"); + + let name = match symbol.name.as_ref() { + Some(name) if !name.is_empty() && symbol.kind == SymbolKind::Data => name, + _ => continue, + }; + + // Determine the struct type name from BTF if available + let struct_type_name = if let Some(btf) = &self.btf { + // Look up the BTF type for this variable + let mut found_type_name = None; + for t in btf.types() { + if let BtfType::Var(var) = t { + if let Ok(var_name) = btf.type_name(t) { + if var_name == *name { + // Follow the type to find the struct + if let Ok(inner_type) = btf.type_by_id(var.btf_type) { + if let Ok(type_name) = btf.type_name(inner_type) { + found_type_name = Some(type_name.to_string()); + } + } + break; + } + } + } + } + found_type_name.unwrap_or_default() + } else { + String::new() + }; + + let start = symbol.address as usize; + let end = start + symbol.size as usize; + let data = if end <= section.data.len() { + section.data[start..end].to_vec() + } else { + section.data.to_vec() + }; + + self.maps.insert( + name.clone(), + Map::StructOps(maps::StructOpsMap { + section_index: section.index.0, + symbol_index: *symbol_index, + struct_type_name, + data, + auto_attach, + }), + ); + } + + Ok(()) + } + fn parse_section(&mut self, section: Section<'_>) -> Result<(), ParseError> { self.section_infos .insert(section.name.to_owned(), (section.index, section.size)); @@ -886,6 +965,9 @@ impl Object { ); } } + EbpfSectionKind::StructOps | EbpfSectionKind::StructOpsLink => { + self.parse_struct_ops_section(§ion)?; + } EbpfSectionKind::Undefined | EbpfSectionKind::License | EbpfSectionKind::Version => {} } @@ -898,6 +980,76 @@ impl Object { function.sanitize(features); } } + + /// Fixes up kfunc call instructions by resolving BTF func IDs. + /// + /// After `relocate_calls` patches extern symbol calls to use + /// `BPF_PSEUDO_KFUNC_CALL`, this method resolves the `imm` field + /// to the correct BTF func type ID by matching the relocation + /// symbol name against vmlinux BTF FUNC entries. + /// + /// `kernel_btf` should be the vmlinux BTF loaded from `/sys/kernel/btf/vmlinux`. + pub fn fixup_kfunc_calls(&mut self, kernel_btf: &Btf) { + use crate::generated::BPF_PSEUDO_KFUNC_CALL; + + // Build a map of kfunc name → vmlinux BTF func type_id + let mut kfunc_vmlinux_ids: BTreeMap = BTreeMap::new(); + for sym in self.symbol_table.values() { + if !sym.is_definition && sym.section_index.is_none() { + if let Some(name) = &sym.name { + // Look up this extern symbol in vmlinux BTF + if let Ok(btf_id) = + kernel_btf.id_by_type_name_kind(name, crate::btf::BtfKind::Func) + { + kfunc_vmlinux_ids.insert(name.clone(), btf_id); + } + } + } + } + + if kfunc_vmlinux_ids.is_empty() { + return; + } + + // Build relocation (section_index, offset) → symbol name map + let mut extern_call_names: BTreeMap<(usize, u64), String> = BTreeMap::new(); + for (section_index, relocations) in &self.relocations { + for (offset, rel) in relocations { + if let Some(sym) = self.symbol_table.get(&rel.symbol_index) { + if !sym.is_definition && sym.section_index.is_none() { + if let Some(name) = &sym.name { + extern_call_names + .insert((section_index.0, *offset), name.clone()); + } + } + } + } + } + + // Patch kfunc call instructions with vmlinux BTF func IDs + for function in self.functions.values_mut() { + for (ins_idx, ins) in function.instructions.iter_mut().enumerate() { + let klass = u32::from(ins.code & 0x07); + let op = u32::from(ins.code & 0xF0); + let src = u32::from(ins.code & 0x08); + + if klass == BPF_JMP + && op == BPF_CALL + && src == BPF_K + && u32::from(ins.src_reg()) == BPF_PSEUDO_KFUNC_CALL + { + let offset = + (function.section_offset + ins_idx * INS_SIZE) as u64; + let key = (function.section_index.0, offset); + if let Some(name) = extern_call_names.get(&key) { + if let Some(&vmlinux_id) = kfunc_vmlinux_ids.get(name) { + ins.imm = vmlinux_id as i32; + } + } + } + } + } + } } fn insn_is_helper_call(ins: bpf_insn) -> bool { @@ -1044,6 +1196,10 @@ pub enum EbpfSectionKind { License, /// `version` Version, + /// `.struct_ops` + StructOps, + /// `.struct_ops.link` + StructOpsLink, } impl EbpfSectionKind { @@ -1068,6 +1224,10 @@ impl EbpfSectionKind { Self::Btf } else if name == ".BTF.ext" { Self::BtfExt + } else if name.starts_with(".struct_ops.link") { + Self::StructOpsLink + } else if name.starts_with(".struct_ops") { + Self::StructOps } else { Self::Undefined } @@ -2828,4 +2988,94 @@ mod tests { assert_eq!(m.def.max_entries, 1); }); } + + #[test] + fn test_section_kind_struct_ops() { + assert_eq!( + EbpfSectionKind::from_name(".struct_ops"), + EbpfSectionKind::StructOps + ); + assert_eq!( + EbpfSectionKind::from_name(".struct_ops.link"), + EbpfSectionKind::StructOpsLink + ); + // .struct_ops.link must be detected before .struct_ops + assert_ne!( + EbpfSectionKind::from_name(".struct_ops.link"), + EbpfSectionKind::StructOps + ); + } + + #[test] + fn test_program_section_struct_ops() { + assert_matches!( + ProgramSection::from_str("struct_ops/enqueue"), + Ok(ProgramSection::StructOps { sleepable: false }) + ); + assert_matches!( + ProgramSection::from_str("struct_ops.s/init"), + Ok(ProgramSection::StructOps { sleepable: true }) + ); + assert_matches!( + ProgramSection::from_str("struct_ops"), + Ok(ProgramSection::StructOps { sleepable: false }) + ); + } + + #[test] + fn test_parse_struct_ops_section() { + use crate::btf::{Btf, BtfType, Int, IntEncoding, Struct, Var, VarLinkage, DataSec, DataSecEntry}; + + let mut obj = fake_obj(); + // Build a minimal BTF with a VAR pointing to a STRUCT + let mut btf = Btf::new(); + + let struct_name = btf.add_string("my_ops"); + let struct_type_id = btf.add_type(BtfType::Struct(Struct::new(struct_name, vec![], 16))); + + let var_name = btf.add_string("_my_ops"); + let var_type_id = btf.add_type(BtfType::Var(Var::new(var_name, struct_type_id, VarLinkage::Global))); + + let sec_name = btf.add_string(".struct_ops.link"); + btf.add_type(BtfType::DataSec(DataSec::new(sec_name, vec![ + DataSecEntry { btf_type: var_type_id, offset: 0, size: 16 }, + ], 16))); + + obj.btf = Some(btf); + + // Add a symbol for the section + let section_index = 5; + let sym_idx = 1; + obj.symbol_table.insert(sym_idx, Symbol { + index: sym_idx, + section_index: Some(section_index), + name: Some("_my_ops".to_string()), + address: 0, + size: 16, + is_definition: true, + kind: SymbolKind::Data, + }); + obj.symbols_by_section + .entry(SectionIndex(section_index)) + .or_default() + .push(sym_idx); + + let data = vec![0u8; 16]; + let section = fake_section( + EbpfSectionKind::StructOpsLink, + ".struct_ops.link", + &data, + Some(section_index), + ); + + obj.parse_struct_ops_section(§ion).unwrap(); + + assert!(obj.maps.contains_key("_my_ops")); + let map = &obj.maps["_my_ops"]; + assert_matches!(map, Map::StructOps(m) => { + assert_eq!(m.struct_type_name, "my_ops"); + assert!(m.auto_attach); + assert_eq!(m.data.len(), 16); + }); + } } diff --git a/aya-obj/src/relocation.rs b/aya-obj/src/relocation.rs index 67a5d8307..ef70cddb4 100644 --- a/aya-obj/src/relocation.rs +++ b/aya-obj/src/relocation.rs @@ -8,8 +8,8 @@ use object::{SectionIndex, SymbolKind}; use crate::{ EbpfSectionKind, generated::{ - BPF_CALL, BPF_JMP, BPF_K, BPF_PSEUDO_CALL, BPF_PSEUDO_FUNC, BPF_PSEUDO_MAP_FD, - BPF_PSEUDO_MAP_VALUE, bpf_insn, + BPF_CALL, BPF_JMP, BPF_K, BPF_PSEUDO_CALL, BPF_PSEUDO_FUNC, BPF_PSEUDO_KFUNC_CALL, + BPF_PSEUDO_MAP_FD, BPF_PSEUDO_MAP_VALUE, bpf_insn, }, maps::Map, obj::{Function, Object}, @@ -356,11 +356,29 @@ impl<'a> FunctionLinker<'a> { let ins = program.instructions[ins_index]; let is_call = insn_is_call(ins); - let rel = relocations - .and_then(|relocations| { - relocations - .get(&((fun.section_offset + (ins_index - start_ins) * INS_SIZE) as u64)) - }) + // Look up the raw relocation for this instruction + let raw_rel = relocations.and_then(|relocations| { + relocations + .get(&((fun.section_offset + (ins_index - start_ins) * INS_SIZE) as u64)) + }); + + // Check if this is a call to an extern/undefined symbol (e.g. kfunc). + // Patch these from BPF_PSEUDO_CALL to BPF_PSEUDO_KFUNC_CALL so the + // kernel verifier resolves them against vmlinux BTF. + if is_call { + if let Some(rel) = raw_rel { + if let Some(sym) = self.symbol_table.get(&rel.symbol_index) { + if !sym.is_definition && sym.section_index.is_none() { + let ins = &mut program.instructions[ins_index]; + ins.set_src_reg(BPF_PSEUDO_KFUNC_CALL as u8); + ins.imm = 0; // kernel resolves by BTF func name + continue; + } + } + } + } + + let rel = raw_rel .and_then(|rel| { // get the symbol for the relocation self.symbol_table diff --git a/aya/src/bpf.rs b/aya/src/bpf.rs index cf314d1cf..6d4613873 100644 --- a/aya/src/bpf.rs +++ b/aya/src/bpf.rs @@ -9,7 +9,7 @@ use std::{ use aya_obj::{ EbpfSectionKind, Features, Object, ParseError, ProgramSection, - btf::{Btf, BtfError, BtfFeatures, BtfRelocationError}, + btf::{Btf, BtfError, BtfFeatures, BtfKind, BtfRelocationError, BtfType}, generated::{BPF_F_SLEEPABLE, BPF_F_XDP_HAS_FRAGS, bpf_map_type}, relocation::EbpfRelocationError, }; @@ -23,15 +23,17 @@ use crate::{ CgroupSockopt, CgroupSysctl, Extension, FEntry, FExit, FlowDissector, Iter, KProbe, LircMode2, Lsm, LsmCgroup, PerfEvent, ProbeKind, Program, ProgramData, ProgramError, RawTracePoint, SchedClassifier, SkLookup, SkMsg, SkSkb, SkSkbKind, SockOps, SocketFilter, - TracePoint, UProbe, Xdp, + StructOps, TracePoint, UProbe, Xdp, + links::FdLink, + struct_ops::StructOpsLink, }, sys::{ - bpf_load_btf, is_bpf_cookie_supported, is_bpf_global_data_supported, - is_btf_datasec_supported, is_btf_datasec_zero_supported, is_btf_decl_tag_supported, - is_btf_enum64_supported, is_btf_float_supported, is_btf_func_global_supported, - is_btf_func_supported, is_btf_supported, is_btf_type_tag_supported, is_perf_link_supported, - is_probe_read_kernel_supported, is_prog_id_supported, is_prog_name_supported, - retry_with_verifier_logs, + bpf_link_create, bpf_load_btf, bpf_map_update_elem_ptr, is_bpf_cookie_supported, + is_bpf_global_data_supported, is_btf_datasec_supported, is_btf_datasec_zero_supported, + is_btf_decl_tag_supported, is_btf_enum64_supported, is_btf_float_supported, + is_btf_func_global_supported, is_btf_func_supported, is_btf_supported, + is_btf_type_tag_supported, is_perf_link_supported, is_probe_read_kernel_supported, + is_prog_id_supported, is_prog_name_supported, retry_with_verifier_logs, LinkTarget, }, util::{bytes_of, bytes_of_slice, nr_cpus, page_size}, }; @@ -432,7 +434,8 @@ impl<'a> EbpfLoader<'a> { let btf_fd = if let Some(features) = &FEATURES.btf() { if let Some(btf) = obj.fixup_and_sanitize_btf(features)? { - match load_btf(btf.to_bytes(), *verifier_log_level) { + let btf_bytes = btf.to_bytes(); + match load_btf(btf_bytes, *verifier_log_level) { Ok(btf_fd) => Some(Arc::new(btf_fd)), // Only report an error here if the BTF is truly needed, otherwise proceed without. Err(err) => { @@ -444,7 +447,8 @@ impl<'a> EbpfLoader<'a> { | ProgramSection::Lsm { sleepable: _ } | ProgramSection::LsmCgroup | ProgramSection::BtfTracePoint - | ProgramSection::Iter { sleepable: _ } => { + | ProgramSection::Iter { sleepable: _ } + | ProgramSection::StructOps { sleepable: _ } => { return Err(EbpfError::BtfError(err)); } ProgramSection::KRetProbe @@ -498,12 +502,21 @@ impl<'a> EbpfLoader<'a> { obj.relocate_btf(btf)?; } let mut maps = HashMap::new(); + let mut struct_ops_maps = HashMap::new(); for (name, mut obj) in obj.maps.drain() { if let (false, EbpfSectionKind::Bss | EbpfSectionKind::Data | EbpfSectionKind::Rodata) = (FEATURES.bpf_global_data(), obj.section_kind()) { continue; } + // struct_ops maps are handled separately after programs are loaded + if matches!( + obj.section_kind(), + EbpfSectionKind::StructOps | EbpfSectionKind::StructOpsLink + ) { + struct_ops_maps.insert(name, obj); + continue; + } let num_cpus = || { Ok(nr_cpus().map_err(|(path, error)| EbpfError::FileError { path: PathBuf::from(path), @@ -557,6 +570,10 @@ impl<'a> EbpfLoader<'a> { &text_sections, )?; obj.relocate_calls(&text_sections)?; + // Resolve kfunc calls against vmlinux BTF + if let Some(kernel_btf) = &btf { + obj.fixup_kfunc_calls(kernel_btf); + } obj.sanitize_functions(&FEATURES); let programs = obj @@ -736,6 +753,14 @@ impl<'a> EbpfLoader<'a> { } Program::Iter(Iter { data }) } + ProgramSection::StructOps { sleepable } => { + let mut data = + ProgramData::new(prog_name, obj, btf_fd, *verifier_log_level); + if *sleepable { + data.flags = BPF_F_SLEEPABLE; + } + Program::StructOps(StructOps { data }) + } } }; (name, program) @@ -746,7 +771,13 @@ impl<'a> EbpfLoader<'a> { .map(|data| parse_map(data, *allow_unsupported_maps)) .collect::, EbpfError>>()?; - Ok(Ebpf { maps, programs }) + Ok(Ebpf { + maps, + programs, + struct_ops_maps, + btf_fd, + kernel_btf: btf.as_deref().cloned(), + }) } } @@ -910,6 +941,9 @@ impl Default for EbpfLoader<'_> { pub struct Ebpf { maps: HashMap, programs: HashMap, + struct_ops_maps: HashMap, + btf_fd: Option>, + kernel_btf: Option, } /// The main entry point into the library, used to work with eBPF programs and maps. @@ -1155,6 +1189,198 @@ impl Ebpf { pub fn programs_mut(&mut self) -> impl Iterator { self.programs.iter_mut().map(|(s, p)| (s.as_str(), p)) } + + /// Attaches a `struct_ops` map, loading its associated programs and creating + /// a BPF link. + /// + /// This method handles the complete `struct_ops` lifecycle: + /// 1. Loads each `struct_ops` program referenced by the map + /// 2. Creates the `struct_ops` BPF map with program FDs + /// 3. Attaches the map via `BPF_LINK_CREATE` + /// + /// The `map_name` should match the symbol name from the `.struct_ops` or + /// `.struct_ops.link` ELF section (e.g., `"_scx_ops"`). + /// + /// Returns a [`StructOpsLink`] that keeps the `struct_ops` attached. Dropping + /// the link detaches the `struct_ops`. + pub fn attach_struct_ops(&mut self, map_name: &str) -> Result { + use aya_obj::generated::{bpf_attach_type, bpf_attr}; + + let struct_ops_obj = self + .struct_ops_maps + .get(map_name) + .ok_or_else(|| EbpfError::StructOpsMapNotFound { + name: map_name.to_owned(), + })?; + + let (struct_type_name, section_data) = match struct_ops_obj { + aya_obj::Map::StructOps(m) => (m.struct_type_name.clone(), m.data.clone()), + _ => { + return Err(EbpfError::StructOpsMapNotFound { + name: map_name.to_owned(), + }) + } + }; + + // Clone kernel BTF before taking &mut self for program loading + let kernel_btf = self.kernel_btf.clone().ok_or(EbpfError::NoBTF)?; + + // Look up inner struct and wrapper struct in vmlinux BTF + let vmlinux_type_id = kernel_btf + .id_by_type_name_kind(&struct_type_name, BtfKind::Struct) + .map_err(EbpfError::BtfError)?; + + let wrapper_name = format!("bpf_struct_ops_{struct_type_name}"); + let vmlinux_value_type_id = kernel_btf + .id_by_type_name_kind(&wrapper_name, BtfKind::Struct) + .map_err(|e| { + EbpfError::StructOpsError(format!( + "wrapper struct `{wrapper_name}` not found in vmlinux BTF: {e}" + )) + })?; + + let members = match kernel_btf.type_by_id(vmlinux_type_id).map_err(EbpfError::BtfError)? { + BtfType::Struct(s) => &s.members, + other => { + return Err(EbpfError::StructOpsError(format!( + "expected struct type for {struct_type_name}, got {other:?}", + ))); + } + }; + + // Get wrapper struct size and data field offset + let (wrapper_size, data_offset) = match kernel_btf + .type_by_id(vmlinux_value_type_id) + .map_err(EbpfError::BtfError)? + { + BtfType::Struct(s) => { + let data_off = s.members.iter().find_map(|m| { + let name = kernel_btf.string_at(m.name_offset).unwrap_or_default(); + (name == "data").then_some(m.offset / 8) + }).unwrap_or(0); + (s.size, data_off as usize) + } + _ => return Err(EbpfError::StructOpsError(format!( + "wrapper struct {wrapper_name} is not a struct" + ))), + }; + + // Load struct_ops programs, matching each to a struct member by name + let prog_fds = self.load_struct_ops_programs( + &kernel_btf, members, vmlinux_type_id, + )?; + + // Build the map value: wrapper-sized buffer with section data at data_offset + let mut value = vec![0u8; wrapper_size as usize]; + let copy_len = section_data.len().min(value.len() - data_offset); + value[data_offset..data_offset + copy_len] + .copy_from_slice(§ion_data[..copy_len]); + + // Write loaded program FDs into function pointer field positions + for member in members { + let member_name = kernel_btf.string_at(member.name_offset).unwrap_or_default(); + if let Some(&fd) = prog_fds.get(member_name.as_ref()) { + let offset = data_offset + (member.offset / 8) as usize; + if let Ok(member_type) = kernel_btf.type_by_id(member.btf_type) { + if matches!(member_type, BtfType::Ptr(_)) && offset + 4 <= value.len() { + value[offset..offset + 4] + .copy_from_slice(&(fd as u32).to_ne_bytes()); + } + } + } + } + + // Create the struct_ops BPF map + let c_name = std::ffi::CString::new(map_name).map_err(|e| { + EbpfError::StructOpsError(format!("invalid map name: {e}")) + })?; + let map_fd = { + let mut attr = unsafe { std::mem::zeroed::() }; + let u = unsafe { &mut attr.__bindgen_anon_1 }; + u.map_type = bpf_map_type::BPF_MAP_TYPE_STRUCT_OPS as u32; + u.key_size = 4; + u.value_size = wrapper_size; + u.max_entries = 1; + u.map_flags = aya_obj::generated::BPF_F_LINK; + u.btf_vmlinux_value_type_id = vmlinux_value_type_id; + if let Some(btf_fd) = &self.btf_fd { + u.btf_fd = btf_fd.as_fd().as_raw_fd() as u32; + } + let name_bytes = c_name.to_bytes(); + let len = name_bytes.len().min(u.map_name.len() - 1); + u.map_name[..len].copy_from_slice(unsafe { + std::mem::transmute::<&[u8], &[std::ffi::c_char]>(&name_bytes[..len]) + }); + crate::sys::bpf_map_create(&mut attr).map_err(|io_error| { + EbpfError::StructOpsError(format!("failed to create struct_ops map: {io_error}")) + })? + }; + + // Populate the map value + bpf_map_update_elem_ptr(map_fd.as_fd(), &0u32, value.as_mut_ptr(), 0).map_err( + |io_error| { + EbpfError::StructOpsError(format!("failed to update struct_ops map: {io_error}")) + }, + )?; + + // Attach via BPF_LINK_CREATE (map fd goes in prog_fd position) + let link_fd = bpf_link_create( + map_fd.as_fd(), + LinkTarget::Iter, + bpf_attach_type::BPF_STRUCT_OPS, + 0, + None, + ) + .map_err(|io_error| { + EbpfError::StructOpsError(format!("failed to attach struct_ops: {io_error}")) + })?; + + Ok(StructOpsLink::wrap(FdLink::new(link_fd))) + } + + /// Loads `struct_ops` programs, matching each to a kernel struct member by + /// name. Returns a map of member name to loaded program fd. + fn load_struct_ops_programs( + &mut self, + kernel_btf: &Btf, + members: &[aya_obj::btf::BtfMember], + vmlinux_type_id: u32, + ) -> Result, EbpfError> { + let mut prog_fds: HashMap = HashMap::new(); + for (prog_name, program) in &mut self.programs { + if let Program::StructOps(struct_ops_prog) = program { + for (member_idx, member) in members.iter().enumerate() { + let member_name = kernel_btf + .string_at(member.name_offset) + .unwrap_or_default(); + if member_name != *prog_name { + continue; + } + if let Ok(member_type) = kernel_btf.type_by_id(member.btf_type) { + if matches!(member_type, BtfType::Ptr(_)) { + // expected_attach_type = member index (u32 reinterpreted + // as bpf_attach_type by the kernel) + struct_ops_prog.data.expected_attach_type = Some( + // SAFETY: the kernel interprets this field as a raw u32 + // member index for struct_ops programs, not as a real + // bpf_attach_type enum variant. + unsafe { + core::mem::transmute::( + member_idx as u32, + ) + }, + ); + struct_ops_prog.load(vmlinux_type_id)?; + let fd = struct_ops_prog.fd()?; + prog_fds.insert(prog_name.clone(), fd.as_fd().as_raw_fd()); + } + } + break; + } + } + } + Ok(prog_fds) + } } /// The error type returned by [`Ebpf::load_file`] and [`Ebpf::load`]. @@ -1204,6 +1430,17 @@ pub enum EbpfError { #[error("program error: {0}")] /// A program error ProgramError(#[from] ProgramError), + + /// Struct ops map not found + #[error("struct_ops map `{name}` not found")] + StructOpsMapNotFound { + /// The map name + name: String, + }, + + /// Struct ops error + #[error("struct_ops error: {0}")] + StructOpsError(String), } /// The error type returned by [`Bpf::load_file`] and [`Bpf::load`]. diff --git a/aya/src/programs/mod.rs b/aya/src/programs/mod.rs index 65185df20..4ddd28c54 100644 --- a/aya/src/programs/mod.rs +++ b/aya/src/programs/mod.rs @@ -66,6 +66,7 @@ pub mod sk_msg; pub mod sk_skb; pub mod sock_ops; pub mod socket_filter; +pub mod struct_ops; pub mod tc; pub mod tp_btf; pub mod trace_point; @@ -119,6 +120,7 @@ pub use crate::programs::{ sk_skb::{SkSkb, SkSkbKind}, sock_ops::SockOps, socket_filter::{SocketFilter, SocketFilterError}, + struct_ops::StructOps, tc::{SchedClassifier, TcAttachType, TcError}, tp_btf::BtfTracePoint, trace_point::{TracePoint, TracePointError}, @@ -336,6 +338,8 @@ pub enum Program { CgroupDevice(CgroupDevice), /// An [`Iter`] program Iter(Iter), + /// A [`StructOps`] program + StructOps(StructOps), } impl Program { @@ -376,6 +380,7 @@ impl Program { Self::CgroupSock(_) => ProgramType::CgroupSock, Self::CgroupDevice(_) => ProgramType::CgroupDevice, Self::FlowDissector(_) => ProgramType::FlowDissector, + Self::StructOps(_) => ProgramType::StructOps, } } @@ -409,6 +414,7 @@ impl Program { Self::CgroupSock(p) => p.pin(path), Self::CgroupDevice(p) => p.pin(path), Self::Iter(p) => p.pin(path), + Self::StructOps(p) => p.pin(path), } } @@ -442,6 +448,7 @@ impl Program { Self::CgroupSock(mut p) => p.unload(), Self::CgroupDevice(mut p) => p.unload(), Self::Iter(mut p) => p.unload(), + Self::StructOps(mut p) => p.unload(), } } @@ -477,6 +484,7 @@ impl Program { Self::CgroupSock(p) => p.fd(), Self::CgroupDevice(p) => p.fd(), Self::Iter(p) => p.fd(), + Self::StructOps(p) => p.fd(), } } @@ -513,6 +521,7 @@ impl Program { Self::CgroupSock(p) => p.info(), Self::CgroupDevice(p) => p.info(), Self::Iter(p) => p.info(), + Self::StructOps(p) => p.info(), } } } @@ -823,6 +832,7 @@ impl_program_unload!( CgroupSock, CgroupDevice, Iter, + StructOps, ); macro_rules! impl_fd { @@ -866,6 +876,7 @@ impl_fd!( CgroupSock, CgroupDevice, Iter, + StructOps, ); /// Trait implemented by the [`Program`] types which support the kernel's @@ -974,6 +985,7 @@ impl_program_pin!( CgroupSock, CgroupDevice, Iter, + StructOps, ); macro_rules! impl_from_pin { @@ -1014,6 +1026,7 @@ impl_from_pin!( SockOps, CgroupDevice, Iter, + StructOps, ); macro_rules! impl_from_prog_info { @@ -1122,6 +1135,7 @@ impl_from_prog_info!( SkLookup, CgroupDevice, Iter, + StructOps, ); macro_rules! impl_try_from_program { @@ -1180,6 +1194,7 @@ impl_try_from_program!( CgroupSock, CgroupDevice, Iter, + StructOps, ); impl_info!( @@ -1210,6 +1225,7 @@ impl_info!( CgroupSock, CgroupDevice, Iter, + StructOps, ); /// Returns an iterator over all loaded links. diff --git a/aya/src/programs/struct_ops.rs b/aya/src/programs/struct_ops.rs new file mode 100644 index 000000000..e1359d7ad --- /dev/null +++ b/aya/src/programs/struct_ops.rs @@ -0,0 +1,52 @@ +//! Struct ops programs. + +use aya_obj::generated::bpf_prog_type::BPF_PROG_TYPE_STRUCT_OPS; + +use crate::programs::{ + FdLink, FdLinkId, ProgramData, ProgramError, ProgramType, define_link_wrapper, load_program, +}; + +/// A program that implements a kernel struct ops interface. +/// +/// Struct ops programs are used to implement kernel-defined structures +/// containing function pointers, such as `sched_ext_ops` for custom +/// schedulers. Individual struct ops programs correspond to methods +/// in the kernel struct. +/// +/// Unlike most program types, struct ops programs are not individually +/// attached. Instead, they are associated with a struct ops map that +/// is created and attached as a unit. Use [`Ebpf::attach_struct_ops`] +/// to attach the struct ops map after loading. +/// +/// [`Ebpf::attach_struct_ops`]: crate::Ebpf::attach_struct_ops +/// +/// # Minimum kernel version +/// +/// The minimum kernel version required to use this feature is 5.3. +#[derive(Debug)] +#[doc(alias = "BPF_PROG_TYPE_STRUCT_OPS")] +pub struct StructOps { + pub(crate) data: ProgramData, +} + +impl StructOps { + /// The type of the program according to the kernel. + pub const PROGRAM_TYPE: ProgramType = ProgramType::StructOps; + + /// Loads the program inside the kernel. + /// + /// Struct ops programs use `attach_btf_id` to identify the function + /// prototype in the kernel BTF. + pub fn load(&mut self, attach_btf_id: u32) -> Result<(), ProgramError> { + self.data.attach_btf_id = Some(attach_btf_id); + load_program(BPF_PROG_TYPE_STRUCT_OPS, &mut self.data) + } +} + +define_link_wrapper!(StructOpsLink, StructOpsLinkId, FdLink, FdLinkId, StructOps); + +impl StructOpsLink { + pub(crate) const fn wrap(base: FdLink) -> Self { + Self(Some(base)) + } +} diff --git a/aya/src/sys/bpf.rs b/aya/src/sys/bpf.rs index c9153ca4c..f78565ca2 100644 --- a/aya/src/sys/bpf.rs +++ b/aya/src/sys/bpf.rs @@ -756,7 +756,7 @@ fn with_raised_rlimit_retry io::Result>(mut op: F) -> io::Re result } -pub(super) fn bpf_map_create(attr: &mut bpf_attr) -> io::Result { +pub(crate) fn bpf_map_create(attr: &mut bpf_attr) -> io::Result { // SAFETY: BPF_MAP_CREATE returns a new file descriptor. with_raised_rlimit_retry(|| unsafe { fd_sys_bpf(bpf_cmd::BPF_MAP_CREATE, attr) }) }