From ebce817b24f9a558b4cb458c3d45e9f2cb4cea8d Mon Sep 17 00:00:00 2001 From: swananan Date: Tue, 21 Apr 2026 22:02:34 +0800 Subject: [PATCH] aya: support multi-point uprobe attachments Opening many uprobes through perf_event_open is slow because each attach point requires its own perf event and link setup. Kernels with uprobe_multi (Linux 6.6+) can attach one BPF program to many user-space locations with a single BPF link, which makes broad uprobe attachment much cheaper. Allow UProbe::attach to attach one program to multiple uprobe locations. Use uprobe_multi links for programs loaded from uprobe.multi sections. For handles reconstructed from program info or pinned programs, probe the attach mode at attach time: try uprobe_multi first, then fall back to legacy per-point perf links if the kernel does not support uprobe_multi or the loaded program was not loaded for the uprobe_multi attach type. Add per-point attach cookies, grouped perf-link detach handling, and fd-link conversion support for logical links backed by multiple perf attachments. Cover the multi path and unknown-mode fallback in integration tests. Fixes #992. --- aya-obj/Cargo.toml | 1 + aya-obj/src/obj.rs | 150 ++-- aya/src/bpf.rs | 28 +- aya/src/programs/iter.rs | 28 +- aya/src/programs/kprobe.rs | 14 +- aya/src/programs/links.rs | 31 +- aya/src/programs/mod.rs | 68 +- aya/src/programs/perf_attach.rs | 182 +++- aya/src/programs/perf_event.rs | 17 +- aya/src/programs/probe.rs | 21 +- aya/src/programs/trace_point.rs | 17 +- aya/src/programs/uprobe.rs | 820 ++++++++++++++++-- aya/src/sys/bpf.rs | 74 +- test/integration-ebpf/Cargo.toml | 4 + test/integration-ebpf/src/uprobe_multi.rs | 32 + test/integration-test/src/lib.rs | 1 + test/integration-test/src/tests.rs | 1 + test/integration-test/src/tests/array.rs | 2 +- .../src/tests/bloom_filter.rs | 2 +- .../src/tests/bpf_probe_read.rs | 2 +- .../src/tests/btf_relocations.rs | 2 +- test/integration-test/src/tests/info.rs | 2 +- .../src/tests/linear_data_structures.rs | 2 +- test/integration-test/src/tests/load.rs | 10 +- test/integration-test/src/tests/log.rs | 6 +- test/integration-test/src/tests/lpm_trie.rs | 2 +- .../src/tests/maps_disjoint.rs | 2 +- .../src/tests/per_cpu_array.rs | 2 +- .../src/tests/perf_event_array.rs | 2 +- test/integration-test/src/tests/printk.rs | 2 +- test/integration-test/src/tests/prog_array.rs | 4 +- .../integration-test/src/tests/relocations.rs | 2 +- test/integration-test/src/tests/ring_buf.rs | 10 +- .../integration-test/src/tests/stack_trace.rs | 2 +- test/integration-test/src/tests/strncmp.rs | 2 +- .../src/tests/uprobe_cookie.rs | 11 +- .../src/tests/uprobe_multi.rs | 311 +++++++ xtask/public-api/aya-obj.txt | 4 + xtask/public-api/aya.txt | 64 +- 39 files changed, 1631 insertions(+), 306 deletions(-) create mode 100644 test/integration-ebpf/src/uprobe_multi.rs create mode 100644 test/integration-test/src/tests/uprobe_multi.rs diff --git a/aya-obj/Cargo.toml b/aya-obj/Cargo.toml index bbdc063cb6..34d5efd873 100644 --- a/aya-obj/Cargo.toml +++ b/aya-obj/Cargo.toml @@ -25,3 +25,4 @@ thiserror = { workspace = true, features = ["std"] } [dev-dependencies] assert_matches = { workspace = true } rbpf = { workspace = true } +test-case = { workspace = true } diff --git a/aya-obj/src/obj.rs b/aya-obj/src/obj.rs index 8d7defb819..0e89d1f1b2 100644 --- a/aya-obj/src/obj.rs +++ b/aya-obj/src/obj.rs @@ -231,9 +231,11 @@ pub enum ProgramSection { KProbe, UProbe { sleepable: bool, + multi: bool, }, URetProbe { sleepable: bool, + multi: bool, }, TracePoint, SocketFilter, @@ -305,10 +307,38 @@ impl FromStr for ProgramSection { Ok(match kind { "kprobe" => Self::KProbe, "kretprobe" => Self::KRetProbe, - "uprobe" => Self::UProbe { sleepable: false }, - "uprobe.s" => Self::UProbe { sleepable: true }, - "uretprobe" => Self::URetProbe { sleepable: false }, - "uretprobe.s" => Self::URetProbe { sleepable: true }, + "uprobe" => Self::UProbe { + sleepable: false, + multi: false, + }, + "uprobe.s" => Self::UProbe { + sleepable: true, + multi: false, + }, + "uprobe.multi" => Self::UProbe { + sleepable: false, + multi: true, + }, + "uprobe.multi.s" => Self::UProbe { + sleepable: true, + multi: true, + }, + "uretprobe" => Self::URetProbe { + sleepable: false, + multi: false, + }, + "uretprobe.s" => Self::URetProbe { + sleepable: true, + multi: false, + }, + "uretprobe.multi" => Self::URetProbe { + sleepable: false, + multi: true, + }, + "uretprobe.multi.s" => Self::URetProbe { + sleepable: true, + multi: true, + }, "xdp" | "xdp.frags" => Self::Xdp { frags: kind == "xdp.frags", attach_type: match pieces.next() { @@ -1457,6 +1487,7 @@ fn get_func_and_line_info( #[cfg(test)] mod tests { use assert_matches::assert_matches; + use test_case::test_case; use super::*; use crate::generated::{bpf_map_type::BPF_MAP_TYPE_BLOOM_FILTER, btf_ext_header}; @@ -1971,102 +2002,47 @@ mod tests { ); } - #[test] - fn test_parse_section_uprobe() { - let mut obj = fake_obj(); - fake_sym(&mut obj, 0, 0, "foo", FAKE_INS_LEN); - - assert_matches!( - obj.parse_section(fake_section( - EbpfSectionKind::Program, - "uprobe/foo", - bytes_of(&fake_ins()), - None - )), - Ok(()) - ); - assert_matches!( - obj.programs.get("foo"), - Some(Program { - section: ProgramSection::UProbe { .. }, - .. - }) - ); - } - - #[test] - fn test_parse_section_uprobe_sleepable() { + #[test_case("uprobe/foo", ProgramSection::UProbe { sleepable: false, multi: false }; "uprobe_plain")] + #[test_case("uprobe.s/foo", ProgramSection::UProbe { sleepable: true, multi: false }; "uprobe_sleepable")] + #[test_case("uprobe.multi/foo", ProgramSection::UProbe { sleepable: false, multi: true }; "uprobe_multi")] + #[test_case("uprobe.multi.s/foo", ProgramSection::UProbe { sleepable: true, multi: true }; "uprobe_multi_sleepable")] + #[test_case("uretprobe/foo", ProgramSection::URetProbe { sleepable: false, multi: false }; "uretprobe_plain")] + #[test_case("uretprobe.s/foo", ProgramSection::URetProbe { sleepable: true, multi: false }; "uretprobe_sleepable")] + #[test_case("uretprobe.multi/foo", ProgramSection::URetProbe { sleepable: false, multi: true }; "uretprobe_multi")] + #[test_case("uretprobe.multi.s/foo", ProgramSection::URetProbe { sleepable: true, multi: true }; "uretprobe_multi_sleepable")] + fn test_parse_section_user_probe(section: &str, expected_section: ProgramSection) { let mut obj = fake_obj(); fake_sym(&mut obj, 0, 0, "foo", FAKE_INS_LEN); assert_matches!( obj.parse_section(fake_section( EbpfSectionKind::Program, - "uprobe.s/foo", + section, bytes_of(&fake_ins()), None )), Ok(()) ); - assert_matches!( - obj.programs.get("foo"), - Some(Program { - section: ProgramSection::UProbe { - sleepable: true, - .. + let program = obj.programs.remove("foo").unwrap(); + match (program.section, expected_section) { + ( + ProgramSection::UProbe { + sleepable: actual_sleepable, + multi: actual_multi, }, - .. - }) - ); - } - - #[test] - fn test_parse_section_uretprobe() { - let mut obj = fake_obj(); - fake_sym(&mut obj, 0, 0, "foo", FAKE_INS_LEN); - - assert_matches!( - obj.parse_section(fake_section( - EbpfSectionKind::Program, - "uretprobe/foo", - bytes_of(&fake_ins()), - None - )), - Ok(()) - ); - assert_matches!( - obj.programs.get("foo"), - Some(Program { - section: ProgramSection::URetProbe { .. }, - .. - }) - ); - } - - #[test] - fn test_parse_section_uretprobe_sleepable() { - let mut obj = fake_obj(); - fake_sym(&mut obj, 0, 0, "foo", FAKE_INS_LEN); - - assert_matches!( - obj.parse_section(fake_section( - EbpfSectionKind::Program, - "uretprobe.s/foo", - bytes_of(&fake_ins()), - None - )), - Ok(()) - ); - assert_matches!( - obj.programs.get("foo"), - Some(Program { - section: ProgramSection::URetProbe { - sleepable: true, - .. + ProgramSection::UProbe { sleepable, multi }, + ) + | ( + ProgramSection::URetProbe { + sleepable: actual_sleepable, + multi: actual_multi, }, - .. - }) - ); + ProgramSection::URetProbe { sleepable, multi }, + ) => assert_eq!((actual_sleepable, actual_multi), (sleepable, multi)), + (section, expected_section) => { + panic!("unexpected section: {section:?}, expected: {expected_section:?}") + } + } } #[test] diff --git a/aya/src/bpf.rs b/aya/src/bpf.rs index eeaccedd8a..ddc35b7255 100644 --- a/aya/src/bpf.rs +++ b/aya/src/bpf.rs @@ -23,7 +23,7 @@ use crate::{ CgroupSysctl, Extension, FEntry, FExit, FlowDissector, Iter, KProbe, LircMode2, Lsm, LsmCgroup, PerfEvent, ProbeKind, Program, ProgramData, ProgramError, RawTracePoint, SchedClassifier, SkLookup, SkMsg, SkReuseport, SkSkb, SockOps, SocketFilter, TracePoint, - UProbe, Xdp, + UProbe, Xdp, uprobe::AttachMode, }, sys::{ bpf_load_btf, is_bpf_cookie_supported, is_bpf_global_data_supported, @@ -449,8 +449,14 @@ impl<'a> EbpfLoader<'a> { } ProgramSection::KRetProbe | ProgramSection::KProbe - | ProgramSection::UProbe { sleepable: _ } - | ProgramSection::URetProbe { sleepable: _ } + | ProgramSection::UProbe { + sleepable: _, + multi: _, + } + | ProgramSection::URetProbe { + sleepable: _, + multi: _, + } | ProgramSection::TracePoint | ProgramSection::SocketFilter | ProgramSection::Xdp { @@ -582,7 +588,7 @@ impl<'a> EbpfLoader<'a> { data: ProgramData::new(prog_name, obj, btf_fd, *verifier_log_level), kind: ProbeKind::Return, }), - ProgramSection::UProbe { sleepable } => { + ProgramSection::UProbe { sleepable, multi } => { let mut data = ProgramData::new(prog_name, obj, btf_fd, *verifier_log_level); if *sleepable { @@ -591,9 +597,14 @@ impl<'a> EbpfLoader<'a> { Program::UProbe(UProbe { data, kind: ProbeKind::Entry, + attach_mode: if *multi { + AttachMode::Multi + } else { + AttachMode::Single + }, }) } - ProgramSection::URetProbe { sleepable } => { + ProgramSection::URetProbe { sleepable, multi } => { let mut data = ProgramData::new(prog_name, obj, btf_fd, *verifier_log_level); if *sleepable { @@ -602,6 +613,11 @@ impl<'a> EbpfLoader<'a> { Program::UProbe(UProbe { data, kind: ProbeKind::Return, + attach_mode: if *multi { + AttachMode::Multi + } else { + AttachMode::Single + }, }) } ProgramSection::TracePoint => Program::TracePoint(TracePoint { @@ -1103,7 +1119,7 @@ impl Ebpf { /// /// let program: &mut UProbe = bpf.program_mut("SSL_read").unwrap().try_into()?; /// program.load()?; - /// program.attach("SSL_read", "libssl", UProbeScope::AllProcesses)?; + /// program.attach(["SSL_read"], "libssl", UProbeScope::AllProcesses)?; /// # Ok::<(), aya::EbpfError>(()) /// ``` pub fn program_mut(&mut self, name: &str) -> Option<&mut Program> { diff --git a/aya/src/programs/iter.rs b/aya/src/programs/iter.rs index f6e3473276..0a4daade5c 100644 --- a/aya/src/programs/iter.rs +++ b/aya/src/programs/iter.rs @@ -87,7 +87,7 @@ impl Iter { self.data .links - .insert(IterLink::new(PerfLinkInner::Fd(FdLink::new(link_fd)))) + .insert(IterLink::new(FdLink::new(link_fd).into())) } } @@ -104,8 +104,8 @@ impl AsFd for IterFd { } } -impl_try_into_fdlink!(IterLink, PerfLinkInner); -impl_try_from_fdlink!(IterLink, PerfLinkInner, BPF_LINK_TYPE_ITER); +impl_try_into_fdlink!(IterLink, method = into_fd_link); +impl_try_from_fdlink!(IterLink, BPF_LINK_TYPE_ITER); define_link_wrapper!(IterLink, IterLinkId, PerfLinkInner, PerfLinkIdInner, Iter); @@ -113,16 +113,16 @@ impl IterLink { /// Converts [`IterLink`] into a [`File`] that can be used to retrieve the /// outputs of the iterator program. pub fn into_file(self) -> Result { - if let PerfLinkInner::Fd(fd) = self.into_inner() { - let fd = bpf_create_iter(fd.fd.as_fd()).map_err(|io_error| { - LinkError::SyscallError(SyscallError { - call: "bpf_iter_create", - io_error, - }) - })?; - Ok(fd.into_inner().into()) - } else { - Err(LinkError::InvalidLink) - } + let fd = self + .into_inner() + .into_fd_link() + .map_err(|_inner| LinkError::InvalidLink)?; + let fd = bpf_create_iter(fd.fd.as_fd()).map_err(|io_error| { + LinkError::SyscallError(SyscallError { + call: "bpf_iter_create", + io_error, + }) + })?; + Ok(fd.into_inner().into()) } } diff --git a/aya/src/programs/kprobe.rs b/aya/src/programs/kprobe.rs index b21d6fa511..43d2e37b43 100644 --- a/aya/src/programs/kprobe.rs +++ b/aya/src/programs/kprobe.rs @@ -12,9 +12,9 @@ use thiserror::Error; use crate::{ VerifierLogLevel, programs::{ - ProgramData, ProgramError, ProgramType, define_link_wrapper, impl_try_from_fdlink, - impl_try_into_fdlink, load_program_without_attach_type, - perf_attach::{PerfLinkIdInner, PerfLinkInner}, + PerfLinkIdInner, PerfLinkInner, ProgramData, ProgramError, ProgramType, + define_link_wrapper, impl_try_from_fdlink, impl_try_into_fdlink, + load_program_without_attach_type, probe::{Probe, ProbeKind, attach}, }, }; @@ -143,9 +143,5 @@ pub enum KProbeError { }, } -impl_try_into_fdlink!(KProbeLink, PerfLinkInner); -impl_try_from_fdlink!( - KProbeLink, - PerfLinkInner, - bpf_link_type::BPF_LINK_TYPE_PERF_EVENT -); +impl_try_into_fdlink!(KProbeLink, method = into_fd_link); +impl_try_from_fdlink!(KProbeLink, bpf_link_type::BPF_LINK_TYPE_PERF_EVENT); diff --git a/aya/src/programs/links.rs b/aya/src/programs/links.rs index 62f40102d4..9a9a3f5c39 100644 --- a/aya/src/programs/links.rs +++ b/aya/src/programs/links.rs @@ -579,26 +579,47 @@ macro_rules! impl_try_into_fdlink { } } }; + ($wrapper:ident, method = $method:ident) => { + impl TryFrom<$wrapper> for $crate::programs::FdLink { + type Error = $crate::programs::LinkError; + + fn try_from(value: $wrapper) -> Result { + match value.into_inner().$method() { + Ok(fd) => Ok(fd), + Err(_) => Err($crate::programs::LinkError::InvalidLink), + } + } + } + }; } pub(crate) use impl_try_into_fdlink; macro_rules! impl_try_from_fdlink { - ($wrapper:ident, $inner:ident, $link_type:expr) => { + (@impl $wrapper:ident, [$($link_type:expr),+ $(,)?], $fd_link:ident => $inner:expr) => { impl TryFrom<$crate::programs::FdLink> for $wrapper { type Error = $crate::programs::LinkError; - fn try_from(fd_link: $crate::programs::FdLink) -> Result { + fn try_from($fd_link: $crate::programs::FdLink) -> Result { use std::os::fd::AsFd as _; - let info = $crate::sys::bpf_link_get_info_by_fd(fd_link.fd.as_fd())?; - if info.type_ == ($link_type as u32) { - return Ok(Self::new($inner::Fd(fd_link))); + let info = $crate::sys::bpf_link_get_info_by_fd($fd_link.fd.as_fd())?; + if [$(($link_type as u32)),+].contains(&info.type_) { + return Ok(Self::new($inner)); } Err($crate::programs::LinkError::InvalidLink) } } }; + ($wrapper:ident, $inner:ident, $link_type:expr) => { + impl_try_from_fdlink!(@impl $wrapper, [$link_type], fd_link => $inner::Fd(fd_link)); + }; + ($wrapper:ident, link_types = [$($link_type:expr),+ $(,)?]) => { + impl_try_from_fdlink!(@impl $wrapper, [$($link_type),+], fd_link => fd_link.into()); + }; + ($wrapper:ident, $link_type:expr) => { + impl_try_from_fdlink!(@impl $wrapper, [$link_type], fd_link => fd_link.into()); + }; } pub(crate) use impl_try_from_fdlink; diff --git a/aya/src/programs/mod.rs b/aya/src/programs/mod.rs index bf6445a980..d41925b205 100644 --- a/aya/src/programs/mod.rs +++ b/aya/src/programs/mod.rs @@ -136,7 +136,9 @@ use crate::{ FdLink, FdLinkId, LinkError, LinkInfo, Links, ProgAttachLink, ProgAttachLinkId, define_link_wrapper, id_as_key, impl_try_from_fdlink, impl_try_into_fdlink, }, - perf_attach::{PerfLinkIdInner, PerfLinkInner, perf_attach, perf_attach_debugfs}, + perf_attach::{ + PerfLinkIdInner, PerfLinkInner, PerfLinkLeaf, perf_attach, perf_attach_debugfs, + }, }, sys::{ EbpfLoadProgramAttrs, NetlinkError, ProgQueryTarget, SyscallError, bpf_btf_get_fd_by_id, @@ -179,6 +181,10 @@ pub enum ProgramError { #[error(transparent)] SyscallError(#[from] SyscallError), + /// One or more perf-style links failed to detach. + #[error(transparent)] + PerfLinkDetachError(#[from] perf_attach::PerfLinkDetachError), + /// The network interface does not exist. #[error("unknown network interface {name}")] UnknownInterface { @@ -1046,11 +1052,17 @@ impl_from_pin!( macro_rules! impl_from_prog_info { ( - $(#[$doc:meta])* + @docs + [$($doc:meta)*] + @safety_docs + [$($safety_doc:meta)*] @safety [$($safety:tt)?] @rest - $struct_name:ident $($var:ident : $var_ty:ty)? + $struct_name:ident + $($var:ident : $var_ty:ty,)? + @extra_fields + [$($extra_field:ident : $extra_value:expr),* $(,)?] ) => { impl $struct_name { /// Constructs an instance of a [`Self`] from a [`ProgramInfo`]. @@ -1058,12 +1070,14 @@ macro_rules! impl_from_prog_info { /// This allows the caller to get a handle to an already loaded /// program from the kernel without having to load it again. /// + $(#[$doc])* + /// /// # Errors /// /// - If the program type reported by the kernel does not match /// [`Self::PROGRAM_TYPE`]. /// - If the file descriptor of the program cannot be cloned. - $(#[$doc])* + $(#[$safety_doc])* pub $($safety)? fn from_program_info( info: ProgramInfo, @@ -1086,6 +1100,7 @@ macro_rules! impl_from_prog_info { VerifierLogLevel::default(), )?, $($var,)? + $($extra_field: $extra_value,)* }) } } @@ -1093,30 +1108,45 @@ macro_rules! impl_from_prog_info { // Handle unsafe cases and pass a safety doc section ( - unsafe $struct_name:ident $($var:ident : $var_ty:ty)? $(, $($rest:tt)*)? + $(#[$doc:meta])* + unsafe $struct_name:ident + $($var:ident : $var_ty:ty)? + $(=> { $($extra_field:ident : $extra_value:expr),+ $(,)? })? + $(, $($rest:tt)*)? ) => { impl_from_prog_info! { - /// - /// # Safety - /// - /// The runtime type of this program, as used by the kernel, is - /// overloaded. We assert the program type matches the runtime type - /// but we're unable to perform further checks. Therefore, the caller - /// must ensure that the program type is correct or the behavior is - /// undefined. + @docs [$($doc)*] + @safety_docs [ + doc = "" + doc = "# Safety" + doc = "" + doc = "The runtime type of this program, as used by the kernel, is" + doc = "overloaded. We assert the program type matches the runtime type" + doc = "but we're unable to perform further checks. Therefore, the caller" + doc = "must ensure that the program type is correct or the behavior is" + doc = "undefined." + ] @safety [unsafe] - @rest $struct_name $($var : $var_ty)? + @rest $struct_name $($var : $var_ty,)? + @extra_fields [$($($extra_field : $extra_value),+)?] } $( impl_from_prog_info!($($rest)*); )? }; // Handle non-unsafe cases and omit safety doc section ( - $struct_name:ident $($var:ident : $var_ty:ty)? $(, $($rest:tt)*)? + $(#[$doc:meta])* + $struct_name:ident + $($var:ident : $var_ty:ty)? + $(=> { $($extra_field:ident : $extra_value:expr),+ $(,)? })? + $(, $($rest:tt)*)? ) => { impl_from_prog_info! { + @docs [$($doc)*] + @safety_docs [] @safety [] - @rest $struct_name $($var : $var_ty)? + @rest $struct_name $($var : $var_ty,)? + @extra_fields [$($($extra_field : $extra_value),+)?] } $( impl_from_prog_info!($($rest)*); )? }; @@ -1129,7 +1159,11 @@ macro_rules! impl_from_prog_info { impl_from_prog_info!( unsafe KProbe kind : ProbeKind, - unsafe UProbe kind : ProbeKind, + /// As with [`Self::from_pin`], this constructor starts in unknown mode + /// because it does not know whether the original program came from an + /// `uprobe` or `uprobe.multi` section. As a result, [`Self::attach`] + /// performs runtime mode selection. + unsafe UProbe kind : ProbeKind => { attach_mode: uprobe::AttachMode::Unknown }, TracePoint, Xdp attach_type : XdpAttachType, SkMsg, diff --git a/aya/src/programs/perf_attach.rs b/aya/src/programs/perf_attach.rs index 126332b310..6dea2843fe 100644 --- a/aya/src/programs/perf_attach.rs +++ b/aya/src/programs/perf_attach.rs @@ -5,6 +5,7 @@ use std::{ }; use aya_obj::generated::bpf_attach_type::BPF_PERF_EVENT; +use thiserror::Error; use crate::{ FEATURES, @@ -15,25 +16,64 @@ use crate::{ }, }; +/// Errors returned while detaching multiple perf-style links. +#[derive(Debug, Error)] +#[error("perf link detach errors: {0:?}")] +pub struct PerfLinkDetachError(Vec); + +impl PerfLinkDetachError { + pub(crate) const fn new(errors: Vec) -> Self { + Self(errors) + } + + /// Returns the collected detach errors as a slice. + pub fn as_slice(&self) -> &[ProgramError] { + let Self(errors) = self; + errors + } +} + +/// Internal identifier for a logical link backed by one or more perf-style +/// links. #[derive(Debug, Hash, Eq, PartialEq)] pub(crate) enum PerfLinkIdInner { + /// A single underlying perf-style link id. + One(PerfLinkIdInnerInner), + /// Many underlying perf-style link ids grouped under one logical link. + /// + /// This is used by probe types that group several perf-style links under + /// one logical link. + Many(Vec), +} + +#[derive(Debug, Hash, Eq, PartialEq)] +pub(crate) enum PerfLinkIdInnerInner { FdLinkId(::Id), PerfLinkId(::Id), } #[derive(Debug)] -pub(crate) enum PerfLinkInner { +pub(crate) enum PerfLinkLeaf { Fd(FdLink), PerfLink(PerfLink), } -impl Link for PerfLinkInner { - type Id = PerfLinkIdInner; +impl PerfLinkLeaf { + pub(crate) fn into_fd_link(self) -> Result { + match self { + Self::Fd(link) => Ok(link), + Self::PerfLink(link) => Err(Self::PerfLink(link)), + } + } +} + +impl Link for PerfLinkLeaf { + type Id = PerfLinkIdInnerInner; fn id(&self) -> Self::Id { match self { - Self::Fd(link) => PerfLinkIdInner::FdLinkId(link.id()), - Self::PerfLink(link) => PerfLinkIdInner::PerfLinkId(link.id()), + Self::Fd(link) => PerfLinkIdInnerInner::FdLinkId(link.id()), + Self::PerfLink(link) => PerfLinkIdInnerInner::PerfLinkId(link.id()), } } @@ -45,6 +85,116 @@ impl Link for PerfLinkInner { } } +id_as_key!(PerfLinkLeaf, PerfLinkIdInnerInner); + +/// Internal representation of a logical link backed by one or more perf-style +/// links. +#[derive(Debug)] +pub(crate) enum PerfLinkInner { + /// A single perf-style link. + One(PerfLinkLeaf), + /// Many perf-style links grouped under one logical link. + /// + /// This is used by probe types that group several perf-style links under + /// one logical link. + Many(Vec), +} + +impl PerfLinkInner { + pub(crate) fn into_fd_link(self) -> Result { + match self { + Self::One(PerfLinkLeaf::Fd(link)) => Ok(link), + Self::One(link) => Err(Self::One(link)), + Self::Many(links) => Err(Self::Many(links)), + } + } + + pub(crate) fn into_fd_links(self) -> Result, Self> { + match self { + Self::One(link) => link + .into_fd_link() + .map(|link| vec![link]) + .map_err(Self::One), + Self::Many(links) => { + let mut fd_links = Vec::with_capacity(links.len()); + let mut pending = links.into_iter(); + + while let Some(link) = pending.next() { + match link.into_fd_link() { + Ok(link) => fd_links.push(link), + Err(link) => { + let mut links = fd_links + .into_iter() + .map(PerfLinkLeaf::from) + .collect::>(); + links.push(link); + links.extend(pending); + return Err(Self::Many(links)); + } + } + } + + Ok(fd_links) + } + } + } +} + +impl From for PerfLinkIdInner { + fn from(link_id: PerfLinkIdInnerInner) -> Self { + Self::One(link_id) + } +} + +impl From for PerfLinkLeaf { + fn from(link: FdLink) -> Self { + Self::Fd(link) + } +} + +impl From for PerfLinkInner { + fn from(link: PerfLinkLeaf) -> Self { + Self::One(link) + } +} + +impl From for PerfLinkInner { + fn from(link: FdLink) -> Self { + PerfLinkLeaf::from(link).into() + } +} + +impl Link for PerfLinkInner { + type Id = PerfLinkIdInner; + + fn id(&self) -> Self::Id { + match self { + Self::One(link) => link.id().into(), + Self::Many(links) => PerfLinkIdInner::Many(links.iter().map(Link::id).collect()), + } + } + + fn detach(self) -> Result<(), ProgramError> { + match self { + Self::One(link) => link.detach(), + Self::Many(links) => { + // Best-effort cleanup: keep detaching remaining links even if one fails. + let mut errors = Vec::new(); + for link in links { + if let Err(error) = link.detach() { + errors.push(error); + } + } + if errors.is_empty() { + Ok(()) + } else { + Err(PerfLinkDetachError::new(errors).into()) + } + } + } + } +} + id_as_key!(PerfLinkInner, PerfLinkIdInner); /// The identifier of a `PerfLink`. @@ -81,11 +231,23 @@ impl Link for PerfLink { id_as_key!(PerfLink, PerfLinkId); +impl From for PerfLinkLeaf { + fn from(link: PerfLink) -> Self { + Self::PerfLink(link) + } +} + +impl From for PerfLinkInner { + fn from(link: PerfLink) -> Self { + PerfLinkLeaf::from(link).into() + } +} + pub(crate) fn perf_attach( prog_fd: BorrowedFd<'_>, perf_fd: crate::MockableFd, cookie: Option, -) -> Result { +) -> Result { if cookie.is_some() && (!is_bpf_cookie_supported() || !FEATURES.bpf_perf_link()) { return Err(ProgramError::AttachCookieNotSupported); } @@ -101,7 +263,7 @@ pub(crate) fn perf_attach( call: "bpf_link_create", io_error, })?; - Ok(PerfLinkInner::Fd(FdLink::new(link_fd))) + Ok(FdLink::new(link_fd).into()) } else { perf_attach_either(prog_fd, perf_fd, None) } @@ -111,7 +273,7 @@ pub(crate) fn perf_attach_debugfs( prog_fd: BorrowedFd<'_>, perf_fd: crate::MockableFd, event: ProbeEvent, -) -> Result { +) -> Result { perf_attach_either(prog_fd, perf_fd, Some(event)) } @@ -119,7 +281,7 @@ fn perf_attach_either( prog_fd: BorrowedFd<'_>, perf_fd: crate::MockableFd, mut event: Option, -) -> Result { +) -> Result { perf_event_ioctl(perf_fd.as_fd(), PerfEventIoctlRequest::SetBpf(prog_fd)).map_err( |io_error| SyscallError { call: "PERF_EVENT_IOC_SET_BPF", @@ -137,5 +299,5 @@ fn perf_attach_either( event.disarm(); } - Ok(PerfLinkInner::PerfLink(PerfLink { perf_fd, event })) + Ok(PerfLink { perf_fd, event }.into()) } diff --git a/aya/src/programs/perf_event.rs b/aya/src/programs/perf_event.rs index 0a8eb2728d..fb54ce79f1 100644 --- a/aya/src/programs/perf_event.rs +++ b/aya/src/programs/perf_event.rs @@ -11,10 +11,9 @@ use aya_obj::generated::{ use crate::{ programs::{ - ProgramData, ProgramError, ProgramType, impl_try_from_fdlink, impl_try_into_fdlink, - links::define_link_wrapper, - load_program_without_attach_type, - perf_attach::{PerfLinkIdInner, PerfLinkInner, perf_attach}, + PerfLinkIdInner, PerfLinkInner, ProgramData, ProgramError, ProgramType, + impl_try_from_fdlink, impl_try_into_fdlink, links::define_link_wrapper, + load_program_without_attach_type, perf_attach, }, sys::{SyscallError, perf_event_open}, }; @@ -471,16 +470,12 @@ impl PerfEvent { })?; let link = perf_attach(prog_fd, perf_fd, None /* cookie */)?; - self.data.links.insert(PerfEventLink::new(link)) + self.data.links.insert(PerfEventLink::new(link.into())) } } -impl_try_into_fdlink!(PerfEventLink, PerfLinkInner); -impl_try_from_fdlink!( - PerfEventLink, - PerfLinkInner, - bpf_link_type::BPF_LINK_TYPE_PERF_EVENT -); +impl_try_into_fdlink!(PerfEventLink, method = into_fd_link); +impl_try_from_fdlink!(PerfEventLink, bpf_link_type::BPF_LINK_TYPE_PERF_EVENT); define_link_wrapper!( PerfEventLink, diff --git a/aya/src/programs/probe.rs b/aya/src/programs/probe.rs index e6de2e382a..9454043589 100644 --- a/aya/src/programs/probe.rs +++ b/aya/src/programs/probe.rs @@ -3,7 +3,7 @@ use std::{ fmt::{self, Write}, fs::{self, OpenOptions}, io::{self, Write as _}, - os::fd::AsFd as _, + os::fd::{AsFd as _, BorrowedFd}, path::{Path, PathBuf}, process, sync::atomic::{AtomicUsize, Ordering}, @@ -11,7 +11,7 @@ use std::{ use crate::{ programs::{ - Link, ProgramData, ProgramError, perf_attach, perf_attach::PerfLinkInner, + Link, PerfLinkInner, PerfLinkLeaf, ProgramData, ProgramError, perf_attach, perf_attach_debugfs, trace_point::read_sys_fs_trace_point_id, utils::find_tracefs_path, }, sys::{SyscallError, perf_event_open_probe, perf_event_open_trace_point}, @@ -152,7 +152,19 @@ pub(crate) fn attach>( // Use debugfs to create probe let prog_fd = program_data.fd()?; let prog_fd = prog_fd.as_fd(); - let link = if KernelVersion::at_least(4, 17, 0) { + let link = attach_perf_link::

(prog_fd, kind, fn_name, offset, pid, cookie)?; + program_data.links.insert(T::from(link.into())) +} + +pub(crate) fn attach_perf_link( + prog_fd: BorrowedFd<'_>, + kind: ProbeKind, + fn_name: &OsStr, + offset: u64, + pid: Option, + cookie: Option, +) -> Result { + if KernelVersion::at_least(4, 17, 0) { let perf_fd = create_as_probe::

(kind, fn_name, offset, pid)?; perf_attach(prog_fd, perf_fd, cookie) } else { @@ -161,8 +173,7 @@ pub(crate) fn attach>( } let (perf_fd, event) = create_as_trace_point::

(kind, fn_name, offset, pid)?; perf_attach_debugfs(prog_fd, perf_fd, event) - }?; - program_data.links.insert(T::from(link)) + } } fn detach_debug_fs(event_alias: &OsStr) -> Result<(), ProgramError> { diff --git a/aya/src/programs/trace_point.rs b/aya/src/programs/trace_point.rs index 0dad511a22..296e7b28ef 100644 --- a/aya/src/programs/trace_point.rs +++ b/aya/src/programs/trace_point.rs @@ -10,10 +10,9 @@ use thiserror::Error; use crate::{ programs::{ - ProgramData, ProgramError, ProgramType, define_link_wrapper, impl_try_from_fdlink, - impl_try_into_fdlink, load_program_without_attach_type, - perf_attach::{PerfLinkIdInner, PerfLinkInner, perf_attach}, - utils::find_tracefs_path, + PerfLinkIdInner, PerfLinkInner, ProgramData, ProgramError, ProgramType, + define_link_wrapper, impl_try_from_fdlink, impl_try_into_fdlink, + load_program_without_attach_type, perf_attach, utils::find_tracefs_path, }, sys::{SyscallError, perf_event_open_trace_point}, }; @@ -86,7 +85,7 @@ impl TracePoint { })?; let link = perf_attach(prog_fd, perf_fd, None /* cookie */)?; - self.data.links.insert(TracePointLink::new(link)) + self.data.links.insert(TracePointLink::new(link.into())) } } @@ -98,12 +97,8 @@ define_link_wrapper!( TracePoint, ); -impl_try_into_fdlink!(TracePointLink, PerfLinkInner); -impl_try_from_fdlink!( - TracePointLink, - PerfLinkInner, - bpf_link_type::BPF_LINK_TYPE_PERF_EVENT -); +impl_try_into_fdlink!(TracePointLink, method = into_fd_link); +impl_try_from_fdlink!(TracePointLink, bpf_link_type::BPF_LINK_TYPE_PERF_EVENT); pub(crate) fn read_sys_fs_trace_point_id( tracefs: &Path, diff --git a/aya/src/programs/uprobe.rs b/aya/src/programs/uprobe.rs index 8dc138de98..bb77466f17 100644 --- a/aya/src/programs/uprobe.rs +++ b/aya/src/programs/uprobe.rs @@ -1,30 +1,40 @@ //! User space probes. use std::{ borrow::Cow, + collections::HashMap, error::Error, - ffi::{CStr, OsStr, OsString}, + ffi::{CStr, CString, OsStr, OsString}, fmt::{self, Write}, fs, io::{self, BufRead as _, Cursor, Read as _}, + iter::once, num::NonZeroU32, - os::unix::ffi::{OsStrExt as _, OsStringExt as _}, + os::{ + fd::{AsFd as _, BorrowedFd}, + unix::ffi::{OsStrExt as _, OsStringExt as _}, + }, path::{Path, PathBuf}, + slice::from_ref, sync::LazyLock, }; -use aya_obj::generated::{bpf_link_type, bpf_prog_type::BPF_PROG_TYPE_KPROBE}; +use aya_obj::generated::{ + bpf_attach_type::BPF_TRACE_UPROBE_MULTI, bpf_link_type, bpf_prog_type::BPF_PROG_TYPE_KPROBE, +}; +use libc::{EINVAL, ENOTSUP, EOPNOTSUPP}; use object::{Object as _, ObjectSection as _, ObjectSymbol as _, Symbol}; use thiserror::Error; use crate::{ VerifierLogLevel, programs::{ - ProgramData, ProgramError, ProgramType, define_link_wrapper, impl_try_from_fdlink, - impl_try_into_fdlink, load_program_without_attach_type, - perf_attach::{PerfLinkIdInner, PerfLinkInner}, - probe::{OsStringExt as _, Probe, ProbeKind, attach}, + FdLink, Link as _, PerfLinkIdInner, PerfLinkInner, PerfLinkLeaf, ProgramData, ProgramError, + ProgramType, define_link_wrapper, impl_try_from_fdlink, impl_try_into_fdlink, + load_program_with_attach_type, load_program_without_attach_type, + probe::{self, OsStringExt as _, Probe, ProbeKind}, }, - util::MMap, + sys::{SyscallError, bpf_link_create_uprobe_multi}, + util::{KernelVersion, MMap}, }; const LD_SO_CACHE_FILE: &str = "/etc/ld.so.cache"; @@ -46,10 +56,19 @@ const LD_SO_CACHE_HEADER_NEW: &str = "glibc-ld.so.cache1.1"; pub struct UProbe { pub(crate) data: ProgramData, pub(crate) kind: ProbeKind, + pub(crate) attach_mode: AttachMode, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) enum AttachMode { + Single, + Multi, + Unknown, } /// The location in the target object file to which the uprobe is to be /// attached. +#[derive(Debug, Clone, Copy)] pub enum UProbeAttachLocation<'a> { /// The location of the target function in the target object file. Symbol(&'a str), @@ -66,13 +85,26 @@ impl<'a> From<&'a str> for UProbeAttachLocation<'a> { } } +impl<'a> From<&&'a str> for UProbeAttachLocation<'a> { + fn from(s: &&'a str) -> Self { + Self::Symbol(s) + } +} + impl From for UProbeAttachLocation<'static> { fn from(offset: u64) -> Self { Self::AbsoluteOffset(offset) } } +impl From<&u64> for UProbeAttachLocation<'static> { + fn from(offset: &u64) -> Self { + Self::AbsoluteOffset(*offset) + } +} + /// Describes a single attachment point along with its optional cookie. +#[derive(Debug, Clone, Copy)] pub struct UProbeAttachPoint<'a> { /// The actual target location. pub location: UProbeAttachLocation<'a>, @@ -89,6 +121,12 @@ impl<'a, L: Into>> From for UProbeAttachPoint<'a> { } } +impl From<&Self> for UProbeAttachPoint<'_> { + fn from(point: &Self) -> Self { + *point + } +} + /// Specifies which processes a uprobe should fire for. #[derive(Debug, Clone, Copy)] pub enum UProbeScope { @@ -100,14 +138,117 @@ pub enum UProbeScope { OneProcess(NonZeroU32), } +enum ResolvedPoints { + One { + offset: u64, + cookie: Option, + }, + // Keep these as separate arrays to match the uprobe_multi kernel ABI, + // which takes offsets and cookies separately. This is private internal + // state, so keeping the arrays aligned is controlled here and avoids + // splitting a point-oriented representation before link_create. + Many { + offsets: Vec, + cookies: Option>, + }, +} + +impl ResolvedPoints { + fn offsets(&self) -> &[u64] { + match self { + Self::One { offset, .. } => from_ref(offset), + Self::Many { offsets, .. } => offsets, + } + } + + fn cookies(&self) -> Option<&[u64]> { + match self { + Self::One { + cookie: Some(cookie), + .. + } => Some(from_ref(cookie)), + Self::One { cookie: None, .. } => None, + Self::Many { cookies, .. } => cookies.as_deref(), + } + } +} + +fn resolve_points<'a, I>( + path: &Path, + first: I::Item, + rest: I, +) -> Result +where + I: IntoIterator, + I::Item: Into>, +{ + let points = once(first).chain(rest); + let (lower, _) = points.size_hint(); + let mut offsets = Vec::with_capacity(lower); + // `bpf_link_create()` expects either no cookie array at all or a full `u64` + // array aligned with the requested attach points (`cnt == offsets.len()`). + // Once any point has a cookie, fill in `0` for points that do not. + let mut cookies: Option> = None; + // Defer symbol resolution until all points are collected so the object + // file is mapped once and additional symbol offsets are applied in one pass. + let mut requests = Vec::new(); + + for (index, UProbeAttachPoint { location, cookie }) in points.map(Into::into).enumerate() { + let offset = match location { + UProbeAttachLocation::Symbol(symbol) => { + requests.push((index, symbol, 0)); + 0 + } + UProbeAttachLocation::SymbolOffset(symbol, additional) => { + requests.push((index, symbol, additional)); + 0 + } + UProbeAttachLocation::AbsoluteOffset(offset) => offset, + }; + + offsets.push(offset); + match (&mut cookies, cookie) { + (Some(values), Some(cookie)) => values.push(cookie), + (Some(values), None) => values.push(0), + (slot @ None, Some(cookie)) => { + let mut values = vec![0; index]; + values.push(cookie); + *slot = Some(values); + } + (None, None) => {} + } + } + + resolve_pending_symbols(path, &requests, &mut offsets)?; + + if let [offset] = offsets.as_slice() { + let offset = *offset; + let cookie = cookies.and_then(|mut cookies| cookies.pop()); + Ok(ResolvedPoints::One { offset, cookie }) + } else { + Ok(ResolvedPoints::Many { offsets, cookies }) + } +} + impl UProbe { /// The type of the program according to the kernel. pub const PROGRAM_TYPE: ProgramType = ProgramType::KProbe; /// Loads the program inside the kernel. pub fn load(&mut self) -> Result<(), ProgramError> { - let Self { data, kind: _ } = self; - load_program_without_attach_type(BPF_PROG_TYPE_KPROBE, data) + let Self { + data, + kind: _, + attach_mode, + } = self; + match attach_mode { + AttachMode::Multi => { + load_program_with_attach_type(BPF_PROG_TYPE_KPROBE, BPF_TRACE_UPROBE_MULTI, data) + } + AttachMode::Single | AttachMode::Unknown => { + load_program_without_attach_type(BPF_PROG_TYPE_KPROBE, data) + } + } } /// Returns [`ProbeKind::Entry`] if the program is a `uprobe`, or @@ -116,45 +257,74 @@ impl UProbe { self.kind } - /// Attaches the program. + /// Attaches the program to one or more locations. + /// + /// `points` accepts any `IntoIterator` of attachment points. For a single + /// point, pass a one-element array such as `["malloc"]`, `[0x10]`, or + /// `[UProbeAttachPoint { .. }]`. Each point can carry an optional cookie + /// exposed to eBPF through `bpf_get_attach_cookie()`. Empty input is + /// rejected with [`UProbeError::EmptyPoints`]. /// - /// Attaches the uprobe to the function `fn_name` defined in the `target`. - /// If the attach point specifies an offset, it is added to the address of - /// the target function. `scope` specifies which processes should trigger - /// the uprobe. + /// On the multi-attach path, if any point has a cookie, Aya passes a full + /// cookie array to the kernel; points without cookies are filled with `0`. + /// `0` is also what `bpf_get_attach_cookie()` returns when the kernel has + /// no cookie for a point. /// - /// The `target` argument can be an absolute or relative path to a binary or - /// shared library, or a library name (eg: `"libc"`). + /// `target` can be an absolute or relative path to a binary or library, or + /// a library name (for example `"libc"`). `scope` specifies which processes + /// should trigger the uprobe. /// - /// If the program is an `uprobe`, it is attached to the *start* address of - /// the target function. Instead if the program is a `uretprobe`, it is - /// attached to the return address of the target function. + /// For handles created via `Ebpf::load*`, `attach_mode` is initialized + /// from the ELF section kind (`uprobe` or `uprobe.multi`). Handles created + /// via `from_pin` or `from_program_info` start in unknown mode, so this + /// method first attempts the multi path and falls back to the per-point + /// path on mode-related failures. /// /// The returned value can be used to detach, see [`UProbe::detach`]. /// /// The cookie is supported since kernel 5.15, and it is made available to - /// the eBPF program via the `bpf_get_attach_cookie()` helper. The `point` - /// argument may be just a location (no cookie) or a [`UProbeAttachPoint`], - /// only the latter sets the cookie explicitly. - pub fn attach<'a, T: AsRef, Point: Into>>( + /// the eBPF program via the `bpf_get_attach_cookie()` helper. + /// + /// On the legacy per-point attach path, each point is attached separately and + /// one logical link id manages all of them. + pub fn attach<'a, T, I>( &mut self, - point: Point, + points: I, target: T, scope: UProbeScope, - ) -> Result { - let UProbeAttachPoint { location, cookie } = point.into(); + ) -> Result + where + T: AsRef, + I: IntoIterator, + I::Item: Into>, + { let target = target.as_ref(); - let (proc_map_pid, perf_event_pid) = match scope { - UProbeScope::AllProcesses => (None, None), - // /proc/0/maps does not exist, so use the real pid for ProcMap - // resolution while keeping the kernel's pid=0 sentinel for attach. - UProbeScope::CallingProcess => (Some(std::process::id()), Some(0)), + let mut points = points.into_iter(); + let Some(first) = points.next() else { + return Err(UProbeError::EmptyPoints { + target: target.to_path_buf(), + } + .into()); + }; + + let (proc_map_pid, perf_event_pid, multi_link_pid) = match scope { + // For BPF_TRACE_UPROBE_MULTI link_create, pid=0 means no PID filter + // (all processes), so pass 0 for the multi-attach path. + UProbeScope::AllProcesses => (None, None, 0), + UProbeScope::CallingProcess => { + let pid = std::process::id(); + // /proc/0/maps does not exist, so use the real pid for ProcMap + // resolution. Preserve pid=0 only for the legacy single-attach + // perf_event_open path, where it means calling process/thread. + // For uprobe_multi links, pid=0 means all processes; pass the + // real pid to scope the link to this process. + (Some(pid), Some(0), pid) + } UProbeScope::OneProcess(pid) => { let pid = pid.get(); - (Some(pid), Some(pid)) + (Some(pid), Some(pid), pid) } }; - // Keep ProcMap in this scope so resolve_attach_target_basename can return // a path borrowed from the maps buffer without cloning the matched path. // This keeps the borrow alive until attach uses it. @@ -162,43 +332,138 @@ impl UProbe { // /proc//maps entries are matched by basename, so only bare-basename // targets can benefit from the lookup; paths with a directory separator // are passed through unchanged. - let path = if is_basename_only(target) { + let resolved_path = if is_basename_only(target) { proc_map = proc_map_pid.map(ProcMap::new).transpose()?; resolve_attach_target_basename(target, proc_map.as_ref())? } else { target }; - - let (symbol, offset) = match location { - UProbeAttachLocation::Symbol(s) => (Some(s), 0), - UProbeAttachLocation::SymbolOffset(s, offset) => (Some(s), offset), - UProbeAttachLocation::AbsoluteOffset(offset) => (None, offset), - }; - let offset = if let Some(symbol) = symbol { - let symbol_offset = - resolve_symbol(path, symbol).map_err(|error| UProbeError::SymbolError { - symbol: symbol.to_string(), - error: Box::new(error), - })?; - symbol_offset + offset - } else { - offset - }; - - let Self { data, kind } = self; - let path = path.as_os_str(); - attach::(data, *kind, path, offset, perf_event_pid, cookie) + let resolved = resolve_points(resolved_path, first, points)?; + self.attach_impl(resolved_path, &resolved, perf_event_pid, multi_link_pid) } - /// Creates a program from a pinned entry on a bpffs. + /// Creates a program handle from a pinned entry on bpffs. /// - /// Existing links will not be populated. To work with existing links you should use [`crate::programs::links::PinnedLink`]. + /// Existing links are not populated. To work with existing links, use + /// [`crate::programs::links::PinnedLink`]. + /// + /// This constructor starts in unknown mode because it does not know + /// whether the original program came from an `uprobe` or `uprobe.multi` + /// section. As a result, [`Self::attach`] performs runtime mode selection. /// /// On drop, any managed links are detached and the program is unloaded. This will not result in /// the program being unloaded from the kernel if it is still pinned. pub fn from_pin>(path: P, kind: ProbeKind) -> Result { let data = ProgramData::from_pinned_path(path, VerifierLogLevel::default())?; - Ok(Self { data, kind }) + Ok(Self { + data, + kind, + attach_mode: AttachMode::Unknown, + }) + } + + fn attach_impl( + &mut self, + path: &Path, + resolved: &ResolvedPoints, + perf_event_pid: Option, + multi_link_pid: u32, + ) -> Result { + match self.attach_mode { + AttachMode::Single => self.attach_single_impl(path, resolved, perf_event_pid), + AttachMode::Multi => self.attach_multi_impl(path, resolved, multi_link_pid), + AttachMode::Unknown => { + let multi_result = self.attach_multi_impl(path, resolved, multi_link_pid); + match multi_result { + Ok(link_id) => { + self.attach_mode = AttachMode::Multi; + Ok(link_id) + } + Err(multi_error) if should_fallback_to_single(&multi_error) => { + match self.attach_single_impl(path, resolved, perf_event_pid) { + Ok(link_id) => { + self.attach_mode = AttachMode::Single; + Ok(link_id) + } + Err(single_error) => Err(UProbeError::AttachModeSelectionFailed { + multi_error: Box::new(multi_error), + single_error: Box::new(single_error), + } + .into()), + } + } + Err(error) => Err(error), + } + } + } + } + + fn attach_single_impl( + &mut self, + path: &Path, + resolved: &ResolvedPoints, + pid: Option, + ) -> Result { + let Self { data, kind, .. } = self; + let path = path.as_os_str(); + match resolved { + ResolvedPoints::One { offset, cookie } => { + let link = attach_single_point::(data, *kind, path, *offset, pid, *cookie)?; + data.links + .insert(UProbeLink::from(PerfLinkInner::from(link))) + } + ResolvedPoints::Many { offsets, cookies } => { + let mut links = Vec::with_capacity(offsets.len()); + + for (index, offset) in offsets.iter().copied().enumerate() { + let cookie = cookies + .as_ref() + .and_then(|cookies| cookies.get(index).copied()); + let link = attach_single_point::(data, *kind, path, offset, pid, cookie); + match link { + Ok(link) => links.push(link), + Err(error) => { + let cleanup_error = + PerfLinkInner::Many(links).detach().err().map(Box::new); + return Err(UProbeError::LegacyPerfAttachPointError { + index, + resolved_offset: offset, + attach_error: Box::new(error), + cleanup_error, + } + .into()); + } + } + } + + data.links + .insert(UProbeLink::from(PerfLinkInner::Many(links))) + } + } + } + + fn attach_multi_impl( + &mut self, + path: &Path, + resolved: &ResolvedPoints, + pid: u32, + ) -> Result { + let Self { data, kind, .. } = self; + let prog_fd = data.fd()?; + let prog_fd = prog_fd.as_fd(); + let path_cstr = CString::new(path.as_os_str().as_bytes()).map_err(|error| { + ProgramError::IOError(io::Error::new(io::ErrorKind::InvalidInput, error)) + })?; + + let link = try_attach_uprobe_multi_link( + prog_fd, + &path_cstr, + resolved.offsets(), + pid, + *kind, + resolved.cookies(), + )?; + data.links.insert(UProbeLink::from(link)) } } @@ -264,16 +529,251 @@ define_link_wrapper!( UProbe, ); -impl_try_into_fdlink!(UProbeLink, PerfLinkInner); +impl_try_into_fdlink!(UProbeLink, method = into_fd_link); impl_try_from_fdlink!( UProbeLink, - PerfLinkInner, - bpf_link_type::BPF_LINK_TYPE_PERF_EVENT + link_types = [ + bpf_link_type::BPF_LINK_TYPE_PERF_EVENT, + bpf_link_type::BPF_LINK_TYPE_UPROBE_MULTI, + ] ); +impl UProbeLink { + /// Returns the underlying fd-backed links when available. + /// + /// A single [`UProbeLink`] may correspond to multiple [`FdLink`] values + /// when [`UProbe::attach`] falls back to the legacy single-point attach + /// path for multiple attachment points. + /// + /// If the underlying link representation is not fd-backed, the original + /// [`UProbeLink`] is returned. + pub fn into_fd_links(self) -> Result, Self> { + self.into_inner().into_fd_links().map_err(Self::from) + } +} + +fn attach_single_point( + data: &ProgramData, + kind: ProbeKind, + path: &OsStr, + offset: u64, + pid: Option, + cookie: Option, +) -> Result { + let prog_fd = data.fd()?; + let prog_fd = prog_fd.as_fd(); + probe::attach_perf_link::

(prog_fd, kind, path, offset, pid, cookie) +} + +type PendingSymbol<'a> = (usize, &'a str, u64); + +fn resolve_pending_symbols( + path: &Path, + requests: &[PendingSymbol<'_>], + offsets: &mut [u64], +) -> Result<(), UProbeError> { + if requests.is_empty() { + return Ok(()); + } + + let first_symbol = requests + .first() + .map(|(_, symbol, _)| *symbol) + .unwrap_or_default(); + let symbol_error = |error: ResolveSymbolError| { + let symbol = match &error { + ResolveSymbolError::Unknown(symbol) + | ResolveSymbolError::NotInSection(symbol) + | ResolveSymbolError::SectionFileRangeNone(symbol, _) + | ResolveSymbolError::BuildIdMismatch(symbol) => symbol.as_str(), + _ => first_symbol, + } + .to_owned(); + UProbeError::SymbolError { + symbol, + error: Box::new(error), + } + }; + + // Index pending requests by symbol name so each object symbol table is + // scanned once. The value stores indices into `requests` and + // `resolved_offsets`, allowing duplicate symbol requests to share the same + // resolved base offset before per-request additional offsets are applied. + let symbol_requests = build_symbol_requests(requests); + let mut resolved_offsets = vec![None; requests.len()]; + let data = MMap::map_copy_read_only(path) + .map_err(|error| symbol_error(ResolveSymbolError::Io(error)))?; + let obj = object::read::File::parse(data.as_ref()) + .map_err(|error| symbol_error(ResolveSymbolError::Object(error)))?; + resolve_symbols_in_object(&obj, &symbol_requests, &mut resolved_offsets) + .map_err(&symbol_error)?; + + let unresolved_first_symbol = requests + .iter() + .zip(&resolved_offsets) + .find_map(|((_, symbol, _), offset)| offset.is_none().then_some(*symbol)); + + if let Some(unresolved_first_symbol) = unresolved_first_symbol { + let debug_path = find_debug_path_in_object(&obj, path, unresolved_first_symbol) + .map_err(&symbol_error)?; + let data = MMap::map_copy_read_only(&debug_path).map_err(|e| { + symbol_error(ResolveSymbolError::DebuglinkAccessError( + debug_path.clone().into_owned(), + e, + )) + })?; + let debug_obj = object::read::File::parse(data.as_ref()) + .map_err(|error| symbol_error(ResolveSymbolError::Object(error)))?; + verify_build_ids(&obj, &debug_obj, unresolved_first_symbol).map_err(&symbol_error)?; + resolve_symbols_in_object(&debug_obj, &symbol_requests, &mut resolved_offsets) + .map_err(symbol_error)?; + } + + for ((index, symbol, additional), offset) in requests.iter().zip(&resolved_offsets) { + let offset = offset.ok_or_else(|| UProbeError::SymbolError { + symbol: (*symbol).to_string(), + error: Box::new(ResolveSymbolError::Unknown((*symbol).to_string())), + })?; + offsets[*index] = offset + additional; + } + + Ok(()) +} + +fn build_symbol_requests<'a>(requests: &[PendingSymbol<'a>]) -> HashMap<&'a str, Vec> { + let mut symbol_requests: HashMap<&'a str, Vec> = HashMap::new(); + for (request_index, (_, symbol, _)) in requests.iter().enumerate() { + symbol_requests + .entry(*symbol) + .or_default() + .push(request_index); + } + symbol_requests +} + +fn resolve_symbols_in_object( + obj: &object::File<'_>, + requests: &HashMap<&str, Vec>, + offsets: &mut [Option], +) -> Result<(), ResolveSymbolError> { + if requests.is_empty() { + return Ok(()); + } + + let mut remaining = requests + .values() + .flat_map(|indices| indices.iter()) + .filter(|&&index| offsets[index].is_none()) + .count(); + + for sym in obj.dynamic_symbols().chain(obj.symbols()) { + let Ok(symbol_name) = sym.name() else { + continue; + }; + + let Some(indices) = requests.get(symbol_name) else { + continue; + }; + + let offset = symbol_translated_address(obj, sym, symbol_name)?; + for &index in indices { + if offsets[index].is_none() { + offsets[index] = Some(offset); + remaining -= 1; + if remaining == 0 { + return Ok(()); + } + } + } + } + + Ok(()) +} + +fn try_attach_uprobe_multi_link( + prog_fd: BorrowedFd<'_>, + path: &CStr, + offsets: &[u64], + pid: u32, + kind: ProbeKind, + cookies: Option<&[u64]>, +) -> Result { + let link_fd = bpf_link_create_uprobe_multi( + prog_fd, + path, + offsets, + None, + cookies, + pid, + matches!(kind, ProbeKind::Return), + ) + .map_err(|io_error| { + let errno = io_error.raw_os_error(); + let is_unsupported = match errno { + Some(code) if code == ENOTSUP || code == EOPNOTSUPP => true, + // Multi-uprobe landed in Linux 6.6, older kernels may return EINVAL + // for the unknown attach type (see BPF_TRACE_UPROBE_MULTI in + // https://elixir.bootlin.com/linux/v6.6/source/include/uapi/linux/bpf.h#L1042). + Some(code) if code == EINVAL => { + KernelVersion::current().is_ok_and(|kv| kv < KernelVersion::new(6, 6, 0)) + } + _ => false, + }; + + if is_unsupported { + ProgramError::UProbeError(UProbeError::UProbeMultiNotSupported) + } else { + ProgramError::SyscallError(SyscallError { + call: "bpf_link_create", + io_error, + }) + } + })?; + Ok(FdLink::new(link_fd).into()) +} + +// In AttachMode::Unknown we probe whether the loaded program should attach via +// BPF_TRACE_UPROBE_MULTI or the legacy single-point perf path. +// +// Fallback is appropriate in two cases: +// - UProbeMultiNotSupported: try_attach_uprobe_multi_link() uses this for +// kernels that do not implement multi-uprobe links, including older kernels +// (< 6.6) that may report the unknown attach type as EINVAL. +// - bpf_link_create(...)=EINVAL: on kernels that do support multi-uprobe links, +// a handle loaded from pin/program info can still hit EINVAL when the loaded +// program expects the legacy per-point attach path rather than the multi +// attach type. In that case we intentionally retry via the per-point path +// instead of surfacing the mode-probing error. +// +// Other errors indicate a real multi-attach failure and are propagated. +fn should_fallback_to_single(error: &ProgramError) -> bool { + match error { + ProgramError::UProbeError(UProbeError::UProbeMultiNotSupported) => true, + ProgramError::SyscallError(SyscallError { + call: "bpf_link_create", + io_error, + }) => io_error.raw_os_error() == Some(EINVAL), + _ => false, + } +} + /// The type returned when attaching an [`UProbe`] fails. #[derive(Debug, Error)] pub enum UProbeError { + /// Automatic attach-mode selection failed in both multi and legacy perf paths. + #[error( + "automatic uprobe attach-mode selection failed: multi={multi_error}; \ + legacy perf={single_error}" + )] + // Box the nested ProgramError values to avoid the recursive + // `ProgramError -> UProbeError -> ProgramError` type. + AttachModeSelectionFailed { + /// Error returned by the multi-uprobe path. + multi_error: Box, + /// Error returned by the legacy perf path. + single_error: Box, + }, + /// There was an error parsing `/etc/ld.so.cache`. #[error("error reading `{}` file", LD_SO_CACHE_FILE)] InvalidLdSoCache { @@ -309,7 +809,7 @@ pub enum UProbeError { io_error: io::Error, }, - /// There was en error fetching the memory map for `pid`. + /// There was an error fetching the memory map for `pid`. #[error("error fetching libs for {pid}")] ProcMap { /// The pid. @@ -318,6 +818,40 @@ pub enum UProbeError { #[source] source: ProcMapError, }, + + /// The kernel does not support multi-uprobe links. + #[error("uprobe_multi links are not supported by the running kernel")] + UProbeMultiNotSupported, + + /// No attach points were provided. + #[error("no uprobe attach points provided for target `{target}`")] + EmptyPoints { + /// Target provided by the caller. + target: PathBuf, + }, + + /// The legacy perf attach path failed for a specific point. + #[error( + "legacy perf attach failed at point #{index}, resolved offset \ + {resolved_offset:#x}: {attach_error}{cleanup}", + cleanup = .cleanup_error + .as_deref() + .map(|error| format!("; cleanup={error}")) + .unwrap_or_default() + )] + // Box the nested ProgramError values to avoid the recursive + // `ProgramError -> UProbeError -> ProgramError` type. + LegacyPerfAttachPointError { + /// Index of the attach point within the caller input slice. + index: usize, + /// Resolved file offset used by the failing perf attach operation. + resolved_offset: u64, + /// Original error returned by the attach syscall path. + #[source] + attach_error: Box, + /// Error returned while detaching already attached points, if cleanup failed. + cleanup_error: Option>, + }, } /// Error reading from /proc/pid/maps. @@ -335,7 +869,7 @@ pub enum ProcMapError { }, } -/// A entry that has been parsed from /proc/`pid`/maps. +/// An entry parsed from /proc/`pid`/maps. /// /// This contains information about a mapped portion of memory /// for the process, ranging from address to `address_end`. @@ -716,34 +1250,6 @@ fn find_debug_path_in_object<'a>( } } -fn find_symbol_in_object<'a>(obj: &'a object::File<'a>, symbol: &str) -> Option> { - obj.dynamic_symbols() - .chain(obj.symbols()) - .find(|sym| sym.name().is_ok_and(|name| name == symbol)) -} - -fn resolve_symbol(path: &Path, symbol: &str) -> Result { - let data = MMap::map_copy_read_only(path)?; - let obj = object::read::File::parse(data.as_ref())?; - - if let Some(sym) = find_symbol_in_object(&obj, symbol) { - symbol_translated_address(&obj, sym, symbol) - } else { - // Only search in the debug object if the symbol was not found in the main object - let debug_path = find_debug_path_in_object(&obj, path, symbol)?; - let debug_data = MMap::map_copy_read_only(&debug_path) - .map_err(|e| ResolveSymbolError::DebuglinkAccessError(debug_path.into_owned(), e))?; - let debug_obj = object::read::File::parse(debug_data.as_ref())?; - - verify_build_ids(&obj, &debug_obj, symbol)?; - - let sym = find_symbol_in_object(&debug_obj, symbol) - .ok_or_else(|| ResolveSymbolError::Unknown(symbol.to_string()))?; - - symbol_translated_address(&debug_obj, sym, symbol) - } -} - fn symbol_translated_address( obj: &object::File<'_>, sym: Symbol<'_, '_>, @@ -772,11 +1278,141 @@ fn symbol_translated_address( #[cfg(test)] mod tests { + use std::os::fd::FromRawFd as _; + use assert_matches::assert_matches; - use object::{Architecture, BinaryFormat, Endianness, write::SectionKind}; + use object::{ + Architecture, BinaryFormat, Endianness, SymbolKind, SymbolScope, + write::{SectionKind, StandardSection, Symbol, SymbolFlags, SymbolSection}, + }; use test_case::test_case; use super::*; + use crate::programs::FdLinkId; + + fn mock_fd_link(raw_fd: i32) -> FdLink { + let fd = unsafe { crate::MockableFd::from_raw_fd(raw_fd) }; + FdLink::new(fd) + } + + #[test] + fn test_resolve_points_one_preserves_cookie() { + let mut points = once(UProbeAttachPoint { + location: UProbeAttachLocation::AbsoluteOffset(0x42), + cookie: Some(0x42), + }); + match resolve_points(Path::new("/proc/self/exe"), points.next().unwrap(), points).unwrap() { + ResolvedPoints::One { + offset: 0x42, + cookie: Some(0x42), + } => {} + ResolvedPoints::One { .. } => panic!("unexpected points"), + ResolvedPoints::Many { .. } => panic!("unexpected points"), + } + } + + #[test] + fn test_resolve_points_many_fills_missing_cookies_with_zero() { + let mut points = [ + UProbeAttachPoint { + location: UProbeAttachLocation::AbsoluteOffset(0x11), + cookie: Some(0x22), + }, + UProbeAttachPoint { + location: UProbeAttachLocation::AbsoluteOffset(0x33), + cookie: None, + }, + ] + .into_iter(); + match resolve_points(Path::new("/proc/self/exe"), points.next().unwrap(), points).unwrap() { + ResolvedPoints::Many { offsets, cookies } => { + assert_eq!(offsets, vec![0x11, 0x33]); + assert_eq!(cookies, Some(vec![0x22, 0])); + } + ResolvedPoints::One { .. } => panic!("unexpected points"), + } + } + + fn create_elf_with_text_symbols( + symbols: &[(&[u8], u64)], + ) -> Result, object::write::Error> { + let mut obj = + object::write::Object::new(BinaryFormat::Elf, Architecture::X86_64, Endianness::Little); + let text_section = obj.section_id(StandardSection::Text); + obj.append_section_data(text_section, &[0; 0x40], 1); + + for &(name, value) in symbols { + obj.add_symbol(Symbol { + name: name.to_vec(), + value, + size: 1, + kind: SymbolKind::Text, + scope: SymbolScope::Linkage, + weak: false, + section: SymbolSection::Section(text_section), + flags: SymbolFlags::None, + }); + } + + obj.write() + } + + #[test] + fn test_resolve_symbols_in_object_resolves_mixed_requests() { + let mut bytes = + create_elf_with_text_symbols(&[(b"first", 0x10), (b"second", 0x24)]).unwrap(); + let align_bytes = aligned_slice(&mut bytes); + let obj = object::File::parse(&*align_bytes).unwrap(); + // Model an original attach-point list shaped like: + // 0: symbol "first" + 0x4 + // 1: absolute offset 0x88 + // 2: symbol "second" + // 3: symbol "first" + 0x8 + // + // Each request is `(attach point index, symbol, additional offset)`. + // There is no request for index 1 because absolute offsets do not need + // symbol resolution. + let requests = [(0, "first", 0x4), (2, "second", 0), (3, "first", 0x8)]; + let symbol_requests = build_symbol_requests(&requests); + let mut resolved_offsets = vec![None; requests.len()]; + + resolve_symbols_in_object(&obj, &symbol_requests, &mut resolved_offsets).unwrap(); + assert_eq!(resolved_offsets, vec![Some(0x10), Some(0x24), Some(0x10)]); + } + + #[test] + fn test_uprobe_link_into_fd_links_single() { + let fd = crate::MockableFd::mock_signed_fd(); + let link = UProbeLink::from(PerfLinkInner::from(mock_fd_link(fd))); + + let fd_links = link.into_fd_links().unwrap(); + assert_eq!( + fd_links + .into_iter() + .map(|link| link.id()) + .collect::>(), + vec![FdLinkId(fd)] + ); + } + + #[test] + fn test_uprobe_link_into_fd_links_many() { + let first_fd = crate::MockableFd::mock_signed_fd(); + let second_fd = first_fd + 1; + let link = UProbeLink::from(PerfLinkInner::Many(vec![ + PerfLinkLeaf::from(mock_fd_link(first_fd)), + PerfLinkLeaf::from(mock_fd_link(second_fd)), + ])); + + let fd_links = link.into_fd_links().unwrap(); + assert_eq!( + fd_links + .into_iter() + .map(|link| link.id()) + .collect::>(), + vec![FdLinkId(first_fd), FdLinkId(second_fd)] + ); + } // Only run this test on with libc dynamically linked so that it can // exercise resolving the path to libc via the current process's memory map. diff --git a/aya/src/sys/bpf.rs b/aya/src/sys/bpf.rs index 91e2d751c4..af307abde8 100644 --- a/aya/src/sys/bpf.rs +++ b/aya/src/sys/bpf.rs @@ -16,10 +16,10 @@ use aya_obj::{ VarLinkage, }, generated::{ - BPF_ADD, BPF_ALU64, BPF_CALL, BPF_DW, BPF_EXIT, BPF_F_REPLACE, BPF_IMM, BPF_JMP, BPF_K, - BPF_LD, BPF_MEM, BPF_MOV, BPF_PSEUDO_MAP_VALUE, BPF_ST, BPF_X, bpf_attach_type, bpf_attr, - bpf_btf_info, bpf_cmd, bpf_func_id, bpf_insn, bpf_link_info, bpf_map_info, bpf_map_type, - bpf_prog_info, bpf_prog_type, bpf_stats_type, + BPF_ADD, BPF_ALU64, BPF_CALL, BPF_DW, BPF_EXIT, BPF_F_REPLACE, BPF_F_UPROBE_MULTI_RETURN, + BPF_IMM, BPF_JMP, BPF_K, BPF_LD, BPF_MEM, BPF_MOV, BPF_PSEUDO_MAP_VALUE, BPF_ST, BPF_X, + bpf_attach_type, bpf_attr, bpf_btf_info, bpf_cmd, bpf_func_id, bpf_insn, bpf_link_info, + bpf_map_info, bpf_map_type, bpf_prog_info, bpf_prog_type, bpf_stats_type, }, maps::{LegacyMap, bpf_map_def}, }; @@ -411,15 +411,26 @@ pub(crate) enum LinkTarget<'f> { Fd(BorrowedFd<'f>), IfIndex(u32), Iter, + None, } // Models https://github.com/torvalds/linux/blob/2144da25/include/uapi/linux/bpf.h#L1724-L1782. pub(crate) enum BpfLinkCreateArgs<'a> { TargetBtfId(u32), // since kernel 5.15 - PerfEvent { bpf_cookie: u64 }, + PerfEvent { + bpf_cookie: u64, + }, // since kernel 6.6 Tcx(&'a LinkRef), + UProbeMulti { + path: &'a CStr, + offsets: &'a [u64], + ref_ctr_offsets: Option<&'a [u64]>, + cookies: Option<&'a [u64]>, + pid: u32, + flags: u32, + }, } // since kernel 5.7 @@ -445,7 +456,7 @@ pub(crate) fn bpf_link_create( // fact, the kernel explicitly rejects non-zero target FDs for // iterators: // https://github.com/torvalds/linux/blob/v6.12/kernel/bpf/bpf_iter.c#L517-L518 - LinkTarget::Iter => {} + LinkTarget::Iter | LinkTarget::None => {} } attr.link_create.attach_type = attach_type.into() as u32; attr.link_create.flags = flags; @@ -477,6 +488,27 @@ pub(crate) fn bpf_link_create( ); }, }, + BpfLinkCreateArgs::UProbeMulti { + path, + offsets, + ref_ctr_offsets, + cookies, + pid, + flags, + } => { + let multi = unsafe { &mut attr.link_create.__bindgen_anon_3.uprobe_multi }; + multi.path = path.as_ptr() as u64; + multi.offsets = offsets.as_ptr() as u64; + multi.cnt = offsets.len() as u32; + multi.flags = flags; + multi.pid = pid; + multi.ref_ctr_offsets = ref_ctr_offsets + .map(|slice| slice.as_ptr() as u64) + .unwrap_or_default(); + multi.cookies = cookies + .map(|slice| slice.as_ptr() as u64) + .unwrap_or_default(); + } } } @@ -484,6 +516,36 @@ pub(crate) fn bpf_link_create( unsafe { fd_sys_bpf(bpf_cmd::BPF_LINK_CREATE, &mut attr) } } +pub(crate) fn bpf_link_create_uprobe_multi( + prog_fd: BorrowedFd<'_>, + path: &CStr, + offsets: &[u64], + ref_ctr_offsets: Option<&[u64]>, + cookies: Option<&[u64]>, + pid: u32, + retprobe: bool, +) -> io::Result { + let args = BpfLinkCreateArgs::UProbeMulti { + path, + offsets, + ref_ctr_offsets, + cookies, + pid, + flags: if retprobe { + BPF_F_UPROBE_MULTI_RETURN + } else { + 0 + }, + }; + bpf_link_create( + prog_fd, + LinkTarget::None, + bpf_attach_type::BPF_TRACE_UPROBE_MULTI, + 0, + Some(args), + ) +} + // since kernel 5.7 pub(crate) fn bpf_link_update( link_fd: BorrowedFd<'_>, diff --git a/test/integration-ebpf/Cargo.toml b/test/integration-ebpf/Cargo.toml index 2a388829a6..25c61319b8 100644 --- a/test/integration-ebpf/Cargo.toml +++ b/test/integration-ebpf/Cargo.toml @@ -136,6 +136,10 @@ path = "src/xdp_sec.rs" name = "uprobe_cookie" path = "src/uprobe_cookie.rs" +[[bin]] +name = "uprobe_multi" +path = "src/uprobe_multi.rs" + [[bin]] name = "perf_event_bp" path = "src/perf_event_bp.rs" diff --git a/test/integration-ebpf/src/uprobe_multi.rs b/test/integration-ebpf/src/uprobe_multi.rs new file mode 100644 index 0000000000..6f2db572f3 --- /dev/null +++ b/test/integration-ebpf/src/uprobe_multi.rs @@ -0,0 +1,32 @@ +#![no_std] +#![no_main] +#![expect(unused_crate_dependencies, reason = "used in other bins")] + +use aya_ebpf::{ + EbpfContext as _, helpers, + macros::{map, uprobe}, + maps::RingBuf, + programs::ProbeContext, +}; +#[cfg(not(test))] +extern crate ebpf_panic; + +#[map] +static RING_BUF: RingBuf = RingBuf::with_byte_size(0, 0); + +#[inline(always)] +fn output_cookie(ctx: &ProbeContext) { + let cookie = unsafe { helpers::bpf_get_attach_cookie(ctx.as_ptr()) }; + let cookie_bytes = cookie.to_ne_bytes(); + let _res = RING_BUF.output::<[u8]>(cookie_bytes, 0); +} + +#[uprobe(multi)] +fn uprobe_multi(ctx: ProbeContext) { + output_cookie(&ctx); +} + +#[uprobe] +fn uprobe_single(ctx: ProbeContext) { + output_cookie(&ctx); +} diff --git a/test/integration-test/src/lib.rs b/test/integration-test/src/lib.rs index 54f07e6cc8..562ebdca39 100644 --- a/test/integration-test/src/lib.rs +++ b/test/integration-test/src/lib.rs @@ -72,6 +72,7 @@ bpf_file!( PROG_ARRAY => "prog_array", STACK_TRACE => "stack_trace", STACK_TRACE_LSM => "stack_trace_lsm", + UPROBE_MULTI => "uprobe_multi", ); #[cfg(test)] diff --git a/test/integration-test/src/tests.rs b/test/integration-test/src/tests.rs index bf8f95f55f..d2532bfeff 100644 --- a/test/integration-test/src/tests.rs +++ b/test/integration-test/src/tests.rs @@ -42,4 +42,5 @@ mod stack_trace_lsm; mod strncmp; mod tcx; mod uprobe_cookie; +mod uprobe_multi; mod xdp; diff --git a/test/integration-test/src/tests/array.rs b/test/integration-test/src/tests/array.rs index d3f2261d00..e3c3e44d43 100644 --- a/test/integration-test/src/tests/array.rs +++ b/test/integration-test/src/tests/array.rs @@ -72,7 +72,7 @@ fn test_array( .unwrap_or_else(|err| panic!("program {prog_name} is not a uprobe: {err}")); prog.load() .unwrap_or_else(|err| panic!("load {prog_name}: {err}")); - prog.attach(symbol, "/proc/self/exe", UProbeScope::AllProcesses) + prog.attach([symbol], "/proc/self/exe", UProbeScope::AllProcesses) .unwrap_or_else(|err| panic!("attach {prog_name}: {err}")); } diff --git a/test/integration-test/src/tests/bloom_filter.rs b/test/integration-test/src/tests/bloom_filter.rs index fa0a1023f3..33329e3923 100644 --- a/test/integration-test/src/tests/bloom_filter.rs +++ b/test/integration-test/src/tests/bloom_filter.rs @@ -57,7 +57,7 @@ fn bloom_filter_basic(result_map: &str, filter_map: &str, insert_prog: &str, con .unwrap_or_else(|err| panic!("program {prog_name} is not a uprobe: {err}")); prog.load() .unwrap_or_else(|err| panic!("load {prog_name}: {err}")); - prog.attach(symbol, "/proc/self/exe", UProbeScope::AllProcesses) + prog.attach([symbol], "/proc/self/exe", UProbeScope::AllProcesses) .unwrap_or_else(|err| panic!("attach {prog_name}: {err}")); } diff --git a/test/integration-test/src/tests/bpf_probe_read.rs b/test/integration-test/src/tests/bpf_probe_read.rs index cc324ab738..4f473bea01 100644 --- a/test/integration-test/src/tests/bpf_probe_read.rs +++ b/test/integration-test/src/tests/bpf_probe_read.rs @@ -103,7 +103,7 @@ fn load_and_attach_uprobe(prog_name: &str, func_name: &str, bytes: &[u8]) -> Ebp let prog: &mut UProbe = bpf.program_mut(prog_name).unwrap().try_into().unwrap(); prog.load().unwrap(); - prog.attach(func_name, "/proc/self/exe", UProbeScope::AllProcesses) + prog.attach([func_name], "/proc/self/exe", UProbeScope::AllProcesses) .unwrap(); bpf diff --git a/test/integration-test/src/tests/btf_relocations.rs b/test/integration-test/src/tests/btf_relocations.rs index 3d7757bff7..4361383984 100644 --- a/test/integration-test/src/tests/btf_relocations.rs +++ b/test/integration-test/src/tests/btf_relocations.rs @@ -95,7 +95,7 @@ fn relocation_tests( program.load().unwrap(); program .attach( - "trigger_btf_relocations_program", + ["trigger_btf_relocations_program"], "/proc/self/exe", UProbeScope::AllProcesses, ) diff --git a/test/integration-test/src/tests/info.rs b/test/integration-test/src/tests/info.rs index 44fc95c830..5801a7cd2a 100644 --- a/test/integration-test/src/tests/info.rs +++ b/test/integration-test/src/tests/info.rs @@ -70,7 +70,7 @@ fn test_loaded_programs() { // Ensure we can perform basic operations on the re-created program. let res = p .attach( - "uprobe_function", + ["uprobe_function"], "/proc/self/exe", UProbeScope::AllProcesses, ) diff --git a/test/integration-test/src/tests/linear_data_structures.rs b/test/integration-test/src/tests/linear_data_structures.rs index bf403d425a..eb9ba30760 100644 --- a/test/integration-test/src/tests/linear_data_structures.rs +++ b/test/integration-test/src/tests/linear_data_structures.rs @@ -51,7 +51,7 @@ macro_rules! define_linear_ds_host_test { ] { let prog: &mut UProbe = bpf.program_mut(prog_name).unwrap().try_into().unwrap(); prog.load().unwrap(); - prog.attach(symbol, "/proc/self/exe", UProbeScope::AllProcesses) + prog.attach([symbol], "/proc/self/exe", UProbeScope::AllProcesses) .unwrap(); } let array_map = bpf.map("RESULT").unwrap(); diff --git a/test/integration-test/src/tests/load.rs b/test/integration-test/src/tests/load.rs index d42eee9b7a..8ead9e33ef 100644 --- a/test/integration-test/src/tests/load.rs +++ b/test/integration-test/src/tests/load.rs @@ -59,7 +59,7 @@ fn ringbuffer_btf_map() { let prog: &mut UProbe = bpf.program_mut("bpf_prog").unwrap().try_into().unwrap(); prog.load().unwrap(); prog.attach( - "trigger_bpf_program", + ["trigger_bpf_program"], "/proc/self/exe", UProbeScope::AllProcesses, ) @@ -85,7 +85,7 @@ fn multiple_btf_maps() { let prog: &mut UProbe = bpf.program_mut("bpf_prog").unwrap().try_into().unwrap(); prog.load().unwrap(); prog.attach( - "trigger_bpf_program", + ["trigger_bpf_program"], "/proc/self/exe", UProbeScope::AllProcesses, ) @@ -139,7 +139,7 @@ fn pin_lifecycle_multiple_btf_maps() { let prog: &mut UProbe = bpf.program_mut("bpf_prog").unwrap().try_into().unwrap(); prog.load().unwrap(); prog.attach( - "trigger_bpf_program", + ["trigger_bpf_program"], "/proc/self/exe", UProbeScope::AllProcesses, ) @@ -379,7 +379,7 @@ fn basic_uprobe_scopes(scope: UProbeScope) { let program_name = "test_uprobe"; let attach = |prog: &mut P| { - prog.attach("uprobe_function", "/proc/self/exe", scope) + prog.attach(["uprobe_function"], "/proc/self/exe", scope) .unwrap() }; @@ -683,7 +683,7 @@ fn pin_lifecycle_uprobe() { let program_name = "test_uprobe"; let attach = |prog: &mut P| { prog.attach( - "uprobe_function", + ["uprobe_function"], "/proc/self/exe", UProbeScope::AllProcesses, ) diff --git a/test/integration-test/src/tests/log.rs b/test/integration-test/src/tests/log.rs index 133b97948b..5ff9808675 100644 --- a/test/integration-test/src/tests/log.rs +++ b/test/integration-test/src/tests/log.rs @@ -74,7 +74,7 @@ fn log() { let prog: &mut UProbe = bpf.program_mut("test_log").unwrap().try_into().unwrap(); prog.load().unwrap(); prog.attach( - "trigger_ebpf_program", + ["trigger_ebpf_program"], "/proc/self/exe", UProbeScope::AllProcesses, ) @@ -257,7 +257,7 @@ fn log_level_only_error_warn() { let prog: &mut UProbe = bpf.program_mut("test_log").unwrap().try_into().unwrap(); prog.load().unwrap(); prog.attach( - "trigger_ebpf_program", + ["trigger_ebpf_program"], "/proc/self/exe", UProbeScope::AllProcesses, ) @@ -316,7 +316,7 @@ fn log_level_prevents_verif_fail() { .unwrap(); prog.load().unwrap(); prog.attach( - "trigger_ebpf_program", + ["trigger_ebpf_program"], "/proc/self/exe", UProbeScope::AllProcesses, ) diff --git a/test/integration-test/src/tests/lpm_trie.rs b/test/integration-test/src/tests/lpm_trie.rs index bbe28e087f..425b76a466 100644 --- a/test/integration-test/src/tests/lpm_trie.rs +++ b/test/integration-test/src/tests/lpm_trie.rs @@ -37,7 +37,7 @@ fn lpm_trie_basic(prog_name: &str, routes_map: &str, results_map: &str) { prog.load() .unwrap_or_else(|err| panic!("load {prog_name}: {err}")); prog.attach( - "trigger_lpm_trie", + ["trigger_lpm_trie"], "/proc/self/exe", UProbeScope::AllProcesses, ) diff --git a/test/integration-test/src/tests/maps_disjoint.rs b/test/integration-test/src/tests/maps_disjoint.rs index 4aae76b4ae..ab9a2de4bb 100644 --- a/test/integration-test/src/tests/maps_disjoint.rs +++ b/test/integration-test/src/tests/maps_disjoint.rs @@ -21,7 +21,7 @@ fn test_maps_disjoint() { prog.load().unwrap(); prog.attach( - "trigger_ebpf_program_maps_disjoint", + ["trigger_ebpf_program_maps_disjoint"], "/proc/self/exe", UProbeScope::AllProcesses, ) diff --git a/test/integration-test/src/tests/per_cpu_array.rs b/test/integration-test/src/tests/per_cpu_array.rs index 9f217c9a2a..e6a8da0e27 100644 --- a/test/integration-test/src/tests/per_cpu_array.rs +++ b/test/integration-test/src/tests/per_cpu_array.rs @@ -81,7 +81,7 @@ fn per_cpu_array_basic( .unwrap_or_else(|err| panic!("program {prog_name} is not a uprobe: {err}")); prog.load() .unwrap_or_else(|err| panic!("load {prog_name}: {err}")); - prog.attach(symbol, "/proc/self/exe", UProbeScope::AllProcesses) + prog.attach([symbol], "/proc/self/exe", UProbeScope::AllProcesses) .unwrap_or_else(|err| panic!("attach {prog_name}: {err}")); } diff --git a/test/integration-test/src/tests/perf_event_array.rs b/test/integration-test/src/tests/perf_event_array.rs index 4c3affc04f..e9fe14acf4 100644 --- a/test/integration-test/src/tests/perf_event_array.rs +++ b/test/integration-test/src/tests/perf_event_array.rs @@ -40,7 +40,7 @@ fn emit_event(bpf_obj: &[u8], events_map: &str, prog: &str) { .unwrap_or_else(|err| panic!("load {prog}: {err}")); uprobe .attach( - "trigger_emit_event", + ["trigger_emit_event"], "/proc/self/exe", UProbeScope::AllProcesses, ) diff --git a/test/integration-test/src/tests/printk.rs b/test/integration-test/src/tests/printk.rs index 02f3c4bf66..2a8a97713d 100644 --- a/test/integration-test/src/tests/printk.rs +++ b/test/integration-test/src/tests/printk.rs @@ -82,7 +82,7 @@ async fn bpf_printk( .unwrap(); prog.load().unwrap(); prog.attach( - "trigger_bpf_printk", + ["trigger_bpf_printk"], "/proc/self/exe", UProbeScope::AllProcesses, ) diff --git a/test/integration-test/src/tests/prog_array.rs b/test/integration-test/src/tests/prog_array.rs index fd4d488cfb..4e086e59e9 100644 --- a/test/integration-test/src/tests/prog_array.rs +++ b/test/integration-test/src/tests/prog_array.rs @@ -50,7 +50,7 @@ fn tail_call_empty(result_map: &str, entry_prog: &str) { prog.load() .unwrap_or_else(|err| panic!("load {entry_prog}: {err}")); prog.attach( - "trigger_tail_call_empty", + ["trigger_tail_call_empty"], "/proc/self/exe", UProbeScope::AllProcesses, ) @@ -114,7 +114,7 @@ fn tail_call_success(result_map: &str, array_map: &str, entry_prog: &str, target .unwrap_or_else(|err| panic!("load {entry_prog}: {err}")); entry .attach( - "trigger_tail_call_success", + ["trigger_tail_call_success"], "/proc/self/exe", UProbeScope::AllProcesses, ) diff --git a/test/integration-test/src/tests/relocations.rs b/test/integration-test/src/tests/relocations.rs index cb5e27164e..24af1f0f82 100644 --- a/test/integration-test/src/tests/relocations.rs +++ b/test/integration-test/src/tests/relocations.rs @@ -56,7 +56,7 @@ fn load_and_attach(name: &str, bytes: &[u8]) -> Ebpf { prog.load().unwrap(); prog.attach( - "trigger_relocations_program", + ["trigger_relocations_program"], "/proc/self/exe", UProbeScope::AllProcesses, ) diff --git a/test/integration-test/src/tests/ring_buf.rs b/test/integration-test/src/tests/ring_buf.rs index 613dd04648..9c9e6658e8 100644 --- a/test/integration-test/src/tests/ring_buf.rs +++ b/test/integration-test/src/tests/ring_buf.rs @@ -91,7 +91,7 @@ impl RingBufTest { let prog: &mut UProbe = bpf.program_mut(variant.prog).unwrap().try_into().unwrap(); prog.load().unwrap(); prog.attach( - "ring_buf_trigger_ebpf_program", + ["ring_buf_trigger_ebpf_program"], "/proc/self/exe", UProbeScope::AllProcesses, ) @@ -203,8 +203,12 @@ fn ring_buf_mismatch_size( let mut ring_buf = RingBuf::try_from(ring_buf).unwrap(); let prog: &mut UProbe = bpf.program_mut(prog).unwrap().try_into().unwrap(); prog.load().unwrap(); - prog.attach(trigger_symbol, "/proc/self/exe", UProbeScope::AllProcesses) - .unwrap(); + prog.attach( + [trigger_symbol], + "/proc/self/exe", + UProbeScope::AllProcesses, + ) + .unwrap(); trigger(value.into()); { diff --git a/test/integration-test/src/tests/stack_trace.rs b/test/integration-test/src/tests/stack_trace.rs index 23ca10583e..f5fd0d3df0 100644 --- a/test/integration-test/src/tests/stack_trace.rs +++ b/test/integration-test/src/tests/stack_trace.rs @@ -36,7 +36,7 @@ fn record_stackid(stacks_map: &str, result_map: &str, prog: &str) { .unwrap_or_else(|err| panic!("load {prog}: {err}")); uprobe .attach( - "trigger_record_stackid", + ["trigger_record_stackid"], "/proc/self/exe", UProbeScope::AllProcesses, ) diff --git a/test/integration-test/src/tests/strncmp.rs b/test/integration-test/src/tests/strncmp.rs index db0fce3e81..de948b0728 100644 --- a/test/integration-test/src/tests/strncmp.rs +++ b/test/integration-test/src/tests/strncmp.rs @@ -30,7 +30,7 @@ fn bpf_strncmp() { prog.load().unwrap(); prog.attach( - "trigger_bpf_strncmp", + ["trigger_bpf_strncmp"], "/proc/self/exe", UProbeScope::AllProcesses, ) diff --git a/test/integration-test/src/tests/uprobe_cookie.rs b/test/integration-test/src/tests/uprobe_cookie.rs index 1a7f08d22c..225f5e64fa 100644 --- a/test/integration-test/src/tests/uprobe_cookie.rs +++ b/test/integration-test/src/tests/uprobe_cookie.rs @@ -17,8 +17,11 @@ fn test_uprobe_cookie() { ); return; } - const RING_BUF_BYTE_SIZE: u32 = 512; // arbitrary, but big enough - + // Ring buffer sizes are rounded up to a page-sized power-of-two multiple when + // the object is loaded. Using 512 here therefore yields a one-page ring + // buffer on supported test systems, which is ample for the handful of `u64` + // cookie records emitted by this test. + const RING_BUF_BYTE_SIZE: u32 = 512; let mut bpf = EbpfLoader::new() .map_max_entries("RING_BUF", RING_BUF_BYTE_SIZE) .load(crate::UPROBE_COOKIE) @@ -35,10 +38,10 @@ fn test_uprobe_cookie() { const PROG_B: &str = "uprobe_cookie_trigger_ebpf_program_b"; let attach = |prog: &mut UProbe, fn_name: &str, cookie| { prog.attach( - UProbeAttachPoint { + [UProbeAttachPoint { location: fn_name.into(), cookie: Some(cookie), - }, + }], "/proc/self/exe", UProbeScope::AllProcesses, ) diff --git a/test/integration-test/src/tests/uprobe_multi.rs b/test/integration-test/src/tests/uprobe_multi.rs new file mode 100644 index 0000000000..f39323314f --- /dev/null +++ b/test/integration-test/src/tests/uprobe_multi.rs @@ -0,0 +1,311 @@ +use std::path::Path; + +use aya::{ + EbpfLoader, + maps::ring_buf::RingBuf, + programs::{ + ProbeKind, ProgramError, UProbe, + uprobe::{UProbeAttachLocation, UProbeAttachPoint, UProbeError, UProbeScope}, + }, + util::KernelVersion, +}; + +const PROG_A: &str = "uprobe_multi_trigger_program_a"; +const PROG_B: &str = "uprobe_multi_trigger_program_b"; +const PROG_NO_COOKIE: &str = "uprobe_multi_trigger_program_no_cookie"; +const PROG_SYMBOL_INVALID: &str = "uprobe_multi_missing_symbol"; +const UPROBE_MULTI: &str = "uprobe_multi"; +const UPROBE_SINGLE: &str = "uprobe_single"; +// Ring buffer sizes are rounded up to a page-sized power-of-two multiple when +// the object is loaded. Using 512 here therefore yields a one-page ring +// buffer on supported test systems, which is ample for the handful of `u64` +// cookie records emitted by these tests. +const RING_BUF_BYTE_SIZE: u32 = 512; + +#[test_log::test] +fn test_uprobe_attach_multi() { + if !aya::features().bpf_cookie() { + eprintln!( + "skipping test: bpf_get_attach_cookie is unsupported so the test program cannot load" + ); + return; + } + let mut bpf = EbpfLoader::new() + .map_max_entries("RING_BUF", RING_BUF_BYTE_SIZE) + .load(crate::UPROBE_MULTI) + .unwrap(); + let ring_buf = bpf.take_map("RING_BUF").unwrap(); + let mut ring_buf = RingBuf::try_from(ring_buf).unwrap(); + let prog: &mut UProbe = bpf.program_mut(UPROBE_MULTI).unwrap().try_into().unwrap(); + prog.load().unwrap(); + + const COOKIE_A: u64 = 0x11; + const COOKIE_B: u64 = 0x22; + + let points = [ + UProbeAttachPoint { + location: UProbeAttachLocation::Symbol(PROG_A), + cookie: Some(COOKIE_A), + }, + UProbeAttachPoint { + location: UProbeAttachLocation::Symbol(PROG_B), + cookie: Some(COOKIE_B), + }, + UProbeAttachPoint { + location: UProbeAttachLocation::Symbol(PROG_NO_COOKIE), + cookie: None, + }, + ]; + let attach_res = prog.attach( + &points, + Path::new("/proc/self/exe"), + UProbeScope::AllProcesses, + ); + let link_id = match attach_res { + Ok(link) => link, + Err(ProgramError::UProbeError(UProbeError::UProbeMultiNotSupported)) => { + let kernel_version = KernelVersion::current().unwrap(); + let multi_min = KernelVersion::new(6, 6, 0); + assert!( + kernel_version < multi_min, + "kernel {kernel_version:?} is >= 6.6 but returned UProbeMultiNotSupported" + ); + return; + } + Err(err) => panic!("attach failed: {err:?}"), + }; + + // Drain any stale events emitted by other tests before we generate fresh ones. + while ring_buf.next().is_some() {} + + uprobe_multi_trigger_program_a(); + uprobe_multi_trigger_program_b(); + uprobe_multi_trigger_program_no_cookie(); + uprobe_multi_trigger_program_a(); + + prog.detach(link_id).unwrap(); + // Call again, expect no events because detach removed all active links. + uprobe_multi_trigger_program_a(); + + const EXP: &[u64] = &[COOKIE_A, COOKIE_B, 0, COOKIE_A]; + let mut seen = Vec::new(); + while let Some(record) = ring_buf.next() { + let data = record.as_ref(); + match data.try_into() { + Ok(bytes) => seen.push(u64::from_ne_bytes(bytes)), + Err(std::array::TryFromSliceError { .. }) => { + panic!("invalid ring buffer data: {data:x?}") + } + } + } + assert_eq!(seen, EXP); +} + +#[test_log::test] +fn test_uprobe_unknown_program_falls_back_to_multiple_single_points() { + if !aya::features().bpf_cookie() { + eprintln!( + "skipping test: bpf_get_attach_cookie is unsupported so the test program cannot load" + ); + return; + } + let mut bpf = EbpfLoader::new() + .map_max_entries("RING_BUF", RING_BUF_BYTE_SIZE) + .load(crate::UPROBE_MULTI) + .unwrap(); + let ring_buf = bpf.take_map("RING_BUF").unwrap(); + let mut ring_buf = RingBuf::try_from(ring_buf).unwrap(); + let info = { + let prog: &mut UProbe = bpf.program_mut(UPROBE_SINGLE).unwrap().try_into().unwrap(); + prog.load().unwrap(); + prog.info().unwrap() + }; + // Handles reconstructed from program info do not know whether the original + // section was `uprobe` or `uprobe.multi`, so attach must probe and fall back. + let mut prog = + unsafe { UProbe::from_program_info(info, UPROBE_SINGLE.into(), ProbeKind::Entry) }.unwrap(); + + const COOKIE_A: u64 = 0x33; + const COOKIE_B: u64 = 0x44; + let points = [ + UProbeAttachPoint { + location: UProbeAttachLocation::Symbol(PROG_A), + cookie: Some(COOKIE_A), + }, + UProbeAttachPoint { + location: UProbeAttachLocation::Symbol(PROG_B), + cookie: Some(COOKIE_B), + }, + ]; + let link_id = prog + .attach( + &points, + Path::new("/proc/self/exe"), + UProbeScope::AllProcesses, + ) + .expect("unknown-mode multi-point attach should fall back to single attach"); + + // Drain any stale events emitted by other tests before we generate fresh ones. + while ring_buf.next().is_some() {} + + uprobe_multi_trigger_program_a(); + uprobe_multi_trigger_program_b(); + + prog.detach(link_id).unwrap(); + // Call again, expect no events because detach removed all underlying links. + uprobe_multi_trigger_program_a(); + uprobe_multi_trigger_program_b(); + + // Attach the same unknown-mode handle again. The first attach should have + // selected the legacy per-point mode for this ordinary `uprobe` program; a + // second attach verifies the handle remains usable after fallback and detach. + let link_id = prog + .attach( + &points, + Path::new("/proc/self/exe"), + UProbeScope::AllProcesses, + ) + .expect("unknown-mode fallback should allow attaching again"); + + uprobe_multi_trigger_program_a(); + uprobe_multi_trigger_program_b(); + + prog.detach(link_id).unwrap(); + + let mut seen = Vec::new(); + while let Some(record) = ring_buf.next() { + let data = record.as_ref(); + match data.try_into() { + Ok(bytes) => seen.push(u64::from_ne_bytes(bytes)), + Err(std::array::TryFromSliceError { .. }) => { + panic!("invalid ring buffer data: {data:x?}") + } + } + } + assert_eq!(seen, vec![COOKIE_A, COOKIE_B, COOKIE_A, COOKIE_B]); +} + +#[test_log::test] +fn test_uprobe_single_program_composite_link_drop_detaches_all_points() { + if !aya::features().bpf_cookie() { + eprintln!( + "skipping test: bpf_get_attach_cookie is unsupported so the test program cannot load" + ); + return; + } + let mut bpf = EbpfLoader::new() + .map_max_entries("RING_BUF", RING_BUF_BYTE_SIZE) + .load(crate::UPROBE_MULTI) + .unwrap(); + let ring_buf = bpf.take_map("RING_BUF").unwrap(); + let mut ring_buf = RingBuf::try_from(ring_buf).unwrap(); + let prog: &mut UProbe = bpf.program_mut(UPROBE_SINGLE).unwrap().try_into().unwrap(); + prog.load().unwrap(); + + const COOKIE_A: u64 = 0x55; + const COOKIE_B: u64 = 0x66; + let points = [ + UProbeAttachPoint { + location: UProbeAttachLocation::Symbol(PROG_A), + cookie: Some(COOKIE_A), + }, + UProbeAttachPoint { + location: UProbeAttachLocation::Symbol(PROG_B), + cookie: Some(COOKIE_B), + }, + ]; + let link_id = prog + .attach( + &points, + Path::new("/proc/self/exe"), + UProbeScope::AllProcesses, + ) + .expect("legacy single-mode multi-point attach should succeed"); + + // Drain any stale events emitted by other tests before we generate fresh ones. + while ring_buf.next().is_some() {} + + uprobe_multi_trigger_program_a(); + uprobe_multi_trigger_program_b(); + + let link = prog.take_link(link_id).unwrap(); + drop(link); + + // Call again, expect no events because dropping the managed composite link + // must detach all underlying single-point links. + uprobe_multi_trigger_program_a(); + uprobe_multi_trigger_program_b(); + + let mut seen = Vec::new(); + while let Some(record) = ring_buf.next() { + let data = record.as_ref(); + match data.try_into() { + Ok(bytes) => seen.push(u64::from_ne_bytes(bytes)), + Err(std::array::TryFromSliceError { .. }) => { + panic!("invalid ring buffer data: {data:x?}") + } + } + } + assert_eq!(seen, vec![COOKIE_A, COOKIE_B]); +} + +#[test_log::test] +fn test_uprobe_attach_multi_invalid_symbol() { + if !aya::features().bpf_cookie() { + eprintln!( + "skipping test: bpf_get_attach_cookie is unsupported so the test program cannot load" + ); + return; + } + let mut bpf = EbpfLoader::new() + .map_max_entries("RING_BUF", RING_BUF_BYTE_SIZE) + .load(crate::UPROBE_MULTI) + .unwrap(); + let prog: &mut UProbe = bpf.program_mut(UPROBE_MULTI).unwrap().try_into().unwrap(); + prog.load().unwrap(); + + let points = [PROG_A, PROG_SYMBOL_INVALID]; + + let attach_res = prog.attach( + &points, + Path::new("/proc/self/exe"), + UProbeScope::AllProcesses, + ); + match attach_res { + Err(ProgramError::UProbeError(UProbeError::SymbolError { symbol, .. })) => { + assert_eq!(symbol, PROG_SYMBOL_INVALID); + } + Err(ProgramError::UProbeError(UProbeError::UProbeMultiNotSupported)) => { + let kernel_version = KernelVersion::current().unwrap(); + // Multi-uprobe landed in Linux 6.6 (see BPF_TRACE_UPROBE_MULTI in + // https://elixir.bootlin.com/linux/v6.6/source/include/uapi/linux/bpf.h#L1042). + let multi_min = KernelVersion::new(6, 6, 0); + assert!( + kernel_version < multi_min, + "kernel {kernel_version:?} is >= 6.6 but returned UProbeMultiNotSupported" + ); + } + Err(err) => panic!("unexpected error for invalid symbol: {err:?}"), + Ok(link) => panic!("attach succeeded for invalid symbol: {link:?}"), + } +} + +// Keep these bodies distinct so link-time ICF does not fold the symbols onto +// the same address; the exact values are otherwise irrelevant to the tests. +#[unsafe(no_mangle)] +#[inline(never)] +extern "C" fn uprobe_multi_trigger_program_a() { + std::hint::black_box(0u64); +} + +#[unsafe(no_mangle)] +#[inline(never)] +extern "C" fn uprobe_multi_trigger_program_b() { + std::hint::black_box(1u64); +} + +#[unsafe(no_mangle)] +#[inline(never)] +extern "C" fn uprobe_multi_trigger_program_no_cookie() { + std::hint::black_box(2u64); +} diff --git a/xtask/public-api/aya-obj.txt b/xtask/public-api/aya-obj.txt index 92988819d5..7873b0bdf4 100644 --- a/xtask/public-api/aya-obj.txt +++ b/xtask/public-api/aya-obj.txt @@ -4738,8 +4738,10 @@ pub aya_obj::obj::ProgramSection::SockOps pub aya_obj::obj::ProgramSection::SocketFilter pub aya_obj::obj::ProgramSection::TracePoint pub aya_obj::obj::ProgramSection::UProbe +pub aya_obj::obj::ProgramSection::UProbe::multi: bool pub aya_obj::obj::ProgramSection::UProbe::sleepable: bool pub aya_obj::obj::ProgramSection::URetProbe +pub aya_obj::obj::ProgramSection::URetProbe::multi: bool pub aya_obj::obj::ProgramSection::URetProbe::sleepable: bool pub aya_obj::obj::ProgramSection::Xdp pub aya_obj::obj::ProgramSection::Xdp::attach_type: aya_obj::programs::xdp::XdpAttachType @@ -5333,8 +5335,10 @@ pub aya_obj::ProgramSection::SockOps pub aya_obj::ProgramSection::SocketFilter pub aya_obj::ProgramSection::TracePoint pub aya_obj::ProgramSection::UProbe +pub aya_obj::ProgramSection::UProbe::multi: bool pub aya_obj::ProgramSection::UProbe::sleepable: bool pub aya_obj::ProgramSection::URetProbe +pub aya_obj::ProgramSection::URetProbe::multi: bool pub aya_obj::ProgramSection::URetProbe::sleepable: bool pub aya_obj::ProgramSection::Xdp pub aya_obj::ProgramSection::Xdp::attach_type: aya_obj::programs::xdp::XdpAttachType diff --git a/xtask/public-api/aya.txt b/xtask/public-api/aya.txt index d6acf99a74..a8cdab4164 100644 --- a/xtask/public-api/aya.txt +++ b/xtask/public-api/aya.txt @@ -3448,6 +3448,23 @@ impl core::marker::UnsafeUnpin for aya::programs::lsm_cgroup::LsmLinkId impl core::panic::unwind_safe::RefUnwindSafe for aya::programs::lsm_cgroup::LsmLinkId impl core::panic::unwind_safe::UnwindSafe for aya::programs::lsm_cgroup::LsmLinkId pub mod aya::programs::perf_attach +pub struct aya::programs::perf_attach::PerfLinkDetachError(_) +impl aya::programs::perf_attach::PerfLinkDetachError +pub fn aya::programs::perf_attach::PerfLinkDetachError::as_slice(&self) -> &[aya::programs::ProgramError] +impl core::convert::From for aya::programs::ProgramError +pub fn aya::programs::ProgramError::from(source: aya::programs::perf_attach::PerfLinkDetachError) -> Self +impl core::error::Error for aya::programs::perf_attach::PerfLinkDetachError +impl core::fmt::Debug for aya::programs::perf_attach::PerfLinkDetachError +pub fn aya::programs::perf_attach::PerfLinkDetachError::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::fmt::Display for aya::programs::perf_attach::PerfLinkDetachError +pub fn aya::programs::perf_attach::PerfLinkDetachError::fmt(&self, __formatter: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::Freeze for aya::programs::perf_attach::PerfLinkDetachError +impl core::marker::Send for aya::programs::perf_attach::PerfLinkDetachError +impl core::marker::Sync for aya::programs::perf_attach::PerfLinkDetachError +impl core::marker::Unpin for aya::programs::perf_attach::PerfLinkDetachError +impl core::marker::UnsafeUnpin for aya::programs::perf_attach::PerfLinkDetachError +impl !core::panic::unwind_safe::RefUnwindSafe for aya::programs::perf_attach::PerfLinkDetachError +impl !core::panic::unwind_safe::UnwindSafe for aya::programs::perf_attach::PerfLinkDetachError pub struct aya::programs::perf_attach::PerfLinkId(_) impl core::cmp::Eq for aya::programs::perf_attach::PerfLinkId impl core::cmp::PartialEq for aya::programs::perf_attach::PerfLinkId @@ -4726,10 +4743,19 @@ pub enum aya::programs::uprobe::UProbeAttachLocation<'a> pub aya::programs::uprobe::UProbeAttachLocation::AbsoluteOffset(u64) pub aya::programs::uprobe::UProbeAttachLocation::Symbol(&'a str) pub aya::programs::uprobe::UProbeAttachLocation::SymbolOffset(&'a str, u64) +impl core::convert::From<&u64> for aya::programs::uprobe::UProbeAttachLocation<'static> +pub fn aya::programs::uprobe::UProbeAttachLocation<'static>::from(offset: &u64) -> Self impl core::convert::From for aya::programs::uprobe::UProbeAttachLocation<'static> pub fn aya::programs::uprobe::UProbeAttachLocation<'static>::from(offset: u64) -> Self +impl<'a> core::clone::Clone for aya::programs::uprobe::UProbeAttachLocation<'a> +pub fn aya::programs::uprobe::UProbeAttachLocation<'a>::clone(&self) -> aya::programs::uprobe::UProbeAttachLocation<'a> +impl<'a> core::convert::From<&&'a str> for aya::programs::uprobe::UProbeAttachLocation<'a> +pub fn aya::programs::uprobe::UProbeAttachLocation<'a>::from(s: &&'a str) -> Self impl<'a> core::convert::From<&'a str> for aya::programs::uprobe::UProbeAttachLocation<'a> pub fn aya::programs::uprobe::UProbeAttachLocation<'a>::from(s: &'a str) -> Self +impl<'a> core::fmt::Debug for aya::programs::uprobe::UProbeAttachLocation<'a> +pub fn aya::programs::uprobe::UProbeAttachLocation<'a>::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl<'a> core::marker::Copy for aya::programs::uprobe::UProbeAttachLocation<'a> impl<'a> core::marker::Freeze for aya::programs::uprobe::UProbeAttachLocation<'a> impl<'a> core::marker::Send for aya::programs::uprobe::UProbeAttachLocation<'a> impl<'a> core::marker::Sync for aya::programs::uprobe::UProbeAttachLocation<'a> @@ -4738,6 +4764,11 @@ impl<'a> core::marker::UnsafeUnpin for aya::programs::uprobe::UProbeAttachLocati impl<'a> core::panic::unwind_safe::RefUnwindSafe for aya::programs::uprobe::UProbeAttachLocation<'a> impl<'a> core::panic::unwind_safe::UnwindSafe for aya::programs::uprobe::UProbeAttachLocation<'a> pub enum aya::programs::uprobe::UProbeError +pub aya::programs::uprobe::UProbeError::AttachModeSelectionFailed +pub aya::programs::uprobe::UProbeError::AttachModeSelectionFailed::multi_error: alloc::boxed::Box +pub aya::programs::uprobe::UProbeError::AttachModeSelectionFailed::single_error: alloc::boxed::Box +pub aya::programs::uprobe::UProbeError::EmptyPoints +pub aya::programs::uprobe::UProbeError::EmptyPoints::target: std::path::PathBuf pub aya::programs::uprobe::UProbeError::FileError pub aya::programs::uprobe::UProbeError::FileError::filename: std::path::PathBuf pub aya::programs::uprobe::UProbeError::FileError::io_error: std::io::error::Error @@ -4745,12 +4776,18 @@ pub aya::programs::uprobe::UProbeError::InvalidLdSoCache pub aya::programs::uprobe::UProbeError::InvalidLdSoCache::io_error: &'static std::io::error::Error pub aya::programs::uprobe::UProbeError::InvalidTarget pub aya::programs::uprobe::UProbeError::InvalidTarget::path: std::path::PathBuf +pub aya::programs::uprobe::UProbeError::LegacyPerfAttachPointError +pub aya::programs::uprobe::UProbeError::LegacyPerfAttachPointError::attach_error: alloc::boxed::Box +pub aya::programs::uprobe::UProbeError::LegacyPerfAttachPointError::cleanup_error: core::option::Option> +pub aya::programs::uprobe::UProbeError::LegacyPerfAttachPointError::index: usize +pub aya::programs::uprobe::UProbeError::LegacyPerfAttachPointError::resolved_offset: u64 pub aya::programs::uprobe::UProbeError::ProcMap pub aya::programs::uprobe::UProbeError::ProcMap::pid: u32 pub aya::programs::uprobe::UProbeError::ProcMap::source: aya::programs::uprobe::ProcMapError pub aya::programs::uprobe::UProbeError::SymbolError pub aya::programs::uprobe::UProbeError::SymbolError::error: alloc::boxed::Box<(dyn core::error::Error + core::marker::Send + core::marker::Sync)> pub aya::programs::uprobe::UProbeError::SymbolError::symbol: alloc::string::String +pub aya::programs::uprobe::UProbeError::UProbeMultiNotSupported impl core::convert::From for aya::programs::ProgramError pub fn aya::programs::ProgramError::from(source: aya::programs::uprobe::UProbeError) -> Self impl core::error::Error for aya::programs::uprobe::UProbeError @@ -4785,7 +4822,7 @@ impl core::panic::unwind_safe::UnwindSafe for aya::programs::uprobe::UProbeScope pub struct aya::programs::uprobe::UProbe impl aya::programs::uprobe::UProbe pub const aya::programs::uprobe::UProbe::PROGRAM_TYPE: aya::programs::ProgramType -pub fn aya::programs::uprobe::UProbe::attach<'a, T: core::convert::AsRef, Point: core::convert::Into>>(&mut self, point: Point, target: T, scope: aya::programs::uprobe::UProbeScope) -> core::result::Result +pub fn aya::programs::uprobe::UProbe::attach<'a, T, I>(&mut self, points: I, target: T, scope: aya::programs::uprobe::UProbeScope) -> core::result::Result where T: core::convert::AsRef, I: core::iter::traits::collect::IntoIterator, ::Item: core::convert::Into> pub fn aya::programs::uprobe::UProbe::from_pin>(path: P, kind: aya::programs::ProbeKind) -> core::result::Result pub const fn aya::programs::uprobe::UProbe::kind(&self) -> aya::programs::ProbeKind pub fn aya::programs::uprobe::UProbe::load(&mut self) -> core::result::Result<(), aya::programs::ProgramError> @@ -4823,8 +4860,15 @@ impl core::panic::unwind_safe::UnwindSafe for aya::programs::uprobe::UProbe pub struct aya::programs::uprobe::UProbeAttachPoint<'a> pub aya::programs::uprobe::UProbeAttachPoint::cookie: core::option::Option pub aya::programs::uprobe::UProbeAttachPoint::location: aya::programs::uprobe::UProbeAttachLocation<'a> +impl core::convert::From<&aya::programs::uprobe::UProbeAttachPoint<'_>> for aya::programs::uprobe::UProbeAttachPoint<'_> +pub fn aya::programs::uprobe::UProbeAttachPoint<'_>::from(point: &Self) -> Self impl<'a, L: core::convert::Into>> core::convert::From for aya::programs::uprobe::UProbeAttachPoint<'a> pub fn aya::programs::uprobe::UProbeAttachPoint<'a>::from(location: L) -> Self +impl<'a> core::clone::Clone for aya::programs::uprobe::UProbeAttachPoint<'a> +pub fn aya::programs::uprobe::UProbeAttachPoint<'a>::clone(&self) -> aya::programs::uprobe::UProbeAttachPoint<'a> +impl<'a> core::fmt::Debug for aya::programs::uprobe::UProbeAttachPoint<'a> +pub fn aya::programs::uprobe::UProbeAttachPoint<'a>::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl<'a> core::marker::Copy for aya::programs::uprobe::UProbeAttachPoint<'a> impl<'a> core::marker::Freeze for aya::programs::uprobe::UProbeAttachPoint<'a> impl<'a> core::marker::Send for aya::programs::uprobe::UProbeAttachPoint<'a> impl<'a> core::marker::Sync for aya::programs::uprobe::UProbeAttachPoint<'a> @@ -4833,6 +4877,8 @@ impl<'a> core::marker::UnsafeUnpin for aya::programs::uprobe::UProbeAttachPoint< impl<'a> core::panic::unwind_safe::RefUnwindSafe for aya::programs::uprobe::UProbeAttachPoint<'a> impl<'a> core::panic::unwind_safe::UnwindSafe for aya::programs::uprobe::UProbeAttachPoint<'a> pub struct aya::programs::uprobe::UProbeLink(_) +impl aya::programs::uprobe::UProbeLink +pub fn aya::programs::uprobe::UProbeLink::into_fd_links(self) -> core::result::Result, Self> impl aya::programs::links::Link for aya::programs::uprobe::UProbeLink pub type aya::programs::uprobe::UProbeLink::Id = aya::programs::uprobe::UProbeLinkId pub fn aya::programs::uprobe::UProbeLink::detach(self) -> core::result::Result<(), aya::programs::ProgramError> @@ -5393,6 +5439,7 @@ pub aya::programs::ProgramError::MapError(aya::maps::MapError) pub aya::programs::ProgramError::NetlinkError(aya::sys::netlink::NetlinkError) pub aya::programs::ProgramError::NotAttached pub aya::programs::ProgramError::NotLoaded +pub aya::programs::ProgramError::PerfLinkDetachError(aya::programs::perf_attach::PerfLinkDetachError) pub aya::programs::ProgramError::SkReuseportError(aya::programs::sk_reuseport::SkReuseportError) pub aya::programs::ProgramError::SocketFilterError(aya::programs::socket_filter::SocketFilterError) pub aya::programs::ProgramError::SyscallError(aya::sys::SyscallError) @@ -5411,6 +5458,8 @@ impl core::convert::From for aya::prog pub fn aya::programs::ProgramError::from(source: aya::programs::extension::ExtensionError) -> Self impl core::convert::From for aya::programs::ProgramError pub fn aya::programs::ProgramError::from(source: aya::programs::kprobe::KProbeError) -> Self +impl core::convert::From for aya::programs::ProgramError +pub fn aya::programs::ProgramError::from(source: aya::programs::perf_attach::PerfLinkDetachError) -> Self impl core::convert::From for aya::programs::ProgramError pub fn aya::programs::ProgramError::from(source: aya::programs::sk_reuseport::SkReuseportError) -> Self impl core::convert::From for aya::programs::ProgramError @@ -5598,6 +5647,11 @@ impl core::marker::UnsafeUnpin for aya::programs::trace_point::TracePointError impl !core::panic::unwind_safe::RefUnwindSafe for aya::programs::trace_point::TracePointError impl !core::panic::unwind_safe::UnwindSafe for aya::programs::trace_point::TracePointError pub enum aya::programs::UProbeError +pub aya::programs::UProbeError::AttachModeSelectionFailed +pub aya::programs::UProbeError::AttachModeSelectionFailed::multi_error: alloc::boxed::Box +pub aya::programs::UProbeError::AttachModeSelectionFailed::single_error: alloc::boxed::Box +pub aya::programs::UProbeError::EmptyPoints +pub aya::programs::UProbeError::EmptyPoints::target: std::path::PathBuf pub aya::programs::UProbeError::FileError pub aya::programs::UProbeError::FileError::filename: std::path::PathBuf pub aya::programs::UProbeError::FileError::io_error: std::io::error::Error @@ -5605,12 +5659,18 @@ pub aya::programs::UProbeError::InvalidLdSoCache pub aya::programs::UProbeError::InvalidLdSoCache::io_error: &'static std::io::error::Error pub aya::programs::UProbeError::InvalidTarget pub aya::programs::UProbeError::InvalidTarget::path: std::path::PathBuf +pub aya::programs::UProbeError::LegacyPerfAttachPointError +pub aya::programs::UProbeError::LegacyPerfAttachPointError::attach_error: alloc::boxed::Box +pub aya::programs::UProbeError::LegacyPerfAttachPointError::cleanup_error: core::option::Option> +pub aya::programs::UProbeError::LegacyPerfAttachPointError::index: usize +pub aya::programs::UProbeError::LegacyPerfAttachPointError::resolved_offset: u64 pub aya::programs::UProbeError::ProcMap pub aya::programs::UProbeError::ProcMap::pid: u32 pub aya::programs::UProbeError::ProcMap::source: aya::programs::uprobe::ProcMapError pub aya::programs::UProbeError::SymbolError pub aya::programs::UProbeError::SymbolError::error: alloc::boxed::Box<(dyn core::error::Error + core::marker::Send + core::marker::Sync)> pub aya::programs::UProbeError::SymbolError::symbol: alloc::string::String +pub aya::programs::UProbeError::UProbeMultiNotSupported impl core::convert::From for aya::programs::ProgramError pub fn aya::programs::ProgramError::from(source: aya::programs::uprobe::UProbeError) -> Self impl core::error::Error for aya::programs::uprobe::UProbeError @@ -6694,7 +6754,7 @@ impl core::panic::unwind_safe::UnwindSafe for aya::programs::trace_point::TraceP pub struct aya::programs::UProbe impl aya::programs::uprobe::UProbe pub const aya::programs::uprobe::UProbe::PROGRAM_TYPE: aya::programs::ProgramType -pub fn aya::programs::uprobe::UProbe::attach<'a, T: core::convert::AsRef, Point: core::convert::Into>>(&mut self, point: Point, target: T, scope: aya::programs::uprobe::UProbeScope) -> core::result::Result +pub fn aya::programs::uprobe::UProbe::attach<'a, T, I>(&mut self, points: I, target: T, scope: aya::programs::uprobe::UProbeScope) -> core::result::Result where T: core::convert::AsRef, I: core::iter::traits::collect::IntoIterator, ::Item: core::convert::Into> pub fn aya::programs::uprobe::UProbe::from_pin>(path: P, kind: aya::programs::ProbeKind) -> core::result::Result pub const fn aya::programs::uprobe::UProbe::kind(&self) -> aya::programs::ProbeKind pub fn aya::programs::uprobe::UProbe::load(&mut self) -> core::result::Result<(), aya::programs::ProgramError>