From 446ba794d163a252770b723f5e79b27a9837350c Mon Sep 17 00:00:00 2001 From: Korenberg Mark Date: Mon, 1 Jun 2026 19:19:35 +0500 Subject: [PATCH] mirror: bpftool: Move LLVM functionality into plugin (fixes #262) Signed-off-by: Korenberg Mark --- README.md | 25 ++++++++--- src/.gitignore | 1 + src/Makefile | 63 +++++++++++++++++++++----- src/jit_disasm.c | 112 ++++++++++++++++++++++++++-------------------- src/llvm_disasm.c | 85 +++++++++++++++++++++++++++++++++++ src/llvm_disasm.h | 38 ++++++++++++++++ 6 files changed, 261 insertions(+), 63 deletions(-) create mode 100644 src/llvm_disasm.c create mode 100644 src/llvm_disasm.h diff --git a/README.md b/README.md index 87d0428c..12f0ef22 100644 --- a/README.md +++ b/README.md @@ -132,8 +132,22 @@ $ cd src $ EXTRA_LDFLAGS=-static make ``` -Note that to use the LLVM disassembler with static builds, we need a static -version of the LLVM library installed on the system: +The LLVM disassembler used for `bpftool prog dump jited` is built as a separate +plugin, at `$(libdir)/bpftool/bpftool-llvm.so`. This keeps the base bpftool free +of the (large) libLLVM dependency. + +The plugin path is baked into `bpftool` at build time, so pass the same `libdir` +to both `make` and `make install`. Otherwise `bpftool` looks for the plugin at +`/usr/local/lib/bpftool/bpftool-llvm.so`. + +There are two independent static-linking knobs: + +- `EXTRA_LDFLAGS=-static` makes the `bpftool` binary itself static. The plugin + is unaffected and stays a shared object. +- `LLVM_LINK_STATIC=1` links libLLVM into the plugin statically. This is also + done automatically when the LLVM installation ships only static libraries. + +To embed a static libLLVM into the plugin, build LLVM statically first: 1. Download a precompiled LLVM release or build it locally. @@ -156,12 +170,13 @@ version of the LLVM library installed on the system: $ make -j -C llvm_build llvm-config llvm-libraries ``` -2. Build bpftool with `EXTRA_LDFLAGS` set to `-static`, and by passing the - path to the relevant `llvm-config`. +2. Build bpftool with `LLVM_LINK_STATIC=1`, passing the path to the relevant + `llvm-config` and the `libdir` the plugin will be installed under (add + `EXTRA_LDFLAGS=-static` too if you also want a static `bpftool` binary). ```console $ cd bpftool - $ LLVM_CONFIG=../../llvm_build/bin/llvm-config EXTRA_LDFLAGS=-static make -j -C src + $ LLVM_CONFIG=../../llvm_build/bin/llvm-config LLVM_LINK_STATIC=1 libdir=/usr/lib make -j -C src ``` ### Build bpftool's man pages diff --git a/src/.gitignore b/src/.gitignore index 712adf5b..48c7a4e9 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -3,6 +3,7 @@ *.d /bootstrap/ /bpftool +/bpftool-llvm.so FEATURE-DUMP.bpftool feature libbpf diff --git a/src/Makefile b/src/Makefile index b840d780..30d4d86e 100644 --- a/src/Makefile +++ b/src/Makefile @@ -60,6 +60,7 @@ $(LIBBPF_BOOTSTRAP)-clean: FORCE | $(LIBBPF_BOOTSTRAP_OUTPUT) $(Q)$(MAKE) -C $(BPF_DIR) OUTPUT=$(LIBBPF_BOOTSTRAP_OUTPUT) clean >/dev/null prefix ?= /usr/local +libdir ?= $(prefix)/lib bash_compdir ?= /usr/share/bash-completion/completions CFLAGS += -O2 @@ -151,6 +152,8 @@ include $(wildcard $(OUTPUT)*.d) all: $(OUTPUT)bpftool SRCS := $(wildcard *.c) +# llvm_disasm.c is compiled separately into the bpftool-llvm.so plugin. +SRCS := $(filter-out llvm_disasm.c,$(SRCS)) ifeq ($(feature-llvm),1) ifneq ($(SKIP_LLVM),1) @@ -159,19 +162,36 @@ endif endif ifeq ($(HAS_LLVM),1) + # The libLLVM-based JIT disassembler is built as a separate plugin, + # bpftool-llvm.so, which is the only object that links against libLLVM. + # bpftool loads it lazily with dlopen() (see jit_disasm.c), so the bpftool + # binary itself keeps no dependency on the large libLLVM shared object. CFLAGS += -DHAVE_LLVM_SUPPORT + CFLAGS += -DLLVM_PLUGIN_DIR='"$(libdir)/bpftool"' + # dlopen() lives in libc on modern glibc, but keep -ldl for portability. + LIBS += -ldl + + # Flags used to build the plugin itself (the only part that needs libLLVM). LLVM_CONFIG_LIB_COMPONENTS := mcdisassembler all-targets - # llvm-config always adds -D_GNU_SOURCE, however, it may already be in CFLAGS - # (e.g. when bpftool build is called from selftests build as selftests - # Makefile includes lib.mk which sets -D_GNU_SOURCE) which would cause - # compilation error due to redefinition. Let's filter it out here. - CFLAGS += $(filter-out -D_GNU_SOURCE,$(shell $(LLVM_CONFIG) --cflags)) - LIBS += $(shell $(LLVM_CONFIG) --libs $(LLVM_CONFIG_LIB_COMPONENTS)) + # llvm-config always adds -D_GNU_SOURCE, which llvm_disasm.c already defines; + # filter it out to avoid a redefinition warning. + LLVM_PLUGIN_CFLAGS := $(filter-out -D_GNU_SOURCE,$(shell $(LLVM_CONFIG) --cflags)) + + # Embed libLLVM into the plugin statically when requested with + # LLVM_LINK_STATIC=1, or when this LLVM install only ships static libraries + # ("llvm-config --shared-mode" reports "static"). Otherwise link the shared + # libLLVM, which is the only runtime dependency of the plugin. ifeq ($(shell $(LLVM_CONFIG) --shared-mode),static) - LIBS += $(shell $(LLVM_CONFIG) --system-libs $(LLVM_CONFIG_LIB_COMPONENTS)) - LIBS += -lstdc++ + LLVM_LINK_STATIC := 1 + endif + ifeq ($(LLVM_LINK_STATIC),1) + LLVM_PLUGIN_LIBS := $(shell $(LLVM_CONFIG) --link-static --libs $(LLVM_CONFIG_LIB_COMPONENTS)) + LLVM_PLUGIN_LIBS += $(shell $(LLVM_CONFIG) --link-static --system-libs $(LLVM_CONFIG_LIB_COMPONENTS)) + LLVM_PLUGIN_LIBS += -lstdc++ + else + LLVM_PLUGIN_LIBS := $(shell $(LLVM_CONFIG) --libs $(LLVM_CONFIG_LIB_COMPONENTS)) endif - LDFLAGS += $(shell $(LLVM_CONFIG) --ldflags) + LLVM_PLUGIN_LDFLAGS := $(shell $(LLVM_CONFIG) --ldflags) else ifneq ($(SKIP_LIBBFD),1) # Fall back on libbfd @@ -270,6 +290,20 @@ $(BPFTOOL_BOOTSTRAP): $(BOOTSTRAP_OBJS) $(LIBBPF_BOOTSTRAP) $(OUTPUT)bpftool: $(OBJS) $(LIBBPF) $(QUIET_LINK)$(CC) $(CFLAGS) $(LDFLAGS) $(OBJS) $(LIBS) -o $@ +ifeq ($(HAS_LLVM),1) +all: $(OUTPUT)bpftool-llvm.so + +$(OUTPUT)llvm_disasm.o: llvm_disasm.c + $(QUIET_CC)$(CC) $(CFLAGS) $(LLVM_PLUGIN_CFLAGS) -fPIC -c -MMD $< -o $@ + +# The plugin is a shared object by definition, so drop a global -static (e.g. +# from EXTRA_LDFLAGS for a static bpftool) which would conflict with -shared. +# Embedding libLLVM statically is controlled separately (see LLVM_LINK_STATIC). +$(OUTPUT)bpftool-llvm.so: $(OUTPUT)llvm_disasm.o + $(QUIET_LINK)$(CC) $(CFLAGS) $(filter-out -static,$(LDFLAGS)) \ + $(LLVM_PLUGIN_LDFLAGS) -shared -o $@ $< $(LLVM_PLUGIN_LIBS) +endif + $(BOOTSTRAP_OUTPUT)%.o: %.c $(LIBBPF_BOOTSTRAP_INTERNAL_HDRS) | $(BOOTSTRAP_OUTPUT) $(QUIET_CC)$(HOSTCC) $(HOST_CFLAGS) -c -MMD $< -o $@ @@ -282,17 +316,25 @@ feature-detect-clean: clean: $(LIBBPF)-clean $(LIBBPF_BOOTSTRAP)-clean $(call QUIET_CLEAN, bpftool) - $(Q)$(RM) -- $(OUTPUT)bpftool $(OUTPUT)*.o $(OUTPUT)*.d + $(Q)$(RM) -- $(OUTPUT)bpftool $(OUTPUT)bpftool-llvm.so $(OUTPUT)*.o $(OUTPUT)*.d $(Q)$(RM) -- $(OUTPUT)*.skel.h $(OUTPUT)vmlinux.h $(Q)$(RM) -r -- $(LIBBPF_OUTPUT) $(BOOTSTRAP_OUTPUT) $(call QUIET_CLEAN, core-gen) $(Q)$(RM) -- $(OUTPUT)FEATURE-DUMP.bpftool $(Q)$(RM) -r -- $(OUTPUT)feature/ +ifeq ($(HAS_LLVM),1) +install-bin: $(OUTPUT)bpftool-llvm.so +endif install-bin: $(OUTPUT)bpftool $(call QUIET_INSTALL, bpftool) $(Q)$(INSTALL) -m 0755 -d $(DESTDIR)$(prefix)/sbin $(Q)$(INSTALL) $(OUTPUT)bpftool $(DESTDIR)$(prefix)/sbin/bpftool +ifeq ($(HAS_LLVM),1) + $(call QUIET_INSTALL, bpftool-llvm.so) + $(Q)$(INSTALL) -m 0755 -d $(DESTDIR)$(libdir)/bpftool + $(Q)$(INSTALL) -m 0755 $(OUTPUT)bpftool-llvm.so $(DESTDIR)$(libdir)/bpftool/bpftool-llvm.so +endif install: install-bin $(Q)$(INSTALL) -m 0755 -d $(DESTDIR)$(bash_compdir) @@ -301,6 +343,7 @@ install: install-bin uninstall: $(call QUIET_UNINST, bpftool) $(Q)$(RM) -- $(DESTDIR)$(prefix)/sbin/bpftool + $(Q)$(RM) -- $(DESTDIR)$(libdir)/bpftool/bpftool-llvm.so $(Q)$(RM) -- $(DESTDIR)$(bash_compdir)/bpftool doc: diff --git a/src/jit_disasm.c b/src/jit_disasm.c index 04541155..e8cef2da 100644 --- a/src/jit_disasm.c +++ b/src/jit_disasm.c @@ -25,10 +25,9 @@ #include #ifdef HAVE_LLVM_SUPPORT -#include -#include -#include -#include +#include + +#include "llvm_disasm.h" #endif #ifdef HAVE_LIBBFD_SUPPORT @@ -45,7 +44,32 @@ static int oper_count; #ifdef HAVE_LLVM_SUPPORT #define DISASM_SPACER -typedef LLVMDisasmContextRef disasm_ctx_t; +/* + * The libLLVM-based disassembler used for "bpftool prog dump jited" lives in a + * separate plugin, bpftool-llvm.so, which is the only object linked against + * libLLVM. This keeps the bpftool binary itself free of a hard dependency on + * the (large) libLLVM shared object: the plugin is loaded lazily with dlopen() + * the first time a JITed image actually needs to be disassembled, with its + * entry points resolved by dlsym(). See llvm_disasm.c for the plugin. + * + * LLVM_PLUGIN_DIR is the install directory baked in at build time + * ($(libdir)/bpftool). When set, the plugin is loaded from that absolute + * location; otherwise only the bare file name is used, i.e. the plugin is + * looked up via the dynamic linker search path (or the current directory). + */ +#ifdef LLVM_PLUGIN_DIR +#define LLVM_PLUGIN_PATH LLVM_PLUGIN_DIR "/bpftool-llvm.so" +#else +#define LLVM_PLUGIN_PATH "bpftool-llvm.so" +#endif + +typedef void *disasm_ctx_t; + +static void *llvm_plugin_handle; +static __typeof__(&bpftool_llvm_init) p_bpftool_llvm_init; +static __typeof__(&bpftool_llvm_create_context) p_bpftool_llvm_create_context; +static __typeof__(&bpftool_llvm_destroy_context) p_bpftool_llvm_destroy_context; +static __typeof__(&bpftool_llvm_disassemble) p_bpftool_llvm_disassemble; static int printf_json(char *s) { @@ -63,48 +87,13 @@ static int printf_json(char *s) return 0; } -/* This callback to set the ref_type is necessary to have the LLVM disassembler - * print PC-relative addresses instead of byte offsets for branch instruction - * targets. - */ -static const char * -symbol_lookup_callback(__maybe_unused void *disasm_info, - __maybe_unused uint64_t ref_value, - uint64_t *ref_type, __maybe_unused uint64_t ref_PC, - __maybe_unused const char **ref_name) -{ - *ref_type = LLVMDisassembler_ReferenceType_InOut_None; - return NULL; -} - static int init_context(disasm_ctx_t *ctx, const char *arch, __maybe_unused const char *disassembler_options, __maybe_unused unsigned char *image, __maybe_unused ssize_t len, __maybe_unused __u64 func_ksym) { - char *triple; - - if (arch) - triple = LLVMNormalizeTargetTriple(arch); - else - triple = LLVMGetDefaultTargetTriple(); - if (!triple) { - p_err("Failed to retrieve triple"); - return -1; - } - - /* - * Enable all aarch64 ISA extensions so the disassembler can handle any - * instruction the kernel JIT might emit (e.g. ARM64 LSE atomics). - */ - if (!strncmp(triple, "aarch64", 7)) - *ctx = LLVMCreateDisasmCPUFeatures(triple, "", "+all", NULL, 0, NULL, - symbol_lookup_callback); - else - *ctx = LLVMCreateDisasm(triple, NULL, 0, NULL, symbol_lookup_callback); - LLVMDisposeMessage(triple); - + *ctx = p_bpftool_llvm_create_context(arch); if (!*ctx) { p_err("Failed to create disassembler"); return -1; @@ -115,7 +104,7 @@ init_context(disasm_ctx_t *ctx, const char *arch, static void destroy_context(disasm_ctx_t *ctx) { - LLVMDisposeMessage(*ctx); + p_bpftool_llvm_destroy_context(*ctx); } static int @@ -125,8 +114,8 @@ disassemble_insn(disasm_ctx_t *ctx, unsigned char *image, ssize_t len, int pc, char buf[256]; int count; - count = LLVMDisasmInstruction(*ctx, image + pc, len - pc, func_ksym + pc, - buf, sizeof(buf)); + count = p_bpftool_llvm_disassemble(*ctx, image, len, pc, func_ksym, + buf, sizeof(buf)); if (json_output) printf_json(buf); else @@ -137,10 +126,37 @@ disassemble_insn(disasm_ctx_t *ctx, unsigned char *image, ssize_t len, int pc, int disasm_init(void) { - LLVMInitializeAllTargetInfos(); - LLVMInitializeAllTargetMCs(); - LLVMInitializeAllDisassemblers(); - return 0; + if (llvm_plugin_handle) + return p_bpftool_llvm_init(); + + /* Load the plugin by its absolute install path. */ + llvm_plugin_handle = dlopen(LLVM_PLUGIN_PATH, RTLD_NOW | RTLD_LOCAL); + if (!llvm_plugin_handle) { + p_err("failed to load %s, install it to disassemble JITed programs: %s", + LLVM_PLUGIN_PATH, dlerror()); + return -1; + } + +#define RESOLVE(name) \ + do { \ + p_##name = (__typeof__(p_##name))dlsym(llvm_plugin_handle, \ + #name); \ + if (!p_##name) { \ + p_err("%s is missing symbol %s: %s", \ + LLVM_PLUGIN_PATH, #name, dlerror()); \ + dlclose(llvm_plugin_handle); \ + llvm_plugin_handle = NULL; \ + return -1; \ + } \ + } while (0) + + RESOLVE(bpftool_llvm_init); + RESOLVE(bpftool_llvm_create_context); + RESOLVE(bpftool_llvm_destroy_context); + RESOLVE(bpftool_llvm_disassemble); +#undef RESOLVE + + return p_bpftool_llvm_init(); } #endif /* HAVE_LLVM_SUPPORT */ diff --git a/src/llvm_disasm.c b/src/llvm_disasm.c new file mode 100644 index 00000000..b8321619 --- /dev/null +++ b/src/llvm_disasm.c @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* + * libLLVM-based BPF JIT disassembler plugin for bpftool. + * + * This translation unit is built into a standalone shared object + * (bpftool-llvm.so) which is the only bpftool component that links against + * libLLVM. bpftool loads it lazily with dlopen() (see jit_disasm.c) so that + * the bpftool binary itself does not depend on the large libLLVM shared + * object. Only the small, stable C ABI declared in llvm_disasm.h is exposed. + */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include +#include +#include + +#include +#include +#include +#include + +#include "llvm_disasm.h" + +/* This callback to set the ref_type is necessary to have the LLVM disassembler + * print PC-relative addresses instead of byte offsets for branch instruction + * targets. + */ +static const char * +symbol_lookup_callback(void *disasm_info, uint64_t ref_value, + uint64_t *ref_type, uint64_t ref_PC, + const char **ref_name) +{ + *ref_type = LLVMDisassembler_ReferenceType_InOut_None; + return NULL; +} + +int bpftool_llvm_init(void) +{ + LLVMInitializeAllTargetInfos(); + LLVMInitializeAllTargetMCs(); + LLVMInitializeAllDisassemblers(); + + return 0; +} + +void *bpftool_llvm_create_context(const char *arch) +{ + LLVMDisasmContextRef ctx; + char *triple; + + if (arch) + triple = LLVMNormalizeTargetTriple(arch); + else + triple = LLVMGetDefaultTargetTriple(); + if (!triple) + return NULL; + + /* + * Enable all aarch64 ISA extensions so the disassembler can handle any + * instruction the kernel JIT might emit (e.g. ARM64 LSE atomics). + */ + if (!strncmp(triple, "aarch64", 7)) + ctx = LLVMCreateDisasmCPUFeatures(triple, "", "+all", NULL, 0, + NULL, symbol_lookup_callback); + else + ctx = LLVMCreateDisasm(triple, NULL, 0, NULL, + symbol_lookup_callback); + LLVMDisposeMessage(triple); + + return ctx; +} + +void bpftool_llvm_destroy_context(void *ctx) +{ + LLVMDisasmDispose(ctx); +} + +int bpftool_llvm_disassemble(void *ctx, unsigned char *image, ssize_t len, + int pc, uint64_t func_ksym, char *buf, + size_t buf_sz) +{ + return LLVMDisasmInstruction(ctx, image + pc, len - pc, func_ksym + pc, + buf, buf_sz); +} diff --git a/src/llvm_disasm.h b/src/llvm_disasm.h new file mode 100644 index 00000000..cd9491ea --- /dev/null +++ b/src/llvm_disasm.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +#ifndef __BPFTOOL_LLVM_DISASM_H +#define __BPFTOOL_LLVM_DISASM_H + +#include +#include +#include + +/* + * Stable C ABI between bpftool and its optional libLLVM-based JIT disassembler + * plugin (bpftool-llvm.so). bpftool resolves these symbols with dlsym() + * after dlopen()ing the plugin; the plugin is the only object that links + * against libLLVM. See jit_disasm.c (loader) and llvm_disasm.c (plugin). + */ + +/* Initialize the libLLVM targets and disassemblers. Returns 0 on success. */ +int bpftool_llvm_init(void); + +/* + * Create a disassembler context for @arch (NULL selects the host + * architecture). Returns an opaque context pointer, or NULL on failure. + */ +void *bpftool_llvm_create_context(const char *arch); + +/* Release a context previously returned by bpftool_llvm_create_context(). */ +void bpftool_llvm_destroy_context(void *ctx); + +/* + * Disassemble the single instruction at @image[@pc] into @buf as a NUL + * terminated string. @func_ksym is the kernel address of @image and is used to + * render absolute branch targets. Returns the instruction length in bytes, or + * 0 if the instruction could not be decoded. + */ +int bpftool_llvm_disassemble(void *ctx, unsigned char *image, ssize_t len, + int pc, uint64_t func_ksym, char *buf, + size_t buf_sz); + +#endif /* __BPFTOOL_LLVM_DISASM_H */