bpftool: Move LLVM functionality into plugin (fixes #262)

README.md

@@ -132,8 +132,22 @@ $ cd src
 $ EXTRA_LDFLAGS=-static make
 ```
 
-Note that to use the LLVM disassembler with static builds, we need a static
-version of the LLVM library installed on the system:
+The LLVM disassembler used for `bpftool prog dump jited` is built as a separate
+plugin, at `$(libdir)/bpftool/bpftool-llvm.so`. This keeps the base bpftool free
+of the (large) libLLVM dependency.
+
+The plugin path is baked into `bpftool` at build time, so pass the same `libdir`
+to both `make` and `make install`. Otherwise `bpftool` looks for the plugin at
+`/usr/local/lib/bpftool/bpftool-llvm.so`.
+
+There are two independent static-linking knobs:
+
+- `EXTRA_LDFLAGS=-static` makes the `bpftool` binary itself static. The plugin
+  is unaffected and stays a shared object.
+- `LLVM_LINK_STATIC=1` links libLLVM into the plugin statically. This is also
+  done automatically when the LLVM installation ships only static libraries.
+
+To embed a static libLLVM into the plugin, build LLVM statically first:
 
 1.  Download a precompiled LLVM release or build it locally.
 
@@ -156,12 +170,13 @@ version of the LLVM library installed on the system:
       $ make -j -C llvm_build llvm-config llvm-libraries
       ```
 
-2.  Build bpftool with `EXTRA_LDFLAGS` set to `-static`, and by passing the
-    path to the relevant `llvm-config`.
+2.  Build bpftool with `LLVM_LINK_STATIC=1`, passing the path to the relevant
+    `llvm-config` and the `libdir` the plugin will be installed under (add
+    `EXTRA_LDFLAGS=-static` too if you also want a static `bpftool` binary).
 
     ```console
     $ cd bpftool
-    $ LLVM_CONFIG=../../llvm_build/bin/llvm-config EXTRA_LDFLAGS=-static make -j -C src
+    $ LLVM_CONFIG=../../llvm_build/bin/llvm-config LLVM_LINK_STATIC=1 libdir=/usr/lib make -j -C src
     ```
 
 ### Build bpftool's man pages

src/.gitignore

@@ -3,6 +3,7 @@
 *.d
 /bootstrap/
 /bpftool
+/bpftool-llvm.so
 FEATURE-DUMP.bpftool
 feature
 libbpf

src/Makefile

@@ -60,6 +60,7 @@ $(LIBBPF_BOOTSTRAP)-clean: FORCE | $(LIBBPF_BOOTSTRAP_OUTPUT)
 	$(Q)$(MAKE) -C $(BPF_DIR) OUTPUT=$(LIBBPF_BOOTSTRAP_OUTPUT) clean >/dev/null
 
 prefix ?= /usr/local
+libdir ?= $(prefix)/lib
 bash_compdir ?= /usr/share/bash-completion/completions
 
 CFLAGS += -O2
@@ -151,6 +152,8 @@ include $(wildcard $(OUTPUT)*.d)
 all: $(OUTPUT)bpftool
 
 SRCS := $(wildcard *.c)
+# llvm_disasm.c is compiled separately into the bpftool-llvm.so plugin.
+SRCS := $(filter-out llvm_disasm.c,$(SRCS))
 
 ifeq ($(feature-llvm),1)
 ifneq ($(SKIP_LLVM),1)
@@ -159,19 +162,36 @@ endif
 endif
 
 ifeq ($(HAS_LLVM),1)
+  # The libLLVM-based JIT disassembler is built as a separate plugin,
+  # bpftool-llvm.so, which is the only object that links against libLLVM.
+  # bpftool loads it lazily with dlopen() (see jit_disasm.c), so the bpftool
+  # binary itself keeps no dependency on the large libLLVM shared object.
   CFLAGS  += -DHAVE_LLVM_SUPPORT
+  CFLAGS  += -DLLVM_PLUGIN_DIR='"$(libdir)/bpftool"'
+  # dlopen() lives in libc on modern glibc, but keep -ldl for portability.
+  LIBS    += -ldl
+
+  # Flags used to build the plugin itself (the only part that needs libLLVM).
   LLVM_CONFIG_LIB_COMPONENTS := mcdisassembler all-targets
-  # llvm-config always adds -D_GNU_SOURCE, however, it may already be in CFLAGS
-  # (e.g. when bpftool build is called from selftests build as selftests
-  # Makefile includes lib.mk which sets -D_GNU_SOURCE) which would cause
-  # compilation error due to redefinition. Let's filter it out here.
-  CFLAGS  += $(filter-out -D_GNU_SOURCE,$(shell $(LLVM_CONFIG) --cflags))
-  LIBS    += $(shell $(LLVM_CONFIG) --libs $(LLVM_CONFIG_LIB_COMPONENTS))
+  # llvm-config always adds -D_GNU_SOURCE, which llvm_disasm.c already defines;
+  # filter it out to avoid a redefinition warning.
+  LLVM_PLUGIN_CFLAGS := $(filter-out -D_GNU_SOURCE,$(shell $(LLVM_CONFIG) --cflags))
+
+  # Embed libLLVM into the plugin statically when requested with
+  # LLVM_LINK_STATIC=1, or when this LLVM install only ships static libraries
+  # ("llvm-config --shared-mode" reports "static"). Otherwise link the shared
+  # libLLVM, which is the only runtime dependency of the plugin.
   ifeq ($(shell $(LLVM_CONFIG) --shared-mode),static)
-    LIBS += $(shell $(LLVM_CONFIG) --system-libs $(LLVM_CONFIG_LIB_COMPONENTS))
-    LIBS += -lstdc++
+    LLVM_LINK_STATIC := 1
+  endif
+  ifeq ($(LLVM_LINK_STATIC),1)
+    LLVM_PLUGIN_LIBS := $(shell $(LLVM_CONFIG) --link-static --libs $(LLVM_CONFIG_LIB_COMPONENTS))
+    LLVM_PLUGIN_LIBS += $(shell $(LLVM_CONFIG) --link-static --system-libs $(LLVM_CONFIG_LIB_COMPONENTS))
+    LLVM_PLUGIN_LIBS += -lstdc++
+  else
+    LLVM_PLUGIN_LIBS := $(shell $(LLVM_CONFIG) --libs $(LLVM_CONFIG_LIB_COMPONENTS))
   endif
-  LDFLAGS += $(shell $(LLVM_CONFIG) --ldflags)
+  LLVM_PLUGIN_LDFLAGS := $(shell $(LLVM_CONFIG) --ldflags)
 else
   ifneq ($(SKIP_LIBBFD),1)
   # Fall back on libbfd
@@ -270,6 +290,20 @@ $(BPFTOOL_BOOTSTRAP): $(BOOTSTRAP_OBJS) $(LIBBPF_BOOTSTRAP)
 $(OUTPUT)bpftool: $(OBJS) $(LIBBPF)
 	$(QUIET_LINK)$(CC) $(CFLAGS) $(LDFLAGS) $(OBJS) $(LIBS) -o $@
 
+ifeq ($(HAS_LLVM),1)
+all: $(OUTPUT)bpftool-llvm.so
+
+$(OUTPUT)llvm_disasm.o: llvm_disasm.c
+	$(QUIET_CC)$(CC) $(CFLAGS) $(LLVM_PLUGIN_CFLAGS) -fPIC -c -MMD $< -o $@
+
+# The plugin is a shared object by definition, so drop a global -static (e.g.
+# from EXTRA_LDFLAGS for a static bpftool) which would conflict with -shared.
+# Embedding libLLVM statically is controlled separately (see LLVM_LINK_STATIC).
+$(OUTPUT)bpftool-llvm.so: $(OUTPUT)llvm_disasm.o
+	$(QUIET_LINK)$(CC) $(CFLAGS) $(filter-out -static,$(LDFLAGS)) \
+		$(LLVM_PLUGIN_LDFLAGS) -shared -o $@ $< $(LLVM_PLUGIN_LIBS)
+endif
+
 $(BOOTSTRAP_OUTPUT)%.o: %.c $(LIBBPF_BOOTSTRAP_INTERNAL_HDRS) | $(BOOTSTRAP_OUTPUT)
 	$(QUIET_CC)$(HOSTCC) $(HOST_CFLAGS) -c -MMD $< -o $@
 
@@ -282,17 +316,25 @@ feature-detect-clean:
 
 clean: $(LIBBPF)-clean $(LIBBPF_BOOTSTRAP)-clean
 	$(call QUIET_CLEAN, bpftool)
-	$(Q)$(RM) -- $(OUTPUT)bpftool $(OUTPUT)*.o $(OUTPUT)*.d
+	$(Q)$(RM) -- $(OUTPUT)bpftool $(OUTPUT)bpftool-llvm.so $(OUTPUT)*.o $(OUTPUT)*.d
 	$(Q)$(RM) -- $(OUTPUT)*.skel.h $(OUTPUT)vmlinux.h
 	$(Q)$(RM) -r -- $(LIBBPF_OUTPUT) $(BOOTSTRAP_OUTPUT)
 	$(call QUIET_CLEAN, core-gen)
 	$(Q)$(RM) -- $(OUTPUT)FEATURE-DUMP.bpftool
 	$(Q)$(RM) -r -- $(OUTPUT)feature/
 
+ifeq ($(HAS_LLVM),1)
+install-bin: $(OUTPUT)bpftool-llvm.so
+endif
 install-bin: $(OUTPUT)bpftool
 	$(call QUIET_INSTALL, bpftool)
 	$(Q)$(INSTALL) -m 0755 -d $(DESTDIR)$(prefix)/sbin
 	$(Q)$(INSTALL) $(OUTPUT)bpftool $(DESTDIR)$(prefix)/sbin/bpftool
+ifeq ($(HAS_LLVM),1)
+	$(call QUIET_INSTALL, bpftool-llvm.so)
+	$(Q)$(INSTALL) -m 0755 -d $(DESTDIR)$(libdir)/bpftool
+	$(Q)$(INSTALL) -m 0755 $(OUTPUT)bpftool-llvm.so $(DESTDIR)$(libdir)/bpftool/bpftool-llvm.so
+endif
 
 install: install-bin
 	$(Q)$(INSTALL) -m 0755 -d $(DESTDIR)$(bash_compdir)
@@ -301,6 +343,7 @@ install: install-bin
 uninstall:
 	$(call QUIET_UNINST, bpftool)
 	$(Q)$(RM) -- $(DESTDIR)$(prefix)/sbin/bpftool
+	$(Q)$(RM) -- $(DESTDIR)$(libdir)/bpftool/bpftool-llvm.so
 	$(Q)$(RM) -- $(DESTDIR)$(bash_compdir)/bpftool
 
 doc:

src/jit_disasm.c

@@ -25,10 +25,9 @@
 #include <bpf/libbpf.h>
 
 #ifdef HAVE_LLVM_SUPPORT
-#include <llvm-c/Core.h>
-#include <llvm-c/Disassembler.h>
-#include <llvm-c/Target.h>
-#include <llvm-c/TargetMachine.h>
+#include <dlfcn.h>
+
+#include "llvm_disasm.h"
 #endif
 
 #ifdef HAVE_LIBBFD_SUPPORT
@@ -45,7 +44,32 @@ static int oper_count;
 #ifdef HAVE_LLVM_SUPPORT
 #define DISASM_SPACER
 
-typedef LLVMDisasmContextRef disasm_ctx_t;
+/*
+ * The libLLVM-based disassembler used for "bpftool prog dump jited" lives in a
+ * separate plugin, bpftool-llvm.so, which is the only object linked against
+ * libLLVM. This keeps the bpftool binary itself free of a hard dependency on
+ * the (large) libLLVM shared object: the plugin is loaded lazily with dlopen()
+ * the first time a JITed image actually needs to be disassembled, with its
+ * entry points resolved by dlsym(). See llvm_disasm.c for the plugin.
+ *
+ * LLVM_PLUGIN_DIR is the install directory baked in at build time
+ * ($(libdir)/bpftool). When set, the plugin is loaded from that absolute
+ * location; otherwise only the bare file name is used, i.e. the plugin is
+ * looked up via the dynamic linker search path (or the current directory).
+ */
+#ifdef LLVM_PLUGIN_DIR
+#define LLVM_PLUGIN_PATH LLVM_PLUGIN_DIR "/bpftool-llvm.so"
+#else
+#define LLVM_PLUGIN_PATH "bpftool-llvm.so"
+#endif
+
+typedef void *disasm_ctx_t;
+
+static void *llvm_plugin_handle;
+static __typeof__(&bpftool_llvm_init) p_bpftool_llvm_init;
+static __typeof__(&bpftool_llvm_create_context) p_bpftool_llvm_create_context;
+static __typeof__(&bpftool_llvm_destroy_context) p_bpftool_llvm_destroy_context;
+static __typeof__(&bpftool_llvm_disassemble) p_bpftool_llvm_disassemble;
 
 static int printf_json(char *s)
 {
@@ -63,48 +87,13 @@ static int printf_json(char *s)
 	return 0;
 }
 
-/* This callback to set the ref_type is necessary to have the LLVM disassembler
- * print PC-relative addresses instead of byte offsets for branch instruction
- * targets.
- */
-static const char *
-symbol_lookup_callback(__maybe_unused void *disasm_info,
-		       __maybe_unused uint64_t ref_value,
-		       uint64_t *ref_type, __maybe_unused uint64_t ref_PC,
-		       __maybe_unused const char **ref_name)
-{
-	*ref_type = LLVMDisassembler_ReferenceType_InOut_None;
-	return NULL;
-}
-
 static int
 init_context(disasm_ctx_t *ctx, const char *arch,
 	     __maybe_unused const char *disassembler_options,
 	     __maybe_unused unsigned char *image, __maybe_unused ssize_t len,
 	     __maybe_unused __u64 func_ksym)
 {
-	char *triple;
-
-	if (arch)
-		triple = LLVMNormalizeTargetTriple(arch);
-	else
-		triple = LLVMGetDefaultTargetTriple();
-	if (!triple) {
-		p_err("Failed to retrieve triple");
-		return -1;
-	}
-
-	/*
-	 * Enable all aarch64 ISA extensions so the disassembler can handle any
-	 * instruction the kernel JIT might emit (e.g. ARM64 LSE atomics).
-	 */
-	if (!strncmp(triple, "aarch64", 7))
-		*ctx = LLVMCreateDisasmCPUFeatures(triple, "", "+all", NULL, 0, NULL,
-						   symbol_lookup_callback);
-	else
-		*ctx = LLVMCreateDisasm(triple, NULL, 0, NULL, symbol_lookup_callback);
-	LLVMDisposeMessage(triple);
-
+	*ctx = p_bpftool_llvm_create_context(arch);
 	if (!*ctx) {
 		p_err("Failed to create disassembler");
 		return -1;
@@ -115,7 +104,7 @@ init_context(disasm_ctx_t *ctx, const char *arch,
 
 static void destroy_context(disasm_ctx_t *ctx)
 {
-	LLVMDisposeMessage(*ctx);
+	p_bpftool_llvm_destroy_context(*ctx);
 }
 
 static int
@@ -125,8 +114,8 @@ disassemble_insn(disasm_ctx_t *ctx, unsigned char *image, ssize_t len, int pc,
 	char buf[256];
 	int count;
 
-	count = LLVMDisasmInstruction(*ctx, image + pc, len - pc, func_ksym + pc,
-				      buf, sizeof(buf));
+	count = p_bpftool_llvm_disassemble(*ctx, image, len, pc, func_ksym,
+					   buf, sizeof(buf));
 	if (json_output)
 		printf_json(buf);
 	else
@@ -137,10 +126,37 @@ disassemble_insn(disasm_ctx_t *ctx, unsigned char *image, ssize_t len, int pc,
 
 int disasm_init(void)
 {
-	LLVMInitializeAllTargetInfos();
-	LLVMInitializeAllTargetMCs();
-	LLVMInitializeAllDisassemblers();
-	return 0;
+	if (llvm_plugin_handle)
+		return p_bpftool_llvm_init();
+
+	/* Load the plugin by its absolute install path. */
+	llvm_plugin_handle = dlopen(LLVM_PLUGIN_PATH, RTLD_NOW | RTLD_LOCAL);
+	if (!llvm_plugin_handle) {
+		p_err("failed to load %s, install it to disassemble JITed programs: %s",
+		      LLVM_PLUGIN_PATH, dlerror());
+		return -1;
+	}
+
+#define RESOLVE(name)							       \
+	do {								       \
+		p_##name = (__typeof__(p_##name))dlsym(llvm_plugin_handle,     \
+						       #name);		       \
+		if (!p_##name) {					       \
+			p_err("%s is missing symbol %s: %s",		       \
+			      LLVM_PLUGIN_PATH, #name, dlerror());	       \
+			dlclose(llvm_plugin_handle);			       \
+			llvm_plugin_handle = NULL;			       \
+			return -1;					       \
+		}							       \
+	} while (0)
+
+	RESOLVE(bpftool_llvm_init);
+	RESOLVE(bpftool_llvm_create_context);
+	RESOLVE(bpftool_llvm_destroy_context);
+	RESOLVE(bpftool_llvm_disassemble);
+#undef RESOLVE
+
+	return p_bpftool_llvm_init();
 }
 #endif /* HAVE_LLVM_SUPPORT */