From 01afe8cf9f8efb2c9d6126f34d701b31b8930c78 Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Wed, 24 Jun 2026 16:21:49 -0400 Subject: [PATCH 1/3] fix(ungrub): replicate GRUB EFI core to all boot-pool members install_grub installed the core to only the target device's ESP, while grub-install rebuilt the shared modules under /boot/grub. On a mirror that left every other member's ESP with a stale core that no longer matched the rebuilt modules, so those members failed to boot with "symbol 'grub_memcpy' not found" -- silently destroying boot redundancy. - install_grub now copies the freshly built core to every other member's ESP (no-op for a single-device pool). - new "mkbootable sync " op re-distributes the core from a known good member to all others, to repair members that already drifted (eg a device added via a bare "zpool attach"). install_efi_core writes safely: - reuse the ESP's existing mount if it has one, else mount it on a temp dir (never stack a second mount on one FAT -- two rw mounts corrupt each other). - stage as BOOTX64.EFI.new then mv over the live file; mv on one filesystem is rename(2), an atomic replace, so an interrupted write can't leave a torn or missing core. The core uses a dynamic ($root) prefix so the same image is valid on every member. Split into member_esp / install_efi_core / replicate_grub. Both the reuse-mount and self-mount paths verified on a loop device. --- ungrub/mkbootable | 90 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/ungrub/mkbootable b/ungrub/mkbootable index 749ca6aa18..2ad425b70d 100755 --- a/ungrub/mkbootable +++ b/ungrub/mkbootable @@ -13,6 +13,9 @@ # Reconfigure grub.cfg and theme.txt after UUID change # mkbootable reconfigure +# Copy the EFI core from a known-good member to every other member's ESP +# mkbootable sync + #set -euo pipefail #set -x source /etc/rc.d/rc.runlog @@ -82,6 +85,72 @@ resize_partitions() { udevadm settle } +# resolve a boot-pool member partition to that disk's EFI System Partition, +# eg /dev/nvme1n1p3 -> /dev/nvme1n1p2, /dev/sdb3 -> /dev/sdb2 +member_esp() { + local part="$1" disk sep + disk="$(lsblk -no pkname "$part" 2>/dev/null | head -1)" + [[ -n "$disk" ]] || return 1 + [[ $disk == *[0-9] ]] && sep="p" || sep="" + printf '/dev/%s%s2\n' "$disk" "$sep" +} + +# write the EFI core ($2) onto a single ESP ($1), via a real mount + atomic +# rename. Staged as BOOTX64.EFI.new then renamed over the live file -- mv on the +# same filesystem is rename(2), an atomic replace, so an interrupted write can +# never leave a torn or missing core. +# If the ESP is already mounted we reuse that mount (never stack a second mount +# on one FAT -- two rw mounts corrupt each other); otherwise we mount it on a +# temp dir and unmount when done. +install_efi_core() { + local esp="$1" src="$2" mnt dir owned=0 rc=0 + mnt="$(findmnt -nro TARGET -S "$esp" 2>/dev/null | head -1)" + if [[ -z "$mnt" ]]; then + mnt="$(mktemp -d)" || return 1 + if ! mount "$esp" "$mnt"; then + log "install_efi_core: cannot mount $esp" + rmdir "$mnt" + return 1 + fi + owned=1 + fi + + dir="$mnt/EFI/BOOT" + if mkdir -p "$dir" \ + && cp -f "$src" "$dir/BOOTX64.EFI.new" \ + && mv -f "$dir/BOOTX64.EFI.new" "$dir/BOOTX64.EFI"; then + sync -f "$mnt" + else + rc=1 + fi + + if (( owned )); then + umount "$mnt" + rmdir "$mnt" + fi + return $rc +} + +# Copy a known-good EFI core to the ESP of every boot pool member except the one +# it came from. grub-install rebuilds the shared modules under $BOOT/grub but +# writes the core to only one ESP; a member left with a stale core no longer +# matches those modules and fails to boot with "symbol 'grub_memcpy' not found", +# silently breaking the mirror's boot redundancy. The core uses a dynamic +# ($root) prefix, so the same image is valid on every member. +# $1 = path to the known-good BOOTX64.EFI to distribute (on a mounted ESP) +replicate_grub() { + local src="$1" part esp + for part in $(zpool list -v -H -P "$POOL" | awk '/\/dev\// {print $1}'); do + esp="$(member_esp "$part")" || { log "replicate_grub: cannot resolve ESP for $part, skipping"; continue; } + [[ "$esp" == "$EFI_PART" || ! -b "$esp" ]] && continue + if install_efi_core "$esp" "$src"; then + log "replicate_grub: synced EFI core to $esp" + else + log "replicate_grub: failed to sync EFI core to $esp" + fi + done +} + install_grub() { # format the EFI System Partition (partition 2) with FAT32 and mount mkfs.fat -F32 -n EFI "$EFI_PART" @@ -103,6 +172,11 @@ install_grub() { --modules="part_gpt zfs zfsinfo search search_fs_uuid configfile normal" \ --no-floppy --no-rs-codes + # replicate the core we just built to every other member's ESP so all mirror + # members stay in lockstep with the shared modules rebuilt above (no-op while + # this is the only member) + replicate_grub "$BOOT/efi/EFI/BOOT/BOOTX64.EFI" + # we don't need this mounted anymore umount "$BOOT/efi" } @@ -162,6 +236,19 @@ remove_device() { fi } +# Re-distribute the EFI core from a known-good member to all other members. +# Use this to repair members whose ESP holds a stale core (eg a device added +# with a bare "zpool attach", or a member missed by an earlier grub refresh). +sync_device() { + mkdir -p -m 0700 "$BOOT/efi" + if ! mount -o ro "$EFI_PART" "$BOOT/efi" 2>/dev/null; then + log "sync: cannot mount source ESP $EFI_PART" + exit 1 + fi + replicate_grub "$BOOT/efi/EFI/BOOT/BOOTX64.EFI" + umount "$BOOT/efi" +} + case "$OPER" in 'add') add_device @@ -175,6 +262,9 @@ case "$OPER" in 'reconfigure') configure_grub ;; +'sync') + sync_device + ;; *) log "error 2" # shouldn't happen exit 2 From 83b995269d608dd9c4b3d11676df8dfed1e9790a Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Thu, 25 Jun 2026 12:00:31 -0400 Subject: [PATCH 2/3] fix(ungrub): reinstall grub per member (UEFI+BIOS) instead of EFI-only copy Replace the file-copy replicate_grub/install_efi_core with install_grub_all_members, which re-runs grub-install for both x86_64-efi and i386-pc on every boot-pool member. The EFI-only copy left the legacy/BIOS boot blocks to drift: install_grub runs grub-install --target=i386-pc on only the added disk, so a pre-existing member's core.img (BIOS Boot Partition) and boot.img (MBR) go stale against the rebuilt i386-pc modules and brick that member under CSM/legacy boot with "symbol 'grub_memcpy' not found" -- the same failure the EFI fix addresses, on the path it didn't cover. grub-install is the canonical mechanism (cf. OpenZFS "repeat grub-install for each disk"): it writes core.img to each member's BIOS Boot Partition and boot.img to each MBR while preserving the partition table, which a raw copy cannot guarantee. The ESP mount-reuse guard is kept for the UEFI install. "mkbootable sync" no longer needs a arg -- it reinstalls from the current shared modules on all members. --- ungrub/mkbootable | 129 +++++++++++++++++++++------------------------- 1 file changed, 60 insertions(+), 69 deletions(-) diff --git a/ungrub/mkbootable b/ungrub/mkbootable index 2ad425b70d..9283e32be4 100755 --- a/ungrub/mkbootable +++ b/ungrub/mkbootable @@ -13,8 +13,8 @@ # Reconfigure grub.cfg and theme.txt after UUID change # mkbootable reconfigure -# Copy the EFI core from a known-good member to every other member's ESP -# mkbootable sync +# Reinstall GRUB on every boot-pool member (repairs drifted/stale cores) +# mkbootable sync #set -euo pipefail #set -x @@ -95,35 +95,46 @@ member_esp() { printf '/dev/%s%s2\n' "$disk" "$sep" } -# write the EFI core ($2) onto a single ESP ($1), via a real mount + atomic -# rename. Staged as BOOTX64.EFI.new then renamed over the live file -- mv on the -# same filesystem is rename(2), an atomic replace, so an interrupted write can -# never leave a torn or missing core. -# If the ESP is already mounted we reuse that mount (never stack a second mount -# on one FAT -- two rw mounts corrupt each other); otherwise we mount it on a -# temp dir and unmount when done. -install_efi_core() { - local esp="$1" src="$2" mnt dir owned=0 rc=0 +# Install both GRUB cores onto ONE pool member: the UEFI core to its ESP +# (removable path /EFI/BOOT/BOOTX64.EFI) and the BIOS core to its MBR (boot.img) +# plus BIOS Boot Partition (core.img). grub-install regenerates each core fresh +# from the shared modules under $BOOT/grub, so the member is always self +# consistent. The ESP must already hold a FAT filesystem (a freshly-added disk +# is formatted by install_grub before this runs). If the ESP is already mounted +# we reuse that mount (never stack a second rw mount on one FAT -- two corrupt +# each other); otherwise we mount it on a temp dir and unmount when done. +# $1 = disk (eg "nvme0n1" or "sdb") $2 = that disk's ESP (eg /dev/nvme0n1p2) +install_member_grub() { + local disk="$1" esp="$2" mnt owned=0 rc=0 mnt="$(findmnt -nro TARGET -S "$esp" 2>/dev/null | head -1)" if [[ -z "$mnt" ]]; then mnt="$(mktemp -d)" || return 1 if ! mount "$esp" "$mnt"; then - log "install_efi_core: cannot mount $esp" + log "install_member_grub: cannot mount $esp" rmdir "$mnt" return 1 fi owned=1 fi - dir="$mnt/EFI/BOOT" - if mkdir -p "$dir" \ - && cp -f "$src" "$dir/BOOTX64.EFI.new" \ - && mv -f "$dir/BOOTX64.EFI.new" "$dir/BOOTX64.EFI"; then - sync -f "$mnt" - else - rc=1 - fi + # UEFI core -> this member's ESP + grub-install \ + --target=x86_64-efi \ + --boot-directory="$BOOT" \ + --efi-directory="$mnt" \ + --modules="part_gpt zfs zfsinfo search search_fs_uuid configfile normal" \ + --removable --no-nvram || rc=1 + # BIOS core -> this member's MBR (boot.img) + BIOS Boot Partition (core.img). + # grub-install rewrites only the boot code in the MBR, preserving the disk's + # partition table. + grub-install \ + --target=i386-pc "/dev/$disk" \ + --boot-directory="$BOOT" \ + --modules="part_gpt zfs zfsinfo search search_fs_uuid configfile normal" \ + --no-floppy --no-rs-codes || rc=1 + + sync -f "$mnt" if (( owned )); then umount "$mnt" rmdir "$mnt" @@ -131,54 +142,39 @@ install_efi_core() { return $rc } -# Copy a known-good EFI core to the ESP of every boot pool member except the one -# it came from. grub-install rebuilds the shared modules under $BOOT/grub but -# writes the core to only one ESP; a member left with a stale core no longer -# matches those modules and fails to boot with "symbol 'grub_memcpy' not found", -# silently breaking the mirror's boot redundancy. The core uses a dynamic -# ($root) prefix, so the same image is valid on every member. -# $1 = path to the known-good BOOTX64.EFI to distribute (on a mounted ESP) -replicate_grub() { - local src="$1" part esp +# (Re)install GRUB on EVERY member of the boot pool. grub-install rebuilds the +# shared modules under $BOOT/grub and writes a fresh, matching core to each +# member -- UEFI (ESP) and BIOS (MBR + BIOS Boot Partition) alike. This is what +# keeps every member in lockstep with the rebuilt modules: a member left with a +# stale core no longer matches them and fails to boot with "symbol 'grub_memcpy' +# not found", silently breaking the mirror's boot redundancy. Reinstalling per +# member is slower than copying one core, but it is the canonical, correct +# operation (cf. the OpenZFS "repeat grub-install for each disk" guidance) and +# covers BIOS, which a bare file copy cannot. Idempotent; runs only on +# add/replace/sync. +install_grub_all_members() { + local part disk esp for part in $(zpool list -v -H -P "$POOL" | awk '/\/dev\// {print $1}'); do - esp="$(member_esp "$part")" || { log "replicate_grub: cannot resolve ESP for $part, skipping"; continue; } - [[ "$esp" == "$EFI_PART" || ! -b "$esp" ]] && continue - if install_efi_core "$esp" "$src"; then - log "replicate_grub: synced EFI core to $esp" + disk="$(lsblk -no pkname "$part" 2>/dev/null | head -1)" + esp="$(member_esp "$part")" || { log "install_grub_all_members: cannot resolve ESP for $part, skipping"; continue; } + [[ -n "$disk" && -b "$esp" ]] || { log "install_grub_all_members: no ESP for $part, skipping"; continue; } + if install_member_grub "$disk" "$esp"; then + log "install_grub_all_members: installed GRUB on $disk ($esp)" else - log "replicate_grub: failed to sync EFI core to $esp" + log "install_grub_all_members: FAILED to install GRUB on $disk ($esp)" fi done } install_grub() { - # format the EFI System Partition (partition 2) with FAT32 and mount + # format the freshly-added disk's EFI System Partition (partition 2) with FAT32 + # (existing members keep their ESP -- install_grub_all_members never reformats) mkfs.fat -F32 -n EFI "$EFI_PART" - mkdir -p -m 0700 "$BOOT/efi" - mount "$EFI_PART" "$BOOT/efi" - # UEFI install - grub-install \ - --target=x86_64-efi \ - --boot-directory="$BOOT" \ - --efi-directory="$BOOT/efi" \ - --modules="part_gpt zfs zfsinfo search search_fs_uuid configfile normal" \ - --removable --no-nvram - - # BIOS install - grub-install \ - --target=i386-pc "/dev/$DEV" \ - --boot-directory="$BOOT" \ - --modules="part_gpt zfs zfsinfo search search_fs_uuid configfile normal" \ - --no-floppy --no-rs-codes - - # replicate the core we just built to every other member's ESP so all mirror - # members stay in lockstep with the shared modules rebuilt above (no-op while - # this is the only member) - replicate_grub "$BOOT/efi/EFI/BOOT/BOOTX64.EFI" - - # we don't need this mounted anymore - umount "$BOOT/efi" + # install UEFI + BIOS GRUB on every pool member -- the new disk plus any + # pre-existing members -- so every core stays in lockstep with the shared + # modules rebuilt here (a single-member pool just does the one disk) + install_grub_all_members } add_device() { @@ -236,17 +232,12 @@ remove_device() { fi } -# Re-distribute the EFI core from a known-good member to all other members. -# Use this to repair members whose ESP holds a stale core (eg a device added -# with a bare "zpool attach", or a member missed by an earlier grub refresh). +# Reinstall GRUB on every boot-pool member from the current shared modules. +# Use this to repair members whose on-disk core has drifted from the modules +# (eg a device added with a bare "zpool attach", or a member missed by an +# earlier grub refresh). Restores UEFI and BIOS cores on all members in lockstep. sync_device() { - mkdir -p -m 0700 "$BOOT/efi" - if ! mount -o ro "$EFI_PART" "$BOOT/efi" 2>/dev/null; then - log "sync: cannot mount source ESP $EFI_PART" - exit 1 - fi - replicate_grub "$BOOT/efi/EFI/BOOT/BOOTX64.EFI" - umount "$BOOT/efi" + install_grub_all_members } case "$OPER" in From 769bab1eae041484c6b66818ddd46733cbba6af9 Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Thu, 25 Jun 2026 13:00:36 -0400 Subject: [PATCH 3/3] fix(ungrub): make grub install fail loudly -- non-zero exit on any member install_grub_all_members logged per-member failures but always returned 0, so mkbootable add/sync could report success while a member was skipped or failed to update -- silently leaving behind the exact stale-core drift this fix is meant to prevent. Now it counts skipped/failed members, logs a "N of M member(s) failed" summary, and returns non-zero if any member did not get a fresh core (or the pool enumerated zero members). install_grub and add_device propagate that status, and the operation dispatch exits with it, so a degraded remediation surfaces as a failure to the caller instead of a false success. --- ungrub/mkbootable | 43 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/ungrub/mkbootable b/ungrub/mkbootable index 9283e32be4..e4fc8f69ee 100755 --- a/ungrub/mkbootable +++ b/ungrub/mkbootable @@ -153,31 +153,52 @@ install_member_grub() { # covers BIOS, which a bare file copy cannot. Idempotent; runs only on # add/replace/sync. install_grub_all_members() { - local part disk esp + local part disk esp count=0 fail=0 for part in $(zpool list -v -H -P "$POOL" | awk '/\/dev\// {print $1}'); do + count=$((count + 1)) disk="$(lsblk -no pkname "$part" 2>/dev/null | head -1)" - esp="$(member_esp "$part")" || { log "install_grub_all_members: cannot resolve ESP for $part, skipping"; continue; } - [[ -n "$disk" && -b "$esp" ]] || { log "install_grub_all_members: no ESP for $part, skipping"; continue; } + esp="$(member_esp "$part")" || { log "install_grub_all_members: cannot resolve ESP for $part"; fail=$((fail + 1)); continue; } + [[ -n "$disk" && -b "$esp" ]] || { log "install_grub_all_members: no ESP block device for $part ($esp)"; fail=$((fail + 1)); continue; } if install_member_grub "$disk" "$esp"; then log "install_grub_all_members: installed GRUB on $disk ($esp)" else log "install_grub_all_members: FAILED to install GRUB on $disk ($esp)" + fail=$((fail + 1)) fi done + # A member we skipped or failed to update is one left with a stale core -- the + # exact drift this fix exists to prevent -- so surface it as a non-zero exit + # instead of letting add/sync report success. + if (( count == 0 )); then + log "install_grub_all_members: no members found in pool $POOL" + return 1 + fi + if (( fail > 0 )); then + log "install_grub_all_members: $fail of $count member(s) failed -- boot redundancy degraded" + return 1 + fi + log "install_grub_all_members: all $count member(s) updated" + return 0 } install_grub() { + local rc=0 + # format the freshly-added disk's EFI System Partition (partition 2) with FAT32 # (existing members keep their ESP -- install_grub_all_members never reformats) - mkfs.fat -F32 -n EFI "$EFI_PART" + mkfs.fat -F32 -n EFI "$EFI_PART" || { log "install_grub: mkfs.fat failed on $EFI_PART"; rc=1; } # install UEFI + BIOS GRUB on every pool member -- the new disk plus any # pre-existing members -- so every core stays in lockstep with the shared # modules rebuilt here (a single-member pool just does the one disk) - install_grub_all_members + install_grub_all_members || rc=1 + + return $rc } add_device() { + local rc=0 + # create gpt partition layout create_partitions @@ -206,21 +227,23 @@ add_device() { zfs mount "$POOL/boot" # install grub - install_grub + install_grub || rc=1 # configure grub.cfg - configure_grub + configure_grub || rc=1 else # attach to first existing device already in the pool EXISTING=$(zpool list -v -H -P "$POOL" | awk '/\/dev\// {print $1; exit}') zpool attach -f "$POOL" "$EXISTING" "$BOOT_PART" # install grub - install_grub + install_grub || rc=1 # wait for resilver to finish zpool wait -t resilver "$POOL" fi + + return $rc } remove_device() { @@ -260,4 +283,6 @@ case "$OPER" in log "error 2" # shouldn't happen exit 2 esac -exit 0 +# propagate the operation's exit status so a partial/failed grub install (a +# member left with a stale core) is never reported as success +exit $?