fix(ungrub): replicate GRUB EFI core to all boot-pool members

ungrub/mkbootable

@@ -13,6 +13,9 @@
 # Reconfigure grub.cfg and theme.txt after UUID change
 # mkbootable reconfigure
 
+# Reinstall GRUB on every boot-pool member (repairs drifted/stale cores)
+# mkbootable sync
+
 #set -euo pipefail
 #set -x
 source /etc/rc.d/rc.runlog
@@ -82,32 +85,120 @@ resize_partitions() {
   udevadm settle
 }
 
-install_grub() {
-  # format the EFI System Partition (partition 2) with FAT32 and mount
-  mkfs.fat -F32 -n EFI "$EFI_PART"
-  mkdir -p -m 0700 "$BOOT/efi"
-  mount "$EFI_PART" "$BOOT/efi"
+# resolve a boot-pool member partition to that disk's EFI System Partition,
+# eg /dev/nvme1n1p3 -> /dev/nvme1n1p2, /dev/sdb3 -> /dev/sdb2
+member_esp() {
+  local part="$1" disk sep
+  disk="$(lsblk -no pkname "$part" 2>/dev/null | head -1)"
+  [[ -n "$disk" ]] || return 1
+  [[ $disk == *[0-9] ]] && sep="p" || sep=""
+  printf '/dev/%s%s2\n' "$disk" "$sep"
+}
 
-  # UEFI install
+# Install both GRUB cores onto ONE pool member: the UEFI core to its ESP
+# (removable path /EFI/BOOT/BOOTX64.EFI) and the BIOS core to its MBR (boot.img)
+# plus BIOS Boot Partition (core.img). grub-install regenerates each core fresh
+# from the shared modules under $BOOT/grub, so the member is always self
+# consistent. The ESP must already hold a FAT filesystem (a freshly-added disk
+# is formatted by install_grub before this runs). If the ESP is already mounted
+# we reuse that mount (never stack a second rw mount on one FAT -- two corrupt
+# each other); otherwise we mount it on a temp dir and unmount when done.
+#   $1 = disk (eg "nvme0n1" or "sdb")   $2 = that disk's ESP (eg /dev/nvme0n1p2)
+install_member_grub() {
+  local disk="$1" esp="$2" mnt owned=0 rc=0
+  mnt="$(findmnt -nro TARGET -S "$esp" 2>/dev/null | head -1)"
+  if [[ -z "$mnt" ]]; then
+    mnt="$(mktemp -d)" || return 1
+    if ! mount "$esp" "$mnt"; then
+      log "install_member_grub: cannot mount $esp"
+      rmdir "$mnt"
+      return 1
+    fi
+    owned=1
+  fi
+
+  # UEFI core -> this member's ESP
   grub-install \
     --target=x86_64-efi \
     --boot-directory="$BOOT" \
-    --efi-directory="$BOOT/efi" \
+    --efi-directory="$mnt" \
     --modules="part_gpt zfs zfsinfo search search_fs_uuid configfile normal" \
-    --removable --no-nvram
+    --removable --no-nvram || rc=1
 
-  # BIOS install
+  # BIOS core -> this member's MBR (boot.img) + BIOS Boot Partition (core.img).
+  # grub-install rewrites only the boot code in the MBR, preserving the disk's
+  # partition table.
   grub-install \
-    --target=i386-pc "/dev/$DEV" \
+    --target=i386-pc "/dev/$disk" \
     --boot-directory="$BOOT" \
     --modules="part_gpt zfs zfsinfo search search_fs_uuid configfile normal" \
-    --no-floppy --no-rs-codes
+    --no-floppy --no-rs-codes || rc=1
 
-  # we don't need this mounted anymore
-  umount "$BOOT/efi"
+  sync -f "$mnt"
+  if (( owned )); then
+    umount "$mnt"
+    rmdir "$mnt"
+  fi
+  return $rc
+}
+
+# (Re)install GRUB on EVERY member of the boot pool. grub-install rebuilds the
+# shared modules under $BOOT/grub and writes a fresh, matching core to each
+# member -- UEFI (ESP) and BIOS (MBR + BIOS Boot Partition) alike. This is what
+# keeps every member in lockstep with the rebuilt modules: a member left with a
+# stale core no longer matches them and fails to boot with "symbol 'grub_memcpy'
+# not found", silently breaking the mirror's boot redundancy. Reinstalling per
+# member is slower than copying one core, but it is the canonical, correct
+# operation (cf. the OpenZFS "repeat grub-install for each disk" guidance) and
+# covers BIOS, which a bare file copy cannot. Idempotent; runs only on
+# add/replace/sync.
+install_grub_all_members() {
+  local part disk esp count=0 fail=0
+  for part in $(zpool list -v -H -P "$POOL" | awk '/\/dev\// {print $1}'); do
+    count=$((count + 1))
+    disk="$(lsblk -no pkname "$part" 2>/dev/null | head -1)"
+    esp="$(member_esp "$part")" || { log "install_grub_all_members: cannot resolve ESP for $part"; fail=$((fail + 1)); continue; }
+    [[ -n "$disk" && -b "$esp" ]] || { log "install_grub_all_members: no ESP block device for $part ($esp)"; fail=$((fail + 1)); continue; }
+    if install_member_grub "$disk" "$esp"; then
+      log "install_grub_all_members: installed GRUB on $disk ($esp)"
+    else
+      log "install_grub_all_members: FAILED to install GRUB on $disk ($esp)"
+      fail=$((fail + 1))
+    fi
+  done
+  # A member we skipped or failed to update is one left with a stale core -- the
+  # exact drift this fix exists to prevent -- so surface it as a non-zero exit
+  # instead of letting add/sync report success.
+  if (( count == 0 )); then
+    log "install_grub_all_members: no members found in pool $POOL"
+    return 1
+  fi
+  if (( fail > 0 )); then
+    log "install_grub_all_members: $fail of $count member(s) failed -- boot redundancy degraded"
+    return 1
+  fi
+  log "install_grub_all_members: all $count member(s) updated"
+  return 0
+}
+
+install_grub() {
+  local rc=0
+
+  # format the freshly-added disk's EFI System Partition (partition 2) with FAT32
+  # (existing members keep their ESP -- install_grub_all_members never reformats)
+  mkfs.fat -F32 -n EFI "$EFI_PART" || { log "install_grub: mkfs.fat failed on $EFI_PART"; rc=1; }
+
+  # install UEFI + BIOS GRUB on every pool member -- the new disk plus any
+  # pre-existing members -- so every core stays in lockstep with the shared
+  # modules rebuilt here (a single-member pool just does the one disk)
+  install_grub_all_members || rc=1
+
+  return $rc
 }
 
 add_device() {
+  local rc=0
+
   # create gpt partition layout
   create_partitions
 
@@ -136,21 +227,23 @@ add_device() {
     zfs mount "$POOL/boot"
 
     # install grub
-    install_grub
+    install_grub || rc=1
 
     # configure grub.cfg
-    configure_grub
+    configure_grub || rc=1
   else
     # attach to first existing device already in the pool
     EXISTING=$(zpool list -v -H -P "$POOL" | awk '/\/dev\// {print $1; exit}')
     zpool attach -f "$POOL" "$EXISTING" "$BOOT_PART"
 
     # install grub
-    install_grub
+    install_grub || rc=1
 
     # wait for resilver to finish
     zpool wait -t resilver "$POOL"
   fi
+
+  return $rc
 }
 
 remove_device() {
@@ -162,6 +255,14 @@ remove_device() {
   fi
 }
 
+# Reinstall GRUB on every boot-pool member from the current shared modules.
+# Use this to repair members whose on-disk core has drifted from the modules
+# (eg a device added with a bare "zpool attach", or a member missed by an
+# earlier grub refresh). Restores UEFI and BIOS cores on all members in lockstep.
+sync_device() {
+  install_grub_all_members
+}
+
 case "$OPER" in
 'add')
   add_device
@@ -175,8 +276,13 @@ case "$OPER" in
 'reconfigure')
   configure_grub
   ;;
+'sync')
+  sync_device
+  ;;
 *)
   log "error 2" # shouldn't happen
   exit 2
 esac
-exit 0
+# propagate the operation's exit status so a partial/failed grub install (a
+# member left with a stale core) is never reported as success
+exit $?