From 892425cae8d096d8811506d40a8df946c3f7bd72 Mon Sep 17 00:00:00 2001 From: Paul Menzel Date: Wed, 29 Jul 2026 19:30:49 +0200 Subject: [PATCH] mxgrub: unload NVIDIA driver before kexec reboot NVIDIA GSP-based GPUs (e.g. the A100 in jabberwocky) are not reset by kexec. The GSP firmware and its WPR2 carveout survive into the next kernel, whose freshly-initialised IOMMU then rejects the GPU's stale DMA. The result is an AMD-Vi IO_PAGE_FAULT storm, a GSP exception (Xid 120, load access fault), and RmInitAdapter failing with 0x62 (NV_ERR_RESET_REQUIRED), so nvidia-smi reports "No devices were found". A cold boot is unaffected because the GPU is power-cycled; only the kexec path leaves the device in this wedged state. Unload the NVIDIA driver in cmd_reboot() before handing off to the new kernel, so the driver shuts the GSP down cleanly and the next kernel enumerates a healthy GPU. Guarded on the driver actually being loaded, so it is a no-op on the rest of the fleet, and best effort otherwise: if a module is missing or busy the reboot still proceeds. Resolves: https://github.molgen.mpg.de/mariux64/mxtools/issues/557 Assisted-by: Claude Opus 4.8 --- mxgrub/mxgrub | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/mxgrub/mxgrub b/mxgrub/mxgrub index da540d6..b3cb485 100755 --- a/mxgrub/mxgrub +++ b/mxgrub/mxgrub @@ -373,10 +373,27 @@ sub cmd_initramfs { sys('bash','-c','cd /project/admin/initramfs ; find . -name ".git*" -prune -or -print | cpio -H newc -o | gzip > /boot/grub/initramfs.igz'); } +# NVIDIA GSP-based GPUs (e.g. A100) are not reset by kexec: the GSP and its +# WPR2 carveout survive into the next kernel, whose freshly-initialised IOMMU +# then rejects the stale DMA (AMD-Vi IO_PAGE_FAULT), the GSP faults (Xid 120), +# and the driver cannot re-init (RmInitAdapter 0x62 = NV_ERR_RESET_REQUIRED), +# so nvidia-smi reports "No devices were found". Unloading the driver first +# lets it shut the GSP down cleanly so the next kernel enumerates a healthy +# GPU. Best effort: if a module is absent or busy, proceed with the reboot +# anyway (no worse than not unloading at all). +sub quiesce_nvidia { + return if system('lsmod | grep -q "^nvidia "'); # no NVIDIA driver loaded + warn "mxgrub: unloading NVIDIA driver before kexec ...\n"; + for my $mod (qw(nvidia_drm nvidia_modeset nvidia_uvm nvidia)) { + system('rmmod',$mod); # ignore failures (not loaded / in use) + } +} + sub cmd_reboot { my $chosen=get_chosen(); my $image=label_to_image($chosen); -e "/boot/$image" or die "/boot/$image: no such file\n"; + quiesce_nvidia(); sys('sync'); sys('kexec',"/boot/$image",'--initrd=/boot/grub/initramfs.igz',"--command-line=root=LABEL=root $KERNEL_PARAMETER"); }