From 5fa45b663efc90b6776823ae0d326d8f95016bb0 Mon Sep 17 00:00:00 2001 From: Paul Menzel Date: Mon, 14 Sep 2026 18:56:28 +0200 Subject: [PATCH 1/4] smoke-tests: Add a smoke test for an updated kernel linux.be0 configures with `make olddefconfig', which answers every new or newly visible symbol with its default and never asks, so a symbol that upstream renames, splits or re-parents turns off without a word and its module is simply not built. The update from 6.12 to 6.18 cost several modules that way, and nothing noticed until the hardware they drive was used. `make oldconfig' would have asked, but it cannot be run unattended. The script is meant to be run on a machine rebooted into the new kernel, before the kernel is rolled out further, and looks at it the way the regression shows up in practice: * every device in /sys is matched against the running kernel's modules.alias and against the reference kernel's, so a device that had a driver and has none now is named, whatever the config says; * the modules of the previous kernel are diffed against this one, with a table of known upstream removals to keep the list short enough to read; * symbols the previous kernel had are split into the ones olddefconfig turned off, which are ours to turn back on, and the ones upstream deleted, which are not; * a new symbol answered n that gates a feature still configured -- the _LEGACY pattern -- is reported on its own; * 113 config symbols the site depends on, each with the reason it is listed, from autofs and NFSv3 to the BMC console and k10temp; * the modules /etc/modprobe.d names have to exist; * NFS, the automounter, cgroup v2, an overlayfs mount in a user namespace and a veth pair in a network namespace are exercised; * dmesg, failed units, sensors, EDAC, IPMI, cpufreq and the clocksource are read for what a wrong kernel leaves behind. --save-baseline writes what a machine looks like before the reboot and --baseline compares after it, which is the only check that knows what the old kernel really did rather than what it could have done. --kernel checks an installed kernel against this machine's hardware without booting it first. Everything works as an ordinary user; what needs root says so and is skipped. A run takes under two seconds. On a 6.12.98-492 machine it is silent, and the kernel we are moving to reports, among others: WARN CONFIG_BRIDGE_NF_EBTABLES_LEGACY is new in 6.18.51.mx64.498 and not set, while CONFIG_BRIDGE_NF_EBTABLES is m: whatever it gates is no longer built which is this bug caught in the act: no ebt module is built since 6.12 although the config still says CONFIG_BRIDGE_NF_EBTABLES=m. Assisted-by: Claude Opus 5 --- smoke-tests/linux-smoke-test | 1298 ++++++++++++++++++++++++++++++++++ 1 file changed, 1298 insertions(+) create mode 100755 smoke-tests/linux-smoke-test diff --git a/smoke-tests/linux-smoke-test b/smoke-tests/linux-smoke-test new file mode 100755 index 000000000..b8dc5f10c --- /dev/null +++ b/smoke-tests/linux-smoke-test @@ -0,0 +1,1298 @@ +#! /usr/bin/env python3 + +# Smoke-test the kernel a machine is running. +# +# smoke-tests/linux-smoke-test +# smoke-tests/linux-smoke-test --reference 6.12.100.mx64.493 +# smoke-tests/linux-smoke-test --save-baseline ~/before.json # old kernel +# smoke-tests/linux-smoke-test --baseline ~/before.json # after boot +# smoke-tests/linux-smoke-test --kernel 6.18.51.mx64.498 # before boot +# +# linux.be0 configures with `make olddefconfig', which answers every new or +# newly visible symbol with its default and never asks, so a symbol that +# upstream renames, splits or re-parents silently turns off -- and the module +# is simply gone from the built kernel. Going from 6.12 to 6.18 that cost us +# several drivers, and nothing noticed until the hardware they drive was used. +# `make oldconfig' would have asked, but it cannot be used unattended. +# +# This script therefore looks at the running kernel the way that regression +# shows up in practice: hardware that no longer has a driver, filesystems and +# network features that no longer work, modules that the reference kernel had +# and this one has not. It is meant to be run on a machine that has been +# rebooted into the new kernel, before the new kernel is rolled out further. +# +# Everything works as an ordinary user; the checks that need root say so and +# are skipped. Run it as root as well if you can, which adds the checks that +# actually load modules and mount filesystems. +# +# FAIL is a defect in the kernel package: something the site needs is missing. +# WARN depends on the machine rather than on the package (hardware absent, a +# device deliberately unbound, a buffer already wrapped) and needs a look. +# SKIP is a check this machine cannot answer. + +import argparse +import collections +import fnmatch +import glob +import gzip +import json +import os +import re +import shlex +import subprocess +import sys + +status = collections.Counter() +verbose = False + +# Config symbols the site depends on, with the reason each one is here, so a +# later reader can tell an obsolete entry from a load-bearing one. Everything +# in `fail' has to be =y or =m or this kernel must not be rolled out; +# everything in `warn' is worth a look but does not stop a machine from +# working. Keep grounded in what mariux actually runs, not in what a kernel +# could conceivably want. +REQUIRED = [ + # Root and scratch filesystems, /etc/fstab and the automounter. + ("EXT4_FS", "fail", "root and local filesystems"), + ("XFS_FS", "fail", "/scratch and most server filesystems"), + ("TMPFS", "fail", "/dev/shm, which MXQ jobs use as their scratch space"), + ("TMPFS_POSIX_ACL", "warn", "ACLs on /dev/shm"), + ("HUGETLBFS", "warn", "hugepage-backed jobs"), + ("AUTOFS_FS", "fail", "/pkg, /project and every other automounted tree"), + ("PROC_FS", "fail", "/proc"), + ("SYSFS", "fail", "/sys"), + ("DEVTMPFS", "fail", "/dev"), + ("DEVTMPFS_MOUNT", "fail", "/dev before the initrd hands over"), + ("BLK_DEV_INITRD", "fail", "the dracut initrd this kernel boots with"), + ("EFIVAR_FS", "warn", "efibootmgr on the UEFI machines"), + ("BLK_DEV_DM", "fail", "the LVM volumes /var and /scratch live on"), + ("MD_RAID1", "warn", "mirrored system disks"), + ("SQUASHFS", "warn", "squashfs images, e.g. under squashfuse"), + ("FUSE_FS", "fail", "fuse-overlayfs, squashfuse and sshfs"), + ("OVERLAY_FS", "fail", "the rootless overlay bee-file builds chain on"), + ("BTRFS_FS", "warn", "btrfs volumes on a few machines"), + ("ISO9660_FS", "warn", "mounting installation images"), + ("VFAT_FS", "fail", "the EFI system partition"), + + # NFS: everything here is an NFS environment. + ("NFS_FS", "fail", "every home directory"), + ("NFS_V3", "fail", "the mx64old exports still served over NFSv3"), + ("NFS_V4", "fail", "the default mount version"), + ("NFS_V4_1", "fail", "sessions and trunking"), + ("NFS_V4_2", "warn", "server-side copy and sparse files"), + ("NFSD", "warn", "the file servers, harmless on a client"), + ("SUNRPC_GSS", "warn", "Kerberised mounts"), + ("RPCSEC_GSS_KRB5", "warn", "Kerberised mounts"), + ("CIFS", "warn", "the few Samba shares"), + + # Namespaces, cgroups and BPF: containers, systemd and the bee overlay. + ("NAMESPACES", "fail", "systemd and every container"), + ("USER_NS", "fail", "rootless overlay builds and unprivileged podman"), + ("PID_NS", "fail", "systemd services"), + ("NET_NS", "fail", "containers and network testing"), + ("CGROUPS", "fail", "systemd"), + ("MEMCG", "fail", "MXQ memory accounting"), + ("CGROUP_SCHED", "fail", "CPU shares"), + ("BLK_CGROUP", "warn", "I/O accounting"), + ("CGROUP_CPUACCT", "warn", "CPU accounting"), + ("CPUSETS", "fail", "MXQ pins jobs to processors with cpusets"), + ("SECCOMP", "fail", "systemd sandboxing and browsers"), + ("BPF_SYSCALL", "fail", "systemd's IP accounting and bpftrace"), + ("PERF_EVENTS", "fail", "perf, which people profile their jobs with"), + ("KPROBES", "warn", "tracing"), + ("FTRACE", "warn", "tracing"), + ("DEBUG_FS", "warn", "tracing and driver debugging"), + ("MAGIC_SYSRQ", "warn", "recovering a hung machine over the BMC console"), + + # Virtualisation. + ("KVM", "fail", "the virtual machines on the compute servers"), + ("KVM_AMD", "fail", "this is an AMD site"), + ("KVM_INTEL", "warn", "the remaining Intel machines"), + ("VHOST_NET", "fail", "virtio networking in those machines"), + ("TUN", "fail", "qemu, OpenVPN and podman"), + ("VIRTIO_PCI", "warn", "running this kernel inside a VM"), + ("VIRTIO_NET", "warn", "running this kernel inside a VM"), + ("VIRTIO_BLK", "warn", "running this kernel inside a VM"), + + # Networking. + ("IPV6", "fail", "the site is dual stack"), + ("VLAN_8021Q", "fail", "the tagged VLANs on the server ports"), + ("BRIDGE", "fail", "VM and container bridges"), + ("VETH", "fail", "containers"), + ("MACVLAN", "warn", "some VM setups"), + ("BONDING", "fail", "the bonded uplinks on the file servers"), + ("NF_TABLES", "fail", "nft, the firewall front end"), + ("IP_NF_IPTABLES", "warn", "iptables-legacy rules still in use"), + ("NF_CONNTRACK", "fail", "stateful filtering and NAT"), + ("NETFILTER_XT_TARGET_MASQUERADE", "fail", "podman and libvirt NAT"), + ("NF_NAT", "fail", "podman and libvirt NAT"), + ("INET_DIAG", "warn", "ss(8)"), + + # Storage and network adapters we buy. + ("BLK_DEV_NVME", "fail", "every recent machine boots off NVMe"), + ("SCSI_MPT3SAS", "fail", "the SAS HBAs in the file servers"), + ("MEGARAID_SAS", "fail", "the LSI/Broadcom RAID controllers"), + ("SCSI_AACRAID", "fail", "the Adaptec controllers " + "/etc/modprobe.d/adaptec-raid-controllers.conf " + "sets options for"), + ("ATA_PIIX", "warn", "old SATA machines"), + ("SATA_AHCI", "fail", "SATA disks"), + ("IGB", "fail", "the Intel 1G ports on the mainboards"), + ("IXGBE", "fail", "the Intel 10G cards " + "/etc/modprobe.d/ixgbe_sfp.conf sets options for"), + ("I40E", "warn", "the Intel 40G cards"), + ("E1000E", "warn", "older Intel 1G ports"), + ("R8169", "warn", "Realtek ports on desktop mainboards"), + ("TIGON3", "warn", "Broadcom ports on the Dell servers"), + ("USB_XHCI_HCD", "fail", "every USB port"), + ("USB_STORAGE", "fail", "USB disks"), + ("HID_GENERIC", "fail", "keyboards and mice"), + ("USB_HID", "fail", "keyboards and mice"), + ("HID_APPLE", "warn", "the Apple keyboards " + "/etc/modprobe.d/hid_apple.conf configures"), + + # Graphics: the desktops, and the BMC chip that gives a server a console. + ("DRM", "fail", "any display at all"), + ("DRM_I915", "warn", "Intel graphics on the desktops"), + ("DRM_AMDGPU", "warn", "AMD graphics on the desktops"), + ("DRM_NOUVEAU", "warn", "the NVIDIA cards not driven by nvidia.ko"), + ("DRM_MGAG200", "warn", "the Matrox BMC chip in the older servers"), + # The console of a server comes from what the firmware set up, not from a + # driver for its BMC chip -- lose these three and a machine that fails to + # boot has nothing to show for it on the remote console. + ("SYSFB_SIMPLEFB", "fail", "the framebuffer the firmware hands over"), + ("DRM_SIMPLEDRM", "fail", "the console on that framebuffer"), + ("FB_EFI", "fail", "the console before DRM takes over"), + ("DRM_FBDEV_EMULATION", "fail", "a text console on any DRM driver"), + ("FRAMEBUFFER_CONSOLE", "fail", "the text console"), + ("SND_HDA_INTEL", "warn", "desktop audio"), + ("SND_USB_AUDIO", "warn", "USB headsets"), + + # Hardware health: a server that cannot report a failing DIMM or a hot CPU + # is the kind of loss nobody notices until the machine dies. + # Not built at this site as of 6.18: the iDRAC and the other BMCs log a + # failing DIMM themselves, so the kernel side has never been switched on. + # Listed as `info' rather than dropped, so the next reader knows it was + # looked at and decided, and can promote it if that ever changes. + ("EDAC", "info", "memory error reporting in the kernel"), + ("EDAC_AMD64", "info", "memory error reporting on AMD"), + ("X86_MCE", "fail", "machine check reporting"), + ("IPMI_HANDLER", "fail", "the BMC, which racadm and ipmitool talk to"), + ("IPMI_SI", "fail", "the BMC system interface"), + ("IPMI_DEVICE_INTERFACE", "fail", "/dev/ipmi0"), + ("SENSORS_K10TEMP", "fail", "CPU temperature on AMD"), + ("SENSORS_CORETEMP", "warn", "CPU temperature on Intel"), + ("I2C_PIIX4", "warn", "the SMBus the sensor chips hang off"), + ("HWMON", "fail", "every temperature and fan reading"), + ("THERMAL", "warn", "thermal throttling"), + ("WATCHDOG", "warn", "the hardware watchdog"), + ("DMIID", "fail", "dmidecode and every udev rule matching a board"), + ("ACPI", "fail", "anything modern"), + ("NUMA", "fail", "the multi-socket compute servers"), + ("TRANSPARENT_HUGEPAGE", "warn", "large-memory jobs"), + ("X86_ACPI_CPUFREQ", "warn", "frequency scaling on the older machines"), + ("X86_AMD_PSTATE", "warn", "frequency scaling on the newer AMD machines"), +] + +# Modules and symbols upstream removed or renamed on purpose. Without this the +# reference diff drowns a real loss in a page of known churn; with it, anything +# printed unannotated is worth reading. Add to it as kernels come and go, and +# say in which version, so the entry can be retired eventually. +UPSTREAM_GONE = { + "dccp": "DCCP removed upstream in 6.16", + "dccp_diag": "DCCP removed upstream in 6.16", + "dccp_ipv4": "DCCP removed upstream in 6.16", + "dccp_ipv6": "DCCP removed upstream in 6.16", + "nf_conntrack_proto_dccp": "DCCP removed upstream in 6.16", + "reiserfs": "reiserfs removed upstream in 6.13", + "zbud": "zbud and zpool removed upstream in 6.16, zsmalloc took over", + "p8022": "802.2 helper removed upstream in 6.14", + "libcrc32c": "folded into the crc32c library in 6.15", + "crc32c_generic": "folded into the crc32c library in 6.15", + "crc32c-intel": "folded into the crc32c library in 6.15", + "crc64-rocksoft": "folded into the crc64 library in 6.14", + "crc64_rocksoft_generic": "folded into the crc64 library in 6.14", + "crct10dif_common": "folded into the crc-t10dif library in 6.15", + "crct10dif_generic": "folded into the crc-t10dif library in 6.15", + "sha1_generic": "folded into the sha1 library in 6.18", + "sha256_generic": "folded into the sha256 library in 6.17", + "sha512_generic": "folded into the sha512 library in 6.18", + "chacha-x86_64": "folded into the chacha library in 6.16", + "poly1305-x86_64": "folded into the poly1305 library in 6.16", + "curve25519-x86_64": "folded into the curve25519 library in 6.16", + "libcurve25519-generic": "folded into the curve25519 library in 6.16", + "cifs_arc4": "folded into the arc4 library in 6.16", + "drm_vram_helper": "last user dropped it in 6.16", + "cirrus": "the QEMU cirrus driver was rewritten as cirrus-qemu in 6.15", + "snd-hda-codec-realtek": "split into per-codec modules " + "(snd-hda-codec-alc*) in 6.17", + "snd-hda-codec-cirrus": "split into per-codec modules in 6.17", + "snd-hda-codec-analog": "split into per-codec modules in 6.17", + "snd-hda-codec-idt": "split into per-codec modules in 6.17", + "snd-hda-codec-via": "split into per-codec modules in 6.17", + "snd-hda-codec-conexant": "split into per-codec modules in 6.17", + "snd-hda-codec-cmedia": "split into per-codec modules in 6.17", + "i2c-amd756-s4882": "removed upstream in 6.15", + "i2c-nforce2-s4985": "removed upstream in 6.15", + "ext3": "ext3 has been served by ext4 since 4.3", +} + +# Filesystems that must be mountable, and how to make one to mount. Only the +# ones a plain user can create without a loop device; the rest are covered by +# the config and module checks. +MOUNTABLE = ["tmpfs", "ramfs"] + +# Bits of /proc/sys/kernel/tainted, `Documentation/admin-guide/tainted-kernels'. +TAINT = { + 0: ("G/P", "proprietary module loaded"), + 1: ("F", "module force loaded"), + 2: ("S", "SMP with a CPU not certified for it"), + 3: ("R", "module force unloaded"), + 4: ("M", "machine check exception"), + 5: ("B", "bad page referenced"), + 6: ("U", "user requested the taint"), + 7: ("D", "kernel oopsed before"), + 8: ("A", "ACPI table overridden"), + 9: ("W", "kernel issued a warning before"), + 10: ("C", "staging driver loaded"), + 11: ("I", "workaround for a firmware bug applied"), + 12: ("O", "out-of-tree module loaded"), + 13: ("E", "unsigned module loaded"), + 14: ("L", "soft lockup occurred"), + 15: ("K", "kernel live patched"), + 16: ("X", "auxiliary taint"), + 17: ("T", "built with struct randomisation"), + 18: ("N", "an in-kernel test ran"), +} + +# What must never be in the ring buffer, and what merely wants a look. +DMESG_FATAL = [ + (r"kernel BUG at", "a BUG() fired"), + (r"Oops:", "the kernel oopsed"), + (r"general protection fault", "a general protection fault"), + (r"BUG: unable to handle", "a bad memory access"), + (r"BUG: kernel NULL pointer", "a NULL pointer dereference"), + (r"soft lockup - CPU", "a soft lockup"), + (r"rcu_sched detected stalls", "an RCU stall"), + (r"Machine Check", "a machine check"), + (r"Hardware Error", "a hardware error"), + (r"Unknown symbol", "a module did not resolve against this kernel"), + (r"module verification failed", "a module was built against another tree"), +] +DMESG_SUSPECT = [ + (r"WARNING: CPU", "a WARN_ON() fired"), + (r"firmware: failed to load", "firmware is missing from /lib/firmware"), + (r"Direct firmware load for .* failed", "firmware is missing"), + (r"failed to load module", "a module could not be loaded"), + (r"I/O error", "an I/O error"), + (r"ACPI Error", "an ACPI error"), +] + +# PCI classes where a device without a driver means something is missing. The +# bridges, memory controllers and the assorted AMD housekeeping functions have +# no driver by design and would bury the interesting cases. +PCI_CLASSES = { + 0x01: "storage", + 0x02: "network", + 0x03: "display", + 0x04: "multimedia", + 0x0c: "serial bus", + 0x0d: "wireless", + 0x10: "encryption", + 0x12: "accelerator", +} + + +def report(state, message): + if state != "ok": + status[state] += 1 + if state != "info" or verbose: + print("%-5s %s" % (state.upper(), message)) + + +def note(message): + """Say something only when asked for details.""" + if verbose: + print(" %s" % message) + + +def run(command, timeout=60, stdin=None): + """Return (returncode, output) with stderr folded in; -1 if it cannot run.""" + try: + finished = subprocess.run(command, input=stdin, text=True, timeout=timeout, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + return finished.returncode, finished.stdout + except (OSError, subprocess.SubprocessError) as error: + return -1, str(error) + + +def read(path): + try: + with open(path) as handle: + return handle.read() + except OSError: + return None + + +def canonical(module): + """modprobe treats dashes and underscores alike; comparisons must too.""" + return module.replace("-", "_") + + +def explained(module): + """Why a module is missing, when it is upstream's doing and not ours.""" + return EXPLANATIONS.get(canonical(module)) + + +# UPSTREAM_GONE is written the way the modules are named, with the dashes. +EXPLANATIONS = {canonical(module): reason + for module, reason in UPSTREAM_GONE.items()} + + +def release_key(release): + """Sort key for `6.12.100.mx64.493': the mariux revision decides ties.""" + numbers = [int(part) for part in re.findall(r"\d+", release)] + return tuple(numbers + [0] * (5 - len(numbers)))[:5] + + +def series(release): + """`6.12' out of `6.12.100.mx64.493' -- what an update moves between.""" + return release_key(release)[:2] + + +def installed_releases(): + return sorted((os.path.basename(path)[len("config-"):] + for path in glob.glob("/boot/config-*")), key=release_key) + + +def default_reference(running): + """The newest installed kernel from the series we are updating away from. + + Comparing 6.18.51 against 6.18.46 finds nothing, because the loss happened + when the series changed. So skip our own series and take the newest kernel + below it, which for a 6.12 -> 6.18 update is the last 6.12 we ran. + """ + candidates = [release for release in installed_releases() + if series(release) < series(running)] + return candidates[-1] if candidates else None + + +def read_config(release=None): + """Symbol -> value for the running kernel, or for an installed one.""" + text = None + if release is None: + try: + with gzip.open("/proc/config.gz", "rt") as handle: + text = handle.read() + except OSError: + release = os.uname().release + if text is None: + text = read("/boot/config-%s" % release) + if text is None: + return {} + + config = {} + for line in text.splitlines(): + if line.startswith("CONFIG_") and "=" in line: + symbol, _, value = line.partition("=") + config[symbol[len("CONFIG_"):]] = value.strip('"') + elif line.startswith("# CONFIG_") and line.endswith(" is not set"): + config[line.split()[1][len("CONFIG_"):]] = "n" + return config + + +def module_directory(release): + return "/lib/modules/%s" % release + + +def _module_names(path): + names = set() + for line in (read(path) or "").splitlines(): + name = line.split(":")[0].strip() + if name: + names.add(canonical(re.sub(r"\.ko(\.(xz|gz|zst))?$", "", + os.path.basename(name)))) + return names + + +def loadable_of(release): + """The modules `release' can load, which is what modprobe can be asked for.""" + return _module_names(os.path.join(module_directory(release), "modules.dep")) + + +def modules_of(release): + """Every module the kernel `release' has, built in ones included.""" + return loadable_of(release) | _module_names( + os.path.join(module_directory(release), "modules.builtin")) + + +def aliases_of(release): + """(pattern, module) for everything `release' can be autoloaded by. + + modules.alias covers the loadable modules; the built-in ones keep their + aliases in modules.builtin.modinfo, a NUL separated `module.field=value' + blob, and a driver that moved from =m to =y would otherwise look gone. + """ + directory = module_directory(release) + aliases = [] + for line in (read(os.path.join(directory, "modules.alias")) or "").splitlines(): + parts = line.split() + if len(parts) == 3 and parts[0] == "alias": + aliases.append((parts[1], canonical(parts[2]))) + + try: + with open(os.path.join(directory, "modules.builtin.modinfo"), "rb") as blob: + for entry in blob.read().split(b"\0"): + field = entry.decode("utf-8", "replace") + match = re.match(r"([\w-]+)\.alias=(.*)$", field) + if match: + aliases.append((match.group(2), canonical(match.group(1)))) + except OSError: + pass + return aliases + + +def loaded_modules(): + modules = set() + for line in (read("/proc/modules") or "").splitlines(): + modules.add(canonical(line.split()[0])) + return modules + + +def blacklisted_modules(): + """What /etc/modprobe.d tells modprobe never to load.""" + blacklist = set() + for path in sorted(glob.glob("/etc/modprobe.d/*.conf")): + for line in (read(path) or "").splitlines(): + parts = line.split() + if len(parts) >= 2 and parts[0] == "blacklist": + blacklist.add(canonical(parts[1])) + return blacklist + + +class Matcher: + """Resolve a modalias to the modules a kernel would autoload for it.""" + + def __init__(self, release): + self.aliases = aliases_of(release) + self.cache = {} + + def __call__(self, modalias): + if modalias not in self.cache: + self.cache[modalias] = sorted( + {module for pattern, module in self.aliases + if fnmatch.fnmatchcase(modalias, pattern)}) + return self.cache[modalias] + + +def devices(): + """Every /sys device that says what driver it wants, and what it has. + + os.walk and not a recursive glob: /sys is full of symlinks that point back + up the tree -- every device has a `subsystem' link into /sys/bus -- and a + `**' pattern walks in circles through them until it gives up on the path + length. os.walk leaves symlinks alone unless told otherwise. + """ + found = [] + for directory, subdirectories, files in os.walk("/sys/devices"): + if "modalias" not in files: + continue + modalias = (read(os.path.join(directory, "modalias")) or "").strip() + if not modalias: + continue + driver = None + if os.path.islink(os.path.join(directory, "driver")): + driver = os.path.basename( + os.readlink(os.path.join(directory, "driver"))) + subsystem = None + if os.path.islink(os.path.join(directory, "subsystem")): + subsystem = os.path.basename( + os.readlink(os.path.join(directory, "subsystem"))) + # The basename alone is not an identity: an ACPI device such as + # PNP0C0C:00 also appears under /sys/devices/platform, and keying on + # it makes one of the two look driverless. + found.append({"path": directory, "modalias": modalias, + "driver": driver, "subsystem": subsystem, + "id": os.path.relpath(directory, "/sys/devices"), + "address": os.path.basename(directory)}) + return sorted(found, key=lambda device: device["path"]) + + +def pci_names(): + """Address -> readable name, so a failure names the card and not a alias.""" + names = {} + code, output = run(["lspci", "-D", "-mm"]) + if code != 0: + return names + for line in output.splitlines(): + try: + fields = shlex.split(line) + except ValueError: + continue + if len(fields) >= 4: + names[fields[0]] = "%s %s [%s]" % (fields[2], fields[3], fields[1]) + return names + + +def check_identity(running, reference): + """Which kernel is this, and is it the one that was meant to be tested.""" + if running == os.uname().release: + report("ok", "running %s, %s" % (running, os.uname().version)) + else: + report("ok", "checking the installed %s against the hardware of this" + " machine, which runs %s" % (running, os.uname().release)) + + if not os.path.isdir(module_directory(running)): + report("fail", "no %s -- the running kernel has no modules installed," + " so nothing can be loaded on demand" + % module_directory(running)) + if not read("/boot/config-%s" % running): + report("warn", "no /boot/config-%s; the package should install it" + % running) + + # A kernel booted from an older bzImage than the package installed is the + # usual reason a "fixed" kernel still misses a module. Only worth asking + # about the kernel that is actually running. + on_disk = read_config(running) if running == os.uname().release else {} + booted = read_config() + if on_disk and booted and on_disk != booted: + differing = sorted(set(on_disk) ^ set(booted)) or ["values"] + report("warn", "/proc/config.gz and /boot/config-%s differ (%s ...) --" + " is this machine running the installed bzImage?" + % (running, ", ".join(differing[:3]))) + + if reference: + report("ok", "comparing against %s" % reference) + else: + report("warn", "no reference kernel installed to compare against;" + " pass --reference to point at one") + + tainted = read("/proc/sys/kernel/tainted") + flags = int(tainted.strip()) if tainted else 0 + if flags: + names = ["%s (%s)" % (TAINT[bit][0], TAINT[bit][1]) + for bit in sorted(TAINT) if flags & (1 << bit)] + # Out-of-tree and proprietary are expected: nvidia.ko is both. + expected = flags & ~((1 << 0) | (1 << 12)) + report("warn" if expected else "ok", + "tainted 0x%x: %s" % (flags, ", ".join(names))) + else: + report("ok", "not tainted") + + +def check_config(config): + """The curated list of symbols the site cannot do without.""" + if not config: + report("fail", "no kernel configuration to check: neither" + " /proc/config.gz nor /boot/config-%s is readable" + % os.uname().release) + return + + missing = collections.Counter() + for symbol, severity, why in REQUIRED: + value = config.get(symbol, "n") + if value in ("y", "m"): + note("CONFIG_%s=%s" % (symbol, value)) + continue + missing[severity] += 1 + report(severity, "CONFIG_%s is not set -- needed for %s" % (symbol, why)) + + report("ok", "%d of %d required symbols set" + % (len(REQUIRED) - sum(missing.values()), len(REQUIRED))) + + +def check_modprobe_d(available): + """Every module /etc/modprobe.d talks about has to exist. + + These files are the site's own record of which drivers it cares about -- + options for the Adaptec controllers, for the ixgbe SFPs, for the Apple + keyboards. A module named here that the kernel no longer has is either a + lost driver or a stale file, and both want fixing. + """ + referenced = {} + for path in sorted(glob.glob("/etc/modprobe.d/*.conf")): + for line in (read(path) or "").splitlines(): + parts = line.split() + if len(parts) >= 2 and parts[0] in ("options", "install", "remove", + "blacklist", "softdep"): + keyword = parts[0] + # `install foo /bin/false' is how a module is really kept out; + # a plain `blacklist' only stops autoloading by alias. Either + # way the site wants the module gone, not present. + if keyword == "install" and parts[-1] in ("/bin/false", + "/bin/true"): + keyword = "blacklist" + referenced.setdefault(canonical(parts[1]), (path, keyword)) + elif len(parts) >= 3 and parts[0] == "alias": + referenced.setdefault(canonical(parts[2]), (path, parts[0])) + + if not referenced: + report("info", "no modules referenced from /etc/modprobe.d") + return + for module, (path, keyword) in sorted(referenced.items()): + if module in available: + note("%s from %s exists" % (module, os.path.basename(path))) + elif keyword == "blacklist": + # Keeping out something that is gone is untidy, not broken. + report("info", "%s is disabled in %s but does not exist" + % (module, path)) + else: + report("fail", "%s: %s %s, but this kernel has no such module" + % (path, keyword, module)) + report("ok", "%d modules referenced from /etc/modprobe.d" % len(referenced)) + + +def check_reference(running, reference, config): + """Diff this kernel against the one it replaces: the olddefconfig net. + + Two lists, because they catch different mistakes: a symbol that went from + =m to unset is a configuration loss, while a module that disappeared while + its symbol stayed =m is upstream moving code around. + """ + if not reference: + report("skip", "no reference kernel; module and symbol diff skipped") + return + + before = read_config(reference) + if not before: + report("skip", "no /boot/config-%s to diff against" % reference) + else: + # A symbol upstream deleted is gone from the new .config altogether; a + # symbol olddefconfig turned off is still there as `# ... is not set'. + # Only the second kind can be turned back on, and only that kind is + # this package's decision rather than upstream's, so separate them + # instead of printing one long list nobody reads twice. + turned_off, disappeared = [], [] + for symbol, value in sorted(before.items()): + if value not in ("y", "m") or config.get(symbol, "n") != "n": + continue + (turned_off if symbol in config else disappeared).append(symbol) + + for symbol in turned_off: + report("warn", "CONFIG_%s was %s in %s and is not set in %s --" + " olddefconfig answered for it" + % (symbol, before[symbol], reference, running)) + for symbol in disappeared: + note("CONFIG_%s: gone from the Kconfig of %s" % (symbol, running)) + report("ok", "%d symbols %s had are gone from the Kconfig of %s" + % (len(disappeared), reference, running)) + + # The other half of the same mistake: a symbol that is new in this + # kernel, answered n by default, and gates something we do have + # enabled. That is how ebtables.ko left a kernel whose + # CONFIG_BRIDGE_NF_EBTABLES still says =m -- 6.18 put the old tables + # behind a new CONFIG_BRIDGE_NF_EBTABLES_LEGACY and olddefconfig said + # no. Only the suffixes that split an existing feature are worth + # looking at, or every new driver in the tree ends up in this list. + for suffix in ("_LEGACY", "_COMPAT", "_DEPRECATED", "_OLD"): + for symbol, value in sorted(config.items()): + base = symbol[:-len(suffix)] + if (value == "n" and symbol.endswith(suffix) + and symbol not in before + and config.get(base, "n") in ("y", "m")): + report("warn", "CONFIG_%s is new in %s and not set, while" + " CONFIG_%s is %s: whatever it gates is no" + " longer built" + % (symbol, running, base, config[base])) + + # The old kernel's loadable modules against everything the new one has: + # a module that went =m to =y is not a loss, and comparing the built-in + # lists instead only turns up Kbuild merging objects differently -- between + # 6.6 and 6.12 that alone renames a dozen built-in "modules" such as unix + # and fscache, which have not gone anywhere. + old = loadable_of(reference) + new = modules_of(running) + if not old: + report("skip", "no %s to diff against" % module_directory(reference)) + return + + unexplained = [] + for module in sorted(old - new): + reason = explained(module) + if reason: + note("%s gone: %s" % (module, reason)) + else: + unexplained.append(module) + + for module in unexplained: + report("warn", "%s exists in %s but not in %s -- a lost driver, or" + " upstream churn to add to UPSTREAM_GONE" + % (module, reference, running)) + report("ok", "%d modules in %s, %d of them not in %s; of the %d %s has and" + " %s has not, %d are known upstream removals" + % (len(new), running, len(new - old), reference, + len(old - new), reference, running, + len(old - new) - len(unexplained))) + + +def check_hardware(running, reference): + """The check that would have caught the 6.12 -> 6.18 loss by itself. + + Every device in /sys says, in its modalias, which driver it wants. Ask the + running kernel's modules.alias what would serve that, and ask the reference + kernel the same thing: a device that the old kernel had a module for and + the new one has not is a lost driver, whatever the config says about it. + """ + here = Matcher(running) + there = Matcher(reference) if reference else None + names = pci_names() + blacklist = blacklisted_modules() + + bound = unbound = lost = 0 + for device in devices(): + subsystem = device["subsystem"] + # acpi and platform devices are unbound by the dozen on every healthy + # machine, and a pci bridge has no driver by design. + if subsystem not in ("pci", "usb", "virtio", "hid", "scsi", "hdaudio"): + continue + if subsystem == "pci": + classfile = read(os.path.join(device["path"], "class")) or "0" + if (int(classfile, 16) >> 16) not in PCI_CLASSES: + continue + + name = names.get(device["address"], device["modalias"]) + if device["driver"]: + bound += 1 + note("%s driven by %s" % (name, device["driver"])) + continue + + candidates = here(device["modalias"]) + previous = there(device["modalias"]) if there else [] + if candidates: + unbound += 1 + blocked = [module for module in candidates if module in blacklist] + if blocked: + report("info", "%s: %s blacklisted, so nothing is bound" + % (name, ", ".join(blocked))) + else: + report("warn", "%s: no driver bound, though %s matches it" + % (name, " or ".join(candidates))) + elif previous: + lost += 1 + report("fail", "%s: no driver in %s -- %s had %s for it" + % (name, running, reference, " or ".join(previous))) + else: + report("info", "%s: no driver in either kernel (%s)" + % (name, device["modalias"])) + + report("ok" if not lost else "fail", + "%d devices driven, %d unbound with a module available, %d without" + " a driver this kernel has" % (bound, unbound, lost)) + + +def check_filesystems(config): + """What /proc/filesystems offers, and whether a mount actually works.""" + supported = set() + for line in (read("/proc/filesystems") or "").splitlines(): + supported.add(line.split()[-1]) + + # Anything mounted right now obviously works; anything in fstab has to. + wanted = {"ext4", "xfs", "tmpfs", "proc", "sysfs", "devtmpfs"} + for line in (read("/proc/mounts") or "").splitlines(): + fields = line.split() + if len(fields) >= 3: + wanted.add(fields[2].split(".")[0]) + for line in (read("/etc/fstab") or "").splitlines(): + fields = line.split() + if len(fields) >= 3 and not line.lstrip().startswith("#"): + if fields[2] not in ("swap", "none", "auto"): + wanted.add(fields[2]) + + available = modules_of(os.uname().release) + for filesystem in sorted(wanted): + if filesystem in supported: + note("%s supported" % filesystem) + elif canonical(filesystem) in available: + report("info", "%s not registered yet, but the module exists" + % filesystem) + else: + report("fail", "%s is mounted or in /etc/fstab, but this kernel" + " does not support it" % filesystem) + report("ok", "%d filesystems registered, %d in use or in /etc/fstab" + % (len(supported), len(wanted))) + + # The automounter is what makes /pkg and the home directories appear, and + # a kernel without it leaves an apparently empty directory rather than an + # error, which is a miserable way to find out. + if "autofs" in supported: + report("ok", "autofs registered") + else: + report("fail", "no autofs -- /pkg and the automounted trees will look" + " empty rather than fail") + + mounts = read("/proc/mounts") or "" + if re.search(r"\bnfs4?\b", mounts): + report("ok", "NFS mounts present") + else: + report("warn", "no NFS mount on this machine; the client side is" + " untested here") + + +def check_automount(): + """Walk into an automounted path and see that something appears.""" + candidates = [path for path in ("/pkg", "/project", "/home") if + os.path.isdir(path)] + if not candidates: + report("skip", "no automounted tree on this machine") + return + for path in candidates: + try: + # Do not glob: /pkg is an automount map, and a glob lists nothing + # until the path below it has been walked into. + entries = os.listdir(path) + except OSError as error: + report("fail", "%s: %s" % (path, error)) + continue + report("ok" if entries is not None else "warn", + "%s reachable (%d entries)" % (path, len(entries))) + + +def check_namespaces(config): + """Unprivileged namespaces, cgroup v2 and an overlay mount in one go. + + Rootless bee builds chain fuse-overlayfs inside a user namespace, so this + is the site's own workload, not a synthetic test. + """ + maximum = read("/proc/sys/user/max_user_namespaces") + if maximum is not None and maximum.strip() == "0": + report("fail", "user.max_user_namespaces is 0 -- rootless overlay" + " builds cannot start") + code, output = run(["unshare", "--user", "--map-root-user", "true"]) + if code == 0: + report("ok", "unprivileged user namespace works") + else: + report("fail", "unprivileged user namespace refused: %s" % output.strip()) + + controllers = (read("/sys/fs/cgroup/cgroup.controllers") or "").split() + if controllers: + for wanted in ("cpu", "cpuset", "memory", "pids"): + if wanted not in controllers: + report("fail", "cgroup v2 controller %s missing" % wanted) + report("ok", "cgroup v2 controllers: %s" % " ".join(controllers)) + elif os.path.isdir("/sys/fs/cgroup/memory"): + report("warn", "cgroup v1 only; systemd and MXQ expect the unified" + " hierarchy") + else: + report("fail", "no cgroup hierarchy mounted") + + # tmpfs over overlay over tmpfs, in a namespace, as ourselves. + script = """set -e +mkdir -p /tmp/smoke/lower /tmp/smoke/upper /tmp/smoke/work /tmp/smoke/merged +mount -t tmpfs tmpfs /tmp/smoke +mkdir -p /tmp/smoke/lower /tmp/smoke/upper /tmp/smoke/work /tmp/smoke/merged +echo lower > /tmp/smoke/lower/file +mount -t overlay overlay -o \ + lowerdir=/tmp/smoke/lower,upperdir=/tmp/smoke/upper,workdir=/tmp/smoke/work \ + /tmp/smoke/merged +cat /tmp/smoke/merged/file +echo upper > /tmp/smoke/merged/file +cat /tmp/smoke/merged/file +""" + code, output = run(["unshare", "--user", "--map-root-user", "--mount", + "sh", "-c", script]) + if code == 0 and output.split() == ["lower", "upper"]: + report("ok", "tmpfs and overlayfs mount and copy up in a user namespace") + else: + report("fail", "overlayfs in a user namespace failed: %s" + % " ".join(output.split())[:200]) + + if os.path.exists("/dev/fuse"): + try: + os.close(os.open("/dev/fuse", os.O_RDWR)) + report("ok", "/dev/fuse opens (fuse-overlayfs, squashfuse, sshfs)") + except OSError as error: + report("fail", "/dev/fuse: %s" % error) + else: + report("fail", "no /dev/fuse -- fuse-overlayfs and squashfuse cannot run") + + +def check_network(running): + """Interfaces have drivers and a link, and the usual virtual devices work.""" + available = modules_of(running) + loaded = loaded_modules() + + interfaces = 0 + for path in sorted(glob.glob("/sys/class/net/*")): + name = os.path.basename(path) + if name == "lo": + continue + interfaces += 1 + driver = None + link = os.path.join(path, "device", "driver") + if os.path.islink(link): + driver = os.path.basename(os.readlink(link)) + state = (read(os.path.join(path, "operstate")) or "?").strip() + if driver: + note("%s driven by %s, %s" % (name, driver, state)) + elif os.path.exists(os.path.join(path, "device")): + report("fail", "%s has no driver bound" % name) + else: + note("%s is virtual, %s" % (name, state)) + if state == "up": + report("ok", "%s up%s" % (name, " (%s)" % driver if driver else "")) + if not interfaces: + report("fail", "no network interface besides lo") + + for module in ("veth", "bridge", "8021q", "tun", "vxlan", "macvlan"): + if canonical(module) in available: + note("%s available" % module) + else: + report("fail", "no %s module -- containers, VLANs or VPNs will" + " break" % module) + + # A network namespace with a veth pair and traffic over it. modprobe from + # an unprivileged user namespace is refused, so this only means anything + # when the modules are already in the kernel. + if "veth" not in loaded and os.geteuid() != 0: + report("skip", "veth is not loaded and an unprivileged user cannot" + " autoload it; run as root for the live network test") + return + script = """set -e +ip link set lo up +ip link add v0 type veth peer name v1 +ip addr add 10.99.0.1/24 dev v0 +ip addr add 10.99.0.2/24 dev v1 +ip link set v0 up +ip link set v1 up +ping -c1 -W2 -I v0 10.99.0.2 > /dev/null +echo pinged +""" + code, output = run(["unshare", "--user", "--map-root-user", "--net", + "sh", "-c", script]) + if code == 0 and "pinged" in output: + report("ok", "veth pair in a network namespace carries traffic") + else: + report("warn", "veth test in a network namespace failed: %s" + % " ".join(output.split())[:200]) + + +def check_storage(running): + """Every disk has a driver, and the stack above it is there.""" + disks = 0 + for path in sorted(glob.glob("/sys/block/*")): + name = os.path.basename(path) + if name.startswith(("loop", "ram", "zram", "dm-", "md")): + continue + disks += 1 + link = os.path.join(path, "device", "driver") + if os.path.islink(link): + note("%s driven by %s" % (name, os.path.basename(os.readlink(link)))) + else: + report("warn", "%s has no driver link" % name) + report("ok" if disks else "warn", "%d physical block devices" % disks) + + for path in ("/dev/kvm", "/dev/net/tun", "/dev/ipmi0"): + if os.path.exists(path): + note("%s present" % path) + + +def check_virtualisation(config): + """KVM, which the compute servers run virtual machines on.""" + vendor = read("/proc/cpuinfo") or "" + if "hypervisor" in vendor: + report("info", "running inside a hypervisor; nested KVM not checked") + if not os.path.exists("/dev/kvm"): + report("warn", "no /dev/kvm -- either the module is not loaded or" + " virtualisation is off in the firmware") + return + try: + os.close(os.open("/dev/kvm", os.O_RDWR)) + report("ok", "/dev/kvm opens") + except OSError as error: + # Permission denied only means this user is not in the kvm group. + report("info" if error.errno == 13 else "fail", "/dev/kvm: %s" % error) + + +def check_health(running, config): + """Sensors, EDAC and the BMC: what tells us a machine is about to die.""" + hwmon = sorted(glob.glob("/sys/class/hwmon/hwmon*")) + names = sorted({(read(os.path.join(path, "name")) or "?").strip() + for path in hwmon}) + temperatures = glob.glob("/sys/class/hwmon/hwmon*/temp*_input") + if temperatures: + readings = [] + for path in sorted(temperatures)[:4]: + value = read(path) + if value and value.strip().lstrip("-").isdigit(): + readings.append("%.1f C" % (int(value.strip()) / 1000.0)) + report("ok", "%d hwmon devices (%s), temperatures %s" + % (len(hwmon), " ".join(names), ", ".join(readings) or "none")) + else: + report("fail", "no temperature reading anywhere under /sys/class/hwmon" + " -- k10temp, coretemp or the SMBus driver is missing") + + controllers = sorted(glob.glob("/sys/devices/system/edac/mc/mc*")) + if controllers: + errors = 0 + for path in controllers: + for counter in ("ce_count", "ue_count"): + value = read(os.path.join(path, counter)) + errors += int(value.strip()) if value and value.strip().isdigit() else 0 + report("warn" if errors else "ok", + "%d EDAC memory controllers, %d errors counted" + % (len(controllers), errors)) + elif config.get("EDAC", "n") in ("y", "m"): + report("warn", "CONFIG_EDAC is set but no memory controller" + " registered; this machine cannot report a failing DIMM") + else: + report("info", "no EDAC in this kernel; the BMC reports failing DIMMs") + + # ipmi_si loads on any machine and finds nothing on most of them; only a + # registered interface without a /dev node means the device interface + # module is missing, which is what would silence racadm and ipmitool. + if os.path.exists("/dev/ipmi0"): + report("ok", "/dev/ipmi0 present") + elif glob.glob("/sys/class/ipmi/ipmi*"): + report("fail", "the BMC registered an IPMI interface but there is no" + " /dev/ipmi0 -- ipmi_devintf is missing") + else: + report("info", "no IPMI interface registered; expected on a desktop") + + online = (read("/sys/devices/system/cpu/online") or "?").strip() + present = (read("/sys/devices/system/cpu/present") or "?").strip() + report("ok" if online == present else "warn", + "CPUs online %s of %s" % (online, present)) + + if os.path.isdir("/sys/devices/system/cpu/cpu0/cpufreq"): + driver = read("/sys/devices/system/cpu/cpu0/cpufreq/scaling_driver") + report("ok", "cpufreq driver %s" % (driver or "?").strip()) + else: + report("warn", "no cpufreq on cpu0 -- the machine will run at a fixed" + " frequency, which costs power or performance") + + clocksource = read("/sys/devices/system/clocksource/clocksource0/" + "current_clocksource") + if clocksource: + clocksource = clocksource.strip() + report("ok" if clocksource == "tsc" else "warn", + "clocksource %s" % clocksource) + + +def check_dmesg(): + """Read the ring buffer for the damage a wrong kernel does at boot.""" + code, output = run(["dmesg", "--notime"]) + if code != 0: + report("skip", "cannot read the ring buffer (%s); try as root" + % output.split("\n")[0][:80]) + return + + # A crashing program logs `traps:' or `segfault at' through the same ring + # buffer as the kernel, and signal-desktop dying is not a kernel defect. + lines = [line for line in output.splitlines() + if not re.match(r"\s*(traps|show_signal_msg):", line) + and "segfault at" not in line] + for patterns, severity in ((DMESG_FATAL, "fail"), (DMESG_SUSPECT, "warn")): + for pattern, description in patterns: + hits = [line for line in lines if re.search(pattern, line)] + if hits: + report(severity, "dmesg: %s (%d lines, first: %s)" + % (description, len(hits), hits[0].strip()[:120])) + report("ok", "%d lines of kernel log read" % len(lines)) + + +def check_services(): + """systemd after a kernel change: a unit that needs a module it lost.""" + code, output = run(["systemctl", "--failed", "--no-legend", "--plain"]) + if code != 0: + report("skip", "systemctl unavailable") + return + failed = [line.split()[0] for line in output.splitlines() if line.strip()] + if failed: + report("fail", "failed units: %s" % " ".join(failed)) + else: + report("ok", "no failed systemd units") + + code, output = run(["journalctl", "-b", "-k", "-p", "err", "--no-pager", + "-q"]) + if code == 0 and output.strip(): + lines = output.strip().splitlines() + report("warn", "%d kernel messages at priority err this boot (first:" + " %s)" % (len(lines), lines[0][:120])) + + +def collect_baseline(running): + """What this machine looks like now, to compare after the next reboot.""" + interfaces = {} + for path in sorted(glob.glob("/sys/class/net/*")): + link = os.path.join(path, "device", "driver") + interfaces[os.path.basename(path)] = ( + os.path.basename(os.readlink(link)) if os.path.islink(link) else None) + + return { + "release": running, + "loaded": sorted(loaded_modules()), + "interfaces": interfaces, + "devices": {device["id"]: {"driver": device["driver"], + "modalias": device["modalias"], + "address": device["address"], + "subsystem": device["subsystem"]} + for device in devices() if device["driver"]}, + "filesystems": sorted({line.split()[2] for line + in (read("/proc/mounts") or "").splitlines() + if len(line.split()) >= 3}), + } + + +def check_baseline(running, path): + """Compare against a baseline taken on the kernel this one replaces. + + This is the only check that knows what the old kernel really did rather + than what it could have done: a module that was loaded before and does not + exist now, or a device that was driven before and is bare now. + """ + try: + with open(path) as handle: + baseline = json.load(handle) + except (OSError, ValueError) as error: + report("fail", "cannot read the baseline %s: %s" % (path, error)) + return + + if baseline.get("release") == running: + report("warn", "the baseline was taken on %s, the kernel running now;" + " take it before rebooting into the new one" % running) + else: + report("ok", "baseline from %s" % baseline.get("release", "?")) + + available = modules_of(running) + for module in baseline.get("loaded", []): + if module in available: + continue + reason = explained(module) + if reason: + report("warn", "%s was loaded on %s and is gone: %s" + % (module, baseline.get("release"), reason)) + else: + report("fail", "%s was loaded on %s and this kernel has no such" + " module" % (module, baseline.get("release"))) + + for name, driver in sorted(baseline.get("interfaces", {}).items()): + if not os.path.exists("/sys/class/net/%s" % name): + report("fail", "interface %s (%s) is gone" % (name, driver)) + + current = {device["id"]: device for device in devices()} + for identity, before in sorted(baseline.get("devices", {}).items()): + name = before.get("address", identity) + now = current.get(identity) + if now is None: + report("warn", "device %s (%s) is no longer in /sys" + % (name, before["modalias"])) + elif not now["driver"]: + report("fail", "device %s was driven by %s and has no driver now" + % (name, before["driver"])) + elif now["driver"] != before["driver"]: + report("warn", "device %s changed driver: %s -> %s" + % (name, before["driver"], now["driver"])) + + for filesystem in baseline.get("filesystems", []): + note("%s was mounted before" % filesystem) + + +CHECKS = ["identity", "config", "modprobe.d", "reference", "hardware", + "filesystems", "automount", "namespaces", "network", "storage", + "virtualisation", "health", "dmesg", "services", "baseline"] + + +def main(): + global verbose + + parser = argparse.ArgumentParser( + description="Smoke-test the kernel this machine is running.") + parser.add_argument("--kernel", metavar="RELEASE", + help="check an installed kernel this machine is not" + " running, e.g. before rebooting into it; only" + " the checks that read /lib/modules and /boot" + " can run then") + parser.add_argument("--reference", metavar="RELEASE", + help="kernel to diff against, e.g. 6.12.100.mx64.493" + " (default: the newest installed kernel from the" + " series below the running one)") + parser.add_argument("--no-reference", action="store_true", + help="do not diff against another kernel") + parser.add_argument("--save-baseline", metavar="FILE", + help="write what this machine looks like now and exit;" + " run this on the old kernel before rebooting") + parser.add_argument("--baseline", metavar="FILE", + help="compare against a baseline written by" + " --save-baseline on the old kernel") + parser.add_argument("--only", metavar="CHECK", action="append", default=[], + choices=CHECKS, + help="run only this check; may be repeated (%s)" + % ", ".join(CHECKS)) + parser.add_argument("--verbose", "-v", action="store_true", + help="also print what passed device by device") + arguments = parser.parse_args() + verbose = arguments.verbose + # Walking /sys and matching every modalias takes a while; a pipe would + # otherwise hold the whole run back and look like a hang. + sys.stdout.reconfigure(line_buffering=True) + + running = arguments.kernel or os.uname().release + if arguments.kernel and not os.path.isdir(module_directory(running)): + sys.exit("no %s: that kernel is not installed here" + % module_directory(running)) + if arguments.save_baseline: + with open(arguments.save_baseline, "w") as handle: + json.dump(collect_baseline(running), handle, indent=1, sort_keys=True) + handle.write("\n") + print("baseline of %s written to %s -- run this script with --baseline" + " %s after rebooting into the new kernel" + % (running, arguments.save_baseline, arguments.save_baseline)) + return 0 + + if arguments.no_reference: + reference = None + else: + reference = arguments.reference or default_reference(running) + if reference and not os.path.isdir(module_directory(reference)): + sys.exit("no %s: the reference kernel is not installed here" + % module_directory(reference)) + + booted = running == os.uname().release + config = read_config() if booted else read_config(running) + wanted = set(arguments.only or CHECKS) + if not arguments.baseline: + if "baseline" in arguments.only: + sys.exit("--only baseline needs --baseline FILE to compare against") + wanted.discard("baseline") + if not booted: + # Everything else asks the machine what it is doing now, and the + # answer would be about the kernel it is running, not the one asked + # about. The hardware check stays: it matches this machine's devices + # against that kernel's modules.alias, which is exactly the question + # worth asking before a reboot. + wanted &= {"identity", "config", "modprobe.d", "reference", "hardware"} + if os.geteuid() != 0: + print(" running as %s; the checks that need root are skipped" + % (os.environ.get("USER") or os.getuid())) + + selected = [(name, check) for name, check in ( + ("identity", lambda: check_identity(running, reference)), + ("config", lambda: check_config(config)), + ("modprobe.d", lambda: check_modprobe_d(modules_of(running))), + ("reference", lambda: check_reference(running, reference, config)), + ("hardware", lambda: check_hardware(running, reference)), + ("filesystems", lambda: check_filesystems(config)), + ("automount", check_automount), + ("namespaces", lambda: check_namespaces(config)), + ("network", lambda: check_network(running)), + ("storage", lambda: check_storage(running)), + ("virtualisation", lambda: check_virtualisation(config)), + ("health", lambda: check_health(running, config)), + ("dmesg", check_dmesg), + ("services", check_services), + ("baseline", lambda: check_baseline(running, arguments.baseline)), + ) if name in wanted] + + for name, check in selected: + print("--- %s" % name) + check() + + print("%s (%d failed, %d warned, %d skipped)" + % ("FAIL" if status["fail"] else "PASS", + status["fail"], status["warn"], status["skip"])) + return 1 if status["fail"] else 0 + + +if __name__ == "__main__": + sys.exit(main()) From 0f83afa20375bad835cc6f65b5dabe56e03923b4 Mon Sep 17 00:00:00 2001 From: Paul Menzel Date: Tue, 15 Sep 2026 00:30:45 +0200 Subject: [PATCH 2/4] smoke-tests: Boot the kernel in a VM with --vm Reading a kernel package tells us which modules were built; it does not tell us whether they load, whether iptables still filters, or whether a filesystem can be made and mounted. Until now that needed a machine rebooted into the new kernel, which is the one thing nobody wants to do before the kernel has been tested. --vm boots the package in qemu instead, on any machine, without touching the running kernel and without root. The guest is an initramfs built out of this machine's own binaries -- bash, mount, ip, iptables, mke2fs and the rest, with their libraries resolved by ldd and the xtables plugins -- so the test is the site's userspace against the new kernel rather than a busybox that mariux does not even have and that has no iptables, no mke2fs and only a stub ip. The modules come out of /lib/modules/ with modules.dep parsed directly, and depmod -b rebuilds the indexes kmod needs. The guest then, as PID 1: * loads the 26 modules the site depends on, from ip_tables to nfsv3; * counts real ICMP packets through an ACCEPT rule, shows a DROP rule dropping, puts a MASQUERADE rule in the nat table, matches -m conntrack, and does the ICMPv6 equivalent with ip6tables; * carries IPv4 and IPv6 over a veth pair into another network namespace, puts a port in a bridge, brings up an 802.1Q interface and reaches the gateway over the virtio NIC; * makes and mounts ext4 and XFS on virtio disks, writes, remounts and reads back, mounts a squashfs image, copies a file up through overlayfs and does it again inside unshare --user --map-root-user, as the rootless bee-file builds do, and checks POSIX ACLs and the cgroup v2 controllers. --vm-sweep additionally insmods every module in the tree -- 747 in 6.18.51.mx64.498 -- and, because a module that fails in a VM usually only means the hardware is absent, boots the reference kernel the same way and reports only what loaded before the update and does not load now. A run takes about fourteen seconds with KVM. On 6.18.51.mx64.498 it is quiet except for the ebtables regression, which it now catches as something that breaks at runtime rather than as a config difference: FAIL ebtable_filter is not in 6.18.51.mx64.498's tree at all; the VM needs it for ebtables, which no module has been built for since 6.12 SKIP netfilter: no ebtables in mariux, only the modules can be tested FAIL (1 failed, 7 warned, 1 skipped) and the sweep agrees with the kernel it replaces: OK sweep: 732 of 747 modules loaded, 15 did not, all of them in 6.12.100.mx64.493 too Assisted-by: Claude Opus 5 --- smoke-tests/linux-smoke-test | 996 ++++++++++++++++++++++++++++++++++- 1 file changed, 993 insertions(+), 3 deletions(-) diff --git a/smoke-tests/linux-smoke-test b/smoke-tests/linux-smoke-test index b8dc5f10c..40f14bf1c 100755 --- a/smoke-tests/linux-smoke-test +++ b/smoke-tests/linux-smoke-test @@ -7,6 +7,8 @@ # smoke-tests/linux-smoke-test --save-baseline ~/before.json # old kernel # smoke-tests/linux-smoke-test --baseline ~/before.json # after boot # smoke-tests/linux-smoke-test --kernel 6.18.51.mx64.498 # before boot +# smoke-tests/linux-smoke-test --kernel 6.18.51.mx64.498 --vm # in a VM +# smoke-tests/linux-smoke-test --vm --vm-sweep # load it all # # linux.be0 configures with `make olddefconfig', which answers every new or # newly visible symbol with its default and never asks, so a symbol that @@ -21,9 +23,18 @@ # and this one has not. It is meant to be run on a machine that has been # rebooted into the new kernel, before the new kernel is rolled out further. # +# --vm goes further and boots the kernel package in qemu, on any machine and +# without rebooting anything: an initramfs of this machine's own binaries, and +# a guest that loads the modules, filters packets with iptables, makes and +# mounts ext4, XFS, squashfs and overlayfs, and puts a veth pair across two +# network namespaces. It takes about ten seconds, and --vm-sweep adds every +# module in the tree, compared against the kernel this one replaces so that a +# module which did not load before the update is not blamed on the update. +# # Everything works as an ordinary user; the checks that need root say so and # are skipped. Run it as root as well if you can, which adds the checks that -# actually load modules and mount filesystems. +# actually load modules and mount filesystems. The VM needs /dev/kvm to be +# usable to be quick, and qemu; it needs no privileges. # # FAIL is a defect in the kernel package: something the site needs is missing. # WARN depends on the machine rather than on the package (hardware absent, a @@ -39,8 +50,11 @@ import json import os import re import shlex +import shutil import subprocess import sys +import tempfile +import time status = collections.Counter() verbose = False @@ -1188,9 +1202,939 @@ def check_baseline(running, path): note("%s was mounted before" % filesystem) +# --------------------------------------------------------------------------- +# Booting the kernel in a virtual machine. +# +# Everything above reads what a kernel says about itself: its config, its +# modules, the devices it has bound. That catches a driver that is no longer +# built, but not a driver that is built and does not work, and it can only ask +# about the kernel this machine has been rebooted into. The VM boots the +# kernel package instead -- straight out of /boot and /lib/modules, on any +# machine, before anybody reboots into it -- and makes it do the things the +# site does with it: load a module, filter a packet, mount a filesystem. +# +# There is no busybox in mariux, and busybox would not do anyway: its applets +# have no iptables, no mke2fs and only a stub `ip'. The initramfs therefore +# carries this machine's own /usr binaries with the libraries `ldd' names for +# them, which is not a workaround but the better test -- the userspace we +# actually ship is what gets run against the new kernel. + +# Programs the guest runs, and what each is there for. A missing one is not +# fatal: the test that needs it says so and is skipped. +VM_TOOLS = [ + ("bash", "the shell that runs the tests as PID 1"), + ("mount", "every filesystem test"), + ("umount", "every filesystem test"), + ("modprobe", "loading modules the way the machine does"), + ("insmod", "loading a module by path when modprobe cannot"), + ("lsmod", "what ended up loaded"), + ("rmmod", "unloading between tests"), + ("uname", "the release the guest really booted"), + ("ls", "looking around"), + ("cat", "reading /proc and /sys"), + ("sync", "flushing before the power goes"), + ("sleep", "waiting for a link to come up"), + ("timeout", "a module that wedges must not wedge the run"), + ("awk", "reading counters out of iptables"), + ("sed", "tidying error messages"), + ("sort", "the module sweep"), + ("tr", "folding multi-line errors into one line"), + ("dmesg", "what the kernel complained about"), + ("mkdir", "mount points"), + ("rm", "cleaning up between tests"), + ("touch", "a file to put an ACL on"), + ("tail", "the last lines of a program that failed"), + ("id", "who the guest is running as, when something says no"), + ("dd", "writing to a filesystem"), + ("stat", "reading back what was written"), + ("ip", "links, addresses, namespaces, bridges and VLANs"), + ("ping", "a real packet through a real rule"), + ("ping6", "the same over IPv6"), + ("iptables", "the site's firewall tool"), + ("ip6tables", "the same over IPv6"), + ("ebtables", "bridge filtering; absent from mariux since the tool went"), + ("bridge", "bridge ports"), + ("unshare", "user and mount namespaces, as rootless bee builds use them"), + ("mke2fs", "an ext4 filesystem to mount"), + ("mkfs.ext4", "an ext4 filesystem to mount"), + ("mkfs.xfs", "an XFS filesystem to mount, as on /scratch"), + ("setfacl", "POSIX ACLs on tmpfs"), + ("getfacl", "POSIX ACLs on tmpfs"), +] + +# Files the guest's libc and tools look up by path. +VM_FILES = ["/etc/protocols", "/etc/services", "/etc/hosts", "/etc/nsswitch.conf", + "/etc/mke2fs.conf", "/etc/xattr.conf"] + +# Directories of plugins that are dlopen'd, so ldd does not know about them. +VM_PLUGIN_DIRS = ["/lib/xtables", "/usr/lib/xtables"] + +# Modules the guest tests need, and what each one is for. A module that is not +# in the kernel's tree is reported before the VM starts -- that is the 6.12 +# ebtables regression, and it is a defect whether or not the VM could run. +VM_MODULES = [ + ("ip_tables", "iptables' filter and nat tables"), + ("iptable_filter", "the filter table itself"), + ("iptable_nat", "NAT, as the VM hosts and the gateways do it"), + ("xt_MASQUERADE", "the MASQUERADE target the VM hosts NAT with"), + ("xt_tcpudp", "-p tcp --dport, in every rule set there is"), + ("ipv6", "the IPv6 stack itself, which is a module here"), + ("ip6_tables", "ip6tables on a dual stack site"), + ("ip6table_filter", "the IPv6 filter table"), + ("nf_conntrack", "connection tracking under every NAT rule"), + ("xt_conntrack", "-m conntrack, which every rule set uses"), + ("ebtable_filter", "ebtables, which no module has been built for since 6.12"), + ("bridge", "the VM and container bridges"), + ("8021q", "the tagged VLANs on the server ports"), + ("veth", "container and namespace networking"), + ("tun", "qemu, OpenVPN and podman"), + ("virtio_net", "networking in every VM on the compute servers"), + ("virtio_blk", "the disks of those VMs"), + ("ext4", "root and local filesystems"), + ("xfs", "/scratch and the servers"), + ("overlay", "the rootless overlay bee-file builds chain on"), + ("squashfs", "squashfs images"), + ("fuse", "fuse-overlayfs, squashfuse and sshfs"), + ("loop", "mounting an image file"), + ("nfs", "every home directory"), + ("nfsv4", "the default mount version"), + ("nfsv3", "the mx64old exports"), +] + +# Modules the sweep leaves alone: they spin a CPU for minutes, or panic on +# purpose, which is not what is being tested here. +VM_SWEEP_SKIP = ["test_*", "*_test", "*torture*", "kunit*", "fail_function", + "nvidia*", "evbug", "*_kunit", "lkdtm", "bpf_preload"] + +# A module failing to load in a VM usually means the hardware is not there. +# These messages mean something else: the module and the kernel do not match, +# which is a defect in the package. +VM_SWEEP_FATAL = ["unknown symbol", "invalid module format", "version magic", + "disagrees about version", "exec format error", + "required key not available", "bad address"] + +# The guest's PID 1. It talks to the host in lines of `@@state|message', which +# is the only thing the host reads out of the console; everything else on the +# console is the kernel's own noise and is kept for --verbose. +VM_INIT = r""" +export PATH=/usr/sbin:/usr/bin:/sbin:/bin +umask 022 + +mount -t proc proc /proc +mount -t sysfs sys /sys +mount -t devtmpfs dev /dev 2>/dev/null +mkdir -p /dev/pts /dev/shm /run /tmp /mnt/a /mnt/b /mnt/c +mount -t devpts devpts /dev/pts 2>/dev/null +mount -t tmpfs tmpfs /run +mount -t tmpfs tmpfs /tmp + +say() { echo "@@$1|$2"; } +ok() { say ok "$1"; } +warn() { say warn "$1"; } +fail() { say fail "$1"; } +skip() { say skip "$1"; } + +# Did the machine get a program for this test at all? +have() { command -v "$1" >/dev/null 2>&1; } + +# Load a module, and say who wanted it if it does not load. +load() { + if timeout 20 modprobe "$1" 2>/tmp/modprobe.err; then + return 0 + fi + fail "$2: modprobe $1: $(tr '\n' ' ' < /tmp/modprobe.err | sed 's/modprobe: //g')" + return 1 +} + +say release "$(uname -r)" +tainted=$(cat /proc/sys/kernel/tainted 2>/dev/null) +if [ "$tainted" = 0 ]; then + ok "boot: the kernel booted untainted" +else + warn "boot: tainted flags $tainted -- see /proc/sys/kernel/tainted" +fi + +### modules ################################################################ +# +# Loading is the part no static check can do: a module can be in the tree and +# still refuse, because it was built against different headers or its symbols +# are gone. +for module in $MODULES; do + if timeout 20 modprobe "$module" 2>/tmp/modprobe.err; then + ok "modules: $module loads" + else + fail "modules: $module does not load: $(tr '\n' ' ' < /tmp/modprobe.err | sed 's/modprobe: //g')" + fi +done + +### netfilter ############################################################## +ip link set lo up +if have iptables && iptables -L -n >/dev/null 2>&1; then + ok "netfilter: iptables can read the filter table" + + if iptables -A OUTPUT -p icmp -d 127.0.0.1 -j ACCEPT 2>/tmp/err; then + timeout 5 ping -c 2 -n 127.0.0.1 >/dev/null 2>&1 + packets=$(iptables -nvxL OUTPUT | awk '/icmp/ {print $1; exit}') + if [ "${packets:-0}" -gt 0 ]; then + ok "netfilter: an ACCEPT rule counted $packets ICMP packets" + else + fail "netfilter: an OUTPUT ACCEPT rule counted no packets, so the"\ +" filter data path does not work" + fi + else + fail "netfilter: iptables -A OUTPUT: $(tr '\n' ' ' < /tmp/err)" + fi + + iptables -A OUTPUT -p icmp -d 127.0.0.2 -j DROP 2>/dev/null + if timeout 5 ping -c 1 -n 127.0.0.2 >/dev/null 2>&1; then + fail "netfilter: a DROP rule did not drop" + else + ok "netfilter: a DROP rule drops" + fi + + if iptables -t nat -A POSTROUTING -s 10.99.0.0/24 -j MASQUERADE 2>/tmp/err; then + ok "netfilter: the nat table takes a MASQUERADE rule" + else + fail "netfilter: the nat table: $(tr '\n' ' ' < /tmp/err)" + fi + + if iptables -A OUTPUT -m conntrack --ctstate NEW -j ACCEPT 2>/tmp/err; then + ok "netfilter: -m conntrack matches" + else + fail "netfilter: -m conntrack: $(tr '\n' ' ' < /tmp/err)" + fi +else + fail "netfilter: iptables cannot read the filter table -- is ip_tables built?" +fi + +if have ip6tables && ip6tables -L -n >/dev/null 2>&1; then + if ip6tables -A OUTPUT -p ipv6-icmp -d ::1 -j ACCEPT 2>/tmp/err; then + timeout 5 ping6 -c 2 -n ::1 >/dev/null 2>&1 + packets=$(ip6tables -nvxL OUTPUT | awk '/icmp/ {print $1; exit}') + if [ "${packets:-0}" -gt 0 ]; then + ok "netfilter: ip6tables counted $packets ICMPv6 packets" + else + warn "netfilter: the IPv6 rule counted no packets" + fi + else + fail "netfilter: ip6tables -A OUTPUT: $(tr '\n' ' ' < /tmp/err)" + fi +else + fail "netfilter: ip6tables cannot read the filter table" +fi + +if have ebtables; then + if ebtables -L >/dev/null 2>&1; then + ok "netfilter: ebtables can read the bridge filter table" + else + fail "netfilter: ebtables cannot read the bridge filter table" + fi +else + skip "netfilter: no ebtables in mariux, only the modules can be tested" +fi + +### network ################################################################ +if ip link add smoke0 type veth peer name smoke1 2>/tmp/err; then + ip netns add smokens 2>/dev/null + ip link set smoke1 netns smokens 2>/dev/null + ip addr add 10.99.0.1/24 dev smoke0 + ip link set smoke0 up + ip netns exec smokens ip link set lo up + ip netns exec smokens ip addr add 10.99.0.2/24 dev smoke1 + ip netns exec smokens ip link set smoke1 up + if timeout 5 ping -c 2 -n 10.99.0.2 >/dev/null 2>&1; then + ok "network: a veth pair carries IPv4 into another network namespace" + else + fail "network: no reply over the veth pair between network namespaces" + fi + + ip addr add fd00:5::1/64 dev smoke0 nodad + ip netns exec smokens ip addr add fd00:5::2/64 dev smoke1 nodad + if timeout 5 ping6 -c 2 -n fd00:5::2 >/dev/null 2>&1; then + ok "network: the same pair carries IPv6" + else + fail "network: no IPv6 reply over the veth pair" + fi +else + fail "network: cannot create a veth pair: $(tr '\n' ' ' < /tmp/err)" +fi + +if ip link add smoke2 type veth peer name smoke3 2>/dev/null; then + if ip link add smokebr type bridge 2>/tmp/err; then + ip link set smoke2 master smokebr + ip link set smokebr up + ip link set smoke2 up + if [ -e /sys/class/net/smoke2/master ]; then + ok "network: a bridge takes a port" + else + fail "network: the port did not join the bridge" + fi + else + fail "network: cannot create a bridge: $(tr '\n' ' ' < /tmp/err)" + fi + + if ip link add link smoke3 name smoke3.100 type vlan id 100 2>/tmp/err; then + ip link set smoke3 up + ip link set smoke3.100 up + if [ -d /sys/class/net/smoke3.100 ]; then + ok "network: an 802.1Q VLAN interface comes up" + else + fail "network: the VLAN interface did not appear" + fi + else + fail "network: cannot create a VLAN interface: $(tr '\n' ' ' < /tmp/err)" + fi +fi + +# The virtio NIC is how every VM on the compute servers is on the network. +virtio_if= +for path in /sys/class/net/*; do + driver=$(cat "$path/device/uevent" 2>/dev/null | sed -n 's/^DRIVER=//p') + if [ "$driver" = virtio_net ]; then + virtio_if=${path##*/} + fi +done +if [ -n "$virtio_if" ]; then + ip link set "$virtio_if" up + ip addr add 10.0.2.15/24 dev "$virtio_if" + ip route add default via 10.0.2.2 2>/dev/null + sleep 1 + if timeout 8 ping -c 2 -n 10.0.2.2 >/dev/null 2>&1; then + ok "network: the virtio NIC $virtio_if reaches the gateway" + else + warn "network: the virtio NIC $virtio_if came up but the gateway did"\ +" not answer" + fi +else + fail "network: no interface is driven by virtio_net" +fi + +if [ -c /dev/net/tun ]; then + ok "network: /dev/net/tun exists (qemu, OpenVPN, podman)" +else + fail "network: no /dev/net/tun after loading tun" +fi + +### filesystems ############################################################ +disk_test() { + # $1 device, $2 filesystem, $3 mkfs program, and the rest its arguments + device=$1; filesystem=$2; maker=$3; shift 3 + if [ ! -b "$device" ]; then + skip "filesystems: no $device, the host gave the VM no disk for $filesystem" + return + fi + if ! have "$maker"; then + skip "filesystems: no $maker on this machine, $filesystem untested" + return + fi + if ! "$maker" "$@" "$device" >/tmp/err 2>&1; then + fail "filesystems: $maker on $device: $(tail -n 2 /tmp/err | tr '\n' ' ')" + return + fi + if ! mount -t "$filesystem" "$device" /mnt/a 2>/tmp/err; then + fail "filesystems: cannot mount $filesystem: $(tr '\n' ' ' < /tmp/err)" + return + fi + dd if=/dev/zero of=/mnt/a/file bs=1M count=8 2>/dev/null + echo written > /mnt/a/marker + sync + umount /mnt/a + if ! mount -t "$filesystem" "$device" /mnt/a 2>/tmp/err; then + fail "filesystems: cannot mount $filesystem again: $(tr '\n' ' ' < /tmp/err)" + return + fi + if [ "$(cat /mnt/a/marker 2>/dev/null)" = written ] && + [ "$(stat -c %s /mnt/a/file 2>/dev/null)" = 8388608 ]; then + ok "filesystems: $filesystem takes a filesystem, a file and a remount" + else + fail "filesystems: $filesystem lost what was written to it" + fi + umount /mnt/a +} + +disk_test /dev/vda ext4 mkfs.ext4 -q -F +disk_test /dev/vdb xfs mkfs.xfs -q -f + +if [ -b /dev/vdc ]; then + if mount -t squashfs /dev/vdc /mnt/b 2>/tmp/err; then + if [ "$(cat /mnt/b/marker 2>/dev/null)" = squashed ]; then + ok "filesystems: squashfs mounts and reads back" + else + fail "filesystems: squashfs mounted but the image reads wrong" + fi + umount /mnt/b + else + fail "filesystems: cannot mount squashfs: $(tr '\n' ' ' < /tmp/err)" + fi +else + skip "filesystems: no squashfs image was attached" +fi + +mkdir -p /tmp/over/lower /tmp/over/upper /tmp/over/work /tmp/over/merged +echo lower > /tmp/over/lower/file +if mount -t overlay overlay -o lowerdir=/tmp/over/lower,upperdir=/tmp/over/upper,workdir=/tmp/over/work /tmp/over/merged 2>/tmp/err; then + echo upper > /tmp/over/merged/file + if [ "$(cat /tmp/over/merged/file)" = upper ] && + [ "$(cat /tmp/over/upper/file 2>/dev/null)" = upper ]; then + ok "filesystems: overlayfs mounts and copies a file up" + else + fail "filesystems: overlayfs mounted but did not copy up" + fi + umount /tmp/over/merged +else + fail "filesystems: cannot mount overlayfs: $(tr '\n' ' ' < /tmp/err)" +fi + +# The same thing rootless bee builds do: an overlay inside a user namespace. +if have unshare; then + if unshare --user --map-root-user --mount sh -c ' + mkdir -p /tmp/ns/lower /tmp/ns/upper /tmp/ns/work /tmp/ns/merged + mount -t tmpfs tmpfs /tmp/ns + mkdir -p /tmp/ns/lower /tmp/ns/upper /tmp/ns/work /tmp/ns/merged + echo lower > /tmp/ns/lower/file + mount -t overlay overlay -o lowerdir=/tmp/ns/lower,upperdir=/tmp/ns/upper,workdir=/tmp/ns/work /tmp/ns/merged + echo upper > /tmp/ns/merged/file + [ "$(cat /tmp/ns/merged/file)" = upper ]' 2>/tmp/err; then + ok "filesystems: overlayfs in a user namespace, as bee builds use it" + else + fail "filesystems: overlayfs in a user namespace: $(tr '\n' ' ' < /tmp/err)" + fi +fi + +if [ -c /dev/fuse ]; then + ok "filesystems: /dev/fuse exists (fuse-overlayfs, squashfuse, sshfs)" +else + fail "filesystems: no /dev/fuse after loading fuse" +fi + +if have setfacl; then + touch /tmp/acl + if setfacl -m u:1000:rwx /tmp/acl 2>/tmp/err && + getfacl -c /tmp/acl 2>/dev/null | awk '/user:1000:rwx/ {found=1} END {exit !found}'; then + ok "filesystems: POSIX ACLs work on tmpfs" + else + fail "filesystems: no POSIX ACLs on tmpfs: $(tr '\n' ' ' < /tmp/err)" + fi +fi + +mkdir -p /sys/fs/cgroup 2>/dev/null +if mount -t cgroup2 none /sys/fs/cgroup 2>/tmp/err || + [ -e /sys/fs/cgroup/cgroup.controllers ]; then + controllers=$(cat /sys/fs/cgroup/cgroup.controllers 2>/dev/null) + missing= + for wanted in cpu cpuset memory pids; do + case " $controllers " in + *" $wanted "*) ;; + *) missing="$missing $wanted" ;; + esac + done + if [ -z "$missing" ]; then + ok "filesystems: cgroup v2 with $controllers" + else + fail "filesystems: cgroup v2 is missing the controllers$missing" + fi +else + fail "filesystems: cannot mount cgroup v2: $(tr '\n' ' ' < /tmp/err)" +fi + +### sweep ################################################################## +# +# Last, because loading every module in the tree is the one thing here that +# can take the machine down with it; everything above has been reported by +# then and survives in the console log. +if [ -s /sweep.list ]; then + total=0 + failed=0 + while read -r module; do + total=$((total + 1)) + if ! timeout 15 modprobe "$module" 2>/tmp/modprobe.err; then + failed=$((failed + 1)) + say sweep "$module|$(tr '\n' ' ' < /tmp/modprobe.err | sed 's/modprobe: //g')" + fi + done < /sweep.list + say swept "$total|$failed" +fi + +say finished "" +sync +echo o > /proc/sysrq-trigger +sleep 60 +""" + + +def vm_qemu(preferred=None): + """qemu 11 if the machine has it, the system one otherwise.""" + for name in ([preferred] if preferred else []) + ["qemu11-system-x86_64", + "qemu-system-x86_64"]: + found = shutil.which(name) or (name if os.path.isfile(name) else None) + if found: + return found + return None + + +def vm_libraries(path): + """The shared libraries a program needs, as ldd names them.""" + code, output = run(["ldd", path], timeout=20) + if code != 0: + return [] # static, or not an ELF file at all + found = [] + for line in output.splitlines(): + match = re.search(r"=>\s+(/\S+)", line) or re.match(r"\s*(/\S+)\s+\(0x", line) + if match: + found.append(match.group(1)) + return found + + +def vm_target(root, path): + """Where a host path belongs in the tree. + + The directories a host symlink points through are resolved here, so a file + is never written through a symlink that leads back out of the tree: /lib + is /usr/lib on mariux, and copying to root + /lib would land in the + machine's own /usr/lib. + """ + directory = os.path.realpath(os.path.dirname(path)) + return os.path.join(root, directory.lstrip("/"), os.path.basename(path)) + + +def vm_mirror(root, path): + """Recreate the host's symlinked directories in the tree. + + /lib -> /usr/lib and /lib64 -> lib are how this machine's loader finds + anything at all; a binary asking for /lib64/ld-linux-x86-64.so.2 has to + find it under that name in the guest too. + """ + prefix = "" + for part in os.path.dirname(path).strip("/").split("/"): + prefix += "/" + part + if os.path.islink(prefix): + link = vm_target(root, prefix) + if not os.path.lexists(link): + os.makedirs(os.path.dirname(link), exist_ok=True) + os.symlink(vm_relative(prefix), link) + + +def vm_relative(path): + """A symlink's target, made relative to the directory the link is in. + + /lib points at /usr/lib here. Copied as it stands, the link in the tree + points at this machine's /usr/lib, and depmod -b or anything else working + on the tree from outside quietly follows it out of the tree. Relative, it + means the same thing to the guest and stays inside. + """ + destination = os.readlink(path) + if not os.path.isabs(destination): + return destination + return os.path.relpath(destination, os.path.dirname(path)) + + +def vm_install(root, path, seen): + """Copy a file into the initramfs tree, with whatever it is linked against.""" + if path in seen or not os.path.lexists(path): + return + seen.add(path) + vm_mirror(root, path) + target = vm_target(root, path) + os.makedirs(os.path.dirname(target), exist_ok=True) + if os.path.islink(path): + vm_install(root, os.path.realpath(path), seen) + if not os.path.lexists(target): + os.symlink(vm_relative(path), target) + return + shutil.copy2(path, target) + # The archive is built by whoever runs this script, so every file in it + # belongs to them; a setuid mount(8) would hand PID 1 that uid and take + # away the privilege it started with. + os.chmod(target, os.stat(path).st_mode & 0o777 & ~0o6000) + for library in vm_libraries(path): + vm_install(root, library, seen) + + +def vm_dependencies(release): + """modules.dep as a dictionary of module name to the files to load for it. + + modprobe would do this, but `modprobe -S RELEASE --show-depends' answers + for half the modules here with `could not insert', because it is asking + the running kernel about a module tree that is not the running kernel's. + The file it reads is two columns of paths and needs no interpreting. + """ + directory = module_directory(release) + dependencies = {} + for line in (read(os.path.join(directory, "modules.dep")) or "").splitlines(): + first, _, rest = line.partition(":") + name = canonical(re.sub(r"\.ko.*$", "", os.path.basename(first))) + dependencies[name] = [os.path.join(directory, path) + for path in [first.strip()] + rest.split()] + return dependencies + + +def vm_install_modules(root, release, modules): + """Copy the named modules and their dependencies out of the kernel's tree. + + Returns the modules the tree has neither as a file nor built in, which is + a finding in itself: a module the site needs and olddefconfig turned off + never gets as far as failing in the VM, it is simply not there. + """ + dependencies = vm_dependencies(release) + builtin = _module_names(os.path.join(module_directory(release), + "modules.builtin")) + missing = [] + seen = set() + for module in modules: + name = canonical(module) + if name in dependencies: + for path in dependencies[name]: + vm_install(root, path, seen) + elif name not in builtin: + missing.append(module) + return missing + + +def vm_sweep_list(release): + """Every loadable module in the tree, minus the ones not worth loading.""" + names = [] + for name in sorted(loadable_of(release)): + if any(fnmatch.fnmatch(name, pattern) for pattern in VM_SWEEP_SKIP): + continue + names.append(name) + return names + + +def vm_build_initramfs(root, release, sweep): + """Lay out the guest's root and pack it into a cpio archive.""" + seen = set() + missing_tools = [] + for name, _ in VM_TOOLS: + path = shutil.which(name, path="/usr/sbin:/usr/bin:/sbin:/bin") + if path: + vm_install(root, path, seen) + else: + missing_tools.append(name) + for path in VM_FILES: + vm_install(root, path, seen) + for directory in VM_PLUGIN_DIRS: + for path in glob.glob(os.path.join(directory, "*.so")): + vm_install(root, path, seen) + # dlopen'd by glibc for /etc/protocols and friends, so ldd never sees it. + for path in glob.glob("/lib*/libnss_files.so*") + glob.glob("/lib*/libnss_dns.so*"): + vm_install(root, path, seen) + + os.makedirs(os.path.join(root, "bin"), exist_ok=True) + for name in ("sh", "bash"): + link = os.path.join(root, "bin", name) + if not os.path.lexists(link): + os.symlink("../usr/bin/bash", link) + os.makedirs(os.path.join(root, "var"), exist_ok=True) + if not os.path.lexists(os.path.join(root, "var", "run")): + os.symlink("/run", os.path.join(root, "var", "run")) + for directory in ("proc", "sys", "dev", "run", "tmp", "mnt", "usr/lib/modules"): + os.makedirs(os.path.join(root, directory), exist_ok=True) + + missing_modules = vm_install_modules(root, release, + [name for name, _ in VM_MODULES]) + if sweep: + shutil.copytree(os.path.join(module_directory(release), "kernel"), + os.path.join(root, module_directory(release).lstrip("/"), + "kernel"), + dirs_exist_ok=True, symlinks=True) + # depmod writes the indexes but not these: they come out of the kernel + # build, and without them modprobe does not know a builtin from a module + # that is missing. + for name in ("modules.builtin", "modules.builtin.modinfo", "modules.order"): + vm_install(root, os.path.join(module_directory(release), name), set()) + with open(os.path.join(root, "sweep.list"), "w") as handle: + if sweep: + handle.write("".join("%s\n" % name for name in vm_sweep_list(release))) + # depmod over what was copied, so modprobe in the guest resolves against + # the initramfs rather than against a tree that is not there. + code, output = run(["depmod", "-b", root, release], timeout=300) + if code != 0: + report("warn", "depmod on the initramfs: %s" % " ".join(output.split())) + + init = os.path.join(root, "init") + with open(init, "w") as handle: + handle.write("#! /bin/sh\n") + handle.write("MODULES='%s'\n" % " ".join(name for name, _ in VM_MODULES + if name not in missing_modules)) + handle.write(VM_INIT) + os.chmod(init, 0o755) + return missing_tools, missing_modules + + +def vm_pack(root, archive): + """find | cpio | gzip, the way every initramfs has always been built. + + -R 0:0 is what makes the guest work at all: the archive is written by an + ordinary user, and a file owned by a uid that is not mapped into a user + namespace cannot even be executed inside one, so the overlay test would + fail on the initramfs rather than on the kernel. + """ + with open(archive, "wb") as handle: + find = subprocess.Popen(["find", ".", "-print0"], cwd=root, + stdout=subprocess.PIPE) + cpio = subprocess.Popen(["cpio", "--quiet", "--null", "-o", "-H", "newc", + "-R", "0:0"], + cwd=root, stdin=find.stdout, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + gzip_process = subprocess.Popen(["gzip", "-1"], stdin=cpio.stdout, + stdout=handle) + find.stdout.close() + cpio.stdout.close() + gzip_process.communicate() + return gzip_process.returncode == 0 and cpio.wait() == 0 + + +def vm_disks(directory): + """The disks the filesystem tests want: one for ext4, one for XFS, and a + squashfs image with something in it to read back.""" + disks = [] + for name, megabytes in (("ext4.img", 256), ("xfs.img", 512)): + path = os.path.join(directory, name) + with open(path, "wb") as handle: + handle.truncate(megabytes * 1024 * 1024) + disks.append(path) + + image = os.path.join(directory, "squashfs.img") + content = os.path.join(directory, "squashfs") + os.makedirs(content, exist_ok=True) + with open(os.path.join(content, "marker"), "w") as handle: + handle.write("squashed\n") + if shutil.which("mksquashfs"): + code, _ = run(["mksquashfs", content, image, "-noappend", "-quiet"], + timeout=120) + if code == 0: + disks.append(image) + return disks + + +def vm_command(qemu, kernel, initrd, disks, memory, accelerated): + command = [qemu, "-machine", "q35", "-smp", "1", "-m", str(memory), + "-display", "none", "-serial", "stdio", "-no-reboot", + "-kernel", kernel, "-initrd", initrd] + command += ["-accel", "kvm", "-cpu", "host"] if accelerated else \ + ["-accel", "tcg", "-cpu", "max"] + for index, disk in enumerate(disks): + command += ["-drive", "file=%s,format=raw,if=virtio,cache=unsafe,index=%d" + % (disk, index)] + command += ["-netdev", "user,id=smoke0", + "-device", "virtio-net-pci,netdev=smoke0"] + # panic=-1 with -no-reboot makes qemu exit the moment the guest panics, + # instead of sitting there until the timeout. + command += ["-append", "console=ttyS0,115200 panic=-1 loglevel=4" + " oops=panic rdinit=/init"] + return command + + +def vm_sweep_verdict(message): + """Did this module fail to load because the hardware is absent, or because + the module and the kernel do not match?""" + lowered = message.lower() + return "fail" if any(pattern in lowered for pattern in VM_SWEEP_FATAL) else "info" + + +def vm_boot(release, kernel, options, qemu, accelerated, directory, sweep, quiet): + """Build an initramfs for `release', boot it, and bring back what it said. + + Nothing is reported from here: the reference kernel is booted through this + same function, and its only job is to say which modules already refused to + load before the update. + """ + root = os.path.join(directory, "root") + os.makedirs(root, exist_ok=True) + archive = os.path.join(directory, "initrd.gz") + console = os.path.join(directory, "console.log") + + missing_tools, missing_modules = vm_build_initramfs(root, release, sweep) + if not vm_pack(root, archive): + report("fail", "could not pack the initramfs in %s" % directory) + return None + if not quiet: + note("initramfs %.0f MiB in %s" + % (os.path.getsize(archive) / 1048576.0, directory)) + + disks = vm_disks(directory) + memory = options.vm_memory or (2048 if sweep else 1024) + timeout = options.vm_timeout or (900 if sweep else 300) + if not accelerated: + timeout *= 4 + command = vm_command(qemu, kernel, archive, disks, memory, accelerated) + if not quiet: + note(" ".join(shlex.quote(word) for word in command)) + + started = time.time() + expired = False + try: + finished = subprocess.run(command, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, timeout=timeout) + output = finished.stdout.decode("utf-8", "replace") + except subprocess.TimeoutExpired as timed_out: + output = (timed_out.stdout or b"").decode("utf-8", "replace") + expired = True + except OSError as error: + report("fail", "cannot run %s: %s" % (qemu, error)) + return None + with open(console, "w") as handle: + handle.write(output) + + outcome = vm_parse(output) + outcome.update({"release": release, "console": console, + "seconds": time.time() - started, "timeout": timeout, + "expired": expired, "missing_tools": missing_tools, + "missing_modules": missing_modules}) + return outcome + + +def vm_parse(output): + """Pick the guest's own lines out of everything the console said.""" + outcome = {"results": [], "sweep": {}, "swept": None, "booted": None, + "finished": False, "panic": None, "console_lines": []} + for line in output.replace("\r", "").splitlines(): + if line.startswith("Kernel panic"): + outcome["panic"] = line.strip() + if not line.startswith("@@"): + if line.strip(): + outcome["console_lines"].append(line) + continue + state, _, message = line[2:].partition("|") + if state == "release": + outcome["booted"] = message.strip() + elif state == "sweep": + module, _, reason = message.partition("|") + outcome["sweep"][module] = reason.strip() + elif state == "swept": + outcome["swept"] = message.split("|") + elif state == "finished": + outcome["finished"] = True + elif state in ("ok", "warn", "fail", "skip"): + outcome["results"].append((state, message)) + else: + outcome["console_lines"].append(line) + return outcome + + +def check_vm(release, options): + """Boot this kernel in a VM and make it do the site's work.""" + qemu = vm_qemu(options.vm_qemu) + if qemu is None: + report("skip", "no qemu-system-x86_64 on this machine") + return + kernel = options.vm_bzimage or "/boot/bzImage-%s" % release + if not os.path.isfile(kernel): + report("fail", "no %s: the VM needs the kernel image that goes with" + " /lib/modules/%s" % (kernel, release)) + return + + accelerated = os.access("/dev/kvm", os.R_OK | os.W_OK) + if not accelerated: + report("warn", "no usable /dev/kvm, falling back to emulation, which is" + " slow enough to need --vm-timeout") + + directory = options.vm_workdir or tempfile.mkdtemp(prefix="linux-smoke-vm.") + os.makedirs(directory, exist_ok=True) + keep = bool(options.vm_keep or options.vm_workdir) + try: + outcome = vm_boot(release, kernel, options, qemu, accelerated, + os.path.join(directory, release), options.vm_sweep, + quiet=False) + if outcome is None: + return + + # The sweep on its own says that a module does not load, not that the + # update broke it: three of them refuse on 6.12 as well. Booting the + # kernel this one replaces costs another twenty seconds and turns the + # sweep into what the rest of this script does -- a comparison. + baseline, reference = None, None + if options.vm_sweep and not options.vm_no_reference_boot: + reference = options.vm_reference or default_reference(release) + if reference and os.path.isfile("/boot/bzImage-%s" % reference) \ + and os.path.isdir(module_directory(reference)): + note("booting %s as well, to see which modules already refused" + " to load before the update" % reference) + # --vm-bzimage is about the kernel under test, never + # about the one it is being compared with. + before = vm_boot(reference, "/boot/bzImage-%s" % reference, + options, qemu, accelerated, + os.path.join(directory, reference), + True, quiet=True) + baseline = before["sweep"] if before else None + else: + reference = None + + vm_report(outcome, release, baseline, reference) + if status["fail"]: + keep = True + report("info", "the guest console is in %s" % outcome["console"]) + finally: + if not keep and os.path.isdir(directory): + shutil.rmtree(directory, ignore_errors=True) + + +def vm_report(outcome, release, baseline, reference): + """Turn what the guest said into this script's verdicts.""" + wanted = dict(VM_MODULES) + for name in outcome["missing_tools"]: + note("no %s on this machine; the guest tests that need it are skipped" + % name) + for name in outcome["missing_modules"]: + reason = explained(name) + if reason: + report("warn", "%s is not in %s's tree; the VM needs it for %s -- %s" + % (name, release, wanted[name], reason)) + else: + report("fail", "%s is not in %s's tree at all; the VM needs it" + " for %s" % (name, release, wanted[name])) + + for line in outcome["console_lines"]: + note(line) + if outcome["panic"]: + report("fail", "the guest panicked: %s" % outcome["panic"]) + if outcome["booted"] is None: + report("fail", "the VM never reached /init -- the kernel did not boot;" + " the console log is in %s" % outcome["console"]) + return + if outcome["booted"] != release: + report("fail", "the VM booted %s, not the %s it was given" + % (outcome["booted"], release)) + else: + report("ok", "the VM booted %s" % release) + + for state, message in outcome["results"]: + report(state, message) + + for module in sorted(outcome["sweep"]): + reason = outcome["sweep"][module] + if baseline is not None and module in baseline: + note("sweep: %s does not load in %s either" % (module, reference)) + elif baseline is not None: + report("fail", "sweep: %s loads in %s and not here: %s" + % (module, reference, reason)) + elif vm_sweep_verdict(reason) == "fail": + report("fail", "sweep: %s does not match this kernel: %s" + % (module, reason)) + else: + note("sweep: %s did not load: %s" % (module, reason)) + if outcome["swept"] and len(outcome["swept"]) == 2: + total, failed = (int(number) for number in outcome["swept"]) + report("ok", "sweep: %d of %d modules loaded, %d did not%s" + % (total - failed, total, failed, + ", all of them in %s too" % reference + if baseline is not None and + set(outcome["sweep"]) <= set(baseline) else "")) + + if outcome["expired"]: + report("fail", "the VM did not finish within %d s; the console log is" + " in %s" % (outcome["timeout"], outcome["console"])) + elif not outcome["finished"]: + report("fail", "the VM stopped before the tests were through") + else: + report("ok", "the VM ran the tests and powered itself off in %.0f s" + % outcome["seconds"]) + + CHECKS = ["identity", "config", "modprobe.d", "reference", "hardware", "filesystems", "automount", "namespaces", "network", "storage", - "virtualisation", "health", "dmesg", "services", "baseline"] + "virtualisation", "health", "dmesg", "services", "baseline", "vm"] def main(): @@ -1215,6 +2159,44 @@ def main(): parser.add_argument("--baseline", metavar="FILE", help="compare against a baseline written by" " --save-baseline on the old kernel") + parser.add_argument("--vm", action="store_true", + help="boot the kernel in a qemu virtual machine and" + " run iptables, the network and the filesystems" + " in it; works for a kernel this machine is not" + " running") + parser.add_argument("--vm-sweep", action="store_true", + help="in the VM, load every module in the tree and" + " report the ones that do not match the kernel;" + " adds the whole module tree to the initramfs and" + " a few minutes to the run") + parser.add_argument("--vm-reference", metavar="RELEASE", + help="kernel to boot as well with --vm-sweep, so that" + " a module which did not load before the update" + " is not reported as if the update broke it" + " (default: the same reference the other checks" + " use)") + parser.add_argument("--vm-no-reference-boot", action="store_true", + help="do not boot the reference kernel with" + " --vm-sweep; every module that does not load is" + " then judged on its own") + parser.add_argument("--vm-memory", metavar="MiB", type=int, + help="memory for the VM (default: 1024, or 2048 with" + " --vm-sweep)") + parser.add_argument("--vm-timeout", metavar="SECONDS", type=int, + help="give up on the VM after this long (default: 300," + " or 900 with --vm-sweep, and four times that" + " without KVM)") + parser.add_argument("--vm-qemu", metavar="COMMAND", + help="qemu to run (default: qemu11-system-x86_64, or" + " qemu-system-x86_64)") + parser.add_argument("--vm-bzimage", metavar="FILE", + help="kernel image to boot (default:" + " /boot/bzImage-RELEASE)") + parser.add_argument("--vm-workdir", metavar="DIRECTORY", + help="build the initramfs here and keep it, instead of" + " in a temporary directory that is removed again") + parser.add_argument("--vm-keep", action="store_true", + help="keep the initramfs and the guest console log") parser.add_argument("--only", metavar="CHECK", action="append", default=[], choices=CHECKS, help="run only this check; may be repeated (%s)" @@ -1255,13 +2237,20 @@ def main(): if "baseline" in arguments.only: sys.exit("--only baseline needs --baseline FILE to compare against") wanted.discard("baseline") + if not arguments.vm: + if "vm" in arguments.only: + sys.exit("--only vm needs --vm to start a virtual machine") + wanted.discard("vm") if not booted: # Everything else asks the machine what it is doing now, and the # answer would be about the kernel it is running, not the one asked # about. The hardware check stays: it matches this machine's devices # against that kernel's modules.alias, which is exactly the question # worth asking before a reboot. - wanted &= {"identity", "config", "modprobe.d", "reference", "hardware"} + # The VM stays: it boots the kernel that was asked about, so its + # answers are about that kernel and not about this machine's. + wanted &= {"identity", "config", "modprobe.d", "reference", "hardware", + "vm"} if os.geteuid() != 0: print(" running as %s; the checks that need root are skipped" % (os.environ.get("USER") or os.getuid())) @@ -1282,6 +2271,7 @@ def main(): ("dmesg", check_dmesg), ("services", check_services), ("baseline", lambda: check_baseline(running, arguments.baseline)), + ("vm", lambda: check_vm(running, arguments)), ) if name in wanted] for name, check in selected: From cfdcc26e79ca290102515183c6c1212731f1f51f Mon Sep 17 00:00:00 2001 From: Paul Menzel Date: Tue, 15 Sep 2026 00:31:56 +0200 Subject: [PATCH 3/4] smoke-tests: Add --vm-no-kvm --vm uses /dev/kvm when it can read and write it, and emulates otherwise. The emulation path had never been taken, because every machine here has a usable /dev/kvm, and there was no way to ask for it; a machine whose KVM is broken rather than absent had no way around it either. $ ./smoke-tests/linux-smoke-test --kernel 6.18.51.mx64.498 --vm \ --vm-no-kvm --only vm WARN no usable /dev/kvm, falling back to emulation, which is slow enough to need --vm-timeout ... OK the VM ran the tests and powered itself off in 21 s FAIL (1 failed, 1 warned, 1 skipped) Everything the guest does passes under TCG as well; it takes 21 seconds instead of 8, and the remaining failure is the ebtables one. Assisted-by: Claude Opus 5 --- smoke-tests/linux-smoke-test | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/smoke-tests/linux-smoke-test b/smoke-tests/linux-smoke-test index 40f14bf1c..857d76160 100755 --- a/smoke-tests/linux-smoke-test +++ b/smoke-tests/linux-smoke-test @@ -2025,7 +2025,8 @@ def check_vm(release, options): " /lib/modules/%s" % (kernel, release)) return - accelerated = os.access("/dev/kvm", os.R_OK | os.W_OK) + accelerated = not options.vm_no_kvm \ + and os.access("/dev/kvm", os.R_OK | os.W_OK) if not accelerated: report("warn", "no usable /dev/kvm, falling back to emulation, which is" " slow enough to need --vm-timeout") @@ -2186,6 +2187,9 @@ def main(): help="give up on the VM after this long (default: 300," " or 900 with --vm-sweep, and four times that" " without KVM)") + parser.add_argument("--vm-no-kvm", action="store_true", + help="emulate instead of using /dev/kvm, for a machine" + " whose KVM is broken rather than absent") parser.add_argument("--vm-qemu", metavar="COMMAND", help="qemu to run (default: qemu11-system-x86_64, or" " qemu-system-x86_64)") From 2ef7a27483dc73e1d415aa55abde1964d3b49a25 Mon Sep 17 00:00:00 2001 From: Paul Menzel Date: Tue, 15 Sep 2026 11:50:05 +0200 Subject: [PATCH 4/4] smoke-tests: Find the driver of an NVMe disk On a machine with an NVMe disk, the storage check warns although the disk is driven perfectly well: --- storage WARN nvme0n1 has no driver link OK 2 physical block devices The check looked for /sys/block//device/driver. That is right for a SCSI or ATA disk, whose parent is the SCSI device carrying the driver link, but the parent of an NVMe namespace is the controller -- a class device nothing binds a driver to -- and the driver sits one level further down on the PCI function: /sys/block/nvme0n1/device -> ../../nvme0 /sys/block/nvme0n1/device/device/driver -> ../../../../bus/pci/drivers/nvme So walk up from the disk's parent instead of looking at that one directory, at most three levels so that a genuinely unbound device still reports nothing rather than its host controller's driver. On sigusr2 running 6.18.51.mx64.498 the check now says --- storage nvme0n1 driven by nvme sda driven by sd OK 2 physical block devices and sd is still reported as before on a SCSI-only machine. Co-Authored-By: Claude Opus 5 --- smoke-tests/linux-smoke-test | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/smoke-tests/linux-smoke-test b/smoke-tests/linux-smoke-test index 857d76160..a01e5d3eb 100755 --- a/smoke-tests/linux-smoke-test +++ b/smoke-tests/linux-smoke-test @@ -981,6 +981,30 @@ echo pinged % " ".join(output.split())[:200]) +def disk_driver(path, levels=3): + """The driver bound for a disk, from /sys/block/. + + A SCSI or ATA disk hangs off its SCSI device, which is what carries the + driver link, but an NVMe namespace hangs off the controller -- a class + device nothing binds a driver to -- and the driver sits one level further + down on the PCI function. So walk up from the disk's parent rather than + looking at that one directory, bounded so that a genuinely unbound device + reports nothing instead of its host controller's driver. + """ + device = os.path.join(path, "device") + if not os.path.islink(device): + return None + device = os.path.realpath(device) + for _ in range(levels): + if not device.startswith("/sys/devices/"): + break + link = os.path.join(device, "driver") + if os.path.islink(link): + return os.path.basename(os.path.realpath(link)) + device = os.path.dirname(device) + return None + + def check_storage(running): """Every disk has a driver, and the stack above it is there.""" disks = 0 @@ -989,11 +1013,11 @@ def check_storage(running): if name.startswith(("loop", "ram", "zram", "dm-", "md")): continue disks += 1 - link = os.path.join(path, "device", "driver") - if os.path.islink(link): - note("%s driven by %s" % (name, os.path.basename(os.readlink(link)))) + driver = disk_driver(path) + if driver: + note("%s driven by %s" % (name, driver)) else: - report("warn", "%s has no driver link" % name) + report("warn", "%s has no driver" % name) report("ok" if disks else "warn", "%d physical block devices" % disks) for path in ("/dev/kvm", "/dev/net/tun", "/dev/ipmi0"):