From caadb40d1e382f6472081b70cf57ae6cec99625d Mon Sep 17 00:00:00 2001 From: Donald Buczek Date: Fri, 24 May 2019 11:43:02 +0200 Subject: [PATCH] Reduce nfs max_session_lots from 64 to 4 The default number of session slots (nfs/max_session_lots) is 64. This is the maximum number of parallel requests a nfs client can have outstanding to a nfs server (per mount). Because we have 64 nfsd threads on our servers, a single client is able to dominate all nsfd threads of a server. There seems to be a bad interaction between nfs and readahead. The readahead size is set to max_session_lots-1 and is doubled when posix_fadvice(..,POSIX_FADV_SEQUENTIAL) is used, which "cp" and "cat" do. Although the exact mechanism is not yet fully understood, the readahead io-requests seem to be handled by multiple nfsd thread in parallel, which are blocking each other while compeeting for the same block device. Because all nfsd threads are used up (by a single client doing a single copy) other nfs clients stall and we've even seen them being pushed over their lease timeouts and losing their state. Reduce this value via modprobe.d to avoid overload situations. This parameter can also be read and written on a running system via /sys/module/nfs/parameters/max_session_slots if nfs is loaded. Changes apply to future mounts only. --- install.sh | 1 + misc_etc_files/modprobe.d/nfs_slotlimit.conf | 1 + 2 files changed, 2 insertions(+) create mode 100644 misc_etc_files/modprobe.d/nfs_slotlimit.conf diff --git a/install.sh b/install.sh index 78f1075..1718ab9 100755 --- a/install.sh +++ b/install.sh @@ -148,4 +148,5 @@ install_data misc_systemd_units/network.service "$DESTDIR$systemdunitd install_data misc_systemd_units/mxvlan.service "$DESTDIR$systemdunitdir/mxvlan.service" mkdir -p "$DESTDIR$usr_prefix/share/applications" install_data applications-defaults/defaults.list "$DESTDIR$usr_prefix/share/applications/defaults.list" +install_data misc_etc_files/modprobe.d/nfs_slotlimit.conf "$DESTDIR$sysconfdir/modprobe.d/nfs_slotlimit.conf" exit diff --git a/misc_etc_files/modprobe.d/nfs_slotlimit.conf b/misc_etc_files/modprobe.d/nfs_slotlimit.conf new file mode 100644 index 0000000..6201cc0 --- /dev/null +++ b/misc_etc_files/modprobe.d/nfs_slotlimit.conf @@ -0,0 +1 @@ +options nfs max_session_slots=4