Skip to content

Commit

Permalink
NFSD: add shrinker to reap courtesy clients on low memory condition
Browse files Browse the repository at this point in the history 
Add courtesy_client_reaper to react to low memory condition triggered
by the system memory shrinker.

The delayed_work for the courtesy_client_reaper is scheduled on
the shrinker's count callback using the laundry_wq.

The shrinker's scan callback is not used for expiring the courtesy
clients due to potential deadlocks.

Signed-off-by: Dai Ngo <dai.ngo@oracle.com>
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
  • Loading branch information
Dai Ngo authored and Chuck Lever committed Sep 26, 2022
1 parent 3a4ea23 commit 7746b32
Show file tree
Hide file tree
Showing 4 changed files with 96 additions and 12 deletions.
2 changes: 2 additions & 0 deletions fs/nfsd/netns.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,8 @@ struct nfsd_net {
int nfs4_max_clients;

atomic_t nfsd_courtesy_clients;
struct shrinker nfsd_client_shrinker;
struct delayed_work nfsd_shrinker_work;
};

/* Simple check to find out if a given net was properly initialized */
Expand Down
94 changes: 86 additions & 8 deletions fs/nfsd/nfs4state.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4349,7 +4349,27 @@ nfsd4_init_slabs(void)
return -ENOMEM;
}

void nfsd4_init_leases_net(struct nfsd_net *nn)
static unsigned long
nfsd_courtesy_client_count(struct shrinker *shrink, struct shrink_control *sc)
{
int cnt;
struct nfsd_net *nn = container_of(shrink,
struct nfsd_net, nfsd_client_shrinker);

cnt = atomic_read(&nn->nfsd_courtesy_clients);
if (cnt > 0)
mod_delayed_work(laundry_wq, &nn->nfsd_shrinker_work, 0);
return (unsigned long)cnt;
}

static unsigned long
nfsd_courtesy_client_scan(struct shrinker *shrink, struct shrink_control *sc)
{
return SHRINK_STOP;
}

int
nfsd4_init_leases_net(struct nfsd_net *nn)
{
struct sysinfo si;
u64 max_clients;
Expand All @@ -4370,6 +4390,16 @@ void nfsd4_init_leases_net(struct nfsd_net *nn)
nn->nfs4_max_clients = max_t(int, max_clients, NFS4_CLIENTS_PER_GB);

atomic_set(&nn->nfsd_courtesy_clients, 0);
nn->nfsd_client_shrinker.scan_objects = nfsd_courtesy_client_scan;
nn->nfsd_client_shrinker.count_objects = nfsd_courtesy_client_count;
nn->nfsd_client_shrinker.seeks = DEFAULT_SEEKS;
return register_shrinker(&nn->nfsd_client_shrinker, "nfsd-client");
}

void
nfsd4_leases_net_shutdown(struct nfsd_net *nn)
{
unregister_shrinker(&nn->nfsd_client_shrinker);
}

static void init_nfs4_replay(struct nfs4_replay *rp)
Expand Down Expand Up @@ -5942,10 +5972,49 @@ nfs4_get_client_reaplist(struct nfsd_net *nn, struct list_head *reaplist,
spin_unlock(&nn->client_lock);
}

static void
nfs4_get_courtesy_client_reaplist(struct nfsd_net *nn,
struct list_head *reaplist)
{
unsigned int maxreap = 0, reapcnt = 0;
struct list_head *pos, *next;
struct nfs4_client *clp;

maxreap = NFSD_CLIENT_MAX_TRIM_PER_RUN;
INIT_LIST_HEAD(reaplist);

spin_lock(&nn->client_lock);
list_for_each_safe(pos, next, &nn->client_lru) {
clp = list_entry(pos, struct nfs4_client, cl_lru);
if (clp->cl_state == NFSD4_ACTIVE)
break;
if (reapcnt >= maxreap)
break;
if (!mark_client_expired_locked(clp)) {
list_add(&clp->cl_lru, reaplist);
reapcnt++;
}
}
spin_unlock(&nn->client_lock);
}

static void
nfs4_process_client_reaplist(struct list_head *reaplist)
{
struct list_head *pos, *next;
struct nfs4_client *clp;

list_for_each_safe(pos, next, reaplist) {
clp = list_entry(pos, struct nfs4_client, cl_lru);
trace_nfsd_clid_purged(&clp->cl_clientid);
list_del_init(&clp->cl_lru);
expire_client(clp);
}
}

static time64_t
nfs4_laundromat(struct nfsd_net *nn)
{
struct nfs4_client *clp;
struct nfs4_openowner *oo;
struct nfs4_delegation *dp;
struct nfs4_ol_stateid *stp;
Expand Down Expand Up @@ -5974,12 +6043,8 @@ nfs4_laundromat(struct nfsd_net *nn)
}
spin_unlock(&nn->s2s_cp_lock);
nfs4_get_client_reaplist(nn, &reaplist, &lt);
list_for_each_safe(pos, next, &reaplist) {
clp = list_entry(pos, struct nfs4_client, cl_lru);
trace_nfsd_clid_purged(&clp->cl_clientid);
list_del_init(&clp->cl_lru);
expire_client(clp);
}
nfs4_process_client_reaplist(&reaplist);

spin_lock(&state_lock);
list_for_each_safe(pos, next, &nn->del_recall_lru) {
dp = list_entry (pos, struct nfs4_delegation, dl_recall_lru);
Expand Down Expand Up @@ -6062,6 +6127,18 @@ laundromat_main(struct work_struct *laundry)
queue_delayed_work(laundry_wq, &nn->laundromat_work, t*HZ);
}

static void
courtesy_client_reaper(struct work_struct *reaper)
{
struct list_head reaplist;
struct delayed_work *dwork = to_delayed_work(reaper);
struct nfsd_net *nn = container_of(dwork, struct nfsd_net,
nfsd_shrinker_work);

nfs4_get_courtesy_client_reaplist(nn, &reaplist);
nfs4_process_client_reaplist(&reaplist);
}

static inline __be32 nfs4_check_fh(struct svc_fh *fhp, struct nfs4_stid *stp)
{
if (!fh_match(&fhp->fh_handle, &stp->sc_file->fi_fhandle))
Expand Down Expand Up @@ -7879,6 +7956,7 @@ static int nfs4_state_create_net(struct net *net)
INIT_LIST_HEAD(&nn->blocked_locks_lru);

INIT_DELAYED_WORK(&nn->laundromat_work, laundromat_main);
INIT_DELAYED_WORK(&nn->nfsd_shrinker_work, courtesy_client_reaper);
get_net(net);

return 0;
Expand Down
6 changes: 4 additions & 2 deletions fs/nfsd/nfsctl.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1481,11 +1481,12 @@ static __net_init int nfsd_init_net(struct net *net)
goto out_idmap_error;
nn->nfsd_versions = NULL;
nn->nfsd4_minorversions = NULL;
retval = nfsd4_init_leases_net(nn);
if (retval)
goto out_drc_error;
retval = nfsd_reply_cache_init(nn);
if (retval)
goto out_drc_error;
nfsd4_init_leases_net(nn);

get_random_bytes(&nn->siphash_key, sizeof(nn->siphash_key));
seqlock_init(&nn->writeverf_lock);

Expand All @@ -1507,6 +1508,7 @@ static __net_exit void nfsd_exit_net(struct net *net)
nfsd_idmap_shutdown(net);
nfsd_export_shutdown(net);
nfsd_netns_free_versions(net_generic(net, nfsd_net_id));
nfsd4_leases_net_shutdown(nn);
}

static struct pernet_operations nfsd_net_ops = {
Expand Down
6 changes: 4 additions & 2 deletions fs/nfsd/nfsd.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -505,15 +505,17 @@ extern void unregister_cld_notifier(void);
extern void nfsd4_ssc_init_umount_work(struct nfsd_net *nn);
#endif

extern void nfsd4_init_leases_net(struct nfsd_net *nn);
extern int nfsd4_init_leases_net(struct nfsd_net *nn);
extern void nfsd4_leases_net_shutdown(struct nfsd_net *nn);

#else /* CONFIG_NFSD_V4 */
static inline int nfsd4_is_junction(struct dentry *dentry)
{
return 0;
}

static inline void nfsd4_init_leases_net(struct nfsd_net *nn) {};
static inline int nfsd4_init_leases_net(struct nfsd_net *nn) { return 0; };
static inline void nfsd4_leases_net_shutdown(struct nfsd_net *nn) {};

#define register_cld_notifier() 0
#define unregister_cld_notifier() do { } while(0)
Expand Down

0 comments on commit 7746b32

Please sign in to comment.