Skip to content

Commit

Permalink
nfsd: make NFSd service structure allocated per net
Browse files Browse the repository at this point in the history 
This patch makes main step in NFSd containerisation.

There could be different approaches to how to make NFSd able to handle
incoming RPC request from different network namespaces.  The two main
options are:

1) Share NFSd kthreads betwween all network namespaces.
2) Create separated pool of threads for each namespace.

While first approach looks more flexible, second one is simpler and
non-racy.  This patch implements the second option.

To make it possible to allocate separate pools of threads, we have to
make it possible to allocate separate NFSd service structures per net.

Signed-off-by: Stanislav Kinsbursky <skinsbursky@parallels.com>
Signed-off-by: J. Bruce Fields <bfields@redhat.com>
  • Loading branch information
Stanislav Kinsbursky authored and J. Bruce Fields committed Dec 10, 2012
1 parent b9c0ef8 commit 9dd9845
Show file tree
Hide file tree
Showing 5 changed files with 110 additions and 78 deletions.
2 changes: 2 additions & 0 deletions fs/nfsd/netns.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ struct nfsd_net {
* Time of server startup
*/
struct timeval nfssvc_boot;

struct svc_serv *nfsd_serv;
};

/* Simple check to find out if a given net was properly initialized */
Expand Down
14 changes: 9 additions & 5 deletions fs/nfsd/nfs4state.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -743,9 +743,12 @@ static struct nfsd4_session *__alloc_session(int slotsize, int numslots)
return NULL;
}

static void init_forechannel_attrs(struct nfsd4_channel_attrs *new, struct nfsd4_channel_attrs *req, int numslots, int slotsize)
static void init_forechannel_attrs(struct nfsd4_channel_attrs *new,
struct nfsd4_channel_attrs *req,
int numslots, int slotsize,
struct nfsd_net *nn)
{
u32 maxrpc = nfsd_serv->sv_max_mesg;
u32 maxrpc = nn->nfsd_serv->sv_max_mesg;

new->maxreqs = numslots;
new->maxresp_cached = min_t(u32, req->maxresp_cached,
Expand Down Expand Up @@ -883,7 +886,8 @@ void nfsd4_put_session(struct nfsd4_session *ses)
spin_unlock(&nn->client_lock);
}

static struct nfsd4_session *alloc_session(struct nfsd4_channel_attrs *fchan)
static struct nfsd4_session *alloc_session(struct nfsd4_channel_attrs *fchan,
struct nfsd_net *nn)
{
struct nfsd4_session *new;
int numslots, slotsize;
Expand All @@ -904,7 +908,7 @@ static struct nfsd4_session *alloc_session(struct nfsd4_channel_attrs *fchan)
nfsd4_put_drc_mem(slotsize, fchan->maxreqs);
return NULL;
}
init_forechannel_attrs(&new->se_fchannel, fchan, numslots, slotsize);
init_forechannel_attrs(&new->se_fchannel, fchan, numslots, slotsize, nn);
return new;
}

Expand Down Expand Up @@ -1776,7 +1780,7 @@ nfsd4_create_session(struct svc_rqst *rqstp,
return nfserr_inval;
if (check_forechannel_attrs(cr_ses->fore_channel))
return nfserr_toosmall;
new = alloc_session(&cr_ses->fore_channel);
new = alloc_session(&cr_ses->fore_channel, nn);
if (!new)
return nfserr_jukebox;
status = nfserr_jukebox;
Expand Down
63 changes: 38 additions & 25 deletions fs/nfsd/nfsctl.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -409,7 +409,7 @@ static ssize_t write_threads(struct file *file, char *buf, size_t size)
if (rv < 0)
return rv;
} else
rv = nfsd_nrthreads();
rv = nfsd_nrthreads(net);

return scnprintf(buf, SIMPLE_TRANSACTION_LIMIT, "%d\n", rv);
}
Expand Down Expand Up @@ -450,7 +450,7 @@ static ssize_t write_pool_threads(struct file *file, char *buf, size_t size)
struct net *net = &init_net;

mutex_lock(&nfsd_mutex);
npools = nfsd_nrpools();
npools = nfsd_nrpools(net);
if (npools == 0) {
/*
* NFS is shut down. The admin can start it by
Expand Down Expand Up @@ -483,7 +483,7 @@ static ssize_t write_pool_threads(struct file *file, char *buf, size_t size)
goto out_free;
}

rv = nfsd_get_nrthreads(npools, nthreads);
rv = nfsd_get_nrthreads(npools, nthreads, net);
if (rv)
goto out_free;

Expand All @@ -510,11 +510,13 @@ static ssize_t __write_versions(struct file *file, char *buf, size_t size)
unsigned minor;
ssize_t tlen = 0;
char *sep;
struct net *net = &init_net;
struct nfsd_net *nn = net_generic(net, nfsd_net_id);

if (size>0) {
if (nfsd_serv)
if (nn->nfsd_serv)
/* Cannot change versions without updating
* nfsd_serv->sv_xdrsize, and reallocing
* nn->nfsd_serv->sv_xdrsize, and reallocing
* rq_argp and rq_resp
*/
return -EBUSY;
Expand Down Expand Up @@ -645,11 +647,13 @@ static ssize_t write_versions(struct file *file, char *buf, size_t size)
* Zero-length write. Return a list of NFSD's current listener
* transports.
*/
static ssize_t __write_ports_names(char *buf)
static ssize_t __write_ports_names(char *buf, struct net *net)
{
if (nfsd_serv == NULL)
struct nfsd_net *nn = net_generic(net, nfsd_net_id);

if (nn->nfsd_serv == NULL)
return 0;
return svc_xprt_names(nfsd_serv, buf, SIMPLE_TRANSACTION_LIMIT);
return svc_xprt_names(nn->nfsd_serv, buf, SIMPLE_TRANSACTION_LIMIT);
}

/*
Expand All @@ -661,6 +665,7 @@ static ssize_t __write_ports_addfd(char *buf, struct net *net)
{
char *mesg = buf;
int fd, err;
struct nfsd_net *nn = net_generic(net, nfsd_net_id);

err = get_int(&mesg, &fd);
if (err != 0 || fd < 0)
Expand All @@ -670,14 +675,14 @@ static ssize_t __write_ports_addfd(char *buf, struct net *net)
if (err != 0)
return err;

err = svc_addsock(nfsd_serv, fd, buf, SIMPLE_TRANSACTION_LIMIT);
err = svc_addsock(nn->nfsd_serv, fd, buf, SIMPLE_TRANSACTION_LIMIT);
if (err < 0) {
nfsd_destroy(net);
return err;
}

/* Decrease the count, but don't shut down the service */
nfsd_serv->sv_nrthreads--;
nn->nfsd_serv->sv_nrthreads--;
return err;
}

Expand All @@ -690,6 +695,7 @@ static ssize_t __write_ports_addxprt(char *buf, struct net *net)
char transport[16];
struct svc_xprt *xprt;
int port, err;
struct nfsd_net *nn = net_generic(net, nfsd_net_id);

if (sscanf(buf, "%15s %5u", transport, &port) != 2)
return -EINVAL;
Expand All @@ -701,21 +707,21 @@ static ssize_t __write_ports_addxprt(char *buf, struct net *net)
if (err != 0)
return err;

err = svc_create_xprt(nfsd_serv, transport, net,
err = svc_create_xprt(nn->nfsd_serv, transport, net,
PF_INET, port, SVC_SOCK_ANONYMOUS);
if (err < 0)
goto out_err;

err = svc_create_xprt(nfsd_serv, transport, net,
err = svc_create_xprt(nn->nfsd_serv, transport, net,
PF_INET6, port, SVC_SOCK_ANONYMOUS);
if (err < 0 && err != -EAFNOSUPPORT)
goto out_close;

/* Decrease the count, but don't shut down the service */
nfsd_serv->sv_nrthreads--;
nn->nfsd_serv->sv_nrthreads--;
return 0;
out_close:
xprt = svc_find_xprt(nfsd_serv, transport, net, PF_INET, port);
xprt = svc_find_xprt(nn->nfsd_serv, transport, net, PF_INET, port);
if (xprt != NULL) {
svc_close_xprt(xprt);
svc_xprt_put(xprt);
Expand All @@ -729,7 +735,7 @@ static ssize_t __write_ports(struct file *file, char *buf, size_t size,
struct net *net)
{
if (size == 0)
return __write_ports_names(buf);
return __write_ports_names(buf, net);

if (isdigit(buf[0]))
return __write_ports_addfd(buf, net);
Expand Down Expand Up @@ -821,6 +827,9 @@ int nfsd_max_blksize;
static ssize_t write_maxblksize(struct file *file, char *buf, size_t size)
{
char *mesg = buf;
struct net *net = &init_net;
struct nfsd_net *nn = net_generic(net, nfsd_net_id);

if (size > 0) {
int bsize;
int rv = get_int(&mesg, &bsize);
Expand All @@ -835,7 +844,7 @@ static ssize_t write_maxblksize(struct file *file, char *buf, size_t size)
bsize = NFSSVC_MAXBLKSIZE;
bsize &= ~(1024-1);
mutex_lock(&nfsd_mutex);
if (nfsd_serv) {
if (nn->nfsd_serv) {
mutex_unlock(&nfsd_mutex);
return -EBUSY;
}
Expand All @@ -848,13 +857,14 @@ static ssize_t write_maxblksize(struct file *file, char *buf, size_t size)
}

#ifdef CONFIG_NFSD_V4
static ssize_t __nfsd4_write_time(struct file *file, char *buf, size_t size, time_t *time)
static ssize_t __nfsd4_write_time(struct file *file, char *buf, size_t size,
time_t *time, struct nfsd_net *nn)
{
char *mesg = buf;
int rv, i;

if (size > 0) {
if (nfsd_serv)
if (nn->nfsd_serv)
return -EBUSY;
rv = get_int(&mesg, &i);
if (rv)
Expand All @@ -879,12 +889,13 @@ static ssize_t __nfsd4_write_time(struct file *file, char *buf, size_t size, tim
return scnprintf(buf, SIMPLE_TRANSACTION_LIMIT, "%ld\n", *time);
}

static ssize_t nfsd4_write_time(struct file *file, char *buf, size_t size, time_t *time)
static ssize_t nfsd4_write_time(struct file *file, char *buf, size_t size,
time_t *time, struct nfsd_net *nn)
{
ssize_t rv;

mutex_lock(&nfsd_mutex);
rv = __nfsd4_write_time(file, buf, size, time);
rv = __nfsd4_write_time(file, buf, size, time, nn);
mutex_unlock(&nfsd_mutex);
return rv;
}
Expand Down Expand Up @@ -913,7 +924,7 @@ static ssize_t nfsd4_write_time(struct file *file, char *buf, size_t size, time_
static ssize_t write_leasetime(struct file *file, char *buf, size_t size)
{
struct nfsd_net *nn = net_generic(&init_net, nfsd_net_id);
return nfsd4_write_time(file, buf, size, &nn->nfsd4_lease);
return nfsd4_write_time(file, buf, size, &nn->nfsd4_lease, nn);
}

/**
Expand All @@ -929,17 +940,18 @@ static ssize_t write_leasetime(struct file *file, char *buf, size_t size)
static ssize_t write_gracetime(struct file *file, char *buf, size_t size)
{
struct nfsd_net *nn = net_generic(&init_net, nfsd_net_id);
return nfsd4_write_time(file, buf, size, &nn->nfsd4_grace);
return nfsd4_write_time(file, buf, size, &nn->nfsd4_grace, nn);
}

static ssize_t __write_recoverydir(struct file *file, char *buf, size_t size)
static ssize_t __write_recoverydir(struct file *file, char *buf, size_t size,
struct nfsd_net *nn)
{
char *mesg = buf;
char *recdir;
int len, status;

if (size > 0) {
if (nfsd_serv)
if (nn->nfsd_serv)
return -EBUSY;
if (size > PATH_MAX || buf[size-1] != '\n')
return -EINVAL;
Expand Down Expand Up @@ -983,9 +995,10 @@ static ssize_t __write_recoverydir(struct file *file, char *buf, size_t size)
static ssize_t write_recoverydir(struct file *file, char *buf, size_t size)
{
ssize_t rv;
struct nfsd_net *nn = net_generic(&init_net, nfsd_net_id);

mutex_lock(&nfsd_mutex);
rv = __write_recoverydir(file, buf, size);
rv = __write_recoverydir(file, buf, size, nn);
mutex_unlock(&nfsd_mutex);
return rv;
}
Expand Down
18 changes: 4 additions & 14 deletions fs/nfsd/nfsd.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ extern struct svc_version nfsd_version2, nfsd_version3,
nfsd_version4;
extern u32 nfsd_supported_minorversion;
extern struct mutex nfsd_mutex;
extern struct svc_serv *nfsd_serv;
extern spinlock_t nfsd_drc_lock;
extern unsigned int nfsd_drc_max_mem;
extern unsigned int nfsd_drc_mem_used;
Expand All @@ -68,23 +67,14 @@ extern const struct seq_operations nfs_exports_op;
int nfsd_svc(int nrservs, struct net *net);
int nfsd_dispatch(struct svc_rqst *rqstp, __be32 *statp);

int nfsd_nrthreads(void);
int nfsd_nrpools(void);
int nfsd_get_nrthreads(int n, int *);
int nfsd_nrthreads(struct net *);
int nfsd_nrpools(struct net *);
int nfsd_get_nrthreads(int n, int *, struct net *);
int nfsd_set_nrthreads(int n, int *, struct net *);
int nfsd_pool_stats_open(struct inode *, struct file *);
int nfsd_pool_stats_release(struct inode *, struct file *);

static inline void nfsd_destroy(struct net *net)
{
int destroy = (nfsd_serv->sv_nrthreads == 1);

if (destroy)
svc_shutdown_net(nfsd_serv, net);
svc_destroy(nfsd_serv);
if (destroy)
nfsd_serv = NULL;
}
void nfsd_destroy(struct net *net);

#if defined(CONFIG_NFSD_V2_ACL) || defined(CONFIG_NFSD_V3_ACL)
#ifdef CONFIG_NFSD_V2_ACL
Expand Down
Loading

0 comments on commit 9dd9845

Please sign in to comment.