Skip to content

Commit

Permalink
NFSv4.1 move deviceid cache to filelayout driver
Browse files Browse the repository at this point in the history 
No need for generic cache with only one user.
Keep a simple hash of deviceids in the filelayout driver.

Signed-off-by: Christoph Hellwig <hch@infradead.org>
Acked-by: Andy Adamson <andros@netapp.com>
Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>
  • Loading branch information
Christoph Hellwig authored and Trond Myklebust committed Mar 11, 2011
1 parent cbdabc7 commit ea8eecd
Show file tree
Hide file tree
Showing 6 changed files with 92 additions and 264 deletions.
46 changes: 8 additions & 38 deletions fs/nfs/nfs4filelayout.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,32 +42,6 @@ MODULE_DESCRIPTION("The NFSv4 file layout driver");

#define FILELAYOUT_POLL_RETRY_MAX (15*HZ)

static int
filelayout_set_layoutdriver(struct nfs_server *nfss)
{
int status = pnfs_alloc_init_deviceid_cache(nfss->nfs_client,
nfs4_fl_free_deviceid_callback);
if (status) {
printk(KERN_WARNING "%s: deviceid cache could not be "
"initialized\n", __func__);
return status;
}
dprintk("%s: deviceid cache has been initialized successfully\n",
__func__);
return 0;
}

/* Clear out the layout by destroying its device list */
static int
filelayout_clear_layoutdriver(struct nfs_server *nfss)
{
dprintk("--> %s\n", __func__);

if (nfss->nfs_client->cl_devid_cache)
pnfs_put_deviceid_cache(nfss->nfs_client);
return 0;
}

static loff_t
filelayout_get_dense_offset(struct nfs4_filelayout_segment *flseg,
loff_t offset)
Expand Down Expand Up @@ -295,7 +269,7 @@ filelayout_check_layout(struct pnfs_layout_hdr *lo,
}

/* find and reference the deviceid */
dsaddr = nfs4_fl_find_get_deviceid(nfss->nfs_client, id);
dsaddr = nfs4_fl_find_get_deviceid(id);
if (dsaddr == NULL) {
dsaddr = get_device_info(lo->plh_inode, id);
if (dsaddr == NULL)
Expand Down Expand Up @@ -330,7 +304,7 @@ filelayout_check_layout(struct pnfs_layout_hdr *lo,
dprintk("--> %s returns %d\n", __func__, status);
return status;
out_put:
pnfs_put_deviceid(nfss->nfs_client->cl_devid_cache, &dsaddr->deviceid);
nfs4_fl_put_deviceid(dsaddr);
goto out;
}

Expand Down Expand Up @@ -439,12 +413,10 @@ filelayout_alloc_lseg(struct pnfs_layout_hdr *layoutid,
static void
filelayout_free_lseg(struct pnfs_layout_segment *lseg)
{
struct nfs_server *nfss = NFS_SERVER(lseg->pls_layout->plh_inode);
struct nfs4_filelayout_segment *fl = FILELAYOUT_LSEG(lseg);

dprintk("--> %s\n", __func__);
pnfs_put_deviceid(nfss->nfs_client->cl_devid_cache,
&fl->dsaddr->deviceid);
nfs4_fl_put_deviceid(fl->dsaddr);
_filelayout_free_lseg(fl);
}

Expand Down Expand Up @@ -474,13 +446,11 @@ filelayout_pg_test(struct nfs_pageio_descriptor *pgio, struct nfs_page *prev,
}

static struct pnfs_layoutdriver_type filelayout_type = {
.id = LAYOUT_NFSV4_1_FILES,
.name = "LAYOUT_NFSV4_1_FILES",
.owner = THIS_MODULE,
.set_layoutdriver = filelayout_set_layoutdriver,
.clear_layoutdriver = filelayout_clear_layoutdriver,
.alloc_lseg = filelayout_alloc_lseg,
.free_lseg = filelayout_free_lseg,
.id = LAYOUT_NFSV4_1_FILES,
.name = "LAYOUT_NFSV4_1_FILES",
.owner = THIS_MODULE,
.alloc_lseg = filelayout_alloc_lseg,
.free_lseg = filelayout_free_lseg,
.pg_test = filelayout_pg_test,
.read_pagelist = filelayout_read_pagelist,
};
Expand Down
8 changes: 5 additions & 3 deletions fs/nfs/nfs4filelayout.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,9 @@ struct nfs4_pnfs_ds {
};

struct nfs4_file_layout_dsaddr {
struct pnfs_deviceid_node deviceid;
struct hlist_node node;
struct nfs4_deviceid deviceid;
atomic_t ref;
u32 stripe_count;
u8 *stripe_indices;
u32 ds_num;
Expand Down Expand Up @@ -86,15 +88,15 @@ FILELAYOUT_LSEG(struct pnfs_layout_segment *lseg)
extern struct nfs_fh *
nfs4_fl_select_ds_fh(struct pnfs_layout_segment *lseg, u32 j);

extern void nfs4_fl_free_deviceid_callback(struct pnfs_deviceid_node *);
extern void print_ds(struct nfs4_pnfs_ds *ds);
extern void print_deviceid(struct nfs4_deviceid *dev_id);
u32 nfs4_fl_calc_j_index(struct pnfs_layout_segment *lseg, loff_t offset);
u32 nfs4_fl_calc_ds_index(struct pnfs_layout_segment *lseg, u32 j);
struct nfs4_pnfs_ds *nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg,
u32 ds_idx);
extern struct nfs4_file_layout_dsaddr *
nfs4_fl_find_get_deviceid(struct nfs_client *, struct nfs4_deviceid *dev_id);
nfs4_fl_find_get_deviceid(struct nfs4_deviceid *dev_id);
extern void nfs4_fl_put_deviceid(struct nfs4_file_layout_dsaddr *dsaddr);
struct nfs4_file_layout_dsaddr *
get_device_info(struct inode *inode, struct nfs4_deviceid *dev_id);

Expand Down
106 changes: 77 additions & 29 deletions fs/nfs/nfs4filelayoutdev.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,30 @@

#define NFSDBG_FACILITY NFSDBG_PNFS_LD

/*
* Device ID RCU cache. A device ID is unique per client ID and layout type.
*/
#define NFS4_FL_DEVICE_ID_HASH_BITS 5
#define NFS4_FL_DEVICE_ID_HASH_SIZE (1 << NFS4_FL_DEVICE_ID_HASH_BITS)
#define NFS4_FL_DEVICE_ID_HASH_MASK (NFS4_FL_DEVICE_ID_HASH_SIZE - 1)

static inline u32
nfs4_fl_deviceid_hash(struct nfs4_deviceid *id)
{
unsigned char *cptr = (unsigned char *)id->data;
unsigned int nbytes = NFS4_DEVICEID4_SIZE;
u32 x = 0;

while (nbytes--) {
x *= 37;
x += *cptr++;
}
return x & NFS4_FL_DEVICE_ID_HASH_MASK;
}

static struct hlist_head filelayout_deviceid_cache[NFS4_FL_DEVICE_ID_HASH_SIZE];
static DEFINE_SPINLOCK(filelayout_deviceid_lock);

/*
* Data server cache
*
Expand Down Expand Up @@ -183,7 +207,7 @@ nfs4_fl_free_deviceid(struct nfs4_file_layout_dsaddr *dsaddr)
struct nfs4_pnfs_ds *ds;
int i;

print_deviceid(&dsaddr->deviceid.de_id);
print_deviceid(&dsaddr->deviceid);

for (i = 0; i < dsaddr->ds_num; i++) {
ds = dsaddr->ds_list[i];
Expand All @@ -200,15 +224,6 @@ nfs4_fl_free_deviceid(struct nfs4_file_layout_dsaddr *dsaddr)
kfree(dsaddr);
}

void
nfs4_fl_free_deviceid_callback(struct pnfs_deviceid_node *device)
{
struct nfs4_file_layout_dsaddr *dsaddr =
container_of(device, struct nfs4_file_layout_dsaddr, deviceid);

nfs4_fl_free_deviceid(dsaddr);
}

static struct nfs4_pnfs_ds *
nfs4_pnfs_ds_add(struct inode *inode, u32 ip_addr, u32 port)
{
Expand Down Expand Up @@ -361,7 +376,7 @@ decode_device(struct inode *ino, struct pnfs_device *pdev)
dsaddr->stripe_count = cnt;
dsaddr->ds_num = num;

memcpy(&dsaddr->deviceid.de_id, &pdev->dev_id, sizeof(pdev->dev_id));
memcpy(&dsaddr->deviceid, &pdev->dev_id, sizeof(pdev->dev_id));

/* Go back an read stripe indices */
p = indicesp;
Expand Down Expand Up @@ -411,28 +426,37 @@ decode_device(struct inode *ino, struct pnfs_device *pdev)
}

/*
* Decode the opaque device specified in 'dev'
* and add it to the list of available devices.
* If the deviceid is already cached, nfs4_add_deviceid will return
* a pointer to the cached struct and throw away the new.
* Decode the opaque device specified in 'dev' and add it to the cache of
* available devices.
*/
static struct nfs4_file_layout_dsaddr*
static struct nfs4_file_layout_dsaddr *
decode_and_add_device(struct inode *inode, struct pnfs_device *dev)
{
struct nfs4_file_layout_dsaddr *dsaddr;
struct pnfs_deviceid_node *d;
struct nfs4_file_layout_dsaddr *d, *new;
long hash;

dsaddr = decode_device(inode, dev);
if (!dsaddr) {
new = decode_device(inode, dev);
if (!new) {
printk(KERN_WARNING "%s: Could not decode or add device\n",
__func__);
return NULL;
}

d = pnfs_add_deviceid(NFS_SERVER(inode)->nfs_client->cl_devid_cache,
&dsaddr->deviceid);
spin_lock(&filelayout_deviceid_lock);
d = nfs4_fl_find_get_deviceid(&new->deviceid);
if (d) {
spin_unlock(&filelayout_deviceid_lock);
nfs4_fl_free_deviceid(new);
return d;
}

INIT_HLIST_NODE(&new->node);
atomic_set(&new->ref, 1);
hash = nfs4_fl_deviceid_hash(&new->deviceid);
hlist_add_head_rcu(&new->node, &filelayout_deviceid_cache[hash]);
spin_unlock(&filelayout_deviceid_lock);

return container_of(d, struct nfs4_file_layout_dsaddr, deviceid);
return new;
}

/*
Expand Down Expand Up @@ -507,14 +531,38 @@ get_device_info(struct inode *inode, struct nfs4_deviceid *dev_id)
return dsaddr;
}

struct nfs4_file_layout_dsaddr *
nfs4_fl_find_get_deviceid(struct nfs_client *clp, struct nfs4_deviceid *id)
void
nfs4_fl_put_deviceid(struct nfs4_file_layout_dsaddr *dsaddr)
{
struct pnfs_deviceid_node *d;
if (atomic_dec_and_lock(&dsaddr->ref, &filelayout_deviceid_lock)) {
hlist_del_rcu(&dsaddr->node);
spin_unlock(&filelayout_deviceid_lock);

synchronize_rcu();
nfs4_fl_free_deviceid(dsaddr);
}
}

d = pnfs_find_get_deviceid(clp->cl_devid_cache, id);
return (d == NULL) ? NULL :
container_of(d, struct nfs4_file_layout_dsaddr, deviceid);
struct nfs4_file_layout_dsaddr *
nfs4_fl_find_get_deviceid(struct nfs4_deviceid *id)
{
struct nfs4_file_layout_dsaddr *d;
struct hlist_node *n;
long hash = nfs4_fl_deviceid_hash(id);


rcu_read_lock();
hlist_for_each_entry_rcu(d, n, &filelayout_deviceid_cache[hash], node) {
if (!memcmp(&d->deviceid, id, sizeof(*id))) {
if (!atomic_inc_not_zero(&d->ref))
goto fail;
rcu_read_unlock();
return d;
}
}
fail:
rcu_read_unlock();
return NULL;
}

/*
Expand Down
Loading

0 comments on commit ea8eecd

Please sign in to comment.