Skip to content

Commit

Permalink
pnfs: factor GETDEVICEINFO implementations
Browse files Browse the repository at this point in the history 
Add support to the common pNFS core to issue GETDEVICEINFO calls on
a device ID cache miss.  The code is taken from the well debugged
file layout implementation and calls out to the layoutdriver through
a new alloc_deviceid_node method.  The calling conventions for
nfs4_find_get_deviceid are changed so that all information needed to
send a GETDEVICEINFO request is passed to the common code.

Signed-off-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Trond Myklebust <trond.myklebust@primarydata.com>
  • Loading branch information
Christoph Hellwig authored and Trond Myklebust committed Sep 10, 2014
1 parent 848746b commit 661373b
Show file tree
Hide file tree
Showing 8 changed files with 182 additions and 312 deletions.
29 changes: 19 additions & 10 deletions fs/nfs/filelayout/filelayout.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -649,18 +649,15 @@ filelayout_check_layout(struct pnfs_layout_hdr *lo,
}

/* find and reference the deviceid */
d = nfs4_find_get_deviceid(NFS_SERVER(lo->plh_inode)->pnfs_curr_ld,
NFS_SERVER(lo->plh_inode)->nfs_client, id);
if (d == NULL) {
dsaddr = filelayout_get_device_info(lo->plh_inode, id,
lo->plh_lc_cred, gfp_flags);
if (dsaddr == NULL)
goto out;
} else
dsaddr = container_of(d, struct nfs4_file_layout_dsaddr, id_node);
d = nfs4_find_get_deviceid(NFS_SERVER(lo->plh_inode), id,
lo->plh_lc_cred, gfp_flags);
if (d == NULL)
goto out;

dsaddr = container_of(d, struct nfs4_file_layout_dsaddr, id_node);
/* Found deviceid is unavailable */
if (filelayout_test_devid_unavailable(&dsaddr->id_node))
goto out_put;
goto out_put;

fl->dsaddr = dsaddr;

Expand Down Expand Up @@ -1371,6 +1368,17 @@ filelayout_commit_pagelist(struct inode *inode, struct list_head *mds_pages,
cinfo->ds->ncommitting = 0;
return PNFS_ATTEMPTED;
}
static struct nfs4_deviceid_node *
filelayout_alloc_deviceid_node(struct nfs_server *server,
struct pnfs_device *pdev, gfp_t gfp_flags)
{
struct nfs4_file_layout_dsaddr *dsaddr;

dsaddr = nfs4_fl_alloc_deviceid_node(server, pdev, gfp_flags);
if (!dsaddr)
return NULL;
return &dsaddr->id_node;
}

static void
filelayout_free_deveiceid_node(struct nfs4_deviceid_node *d)
Expand Down Expand Up @@ -1423,6 +1431,7 @@ static struct pnfs_layoutdriver_type filelayout_type = {
.commit_pagelist = filelayout_commit_pagelist,
.read_pagelist = filelayout_read_pagelist,
.write_pagelist = filelayout_write_pagelist,
.alloc_deviceid_node = filelayout_alloc_deviceid_node,
.free_deviceid_node = filelayout_free_deveiceid_node,
};

Expand Down
7 changes: 4 additions & 3 deletions fs/nfs/filelayout/filelayout.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,10 +147,11 @@ u32 nfs4_fl_calc_j_index(struct pnfs_layout_segment *lseg, loff_t offset);
u32 nfs4_fl_calc_ds_index(struct pnfs_layout_segment *lseg, u32 j);
struct nfs4_pnfs_ds *nfs4_fl_prepare_ds(struct pnfs_layout_segment *lseg,
u32 ds_idx);

extern struct nfs4_file_layout_dsaddr *
nfs4_fl_alloc_deviceid_node(struct nfs_server *server,
struct pnfs_device *pdev, gfp_t gfp_flags);
extern void nfs4_fl_put_deviceid(struct nfs4_file_layout_dsaddr *dsaddr);
extern void nfs4_fl_free_deviceid(struct nfs4_file_layout_dsaddr *dsaddr);
struct nfs4_file_layout_dsaddr *
filelayout_get_device_info(struct inode *inode, struct nfs4_deviceid *dev_id,
struct rpc_cred *cred, gfp_t gfp_flags);

#endif /* FS_NFS_NFS4FILELAYOUT_H */
108 changes: 5 additions & 103 deletions fs/nfs/filelayout/filelayoutdev.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -484,8 +484,9 @@ decode_ds_addr(struct net *net, struct xdr_stream *streamp, gfp_t gfp_flags)
}

/* Decode opaque device data and return the result */
static struct nfs4_file_layout_dsaddr*
decode_device(struct inode *ino, struct pnfs_device *pdev, gfp_t gfp_flags)
struct nfs4_file_layout_dsaddr *
nfs4_fl_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev,
gfp_t gfp_flags)
{
int i;
u32 cnt, num;
Expand Down Expand Up @@ -570,10 +571,7 @@ decode_device(struct inode *ino, struct pnfs_device *pdev, gfp_t gfp_flags)
dsaddr->stripe_indices = stripe_indices;
stripe_indices = NULL;
dsaddr->ds_num = num;
nfs4_init_deviceid_node(&dsaddr->id_node,
NFS_SERVER(ino)->pnfs_curr_ld,
NFS_SERVER(ino)->nfs_client,
&pdev->dev_id);
nfs4_init_deviceid_node(&dsaddr->id_node, server, &pdev->dev_id);

INIT_LIST_HEAD(&dsaddrs);

Expand All @@ -587,7 +585,7 @@ decode_device(struct inode *ino, struct pnfs_device *pdev, gfp_t gfp_flags)

mp_count = be32_to_cpup(p); /* multipath count */
for (j = 0; j < mp_count; j++) {
da = decode_ds_addr(NFS_SERVER(ino)->nfs_client->cl_net,
da = decode_ds_addr(server->nfs_client->cl_net,
&stream, gfp_flags);
if (da)
list_add_tail(&da->da_node, &dsaddrs);
Expand Down Expand Up @@ -637,102 +635,6 @@ decode_device(struct inode *ino, struct pnfs_device *pdev, gfp_t gfp_flags)
return NULL;
}

/*
* Decode the opaque device specified in 'dev' and add it to the cache of
* available devices.
*/
static struct nfs4_file_layout_dsaddr *
decode_and_add_device(struct inode *inode, struct pnfs_device *dev, gfp_t gfp_flags)
{
struct nfs4_deviceid_node *d;
struct nfs4_file_layout_dsaddr *n, *new;

new = decode_device(inode, dev, gfp_flags);
if (!new) {
printk(KERN_WARNING "NFS: %s: Could not decode or add device\n",
__func__);
return NULL;
}

d = nfs4_insert_deviceid_node(&new->id_node);
n = container_of(d, struct nfs4_file_layout_dsaddr, id_node);
if (n != new) {
nfs4_fl_free_deviceid(new);
return n;
}

return new;
}

/*
* Retrieve the information for dev_id, add it to the list
* of available devices, and return it.
*/
struct nfs4_file_layout_dsaddr *
filelayout_get_device_info(struct inode *inode,
struct nfs4_deviceid *dev_id,
struct rpc_cred *cred,
gfp_t gfp_flags)
{
struct pnfs_device *pdev = NULL;
u32 max_resp_sz;
int max_pages;
struct page **pages = NULL;
struct nfs4_file_layout_dsaddr *dsaddr = NULL;
int rc, i;
struct nfs_server *server = NFS_SERVER(inode);

/*
* Use the session max response size as the basis for setting
* GETDEVICEINFO's maxcount
*/
max_resp_sz = server->nfs_client->cl_session->fc_attrs.max_resp_sz;
max_pages = nfs_page_array_len(0, max_resp_sz);
dprintk("%s inode %p max_resp_sz %u max_pages %d\n",
__func__, inode, max_resp_sz, max_pages);

pdev = kzalloc(sizeof(struct pnfs_device), gfp_flags);
if (pdev == NULL)
return NULL;

pages = kcalloc(max_pages, sizeof(struct page *), gfp_flags);
if (pages == NULL) {
kfree(pdev);
return NULL;
}
for (i = 0; i < max_pages; i++) {
pages[i] = alloc_page(gfp_flags);
if (!pages[i])
goto out_free;
}

memcpy(&pdev->dev_id, dev_id, sizeof(*dev_id));
pdev->layout_type = LAYOUT_NFSV4_1_FILES;
pdev->pages = pages;
pdev->pgbase = 0;
pdev->pglen = max_resp_sz;
pdev->mincount = 0;
pdev->maxcount = max_resp_sz - nfs41_maxgetdevinfo_overhead;

rc = nfs4_proc_getdeviceinfo(server, pdev, cred);
dprintk("%s getdevice info returns %d\n", __func__, rc);
if (rc)
goto out_free;

/*
* Found new device, need to decode it and then add it to the
* list of known devices for this mountpoint.
*/
dsaddr = decode_and_add_device(inode, pdev, gfp_flags);
out_free:
for (i = 0; i < max_pages; i++)
__free_page(pages[i]);
kfree(pages);
kfree(pdev);
dprintk("<-- %s dsaddr %p\n", __func__, dsaddr);
return dsaddr;
}

void
nfs4_fl_put_deviceid(struct nfs4_file_layout_dsaddr *dsaddr)
{
Expand Down
113 changes: 41 additions & 72 deletions fs/nfs/objlayout/objio_osd.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,52 +60,6 @@ objio_free_deviceid_node(struct nfs4_deviceid_node *d)
kfree(de);
}

static struct objio_dev_ent *_dev_list_find(const struct nfs_server *nfss,
const struct nfs4_deviceid *d_id)
{
struct nfs4_deviceid_node *d;
struct objio_dev_ent *de;

d = nfs4_find_get_deviceid(nfss->pnfs_curr_ld, nfss->nfs_client, d_id);
if (!d)
return NULL;

de = container_of(d, struct objio_dev_ent, id_node);
return de;
}

static struct objio_dev_ent *
_dev_list_add(const struct nfs_server *nfss,
const struct nfs4_deviceid *d_id, struct osd_dev *od,
gfp_t gfp_flags)
{
struct nfs4_deviceid_node *d;
struct objio_dev_ent *de = kzalloc(sizeof(*de), gfp_flags);
struct objio_dev_ent *n;

if (!de) {
dprintk("%s: -ENOMEM od=%p\n", __func__, od);
return NULL;
}

dprintk("%s: Adding od=%p\n", __func__, od);
nfs4_init_deviceid_node(&de->id_node,
nfss->pnfs_curr_ld,
nfss->nfs_client,
d_id);
de->od.od = od;

d = nfs4_insert_deviceid_node(&de->id_node);
n = container_of(d, struct objio_dev_ent, id_node);
if (n != de) {
dprintk("%s: Race with other n->od=%p\n", __func__, n->od.od);
objio_free_deviceid_node(&de->id_node);
de = n;
}

return de;
}

struct objio_segment {
struct pnfs_layout_segment lseg;

Expand All @@ -130,29 +84,24 @@ struct objio_state {

/* Send and wait for a get_device_info of devices in the layout,
then look them up with the osd_initiator library */
static int objio_devices_lookup(struct pnfs_layout_hdr *pnfslay,
struct objio_segment *objio_seg, unsigned c, struct nfs4_deviceid *d_id,
gfp_t gfp_flags)
struct nfs4_deviceid_node *
objio_alloc_deviceid_node(struct nfs_server *server, struct pnfs_device *pdev,
gfp_t gfp_flags)
{
struct pnfs_osd_deviceaddr *deviceaddr;
struct objio_dev_ent *ode;
struct objio_dev_ent *ode = NULL;
struct osd_dev *od;
struct osd_dev_info odi;
bool retry_flag = true;
u32 *p;
int err;

ode = _dev_list_find(NFS_SERVER(pnfslay->plh_inode), d_id);
if (ode) {
objio_seg->oc.ods[c] = &ode->od; /* must use container_of */
return 0;
}
deviceaddr = kzalloc(sizeof(*deviceaddr), gfp_flags);
if (!deviceaddr)
return NULL;

err = objlayout_get_deviceinfo(pnfslay, d_id, &deviceaddr, gfp_flags);
if (unlikely(err)) {
dprintk("%s: objlayout_get_deviceinfo dev(%llx:%llx) =>%d\n",
__func__, _DEVID_LO(d_id), _DEVID_HI(d_id), err);
return err;
}
p = page_address(pdev->pages[0]);
pnfs_osd_xdr_decode_deviceaddr(deviceaddr, p);

odi.systemid_len = deviceaddr->oda_systemid.len;
if (odi.systemid_len > sizeof(odi.systemid)) {
Expand Down Expand Up @@ -188,14 +137,24 @@ static int objio_devices_lookup(struct pnfs_layout_hdr *pnfslay,
goto out;
}

ode = _dev_list_add(NFS_SERVER(pnfslay->plh_inode), d_id, od,
gfp_flags);
objio_seg->oc.ods[c] = &ode->od; /* must use container_of */
dprintk("Adding new dev_id(%llx:%llx)\n",
_DEVID_LO(d_id), _DEVID_HI(d_id));
_DEVID_LO(&pdev->dev_id), _DEVID_HI(&pdev->dev_id));

ode = kzalloc(sizeof(*ode), gfp_flags);
if (!ode) {
dprintk("%s: -ENOMEM od=%p\n", __func__, od);
goto out;
}

nfs4_init_deviceid_node(&ode->id_node, server, &pdev->dev_id);
kfree(deviceaddr);

ode->od.od = od;
return &ode->id_node;

out:
objlayout_put_deviceinfo(deviceaddr);
return err;
kfree(deviceaddr);
return NULL;
}

static void copy_single_comp(struct ore_components *oc, unsigned c,
Expand Down Expand Up @@ -254,6 +213,7 @@ int objio_alloc_lseg(struct pnfs_layout_segment **outp,
struct xdr_stream *xdr,
gfp_t gfp_flags)
{
struct nfs_server *server = NFS_SERVER(pnfslay->plh_inode);
struct objio_segment *objio_seg;
struct pnfs_osd_xdr_decode_layout_iter iter;
struct pnfs_osd_layout layout;
Expand Down Expand Up @@ -283,13 +243,21 @@ int objio_alloc_lseg(struct pnfs_layout_segment **outp,
objio_seg->oc.first_dev = layout.olo_comps_index;
cur_comp = 0;
while (pnfs_osd_xdr_decode_layout_comp(&src_comp, &iter, xdr, &err)) {
struct nfs4_deviceid_node *d;
struct objio_dev_ent *ode;

copy_single_comp(&objio_seg->oc, cur_comp, &src_comp);
err = objio_devices_lookup(pnfslay, objio_seg, cur_comp,
&src_comp.oc_object_id.oid_device_id,
gfp_flags);
if (err)

d = nfs4_find_get_deviceid(server,
&src_comp.oc_object_id.oid_device_id,
pnfslay->plh_lc_cred, gfp_flags);
if (!d) {
err = -ENXIO;
goto err;
++cur_comp;
}

ode = container_of(d, struct objio_dev_ent, id_node);
objio_seg->oc.ods[cur_comp++] = &ode->od;
}
/* pnfs_osd_xdr_decode_layout_comp returns false on error */
if (unlikely(err))
Expand Down Expand Up @@ -653,6 +621,7 @@ static struct pnfs_layoutdriver_type objlayout_type = {
.flags = PNFS_LAYOUTRET_ON_SETATTR |
PNFS_LAYOUTRET_ON_ERROR,

.max_deviceinfo_size = PAGE_SIZE,
.owner = THIS_MODULE,
.alloc_layout_hdr = objlayout_alloc_layout_hdr,
.free_layout_hdr = objlayout_free_layout_hdr,
Expand Down
Loading

0 comments on commit 661373b

Please sign in to comment.