Skip to content

Commit

Permalink
nvmet: use IOCB_NOWAIT for file-ns buffered I/O
Browse files Browse the repository at this point in the history 
This patch optimizes read command behavior when file-ns configured
with buffered I/O. Instead of offloading the buffered I/O read operations
to the worker threads, we first issue the read operation with IOCB_NOWAIT
and try and access the data from the cache. Here we only offload the
request to the worker thread and complete the request in the worker
thread context when IOCB_NOWAIT request fails.

Signed-off-by: Chaitanya Kulkarni <chaitanya.kulkarni@wdc.com>
Signed-off-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
  • Loading branch information
Chaitanya Kulkarni authored and Jens Axboe committed Dec 8, 2018
1 parent c09305a commit 50a909d
Showing 1 changed file with 84 additions and 46 deletions.
130 changes: 84 additions & 46 deletions drivers/nvme/target/io-cmd-file.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,17 +83,16 @@ static void nvmet_file_init_bvec(struct bio_vec *bv, struct sg_page_iter *iter)
}

static ssize_t nvmet_file_submit_bvec(struct nvmet_req *req, loff_t pos,
unsigned long nr_segs, size_t count)
unsigned long nr_segs, size_t count, int ki_flags)
{
struct kiocb *iocb = &req->f.iocb;
ssize_t (*call_iter)(struct kiocb *iocb, struct iov_iter *iter);
struct iov_iter iter;
int ki_flags = 0, rw;
ssize_t ret;
int rw;

if (req->cmd->rw.opcode == nvme_cmd_write) {
if (req->cmd->rw.control & cpu_to_le16(NVME_RW_FUA))
ki_flags = IOCB_DSYNC;
ki_flags |= IOCB_DSYNC;
call_iter = req->ns->file->f_op->write_iter;
rw = WRITE;
} else {
Expand All @@ -107,12 +106,7 @@ static ssize_t nvmet_file_submit_bvec(struct nvmet_req *req, loff_t pos,
iocb->ki_filp = req->ns->file;
iocb->ki_flags = ki_flags | iocb_flags(req->ns->file);

ret = call_iter(iocb, &iter);

if (ret != -EIOCBQUEUED && iocb->ki_complete)
iocb->ki_complete(iocb, ret, 0);

return ret;
return call_iter(iocb, &iter);
}

static void nvmet_file_io_done(struct kiocb *iocb, long ret, long ret2)
Expand All @@ -130,7 +124,7 @@ static void nvmet_file_io_done(struct kiocb *iocb, long ret, long ret2)
NVME_SC_INTERNAL | NVME_SC_DNR : 0);
}

static void nvmet_file_execute_rw(struct nvmet_req *req)
static bool nvmet_file_execute_io(struct nvmet_req *req, int ki_flags)
{
ssize_t nr_bvec = DIV_ROUND_UP(req->data_len, PAGE_SIZE);
struct sg_page_iter sg_pg_iter;
Expand All @@ -140,30 +134,14 @@ static void nvmet_file_execute_rw(struct nvmet_req *req)
ssize_t ret = 0;
loff_t pos;

if (!req->sg_cnt || !nr_bvec) {
nvmet_req_complete(req, 0);
return;
}

if (req->f.mpool_alloc && nr_bvec > NVMET_MAX_MPOOL_BVEC)
is_sync = true;

pos = le64_to_cpu(req->cmd->rw.slba) << req->ns->blksize_shift;
if (unlikely(pos + req->data_len > req->ns->size)) {
nvmet_req_complete(req, NVME_SC_LBA_RANGE | NVME_SC_DNR);
return;
}

if (nr_bvec > NVMET_MAX_INLINE_BIOVEC)
req->f.bvec = kmalloc_array(nr_bvec, sizeof(struct bio_vec),
GFP_KERNEL);
else
req->f.bvec = req->inline_bvec;

req->f.mpool_alloc = false;
if (unlikely(!req->f.bvec)) {
/* fallback under memory pressure */
req->f.bvec = mempool_alloc(req->ns->bvec_pool, GFP_KERNEL);
req->f.mpool_alloc = true;
if (nr_bvec > NVMET_MAX_MPOOL_BVEC)
is_sync = true;
return true;
}

memset(&req->f.iocb, 0, sizeof(struct kiocb));
Expand All @@ -177,40 +155,103 @@ static void nvmet_file_execute_rw(struct nvmet_req *req)

if (unlikely(is_sync) &&
(nr_bvec - 1 == 0 || bv_cnt == NVMET_MAX_MPOOL_BVEC)) {
ret = nvmet_file_submit_bvec(req, pos, bv_cnt, len);
ret = nvmet_file_submit_bvec(req, pos, bv_cnt, len, 0);
if (ret < 0)
goto out;
goto complete;

pos += len;
bv_cnt = 0;
len = 0;
}
nr_bvec--;
}

if (WARN_ON_ONCE(total_len != req->data_len))
if (WARN_ON_ONCE(total_len != req->data_len)) {
ret = -EIO;
out:
if (unlikely(is_sync || ret)) {
nvmet_file_io_done(&req->f.iocb, ret < 0 ? ret : total_len, 0);
return;
goto complete;
}

if (unlikely(is_sync)) {
ret = total_len;
goto complete;
}
req->f.iocb.ki_complete = nvmet_file_io_done;
nvmet_file_submit_bvec(req, pos, bv_cnt, total_len);

/*
* A NULL ki_complete ask for synchronous execution, which we want
* for the IOCB_NOWAIT case.
*/
if (!(ki_flags & IOCB_NOWAIT))
req->f.iocb.ki_complete = nvmet_file_io_done;

ret = nvmet_file_submit_bvec(req, pos, bv_cnt, total_len, ki_flags);

switch (ret) {
case -EIOCBQUEUED:
return true;
case -EAGAIN:
if (WARN_ON_ONCE(!(ki_flags & IOCB_NOWAIT)))
goto complete;
return false;
case -EOPNOTSUPP:
/*
* For file systems returning error -EOPNOTSUPP, handle
* IOCB_NOWAIT error case separately and retry without
* IOCB_NOWAIT.
*/
if ((ki_flags & IOCB_NOWAIT))
return false;
break;
}

complete:
nvmet_file_io_done(&req->f.iocb, ret, 0);
return true;
}

static void nvmet_file_buffered_io_work(struct work_struct *w)
{
struct nvmet_req *req = container_of(w, struct nvmet_req, f.work);

nvmet_file_execute_rw(req);
nvmet_file_execute_io(req, 0);
}

static void nvmet_file_execute_rw_buffered_io(struct nvmet_req *req)
static void nvmet_file_submit_buffered_io(struct nvmet_req *req)
{
INIT_WORK(&req->f.work, nvmet_file_buffered_io_work);
queue_work(buffered_io_wq, &req->f.work);
}

static void nvmet_file_execute_rw(struct nvmet_req *req)
{
ssize_t nr_bvec = DIV_ROUND_UP(req->data_len, PAGE_SIZE);

if (!req->sg_cnt || !nr_bvec) {
nvmet_req_complete(req, 0);
return;
}

if (nr_bvec > NVMET_MAX_INLINE_BIOVEC)
req->f.bvec = kmalloc_array(nr_bvec, sizeof(struct bio_vec),
GFP_KERNEL);
else
req->f.bvec = req->inline_bvec;

if (unlikely(!req->f.bvec)) {
/* fallback under memory pressure */
req->f.bvec = mempool_alloc(req->ns->bvec_pool, GFP_KERNEL);
req->f.mpool_alloc = true;
} else
req->f.mpool_alloc = false;

if (req->ns->buffered_io) {
if (likely(!req->f.mpool_alloc) &&
nvmet_file_execute_io(req, IOCB_NOWAIT))
return;
nvmet_file_submit_buffered_io(req);
} else
nvmet_file_execute_io(req, 0);
}

u16 nvmet_file_flush(struct nvmet_req *req)
{
if (vfs_fsync(req->ns->file, 1) < 0)
Expand Down Expand Up @@ -320,10 +361,7 @@ u16 nvmet_file_parse_io_cmd(struct nvmet_req *req)
switch (cmd->common.opcode) {
case nvme_cmd_read:
case nvme_cmd_write:
if (req->ns->buffered_io)
req->execute = nvmet_file_execute_rw_buffered_io;
else
req->execute = nvmet_file_execute_rw;
req->execute = nvmet_file_execute_rw;
req->data_len = nvmet_rw_len(req);
return 0;
case nvme_cmd_flush:
Expand Down

0 comments on commit 50a909d

Please sign in to comment.