Skip to content

Commit

Permalink
lightnvm: pblk: control I/O flow also on tear down
Browse files Browse the repository at this point in the history 
When removing a pblk instance, control the write I/O flow to the
controller as we do in the fast path.

Signed-off-by: Javier González <javier@cnexlabs.com>
Signed-off-by: Matias Bjørling <matias@cnexlabs.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
  • Loading branch information
Javier González authored and Jens Axboe committed Jul 7, 2017
1 parent 0e2ff11 commit 3eaa11e
Show file tree
Hide file tree
Showing 4 changed files with 78 additions and 35 deletions.
61 changes: 48 additions & 13 deletions drivers/lightnvm/pblk-core.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1670,13 +1670,10 @@ void pblk_line_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv,
queue_work(wq, &line_ws->ws);
}

void pblk_down_rq(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas,
unsigned long *lun_bitmap)
static void __pblk_down_page(struct pblk *pblk, struct ppa_addr *ppa_list,
int nr_ppas, int pos)
{
struct nvm_tgt_dev *dev = pblk->dev;
struct nvm_geo *geo = &dev->geo;
struct pblk_lun *rlun;
int pos = pblk_ppa_to_pos(geo, ppa_list[0]);
struct pblk_lun *rlun = &pblk->luns[pos];
int ret;

/*
Expand All @@ -1690,14 +1687,8 @@ void pblk_down_rq(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas,
WARN_ON(ppa_list[0].g.lun != ppa_list[i].g.lun ||
ppa_list[0].g.ch != ppa_list[i].g.ch);
#endif
/* If the LUN has been locked for this same request, do no attempt to
* lock it again
*/
if (test_and_set_bit(pos, lun_bitmap))
return;

rlun = &pblk->luns[pos];
ret = down_timeout(&rlun->wr_sem, msecs_to_jiffies(5000));
ret = down_timeout(&rlun->wr_sem, msecs_to_jiffies(30000));
if (ret) {
switch (ret) {
case -ETIME:
Expand All @@ -1710,6 +1701,50 @@ void pblk_down_rq(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas,
}
}

void pblk_down_page(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas)
{
struct nvm_tgt_dev *dev = pblk->dev;
struct nvm_geo *geo = &dev->geo;
int pos = pblk_ppa_to_pos(geo, ppa_list[0]);

__pblk_down_page(pblk, ppa_list, nr_ppas, pos);
}

void pblk_down_rq(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas,
unsigned long *lun_bitmap)
{
struct nvm_tgt_dev *dev = pblk->dev;
struct nvm_geo *geo = &dev->geo;
int pos = pblk_ppa_to_pos(geo, ppa_list[0]);

/* If the LUN has been locked for this same request, do no attempt to
* lock it again
*/
if (test_and_set_bit(pos, lun_bitmap))
return;

__pblk_down_page(pblk, ppa_list, nr_ppas, pos);
}

void pblk_up_page(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas)
{
struct nvm_tgt_dev *dev = pblk->dev;
struct nvm_geo *geo = &dev->geo;
struct pblk_lun *rlun;
int pos = pblk_ppa_to_pos(geo, ppa_list[0]);

#ifdef CONFIG_NVM_DEBUG
int i;

for (i = 1; i < nr_ppas; i++)
WARN_ON(ppa_list[0].g.lun != ppa_list[i].g.lun ||
ppa_list[0].g.ch != ppa_list[i].g.ch);
#endif

rlun = &pblk->luns[pos];
up(&rlun->wr_sem);
}

void pblk_up_rq(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas,
unsigned long *lun_bitmap)
{
Expand Down
31 changes: 21 additions & 10 deletions drivers/lightnvm/pblk-recovery.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -340,9 +340,14 @@ static void pblk_end_io_recov(struct nvm_rq *rqd)
struct pblk *pblk = pad_rq->pblk;
struct nvm_tgt_dev *dev = pblk->dev;

kref_put(&pad_rq->ref, pblk_recov_complete);
pblk_up_page(pblk, rqd->ppa_list, rqd->nr_ppas);

bio_put(rqd->bio);
nvm_dev_dma_free(dev->parent, rqd->meta_list, rqd->dma_meta_list);
pblk_free_rqd(pblk, rqd, WRITE);

atomic_dec(&pblk->inflight_io);
kref_put(&pad_rq->ref, pblk_recov_complete);
}

static int pblk_recov_pad_oob(struct pblk *pblk, struct pblk_line *line,
Expand Down Expand Up @@ -385,15 +390,15 @@ static int pblk_recov_pad_oob(struct pblk *pblk, struct pblk_line *line,
rq_ppas = pblk_calc_secs(pblk, left_ppas, 0);
if (rq_ppas < pblk->min_write_pgs) {
pr_err("pblk: corrupted pad line %d\n", line->id);
goto free_rq;
goto fail_free_pad;
}

rq_len = rq_ppas * geo->sec_size;

meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL, &dma_meta_list);
if (!meta_list) {
ret = -ENOMEM;
goto free_data;
goto fail_free_pad;
}

ppa_list = (void *)(meta_list) + pblk_dma_meta_size;
Expand All @@ -404,9 +409,9 @@ static int pblk_recov_pad_oob(struct pblk *pblk, struct pblk_line *line,
ret = PTR_ERR(rqd);
goto fail_free_meta;
}
memset(rqd, 0, pblk_w_rq_size);

bio = bio_map_kern(dev->q, data, rq_len, GFP_KERNEL);
bio = pblk_bio_map_addr(pblk, data, rq_ppas, rq_len,
PBLK_VMALLOC_META, GFP_KERNEL);
if (IS_ERR(bio)) {
ret = PTR_ERR(bio);
goto fail_free_rqd;
Expand Down Expand Up @@ -453,15 +458,15 @@ static int pblk_recov_pad_oob(struct pblk *pblk, struct pblk_line *line,
}

kref_get(&pad_rq->ref);
pblk_down_page(pblk, rqd->ppa_list, rqd->nr_ppas);

ret = pblk_submit_io(pblk, rqd);
if (ret) {
pr_err("pblk: I/O submission failed: %d\n", ret);
goto free_data;
pblk_up_page(pblk, rqd->ppa_list, rqd->nr_ppas);
goto fail_free_bio;
}

atomic_dec(&pblk->inflight_io);

left_line_ppas -= rq_ppas;
left_ppas -= rq_ppas;
if (left_ppas && left_line_ppas)
Expand All @@ -475,17 +480,23 @@ static int pblk_recov_pad_oob(struct pblk *pblk, struct pblk_line *line,
ret = -ETIME;
}

if (!pblk_line_is_full(line))
pr_err("pblk: corrupted padded line: %d\n", line->id);

vfree(data);
free_rq:
kfree(pad_rq);
free_data:
vfree(data);
return ret;

fail_free_bio:
bio_put(bio);
fail_free_rqd:
pblk_free_rqd(pblk, rqd, WRITE);
fail_free_meta:
nvm_dev_dma_free(dev->parent, meta_list, dma_meta_list);
fail_free_pad:
kfree(pad_rq);
vfree(data);
return ret;
}

Expand Down
19 changes: 7 additions & 12 deletions drivers/lightnvm/pblk-write.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,15 +178,12 @@ static void pblk_end_io_write_meta(struct nvm_rq *rqd)
{
struct pblk *pblk = rqd->private;
struct nvm_tgt_dev *dev = pblk->dev;
struct nvm_geo *geo = &dev->geo;
struct pblk_g_ctx *m_ctx = nvm_rq_to_pdu(rqd);
struct pblk_line *line = m_ctx->private;
struct pblk_emeta *emeta = line->emeta;
int pos = pblk_ppa_to_pos(geo, rqd->ppa_list[0]);
struct pblk_lun *rlun = &pblk->luns[pos];
int sync;

up(&rlun->wr_sem);
pblk_up_page(pblk, rqd->ppa_list, rqd->nr_ppas);

if (rqd->error) {
pblk_log_write_err(pblk, rqd);
Expand All @@ -203,6 +200,7 @@ static void pblk_end_io_write_meta(struct nvm_rq *rqd)
pblk->close_wq);

bio_put(rqd->bio);
nvm_dev_dma_free(dev->parent, rqd->meta_list, rqd->dma_meta_list);
pblk_free_rqd(pblk, rqd, READ);

atomic_dec(&pblk->inflight_io);
Expand Down Expand Up @@ -367,7 +365,6 @@ int pblk_submit_meta_io(struct pblk *pblk, struct pblk_line *meta_line)
struct pblk_line_meta *lm = &pblk->lm;
struct pblk_emeta *emeta = meta_line->emeta;
struct pblk_g_ctx *m_ctx;
struct pblk_lun *rlun;
struct bio *bio;
struct nvm_rq *rqd;
void *data;
Expand Down Expand Up @@ -411,13 +408,6 @@ int pblk_submit_meta_io(struct pblk *pblk, struct pblk_line *meta_line)
rqd->ppa_list[i] = addr_to_gen_ppa(pblk, paddr, id);
}

rlun = &pblk->luns[pblk_ppa_to_pos(geo, rqd->ppa_list[0])];
ret = down_timeout(&rlun->wr_sem, msecs_to_jiffies(5000));
if (ret) {
pr_err("pblk: lun semaphore timed out (%d)\n", ret);
goto fail_free_bio;
}

emeta->mem += rq_len;
if (emeta->mem >= lm->emeta_len[0]) {
spin_lock(&l_mg->close_lock);
Expand All @@ -427,6 +417,8 @@ int pblk_submit_meta_io(struct pblk *pblk, struct pblk_line *meta_line)
spin_unlock(&l_mg->close_lock);
}

pblk_down_page(pblk, rqd->ppa_list, rqd->nr_ppas);

ret = pblk_submit_io(pblk, rqd);
if (ret) {
pr_err("pblk: emeta I/O submission failed: %d\n", ret);
Expand All @@ -436,10 +428,13 @@ int pblk_submit_meta_io(struct pblk *pblk, struct pblk_line *meta_line)
return NVM_IO_OK;

fail_rollback:
pblk_up_page(pblk, rqd->ppa_list, rqd->nr_ppas);
spin_lock(&l_mg->close_lock);
pblk_dealloc_page(pblk, meta_line, rq_ppas);
list_add(&meta_line->list, &meta_line->list);
spin_unlock(&l_mg->close_lock);

nvm_dev_dma_free(dev->parent, rqd->meta_list, rqd->dma_meta_list);
fail_free_bio:
if (likely(l_mg->emeta_alloc_type == PBLK_VMALLOC_META))
bio_put(bio);
Expand Down
2 changes: 2 additions & 0 deletions drivers/lightnvm/pblk.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -739,8 +739,10 @@ u64 pblk_alloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs);
u64 __pblk_alloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs);
int pblk_calc_secs(struct pblk *pblk, unsigned long secs_avail,
unsigned long secs_to_flush);
void pblk_up_page(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas);
void pblk_down_rq(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas,
unsigned long *lun_bitmap);
void pblk_down_page(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas);
void pblk_up_rq(struct pblk *pblk, struct ppa_addr *ppa_list, int nr_ppas,
unsigned long *lun_bitmap);
void pblk_end_bio_sync(struct bio *bio);
Expand Down

0 comments on commit 3eaa11e

Please sign in to comment.