Skip to content

Commit

Permalink
md/raid10: convert resync_lock to use seqlock
Browse files Browse the repository at this point in the history
Currently, wait_barrier() will hold 'resync_lock' to read 'conf->barrier',
and io can't be dispatched until 'barrier' is dropped.

Since holding the 'barrier' is not common, convert 'resync_lock' to use
seqlock so that holding lock can be avoided in fast path.

Signed-off-by: Yu Kuai <yukuai3@huawei.com>
Reviewed-and-Tested-by: Logan Gunthorpe <logang@deltatee.com>
Signed-off-by: Song Liu <song@kernel.org>
  • Loading branch information
Yu Kuai authored and Song Liu committed Sep 22, 2022
1 parent 4f35028 commit b9b083f
Show file tree
Hide file tree
Showing 2 changed files with 59 additions and 30 deletions.
87 changes: 58 additions & 29 deletions drivers/md/raid10.c
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,21 @@ static void end_reshape(struct r10conf *conf);

#include "raid1-10.c"

#define NULL_CMD
#define cmd_before(conf, cmd) \
do { \
write_sequnlock_irq(&(conf)->resync_lock); \
cmd; \
} while (0)
#define cmd_after(conf) write_seqlock_irq(&(conf)->resync_lock)

#define wait_event_barrier_cmd(conf, cond, cmd) \
wait_event_cmd((conf)->wait_barrier, cond, cmd_before(conf, cmd), \
cmd_after(conf))

#define wait_event_barrier(conf, cond) \
wait_event_barrier_cmd(conf, cond, NULL_CMD)

/*
* for resync bio, r10bio pointer can be retrieved from the per-bio
* 'struct resync_pages'.
Expand Down Expand Up @@ -936,30 +951,29 @@ static void flush_pending_writes(struct r10conf *conf)

static void raise_barrier(struct r10conf *conf, int force)
{
spin_lock_irq(&conf->resync_lock);
write_seqlock_irq(&conf->resync_lock);
BUG_ON(force && !conf->barrier);

/* Wait until no block IO is waiting (unless 'force') */
wait_event_lock_irq(conf->wait_barrier, force || !conf->nr_waiting,
conf->resync_lock);
wait_event_barrier(conf, force || !conf->nr_waiting);

/* block any new IO from starting */
conf->barrier++;
WRITE_ONCE(conf->barrier, conf->barrier + 1);

/* Now wait for all pending IO to complete */
wait_event_lock_irq(conf->wait_barrier,
!atomic_read(&conf->nr_pending) && conf->barrier < RESYNC_DEPTH,
conf->resync_lock);
wait_event_barrier(conf, !atomic_read(&conf->nr_pending) &&
conf->barrier < RESYNC_DEPTH);

spin_unlock_irq(&conf->resync_lock);
write_sequnlock_irq(&conf->resync_lock);
}

static void lower_barrier(struct r10conf *conf)
{
unsigned long flags;
spin_lock_irqsave(&conf->resync_lock, flags);
conf->barrier--;
spin_unlock_irqrestore(&conf->resync_lock, flags);

write_seqlock_irqsave(&conf->resync_lock, flags);
WRITE_ONCE(conf->barrier, conf->barrier - 1);
write_sequnlock_irqrestore(&conf->resync_lock, flags);
wake_up(&conf->wait_barrier);
}

Expand Down Expand Up @@ -990,21 +1004,39 @@ static bool stop_waiting_barrier(struct r10conf *conf)
return false;
}

static bool wait_barrier_nolock(struct r10conf *conf)
{
unsigned int seq = read_seqbegin(&conf->resync_lock);

if (READ_ONCE(conf->barrier))
return false;

atomic_inc(&conf->nr_pending);
if (!read_seqretry(&conf->resync_lock, seq))
return true;

if (atomic_dec_and_test(&conf->nr_pending))
wake_up_barrier(conf);

return false;
}

static bool wait_barrier(struct r10conf *conf, bool nowait)
{
bool ret = true;

spin_lock_irq(&conf->resync_lock);
if (wait_barrier_nolock(conf))
return true;

write_seqlock_irq(&conf->resync_lock);
if (conf->barrier) {
/* Return false when nowait flag is set */
if (nowait) {
ret = false;
} else {
conf->nr_waiting++;
raid10_log(conf->mddev, "wait barrier");
wait_event_lock_irq(conf->wait_barrier,
stop_waiting_barrier(conf),
conf->resync_lock);
wait_event_barrier(conf, stop_waiting_barrier(conf));
conf->nr_waiting--;
}
if (!conf->nr_waiting)
Expand All @@ -1013,7 +1045,7 @@ static bool wait_barrier(struct r10conf *conf, bool nowait)
/* Only increment nr_pending when we wait */
if (ret)
atomic_inc(&conf->nr_pending);
spin_unlock_irq(&conf->resync_lock);
write_sequnlock_irq(&conf->resync_lock);
return ret;
}

Expand All @@ -1038,27 +1070,24 @@ static void freeze_array(struct r10conf *conf, int extra)
* must match the number of pending IOs (nr_pending) before
* we continue.
*/
spin_lock_irq(&conf->resync_lock);
write_seqlock_irq(&conf->resync_lock);
conf->array_freeze_pending++;
conf->barrier++;
WRITE_ONCE(conf->barrier, conf->barrier + 1);
conf->nr_waiting++;
wait_event_lock_irq_cmd(conf->wait_barrier,
atomic_read(&conf->nr_pending) == conf->nr_queued+extra,
conf->resync_lock,
flush_pending_writes(conf));

wait_event_barrier_cmd(conf, atomic_read(&conf->nr_pending) ==
conf->nr_queued + extra, flush_pending_writes(conf));
conf->array_freeze_pending--;
spin_unlock_irq(&conf->resync_lock);
write_sequnlock_irq(&conf->resync_lock);
}

static void unfreeze_array(struct r10conf *conf)
{
/* reverse the effect of the freeze */
spin_lock_irq(&conf->resync_lock);
conf->barrier--;
write_seqlock_irq(&conf->resync_lock);
WRITE_ONCE(conf->barrier, conf->barrier - 1);
conf->nr_waiting--;
wake_up(&conf->wait_barrier);
spin_unlock_irq(&conf->resync_lock);
write_sequnlock_irq(&conf->resync_lock);
}

static sector_t choose_data_offset(struct r10bio *r10_bio,
Expand Down Expand Up @@ -4045,7 +4074,7 @@ static struct r10conf *setup_conf(struct mddev *mddev)
INIT_LIST_HEAD(&conf->retry_list);
INIT_LIST_HEAD(&conf->bio_end_io_list);

spin_lock_init(&conf->resync_lock);
seqlock_init(&conf->resync_lock);
init_waitqueue_head(&conf->wait_barrier);
atomic_set(&conf->nr_pending, 0);

Expand Down Expand Up @@ -4364,7 +4393,7 @@ static void *raid10_takeover_raid0(struct mddev *mddev, sector_t size, int devs)
rdev->new_raid_disk = rdev->raid_disk * 2;
rdev->sectors = size;
}
conf->barrier = 1;
WRITE_ONCE(conf->barrier, 1);
}

return conf;
Expand Down
2 changes: 1 addition & 1 deletion drivers/md/raid10.h
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ struct r10conf {
/* queue pending writes and submit them on unplug */
struct bio_list pending_bio_list;

spinlock_t resync_lock;
seqlock_t resync_lock;
atomic_t nr_pending;
int nr_waiting;
int nr_queued;
Expand Down

0 comments on commit b9b083f

Please sign in to comment.