Skip to content

Commit

Permalink
dm: optimize use SRCU and RCU
Browse files Browse the repository at this point in the history 
This patch removes "io_lock" and "map_lock" in struct mapped_device and
"holders" in struct dm_table and replaces these mechanisms with
sleepable-rcu.

Previously, the code would call "dm_get_live_table" and "dm_table_put" to
get and release table. Now, the code is changed to call "dm_get_live_table"
and "dm_put_live_table". dm_get_live_table locks sleepable-rcu and
dm_put_live_table unlocks it.

dm_get_live_table_fast/dm_put_live_table_fast can be used instead of
dm_get_live_table/dm_put_live_table. These *_fast functions use
non-sleepable RCU, so the caller must not block between them.

If the code changes active or inactive dm table, it must call
dm_sync_table before destroying the old table.

Signed-off-by: Mikulas Patocka <mpatocka@redhat.com>
Signed-off-by: Jun'ichi Nomura <j-nomura@ce.jp.nec.com>
Signed-off-by: Alasdair G Kergon <agk@redhat.com>
  • Loading branch information
Mikulas Patocka authored and Alasdair G Kergon committed Jul 10, 2013
1 parent 2480945 commit 83d5e5b
Show file tree
Hide file tree
Showing 5 changed files with 180 additions and 147 deletions.
122 changes: 82 additions & 40 deletions drivers/md/dm-ioctl.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,14 @@ struct hash_cell {
struct dm_table *new_map;
};

/*
* A dummy definition to make RCU happy.
* struct dm_table should never be dereferenced in this file.
*/
struct dm_table {
int undefined__;
};

struct vers_iter {
size_t param_size;
struct dm_target_versions *vers, *old_vers;
Expand Down Expand Up @@ -242,9 +250,10 @@ static int dm_hash_insert(const char *name, const char *uuid, struct mapped_devi
return -EBUSY;
}

static void __hash_remove(struct hash_cell *hc)
static struct dm_table *__hash_remove(struct hash_cell *hc)
{
struct dm_table *table;
int srcu_idx;

/* remove from the dev hash */
list_del(&hc->uuid_list);
Expand All @@ -253,23 +262,26 @@ static void __hash_remove(struct hash_cell *hc)
dm_set_mdptr(hc->md, NULL);
mutex_unlock(&dm_hash_cells_mutex);

table = dm_get_live_table(hc->md);
if (table) {
table = dm_get_live_table(hc->md, &srcu_idx);
if (table)
dm_table_event(table);
dm_table_put(table);
}
dm_put_live_table(hc->md, srcu_idx);

table = NULL;
if (hc->new_map)
dm_table_destroy(hc->new_map);
table = hc->new_map;
dm_put(hc->md);
free_cell(hc);

return table;
}

static void dm_hash_remove_all(int keep_open_devices)
{
int i, dev_skipped;
struct hash_cell *hc;
struct mapped_device *md;
struct dm_table *t;

retry:
dev_skipped = 0;
Expand All @@ -287,10 +299,14 @@ static void dm_hash_remove_all(int keep_open_devices)
continue;
}

__hash_remove(hc);
t = __hash_remove(hc);

up_write(&_hash_lock);

if (t) {
dm_sync_table(md);
dm_table_destroy(t);
}
dm_put(md);
if (likely(keep_open_devices))
dm_destroy(md);
Expand Down Expand Up @@ -356,6 +372,7 @@ static struct mapped_device *dm_hash_rename(struct dm_ioctl *param,
struct dm_table *table;
struct mapped_device *md;
unsigned change_uuid = (param->flags & DM_UUID_FLAG) ? 1 : 0;
int srcu_idx;

/*
* duplicate new.
Expand Down Expand Up @@ -418,11 +435,10 @@ static struct mapped_device *dm_hash_rename(struct dm_ioctl *param,
/*
* Wake up any dm event waiters.
*/
table = dm_get_live_table(hc->md);
if (table) {
table = dm_get_live_table(hc->md, &srcu_idx);
if (table)
dm_table_event(table);
dm_table_put(table);
}
dm_put_live_table(hc->md, srcu_idx);

if (!dm_kobject_uevent(hc->md, KOBJ_CHANGE, param->event_nr))
param->flags |= DM_UEVENT_GENERATED_FLAG;
Expand Down Expand Up @@ -620,11 +636,14 @@ static int check_name(const char *name)
* _hash_lock without first calling dm_table_put, because dm_table_destroy
* waits for this dm_table_put and could be called under this lock.
*/
static struct dm_table *dm_get_inactive_table(struct mapped_device *md)
static struct dm_table *dm_get_inactive_table(struct mapped_device *md, int *srcu_idx)
{
struct hash_cell *hc;
struct dm_table *table = NULL;

/* increment rcu count, we don't care about the table pointer */
dm_get_live_table(md, srcu_idx);

down_read(&_hash_lock);
hc = dm_get_mdptr(md);
if (!hc || hc->md != md) {
Expand All @@ -633,8 +652,6 @@ static struct dm_table *dm_get_inactive_table(struct mapped_device *md)
}

table = hc->new_map;
if (table)
dm_table_get(table);

out:
up_read(&_hash_lock);
Expand All @@ -643,10 +660,11 @@ static struct dm_table *dm_get_inactive_table(struct mapped_device *md)
}

static struct dm_table *dm_get_live_or_inactive_table(struct mapped_device *md,
struct dm_ioctl *param)
struct dm_ioctl *param,
int *srcu_idx)
{
return (param->flags & DM_QUERY_INACTIVE_TABLE_FLAG) ?
dm_get_inactive_table(md) : dm_get_live_table(md);
dm_get_inactive_table(md, srcu_idx) : dm_get_live_table(md, srcu_idx);
}

/*
Expand All @@ -657,6 +675,7 @@ static void __dev_status(struct mapped_device *md, struct dm_ioctl *param)
{
struct gendisk *disk = dm_disk(md);
struct dm_table *table;
int srcu_idx;

param->flags &= ~(DM_SUSPEND_FLAG | DM_READONLY_FLAG |
DM_ACTIVE_PRESENT_FLAG);
Expand All @@ -676,26 +695,27 @@ static void __dev_status(struct mapped_device *md, struct dm_ioctl *param)
param->event_nr = dm_get_event_nr(md);
param->target_count = 0;

table = dm_get_live_table(md);
table = dm_get_live_table(md, &srcu_idx);
if (table) {
if (!(param->flags & DM_QUERY_INACTIVE_TABLE_FLAG)) {
if (get_disk_ro(disk))
param->flags |= DM_READONLY_FLAG;
param->target_count = dm_table_get_num_targets(table);
}
dm_table_put(table);

param->flags |= DM_ACTIVE_PRESENT_FLAG;
}
dm_put_live_table(md, srcu_idx);

if (param->flags & DM_QUERY_INACTIVE_TABLE_FLAG) {
table = dm_get_inactive_table(md);
int srcu_idx;
table = dm_get_inactive_table(md, &srcu_idx);
if (table) {
if (!(dm_table_get_mode(table) & FMODE_WRITE))
param->flags |= DM_READONLY_FLAG;
param->target_count = dm_table_get_num_targets(table);
dm_table_put(table);
}
dm_put_live_table(md, srcu_idx);
}
}

Expand Down Expand Up @@ -796,6 +816,7 @@ static int dev_remove(struct dm_ioctl *param, size_t param_size)
struct hash_cell *hc;
struct mapped_device *md;
int r;
struct dm_table *t;

down_write(&_hash_lock);
hc = __find_device_hash_cell(param);
Expand All @@ -819,9 +840,14 @@ static int dev_remove(struct dm_ioctl *param, size_t param_size)
return r;
}

__hash_remove(hc);
t = __hash_remove(hc);
up_write(&_hash_lock);

if (t) {
dm_sync_table(md);
dm_table_destroy(t);
}

if (!dm_kobject_uevent(md, KOBJ_REMOVE, param->event_nr))
param->flags |= DM_UEVENT_GENERATED_FLAG;

Expand Down Expand Up @@ -986,6 +1012,7 @@ static int do_resume(struct dm_ioctl *param)

old_map = dm_swap_table(md, new_map);
if (IS_ERR(old_map)) {
dm_sync_table(md);
dm_table_destroy(new_map);
dm_put(md);
return PTR_ERR(old_map);
Expand All @@ -1003,6 +1030,10 @@ static int do_resume(struct dm_ioctl *param)
param->flags |= DM_UEVENT_GENERATED_FLAG;
}

/*
* Since dm_swap_table synchronizes RCU, nobody should be in
* read-side critical section already.
*/
if (old_map)
dm_table_destroy(old_map);

Expand Down Expand Up @@ -1125,6 +1156,7 @@ static int dev_wait(struct dm_ioctl *param, size_t param_size)
int r = 0;
struct mapped_device *md;
struct dm_table *table;
int srcu_idx;

md = find_device(param);
if (!md)
Expand All @@ -1145,11 +1177,10 @@ static int dev_wait(struct dm_ioctl *param, size_t param_size)
*/
__dev_status(md, param);

table = dm_get_live_or_inactive_table(md, param);
if (table) {
table = dm_get_live_or_inactive_table(md, param, &srcu_idx);
if (table)
retrieve_status(table, param, param_size);
dm_table_put(table);
}
dm_put_live_table(md, srcu_idx);

out:
dm_put(md);
Expand Down Expand Up @@ -1221,7 +1252,7 @@ static int table_load(struct dm_ioctl *param, size_t param_size)
{
int r;
struct hash_cell *hc;
struct dm_table *t;
struct dm_table *t, *old_map = NULL;
struct mapped_device *md;
struct target_type *immutable_target_type;

Expand Down Expand Up @@ -1277,21 +1308,26 @@ static int table_load(struct dm_ioctl *param, size_t param_size)
hc = dm_get_mdptr(md);
if (!hc || hc->md != md) {
DMWARN("device has been removed from the dev hash table.");
dm_table_destroy(t);
up_write(&_hash_lock);
dm_table_destroy(t);
r = -ENXIO;
goto out;
}

if (hc->new_map)
dm_table_destroy(hc->new_map);
old_map = hc->new_map;
hc->new_map = t;
up_write(&_hash_lock);

param->flags |= DM_INACTIVE_PRESENT_FLAG;
__dev_status(md, param);

out:
if (old_map) {
dm_sync_table(md);
dm_table_destroy(old_map);
}

dm_put(md);

return r;
Expand All @@ -1301,6 +1337,7 @@ static int table_clear(struct dm_ioctl *param, size_t param_size)
{
struct hash_cell *hc;
struct mapped_device *md;
struct dm_table *old_map = NULL;

down_write(&_hash_lock);

Expand All @@ -1312,7 +1349,7 @@ static int table_clear(struct dm_ioctl *param, size_t param_size)
}

if (hc->new_map) {
dm_table_destroy(hc->new_map);
old_map = hc->new_map;
hc->new_map = NULL;
}

Expand All @@ -1321,6 +1358,10 @@ static int table_clear(struct dm_ioctl *param, size_t param_size)
__dev_status(hc->md, param);
md = hc->md;
up_write(&_hash_lock);
if (old_map) {
dm_sync_table(md);
dm_table_destroy(old_map);
}
dm_put(md);

return 0;
Expand Down Expand Up @@ -1370,18 +1411,18 @@ static int table_deps(struct dm_ioctl *param, size_t param_size)
{
struct mapped_device *md;
struct dm_table *table;
int srcu_idx;

md = find_device(param);
if (!md)
return -ENXIO;

__dev_status(md, param);

table = dm_get_live_or_inactive_table(md, param);
if (table) {
table = dm_get_live_or_inactive_table(md, param, &srcu_idx);
if (table)
retrieve_deps(table, param, param_size);
dm_table_put(table);
}
dm_put_live_table(md, srcu_idx);

dm_put(md);

Expand All @@ -1396,18 +1437,18 @@ static int table_status(struct dm_ioctl *param, size_t param_size)
{
struct mapped_device *md;
struct dm_table *table;
int srcu_idx;

md = find_device(param);
if (!md)
return -ENXIO;

__dev_status(md, param);

table = dm_get_live_or_inactive_table(md, param);
if (table) {
table = dm_get_live_or_inactive_table(md, param, &srcu_idx);
if (table)
retrieve_status(table, param, param_size);
dm_table_put(table);
}
dm_put_live_table(md, srcu_idx);

dm_put(md);

Expand Down Expand Up @@ -1443,6 +1484,7 @@ static int target_message(struct dm_ioctl *param, size_t param_size)
struct dm_target_msg *tmsg = (void *) param + param->data_start;
size_t maxlen;
char *result = get_result_buffer(param, param_size, &maxlen);
int srcu_idx;

md = find_device(param);
if (!md)
Expand Down Expand Up @@ -1470,9 +1512,9 @@ static int target_message(struct dm_ioctl *param, size_t param_size)
if (r <= 1)
goto out_argv;

table = dm_get_live_table(md);
table = dm_get_live_table(md, &srcu_idx);
if (!table)
goto out_argv;
goto out_table;

if (dm_deleting_md(md)) {
r = -ENXIO;
Expand All @@ -1491,7 +1533,7 @@ static int target_message(struct dm_ioctl *param, size_t param_size)
}

out_table:
dm_table_put(table);
dm_put_live_table(md, srcu_idx);
out_argv:
kfree(argv);
out:
Expand Down
Loading

0 comments on commit 83d5e5b

Please sign in to comment.