Skip to content

Commit

Permalink
drbd: Move conf_mutex from connection to resource
Browse files Browse the repository at this point in the history 
Signed-off-by: Andreas Gruenbacher <agruen@linbit.com>
Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
  • Loading branch information
Andreas Gruenbacher authored and Philipp Reisner committed Feb 17, 2014
1 parent 3ab706f commit 0500813
Show file tree
Hide file tree
Showing 8 changed files with 160 additions and 161 deletions.
15 changes: 7 additions & 8 deletions drivers/block/drbd/drbd_int.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -518,7 +518,7 @@ struct drbd_backing_dev {
struct block_device *backing_bdev;
struct block_device *md_bdev;
struct drbd_md md;
struct disk_conf *disk_conf; /* RCU, for updates: first_peer_device(device)->connection->conf_update */
struct disk_conf *disk_conf; /* RCU, for updates: resource->conf_update */
sector_t known_size; /* last known size of that backing device */
};

Expand Down Expand Up @@ -578,6 +578,8 @@ struct drbd_resource {
struct list_head connections;
struct list_head resources;
struct res_opts res_opts;
struct mutex conf_update; /* mutex for ready-copy-update of net_conf and disk_conf */
spinlock_t req_lock;
};

struct drbd_connection {
Expand All @@ -594,7 +596,6 @@ struct drbd_connection {

unsigned long flags;
struct net_conf *net_conf; /* content protected by rcu */
struct mutex conf_update; /* mutex for ready-copy-update of net_conf and disk_conf */
wait_queue_head_t ping_wait; /* Woken upon reception of a ping, and a state change */

struct sockaddr_storage my_addr;
Expand All @@ -608,8 +609,6 @@ struct drbd_connection {
unsigned long last_received; /* in jiffies, either socket */
unsigned int ko_count;

spinlock_t req_lock;

struct list_head transfer_log; /* all requests not yet fully processed */

struct crypto_hash *cram_hmac_tfm;
Expand Down Expand Up @@ -1595,9 +1594,9 @@ static inline void drbd_chk_io_error_(struct drbd_device *device,
{
if (error) {
unsigned long flags;
spin_lock_irqsave(&first_peer_device(device)->connection->req_lock, flags);
spin_lock_irqsave(&device->resource->req_lock, flags);
__drbd_chk_io_error_(device, forcedetach, where);
spin_unlock_irqrestore(&first_peer_device(device)->connection->req_lock, flags);
spin_unlock_irqrestore(&device->resource->req_lock, flags);
}
}

Expand Down Expand Up @@ -2069,11 +2068,11 @@ static inline bool inc_ap_bio_cond(struct drbd_device *device)
{
bool rv = false;

spin_lock_irq(&first_peer_device(device)->connection->req_lock);
spin_lock_irq(&device->resource->req_lock);
rv = may_inc_ap_bio(device);
if (rv)
atomic_inc(&device->ap_bio_cnt);
spin_unlock_irq(&first_peer_device(device)->connection->req_lock);
spin_unlock_irq(&device->resource->req_lock);

return rv;
}
Expand Down
36 changes: 18 additions & 18 deletions drivers/block/drbd/drbd_main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ void tl_release(struct drbd_connection *connection, unsigned int barrier_nr,
int expect_epoch = 0;
int expect_size = 0;

spin_lock_irq(&connection->req_lock);
spin_lock_irq(&connection->resource->req_lock);

/* find oldest not yet barrier-acked write request,
* count writes in its epoch. */
Expand Down Expand Up @@ -255,12 +255,12 @@ void tl_release(struct drbd_connection *connection, unsigned int barrier_nr,
break;
_req_mod(req, BARRIER_ACKED);
}
spin_unlock_irq(&connection->req_lock);
spin_unlock_irq(&connection->resource->req_lock);

return;

bail:
spin_unlock_irq(&connection->req_lock);
spin_unlock_irq(&connection->resource->req_lock);
conn_request_state(connection, NS(conn, C_PROTOCOL_ERROR), CS_HARD);
}

Expand All @@ -284,9 +284,9 @@ void _tl_restart(struct drbd_connection *connection, enum drbd_req_event what)

void tl_restart(struct drbd_connection *connection, enum drbd_req_event what)
{
spin_lock_irq(&connection->req_lock);
spin_lock_irq(&connection->resource->req_lock);
_tl_restart(connection, what);
spin_unlock_irq(&connection->req_lock);
spin_unlock_irq(&connection->resource->req_lock);
}

/**
Expand All @@ -311,15 +311,15 @@ void tl_abort_disk_io(struct drbd_device *device)
struct drbd_connection *connection = first_peer_device(device)->connection;
struct drbd_request *req, *r;

spin_lock_irq(&connection->req_lock);
spin_lock_irq(&connection->resource->req_lock);
list_for_each_entry_safe(req, r, &connection->transfer_log, tl_requests) {
if (!(req->rq_state & RQ_LOCAL_PENDING))
continue;
if (req->w.device != device)
continue;
_req_mod(req, ABORT_DISK_IO);
}
spin_unlock_irq(&connection->req_lock);
spin_unlock_irq(&connection->resource->req_lock);
}

static int drbd_thread_setup(void *arg)
Expand Down Expand Up @@ -1836,7 +1836,7 @@ static int drbd_open(struct block_device *bdev, fmode_t mode)
int rv = 0;

mutex_lock(&drbd_main_mutex);
spin_lock_irqsave(&first_peer_device(device)->connection->req_lock, flags);
spin_lock_irqsave(&device->resource->req_lock, flags);
/* to have a stable device->state.role
* and no race with updating open_cnt */

Expand All @@ -1849,7 +1849,7 @@ static int drbd_open(struct block_device *bdev, fmode_t mode)

if (!rv)
device->open_cnt++;
spin_unlock_irqrestore(&first_peer_device(device)->connection->req_lock, flags);
spin_unlock_irqrestore(&device->resource->req_lock, flags);
mutex_unlock(&drbd_main_mutex);

return rv;
Expand Down Expand Up @@ -2546,6 +2546,8 @@ struct drbd_resource *drbd_create_resource(const char *name)
idr_init(&resource->devices);
INIT_LIST_HEAD(&resource->connections);
list_add_tail_rcu(&resource->resources, &drbd_resources);
mutex_init(&resource->conf_update);
spin_lock_init(&resource->req_lock);
return resource;
}

Expand Down Expand Up @@ -2588,8 +2590,6 @@ struct drbd_connection *conn_create(const char *name, struct res_opts *res_opts)

connection->cstate = C_STANDALONE;
mutex_init(&connection->cstate_mutex);
spin_lock_init(&connection->req_lock);
mutex_init(&connection->conf_update);
init_waitqueue_head(&connection->ping_wait);
idr_init(&connection->peer_devices);

Expand Down Expand Up @@ -2720,7 +2720,7 @@ enum drbd_ret_code drbd_create_device(struct drbd_resource *resource, unsigned i
blk_queue_max_hw_sectors(q, DRBD_MAX_BIO_SIZE_SAFE >> 8);
blk_queue_bounce_limit(q, BLK_BOUNCE_ANY);
blk_queue_merge_bvec(q, drbd_merge_bvec);
q->queue_lock = &connection->req_lock;
q->queue_lock = &resource->req_lock;

device->md_io_page = alloc_page(GFP_KERNEL);
if (!device->md_io_page)
Expand Down Expand Up @@ -3281,14 +3281,14 @@ int drbd_md_read(struct drbd_device *device, struct drbd_backing_dev *bdev)

rv = NO_ERROR;

spin_lock_irq(&first_peer_device(device)->connection->req_lock);
spin_lock_irq(&device->resource->req_lock);
if (device->state.conn < C_CONNECTED) {
unsigned int peer;
peer = be32_to_cpu(buffer->la_peer_max_bio_size);
peer = max(peer, DRBD_MAX_BIO_SIZE_SAFE);
device->peer_max_bio_size = peer;
}
spin_unlock_irq(&first_peer_device(device)->connection->req_lock);
spin_unlock_irq(&device->resource->req_lock);

err:
drbd_md_put_buffer(device);
Expand Down Expand Up @@ -3577,13 +3577,13 @@ void drbd_queue_bitmap_io(struct drbd_device *device,
device->bm_io_work.why = why;
device->bm_io_work.flags = flags;

spin_lock_irq(&first_peer_device(device)->connection->req_lock);
spin_lock_irq(&device->resource->req_lock);
set_bit(BITMAP_IO, &device->flags);
if (atomic_read(&device->ap_bio_cnt) == 0) {
if (!test_and_set_bit(BITMAP_IO_QUEUED, &device->flags))
drbd_queue_work(&first_peer_device(device)->connection->sender_work, &device->bm_io_work.w);
}
spin_unlock_irq(&first_peer_device(device)->connection->req_lock);
spin_unlock_irq(&device->resource->req_lock);
}

/**
Expand Down Expand Up @@ -3751,10 +3751,10 @@ int drbd_wait_misc(struct drbd_device *device, struct drbd_interval *i)
/* Indicate to wake up device->misc_wait on progress. */
i->waiting = true;
prepare_to_wait(&device->misc_wait, &wait, TASK_INTERRUPTIBLE);
spin_unlock_irq(&first_peer_device(device)->connection->req_lock);
spin_unlock_irq(&device->resource->req_lock);
timeout = schedule_timeout(timeout);
finish_wait(&device->misc_wait, &wait);
spin_lock_irq(&first_peer_device(device)->connection->req_lock);
spin_lock_irq(&device->resource->req_lock);
if (!timeout || device->state.conn < C_CONNECTED)
return -ETIMEDOUT;
if (signal_pending(current))
Expand Down
50 changes: 25 additions & 25 deletions drivers/block/drbd/drbd_nl.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -443,9 +443,9 @@ bool conn_try_outdate_peer(struct drbd_connection *connection)
return false;
}

spin_lock_irq(&connection->req_lock);
spin_lock_irq(&connection->resource->req_lock);
connect_cnt = connection->connect_cnt;
spin_unlock_irq(&connection->req_lock);
spin_unlock_irq(&connection->resource->req_lock);

fp = highest_fencing_policy(connection);
switch (fp) {
Expand Down Expand Up @@ -510,7 +510,7 @@ bool conn_try_outdate_peer(struct drbd_connection *connection)
conn_request_state(connection, mask, val, CS_VERBOSE);
here, because we might were able to re-establish the connection in the
meantime. */
spin_lock_irq(&connection->req_lock);
spin_lock_irq(&connection->resource->req_lock);
if (connection->cstate < C_WF_REPORT_PARAMS && !test_bit(STATE_SENT, &connection->flags)) {
if (connection->connect_cnt != connect_cnt)
/* In case the connection was established and droped
Expand All @@ -519,7 +519,7 @@ bool conn_try_outdate_peer(struct drbd_connection *connection)
else
_conn_request_state(connection, mask, val, CS_VERBOSE);
}
spin_unlock_irq(&connection->req_lock);
spin_unlock_irq(&connection->resource->req_lock);

return conn_highest_pdsk(connection) <= D_OUTDATED;
}
Expand Down Expand Up @@ -654,11 +654,11 @@ drbd_set_role(struct drbd_device *device, enum drbd_role new_role, int force)
put_ldev(device);
}
} else {
mutex_lock(&first_peer_device(device)->connection->conf_update);
mutex_lock(&device->resource->conf_update);
nc = first_peer_device(device)->connection->net_conf;
if (nc)
nc->discard_my_data = 0; /* without copy; single bit op is atomic */
mutex_unlock(&first_peer_device(device)->connection->conf_update);
mutex_unlock(&device->resource->conf_update);

set_disk_ro(device->vdisk, false);
if (get_ldev(device)) {
Expand Down Expand Up @@ -1188,10 +1188,10 @@ static void conn_reconfig_start(struct drbd_connection *connection)
static void conn_reconfig_done(struct drbd_connection *connection)
{
bool stop_threads;
spin_lock_irq(&connection->req_lock);
spin_lock_irq(&connection->resource->req_lock);
stop_threads = conn_all_vols_unconf(connection) &&
connection->cstate == C_STANDALONE;
spin_unlock_irq(&connection->req_lock);
spin_unlock_irq(&connection->resource->req_lock);
if (stop_threads) {
/* asender is implicitly stopped by receiver
* in conn_disconnect() */
Expand All @@ -1211,10 +1211,10 @@ static void drbd_suspend_al(struct drbd_device *device)
}

drbd_al_shrink(device);
spin_lock_irq(&first_peer_device(device)->connection->req_lock);
spin_lock_irq(&device->resource->req_lock);
if (device->state.conn < C_CONNECTED)
s = !test_and_set_bit(AL_SUSPENDED, &device->flags);
spin_unlock_irq(&first_peer_device(device)->connection->req_lock);
spin_unlock_irq(&device->resource->req_lock);
lc_unlock(device->act_log);

if (s)
Expand Down Expand Up @@ -1285,7 +1285,7 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info)
goto fail;
}

mutex_lock(&first_peer_device(device)->connection->conf_update);
mutex_lock(&device->resource->conf_update);
old_disk_conf = device->ldev->disk_conf;
*new_disk_conf = *old_disk_conf;
if (should_set_defaults(info))
Expand Down Expand Up @@ -1348,7 +1348,7 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info)
rcu_assign_pointer(device->rs_plan_s, new_plan);
}

mutex_unlock(&first_peer_device(device)->connection->conf_update);
mutex_unlock(&device->resource->conf_update);

if (new_disk_conf->al_updates)
device->ldev->md.flags &= ~MDF_AL_DISABLED;
Expand All @@ -1374,7 +1374,7 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info)
goto success;

fail_unlock:
mutex_unlock(&first_peer_device(device)->connection->conf_update);
mutex_unlock(&device->resource->conf_update);
fail:
kfree(new_disk_conf);
kfree(new_plan);
Expand Down Expand Up @@ -1724,7 +1724,7 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
if (_drbd_bm_total_weight(device) == drbd_bm_bits(device))
drbd_suspend_al(device); /* IO is still suspended here... */

spin_lock_irq(&first_peer_device(device)->connection->req_lock);
spin_lock_irq(&device->resource->req_lock);
os = drbd_read_state(device);
ns = os;
/* If MDF_CONSISTENT is not set go into inconsistent state,
Expand Down Expand Up @@ -1776,7 +1776,7 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info)
}

rv = _drbd_set_state(device, ns, CS_VERBOSE, NULL);
spin_unlock_irq(&first_peer_device(device)->connection->req_lock);
spin_unlock_irq(&device->resource->req_lock);

if (rv < SS_SUCCESS)
goto force_diskless_dec;
Expand Down Expand Up @@ -2077,7 +2077,7 @@ int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info)
conn_reconfig_start(connection);

mutex_lock(&connection->data.mutex);
mutex_lock(&connection->conf_update);
mutex_lock(&connection->resource->conf_update);
old_net_conf = connection->net_conf;

if (!old_net_conf) {
Expand Down Expand Up @@ -2141,7 +2141,7 @@ int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info)
crypto_free_hash(connection->cram_hmac_tfm);
connection->cram_hmac_tfm = crypto.cram_hmac_tfm;

mutex_unlock(&connection->conf_update);
mutex_unlock(&connection->resource->conf_update);
mutex_unlock(&connection->data.mutex);
synchronize_rcu();
kfree(old_net_conf);
Expand All @@ -2152,7 +2152,7 @@ int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info)
goto done;

fail:
mutex_unlock(&connection->conf_update);
mutex_unlock(&connection->resource->conf_update);
mutex_unlock(&connection->data.mutex);
free_crypto(&crypto);
kfree(new_net_conf);
Expand Down Expand Up @@ -2243,11 +2243,11 @@ int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info)

conn_flush_workqueue(connection);

mutex_lock(&connection->conf_update);
mutex_lock(&adm_ctx.resource->conf_update);
old_net_conf = connection->net_conf;
if (old_net_conf) {
retcode = ERR_NET_CONFIGURED;
mutex_unlock(&connection->conf_update);
mutex_unlock(&adm_ctx.resource->conf_update);
goto fail;
}
rcu_assign_pointer(connection->net_conf, new_net_conf);
Expand All @@ -2263,7 +2263,7 @@ int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info)
connection->peer_addr_len = nla_len(adm_ctx.peer_addr);
memcpy(&connection->peer_addr, nla_data(adm_ctx.peer_addr), connection->peer_addr_len);

mutex_unlock(&connection->conf_update);
mutex_unlock(&adm_ctx.resource->conf_update);

rcu_read_lock();
idr_for_each_entry(&connection->peer_devices, peer_device, i) {
Expand Down Expand Up @@ -2486,12 +2486,12 @@ int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info)
device->ldev->known_size = drbd_get_capacity(device->ldev->backing_bdev);

if (new_disk_conf) {
mutex_lock(&first_peer_device(device)->connection->conf_update);
mutex_lock(&device->resource->conf_update);
old_disk_conf = device->ldev->disk_conf;
*new_disk_conf = *old_disk_conf;
new_disk_conf->disk_size = (sector_t)rs.resize_size;
rcu_assign_pointer(device->ldev->disk_conf, new_disk_conf);
mutex_unlock(&first_peer_device(device)->connection->conf_update);
mutex_unlock(&device->resource->conf_update);
synchronize_rcu();
kfree(old_disk_conf);
}
Expand Down Expand Up @@ -3248,10 +3248,10 @@ int drbd_adm_new_c_uuid(struct sk_buff *skb, struct genl_info *info)
drbd_send_uuids_skip_initial_sync(device);
_drbd_uuid_set(device, UI_BITMAP, 0);
drbd_print_uuids(device, "cleared bitmap UUID");
spin_lock_irq(&first_peer_device(device)->connection->req_lock);
spin_lock_irq(&device->resource->req_lock);
_drbd_set_state(_NS2(device, disk, D_UP_TO_DATE, pdsk, D_UP_TO_DATE),
CS_VERBOSE, NULL);
spin_unlock_irq(&first_peer_device(device)->connection->req_lock);
spin_unlock_irq(&device->resource->req_lock);
}
}

Expand Down
Loading

0 comments on commit 0500813

Please sign in to comment.