Skip to content

Commit

Permalink
drbd: Delay/reject other state changes while establishing a connection
Browse files Browse the repository at this point in the history 
Changes to the role and disk state should be delayed or rejected
while we establish a connection.

This is necessary, since the peer will base its resync decision
on the UUIDs and the state we sent in the drbd_connect() function.

The most prominent example for this race is becoming primary after
sending state and UUIDs and before the state changes to C_WF_CONNECTION.

Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>
  • Loading branch information
Philipp Reisner committed Nov 8, 2012
1 parent 27eb13e commit a1096a6
Show file tree
Hide file tree
Showing 4 changed files with 30 additions and 10 deletions.
1 change: 1 addition & 0 deletions drivers/block/drbd/drbd_int.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -808,6 +808,7 @@ enum {
CONN_WD_ST_CHG_FAIL,
CONN_DRY_RUN, /* Expect disconnect after resync handshake. */
CREATE_BARRIER, /* next P_DATA is preceded by a P_BARRIER */
STATE_SENT, /* Do not change state/UUIDs while this is set */
};

struct drbd_tconn { /* is a resource from the config file */
Expand Down
2 changes: 1 addition & 1 deletion drivers/block/drbd/drbd_nl.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -502,7 +502,7 @@ bool conn_try_outdate_peer(struct drbd_tconn *tconn)
here, because we might were able to re-establish the connection in the
meantime. */
spin_lock_irq(&tconn->req_lock);
if (tconn->cstate < C_WF_REPORT_PARAMS)
if (tconn->cstate < C_WF_REPORT_PARAMS && !test_bit(STATE_SENT, &tconn->flags))
_conn_request_state(tconn, mask, val, CS_VERBOSE);
spin_unlock_irq(&tconn->req_lock);

Expand Down
8 changes: 7 additions & 1 deletion drivers/block/drbd/drbd_receiver.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -848,6 +848,7 @@ static int conn_connect(struct drbd_tconn *tconn)
struct net_conf *nc;
int vnr, timeout, try, h, ok;
bool discard_my_data;
enum drbd_state_rv rv;

if (conn_request_state(tconn, NS(conn, C_WF_CONNECTION), CS_VERBOSE) < SS_SUCCESS)
return -2;
Expand Down Expand Up @@ -1008,6 +1009,8 @@ static int conn_connect(struct drbd_tconn *tconn)
if (drbd_send_protocol(tconn) == -EOPNOTSUPP)
return -1;

set_bit(STATE_SENT, &tconn->flags);

rcu_read_lock();
idr_for_each_entry(&tconn->volumes, mdev, vnr) {
kref_get(&mdev->kref);
Expand All @@ -1024,8 +1027,11 @@ static int conn_connect(struct drbd_tconn *tconn)
}
rcu_read_unlock();

if (conn_request_state(tconn, NS(conn, C_WF_REPORT_PARAMS), CS_VERBOSE) < SS_SUCCESS)
rv = conn_request_state(tconn, NS(conn, C_WF_REPORT_PARAMS), CS_VERBOSE);
if (rv < SS_SUCCESS) {
clear_bit(STATE_SENT, &tconn->flags);
return 0;
}

drbd_thread_start(&tconn->asender);

Expand Down
29 changes: 21 additions & 8 deletions drivers/block/drbd/drbd_state.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ static int w_after_state_ch(struct drbd_work *w, int unused);
static void after_state_ch(struct drbd_conf *mdev, union drbd_state os,
union drbd_state ns, enum chg_state_flags flags);
static enum drbd_state_rv is_valid_state(struct drbd_conf *, union drbd_state);
static enum drbd_state_rv is_valid_soft_transition(union drbd_state, union drbd_state);
static enum drbd_state_rv is_valid_soft_transition(union drbd_state, union drbd_state, struct drbd_tconn *);
static enum drbd_state_rv is_valid_transition(union drbd_state os, union drbd_state ns);
static union drbd_state sanitize_state(struct drbd_conf *mdev, union drbd_state ns,
enum sanitize_state_warnings *warn);
Expand Down Expand Up @@ -267,7 +267,7 @@ _req_st_cond(struct drbd_conf *mdev, union drbd_state mask,
if (rv == SS_UNKNOWN_ERROR) {
rv = is_valid_state(mdev, ns);
if (rv == SS_SUCCESS) {
rv = is_valid_soft_transition(os, ns);
rv = is_valid_soft_transition(os, ns, mdev->tconn);
if (rv == SS_SUCCESS)
rv = SS_UNKNOWN_ERROR; /* cont waiting, otherwise fail. */
}
Expand Down Expand Up @@ -313,7 +313,7 @@ drbd_req_state(struct drbd_conf *mdev, union drbd_state mask,
if (cl_wide_st_chg(mdev, os, ns)) {
rv = is_valid_state(mdev, ns);
if (rv == SS_SUCCESS)
rv = is_valid_soft_transition(os, ns);
rv = is_valid_soft_transition(os, ns, mdev->tconn);
spin_unlock_irqrestore(&mdev->tconn->req_lock, flags);

if (rv < SS_SUCCESS) {
Expand Down Expand Up @@ -569,7 +569,7 @@ is_valid_state(struct drbd_conf *mdev, union drbd_state ns)
* @os: old state.
*/
static enum drbd_state_rv
is_valid_soft_transition(union drbd_state os, union drbd_state ns)
is_valid_soft_transition(union drbd_state os, union drbd_state ns, struct drbd_tconn *tconn)
{
enum drbd_state_rv rv = SS_SUCCESS;

Expand All @@ -595,6 +595,13 @@ is_valid_soft_transition(union drbd_state os, union drbd_state ns)
/* if (ns.conn == os.conn && ns.conn == C_WF_REPORT_PARAMS)
rv = SS_IN_TRANSIENT_STATE; */

/* While establishing a connection only allow cstate to change.
Delay/refuse role changes, detach attach etc... */
if (test_bit(STATE_SENT, &tconn->flags) &&
!(os.conn == C_WF_REPORT_PARAMS ||
(ns.conn == C_WF_REPORT_PARAMS && os.conn == C_WF_CONNECTION)))
rv = SS_IN_TRANSIENT_STATE;

if ((ns.conn == C_VERIFY_S || ns.conn == C_VERIFY_T) && os.conn < C_CONNECTED)
rv = SS_NEED_CONNECTION;

Expand Down Expand Up @@ -927,9 +934,9 @@ __drbd_set_state(struct drbd_conf *mdev, union drbd_state ns,
this happen...*/

if (is_valid_state(mdev, os) == rv)
rv = is_valid_soft_transition(os, ns);
rv = is_valid_soft_transition(os, ns, mdev->tconn);
} else
rv = is_valid_soft_transition(os, ns);
rv = is_valid_soft_transition(os, ns, mdev->tconn);
}

if (rv < SS_SUCCESS) {
Expand Down Expand Up @@ -1393,6 +1400,12 @@ static void after_state_ch(struct drbd_conf *mdev, union drbd_state os,
if (os.disk < D_UP_TO_DATE && os.conn >= C_SYNC_SOURCE && ns.conn == C_CONNECTED)
drbd_send_state(mdev, ns);

/* Wake up role changes, that were delayed because of connection establishing */
if (os.conn == C_WF_REPORT_PARAMS && ns.conn != C_WF_REPORT_PARAMS) {
if (test_and_clear_bit(STATE_SENT, &mdev->tconn->flags))
wake_up(&mdev->state_wait);
}

/* This triggers bitmap writeout of potentially still unwritten pages
* if the resync finished cleanly, or aborted because of peer disk
* failure, or because of connection loss.
Expand Down Expand Up @@ -1565,9 +1578,9 @@ conn_is_valid_transition(struct drbd_tconn *tconn, union drbd_state mask, union
rv = is_valid_state(mdev, ns);
if (rv < SS_SUCCESS) {
if (is_valid_state(mdev, os) == rv)
rv = is_valid_soft_transition(os, ns);
rv = is_valid_soft_transition(os, ns, tconn);
} else
rv = is_valid_soft_transition(os, ns);
rv = is_valid_soft_transition(os, ns, tconn);
}
if (rv < SS_SUCCESS)
break;
Expand Down

0 comments on commit a1096a6

Please sign in to comment.