Skip to content

Commit

Permalink
ibmvscsis: Enable Logical Partition Migration Support
Browse files Browse the repository at this point in the history 
Changes to support a new mechanism from phyp to better synchronize the
logical partition migration (LPM) of the client partition.
This includes a new VIOCTL to register that we support this new
functionality, and 2 new Transport Event types, and finally another
new VIOCTL to let phyp know once we're ready for the Suspend.

Signed-off-by: Michael Cyr <mikecyr@us.ibm.com>
Signed-off-by: Bryant G. Ly <bryantly@linux.vnet.ibm.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
  • Loading branch information
Michael Cyr authored and Nicholas Bellinger committed Jun 9, 2017
1 parent 12bdcbd commit 464fd64
Show file tree
Hide file tree
Showing 3 changed files with 162 additions and 16 deletions.
148 changes: 135 additions & 13 deletions drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,9 @@ static long ibmvscsis_unregister_command_q(struct scsi_info *vscsi)
qrc = h_free_crq(vscsi->dds.unit_id);
switch (qrc) {
case H_SUCCESS:
spin_lock_bh(&vscsi->intr_lock);
vscsi->flags &= ~PREP_FOR_SUSPEND_FLAGS;
spin_unlock_bh(&vscsi->intr_lock);
break;

case H_HARDWARE:
Expand Down Expand Up @@ -422,6 +425,9 @@ static void ibmvscsis_disconnect(struct work_struct *work)
new_state = vscsi->new_state;
vscsi->new_state = 0;

vscsi->flags |= DISCONNECT_SCHEDULED;
vscsi->flags &= ~SCHEDULE_DISCONNECT;

pr_debug("disconnect: flags 0x%x, state 0x%hx\n", vscsi->flags,
vscsi->state);

Expand Down Expand Up @@ -802,6 +808,13 @@ static long ibmvscsis_establish_new_q(struct scsi_info *vscsi)
long rc = ADAPT_SUCCESS;
uint format;

rc = h_vioctl(vscsi->dds.unit_id, H_ENABLE_PREPARE_FOR_SUSPEND, 30000,
0, 0, 0, 0);
if (rc == H_SUCCESS)
vscsi->flags |= PREP_FOR_SUSPEND_ENABLED;
else if (rc != H_NOT_FOUND)
pr_err("Error from Enable Prepare for Suspend: %ld\n", rc);

vscsi->flags &= PRESERVE_FLAG_FIELDS;
vscsi->rsp_q_timer.timer_pops = 0;
vscsi->debit = 0;
Expand Down Expand Up @@ -950,6 +963,63 @@ static void ibmvscsis_free_cmd_resources(struct scsi_info *vscsi,
}
}

/**
* ibmvscsis_ready_for_suspend() - Helper function to call VIOCTL
* @vscsi: Pointer to our adapter structure
* @idle: Indicates whether we were called from adapter_idle. This
* is important to know if we need to do a disconnect, since if
* we're called from adapter_idle, we're still processing the
* current disconnect, so we can't just call post_disconnect.
*
* This function is called when the adapter is idle when phyp has sent
* us a Prepare for Suspend Transport Event.
*
* EXECUTION ENVIRONMENT:
* Process or interrupt environment called with interrupt lock held
*/
static long ibmvscsis_ready_for_suspend(struct scsi_info *vscsi, bool idle)
{
long rc = 0;
struct viosrp_crq *crq;

/* See if there is a Resume event in the queue */
crq = vscsi->cmd_q.base_addr + vscsi->cmd_q.index;

pr_debug("ready_suspend: flags 0x%x, state 0x%hx crq_valid:%x\n",
vscsi->flags, vscsi->state, (int)crq->valid);

if (!(vscsi->flags & PREP_FOR_SUSPEND_ABORTED) && !(crq->valid)) {
rc = h_vioctl(vscsi->dds.unit_id, H_READY_FOR_SUSPEND, 0, 0, 0,
0, 0);
if (rc) {
pr_err("Ready for Suspend Vioctl failed: %ld\n", rc);
rc = 0;
}
} else if (((vscsi->flags & PREP_FOR_SUSPEND_OVERWRITE) &&
(vscsi->flags & PREP_FOR_SUSPEND_ABORTED)) ||
((crq->valid) && ((crq->valid != VALID_TRANS_EVENT) ||
(crq->format != RESUME_FROM_SUSP)))) {
if (idle) {
vscsi->state = ERR_DISCONNECT_RECONNECT;
ibmvscsis_reset_queue(vscsi);
rc = -1;
} else if (vscsi->state == CONNECTED) {
ibmvscsis_post_disconnect(vscsi,
ERR_DISCONNECT_RECONNECT, 0);
}

vscsi->flags &= ~PREP_FOR_SUSPEND_OVERWRITE;

if ((crq->valid) && ((crq->valid != VALID_TRANS_EVENT) ||
(crq->format != RESUME_FROM_SUSP)))
pr_err("Invalid element in CRQ after Prepare for Suspend");
}

vscsi->flags &= ~(PREP_FOR_SUSPEND_PENDING | PREP_FOR_SUSPEND_ABORTED);

return rc;
}

/**
* ibmvscsis_trans_event() - Handle a Transport Event
* @vscsi: Pointer to our adapter structure
Expand All @@ -974,18 +1044,8 @@ static long ibmvscsis_trans_event(struct scsi_info *vscsi,
case PARTNER_FAILED:
case PARTNER_DEREGISTER:
ibmvscsis_delete_client_info(vscsi, true);
break;

default:
rc = ERROR;
dev_err(&vscsi->dev, "trans_event: invalid format %d\n",
(uint)crq->format);
ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT,
RESPONSE_Q_DOWN);
break;
}

if (rc == ADAPT_SUCCESS) {
if (crq->format == MIGRATED)
vscsi->flags &= ~PREP_FOR_SUSPEND_OVERWRITE;
switch (vscsi->state) {
case NO_QUEUE:
case ERR_DISCONNECTED:
Expand Down Expand Up @@ -1034,6 +1094,60 @@ static long ibmvscsis_trans_event(struct scsi_info *vscsi,
vscsi->flags |= (RESPONSE_Q_DOWN | TRANS_EVENT);
break;
}
break;

case PREPARE_FOR_SUSPEND:
pr_debug("Prep for Suspend, crq status = 0x%x\n",
(int)crq->status);
switch (vscsi->state) {
case ERR_DISCONNECTED:
case WAIT_CONNECTION:
case CONNECTED:
ibmvscsis_ready_for_suspend(vscsi, false);
break;
case SRP_PROCESSING:
vscsi->resume_state = vscsi->state;
vscsi->flags |= PREP_FOR_SUSPEND_PENDING;
if (crq->status == CRQ_ENTRY_OVERWRITTEN)
vscsi->flags |= PREP_FOR_SUSPEND_OVERWRITE;
ibmvscsis_post_disconnect(vscsi, WAIT_IDLE, 0);
break;
case NO_QUEUE:
case UNDEFINED:
case UNCONFIGURING:
case WAIT_ENABLED:
case ERR_DISCONNECT:
case ERR_DISCONNECT_RECONNECT:
case WAIT_IDLE:
pr_err("Invalid state for Prepare for Suspend Trans Event: 0x%x\n",
vscsi->state);
break;
}
break;

case RESUME_FROM_SUSP:
pr_debug("Resume from Suspend, crq status = 0x%x\n",
(int)crq->status);
if (vscsi->flags & PREP_FOR_SUSPEND_PENDING) {
vscsi->flags |= PREP_FOR_SUSPEND_ABORTED;
} else {
if ((crq->status == CRQ_ENTRY_OVERWRITTEN) ||
(vscsi->flags & PREP_FOR_SUSPEND_OVERWRITE)) {
ibmvscsis_post_disconnect(vscsi,
ERR_DISCONNECT_RECONNECT,
0);
vscsi->flags &= ~PREP_FOR_SUSPEND_OVERWRITE;
}
}
break;

default:
rc = ERROR;
dev_err(&vscsi->dev, "trans_event: invalid format %d\n",
(uint)crq->format);
ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT,
RESPONSE_Q_DOWN);
break;
}

rc = vscsi->flags & SCHEDULE_DISCONNECT;
Expand Down Expand Up @@ -1201,6 +1315,7 @@ static struct ibmvscsis_cmd *ibmvscsis_get_free_cmd(struct scsi_info *vscsi)
static void ibmvscsis_adapter_idle(struct scsi_info *vscsi)
{
int free_qs = false;
long rc = 0;

pr_debug("adapter_idle: flags 0x%x, state 0x%hx\n", vscsi->flags,
vscsi->state);
Expand Down Expand Up @@ -1240,7 +1355,14 @@ static void ibmvscsis_adapter_idle(struct scsi_info *vscsi)
vscsi->rsp_q_timer.timer_pops = 0;
vscsi->debit = 0;
vscsi->credit = 0;
if (vscsi->flags & TRANS_EVENT) {
if (vscsi->flags & PREP_FOR_SUSPEND_PENDING) {
vscsi->state = vscsi->resume_state;
vscsi->resume_state = 0;
rc = ibmvscsis_ready_for_suspend(vscsi, true);
vscsi->flags &= ~DISCONNECT_SCHEDULED;
if (rc)
break;
} else if (vscsi->flags & TRANS_EVENT) {
vscsi->state = WAIT_CONNECTION;
vscsi->flags &= PRESERVE_FLAG_FIELDS;
} else {
Expand Down
25 changes: 23 additions & 2 deletions drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,14 @@ struct scsi_info {
#define DISCONNECT_SCHEDULED 0x00800
/* remove function is sleeping */
#define CFG_SLEEPING 0x01000
/* Register for Prepare for Suspend Transport Events */
#define PREP_FOR_SUSPEND_ENABLED 0x02000
/* Prepare for Suspend event sent */
#define PREP_FOR_SUSPEND_PENDING 0x04000
/* Resume from Suspend event sent */
#define PREP_FOR_SUSPEND_ABORTED 0x08000
/* Prepare for Suspend event overwrote another CRQ entry */
#define PREP_FOR_SUSPEND_OVERWRITE 0x10000
u32 flags;
/* adapter lock */
spinlock_t intr_lock;
Expand All @@ -272,6 +280,7 @@ struct scsi_info {
/* used in crq, to tag what iu the response is for */
u64 empty_iu_tag;
uint new_state;
uint resume_state;
/* control block for the response queue timer */
struct timer_cb rsp_q_timer;
/* keep last client to enable proper accounting */
Expand Down Expand Up @@ -324,17 +333,29 @@ struct scsi_info {
#define TARGET_STOP(VSCSI) (long)(((VSCSI)->state & DONT_PROCESS_STATE) | \
((VSCSI)->flags & BLOCK))

#define PREP_FOR_SUSPEND_FLAGS (PREP_FOR_SUSPEND_ENABLED | \
PREP_FOR_SUSPEND_PENDING | \
PREP_FOR_SUSPEND_ABORTED | \
PREP_FOR_SUSPEND_OVERWRITE)

/* flag bit that are not reset during disconnect */
#define PRESERVE_FLAG_FIELDS 0
#define PRESERVE_FLAG_FIELDS (PREP_FOR_SUSPEND_FLAGS)

#define vio_iu(IUE) ((union viosrp_iu *)((IUE)->sbuf->buf))

#define READ_CMD(cdb) (((cdb)[0] & 0x1F) == 8)
#define WRITE_CMD(cdb) (((cdb)[0] & 0x1F) == 0xA)

#ifndef H_GET_PARTNER_INFO
#define H_GET_PARTNER_INFO 0x0000000000000008LL
#define H_GET_PARTNER_INFO 0x0000000000000008LL
#endif
#ifndef H_ENABLE_PREPARE_FOR_SUSPEND
#define H_ENABLE_PREPARE_FOR_SUSPEND 0x000000000000001DLL
#endif
#ifndef H_READY_FOR_SUSPEND
#define H_READY_FOR_SUSPEND 0x000000000000001ELL
#endif


#define h_copy_rdma(l, sa, sb, da, db) \
plpar_hcall_norets(H_COPY_RDMA, l, sa, sb, da, db)
Expand Down
5 changes: 4 additions & 1 deletion drivers/scsi/ibmvscsi_tgt/libsrp.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,13 @@ enum srp_trans_event {
UNUSED_FORMAT = 0,
PARTNER_FAILED = 1,
PARTNER_DEREGISTER = 2,
MIGRATED = 6
MIGRATED = 6,
PREPARE_FOR_SUSPEND = 9,
RESUME_FROM_SUSP = 0xA
};

enum srp_status {
CRQ_ENTRY_OVERWRITTEN = 0x20,
HEADER_DESCRIPTOR = 0xF1,
PING = 0xF5,
PING_RESPONSE = 0xF6
Expand Down

0 comments on commit 464fd64

Please sign in to comment.