ibmvscsis: Enable Logical Partition Migration Support
Changes to support a new mechanism from phyp to better synchronize the
logical partition migration (LPM) of the client partition.
This includes a new VIOCTL to register that we support this new
functionality, and 2 new Transport Event types, and finally another
new VIOCTL to let phyp know once we're ready for the Suspend.
Signed-off-by: Michael Cyr <mikecyr@us.ibm.com>
Signed-off-by: Bryant G. Ly <bryantly@linux.vnet.ibm.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
diff --git a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
index abf6026..3571052 100644
--- a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
+++ b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
@@ -155,6 +155,9 @@ static long ibmvscsis_unregister_command_q(struct scsi_info *vscsi)
qrc = h_free_crq(vscsi->dds.unit_id);
switch (qrc) {
case H_SUCCESS:
+ spin_lock_bh(&vscsi->intr_lock);
+ vscsi->flags &= ~PREP_FOR_SUSPEND_FLAGS;
+ spin_unlock_bh(&vscsi->intr_lock);
break;
case H_HARDWARE:
@@ -422,6 +425,9 @@ static void ibmvscsis_disconnect(struct work_struct *work)
new_state = vscsi->new_state;
vscsi->new_state = 0;
+ vscsi->flags |= DISCONNECT_SCHEDULED;
+ vscsi->flags &= ~SCHEDULE_DISCONNECT;
+
pr_debug("disconnect: flags 0x%x, state 0x%hx\n", vscsi->flags,
vscsi->state);
@@ -802,6 +808,13 @@ static long ibmvscsis_establish_new_q(struct scsi_info *vscsi)
long rc = ADAPT_SUCCESS;
uint format;
+ rc = h_vioctl(vscsi->dds.unit_id, H_ENABLE_PREPARE_FOR_SUSPEND, 30000,
+ 0, 0, 0, 0);
+ if (rc == H_SUCCESS)
+ vscsi->flags |= PREP_FOR_SUSPEND_ENABLED;
+ else if (rc != H_NOT_FOUND)
+ pr_err("Error from Enable Prepare for Suspend: %ld\n", rc);
+
vscsi->flags &= PRESERVE_FLAG_FIELDS;
vscsi->rsp_q_timer.timer_pops = 0;
vscsi->debit = 0;
@@ -951,6 +964,63 @@ static void ibmvscsis_free_cmd_resources(struct scsi_info *vscsi,
}
/**
+ * ibmvscsis_ready_for_suspend() - Helper function to call VIOCTL
+ * @vscsi: Pointer to our adapter structure
+ * @idle: Indicates whether we were called from adapter_idle. This
+ * is important to know if we need to do a disconnect, since if
+ * we're called from adapter_idle, we're still processing the
+ * current disconnect, so we can't just call post_disconnect.
+ *
+ * This function is called when the adapter is idle when phyp has sent
+ * us a Prepare for Suspend Transport Event.
+ *
+ * EXECUTION ENVIRONMENT:
+ * Process or interrupt environment called with interrupt lock held
+ */
+static long ibmvscsis_ready_for_suspend(struct scsi_info *vscsi, bool idle)
+{
+ long rc = 0;
+ struct viosrp_crq *crq;
+
+ /* See if there is a Resume event in the queue */
+ crq = vscsi->cmd_q.base_addr + vscsi->cmd_q.index;
+
+ pr_debug("ready_suspend: flags 0x%x, state 0x%hx crq_valid:%x\n",
+ vscsi->flags, vscsi->state, (int)crq->valid);
+
+ if (!(vscsi->flags & PREP_FOR_SUSPEND_ABORTED) && !(crq->valid)) {
+ rc = h_vioctl(vscsi->dds.unit_id, H_READY_FOR_SUSPEND, 0, 0, 0,
+ 0, 0);
+ if (rc) {
+ pr_err("Ready for Suspend Vioctl failed: %ld\n", rc);
+ rc = 0;
+ }
+ } else if (((vscsi->flags & PREP_FOR_SUSPEND_OVERWRITE) &&
+ (vscsi->flags & PREP_FOR_SUSPEND_ABORTED)) ||
+ ((crq->valid) && ((crq->valid != VALID_TRANS_EVENT) ||
+ (crq->format != RESUME_FROM_SUSP)))) {
+ if (idle) {
+ vscsi->state = ERR_DISCONNECT_RECONNECT;
+ ibmvscsis_reset_queue(vscsi);
+ rc = -1;
+ } else if (vscsi->state == CONNECTED) {
+ ibmvscsis_post_disconnect(vscsi,
+ ERR_DISCONNECT_RECONNECT, 0);
+ }
+
+ vscsi->flags &= ~PREP_FOR_SUSPEND_OVERWRITE;
+
+ if ((crq->valid) && ((crq->valid != VALID_TRANS_EVENT) ||
+ (crq->format != RESUME_FROM_SUSP)))
+ pr_err("Invalid element in CRQ after Prepare for Suspend");
+ }
+
+ vscsi->flags &= ~(PREP_FOR_SUSPEND_PENDING | PREP_FOR_SUSPEND_ABORTED);
+
+ return rc;
+}
+
+/**
* ibmvscsis_trans_event() - Handle a Transport Event
* @vscsi: Pointer to our adapter structure
* @crq: Pointer to CRQ entry containing the Transport Event
@@ -974,18 +1044,8 @@ static long ibmvscsis_trans_event(struct scsi_info *vscsi,
case PARTNER_FAILED:
case PARTNER_DEREGISTER:
ibmvscsis_delete_client_info(vscsi, true);
- break;
-
- default:
- rc = ERROR;
- dev_err(&vscsi->dev, "trans_event: invalid format %d\n",
- (uint)crq->format);
- ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT,
- RESPONSE_Q_DOWN);
- break;
- }
-
- if (rc == ADAPT_SUCCESS) {
+ if (crq->format == MIGRATED)
+ vscsi->flags &= ~PREP_FOR_SUSPEND_OVERWRITE;
switch (vscsi->state) {
case NO_QUEUE:
case ERR_DISCONNECTED:
@@ -1034,6 +1094,60 @@ static long ibmvscsis_trans_event(struct scsi_info *vscsi,
vscsi->flags |= (RESPONSE_Q_DOWN | TRANS_EVENT);
break;
}
+ break;
+
+ case PREPARE_FOR_SUSPEND:
+ pr_debug("Prep for Suspend, crq status = 0x%x\n",
+ (int)crq->status);
+ switch (vscsi->state) {
+ case ERR_DISCONNECTED:
+ case WAIT_CONNECTION:
+ case CONNECTED:
+ ibmvscsis_ready_for_suspend(vscsi, false);
+ break;
+ case SRP_PROCESSING:
+ vscsi->resume_state = vscsi->state;
+ vscsi->flags |= PREP_FOR_SUSPEND_PENDING;
+ if (crq->status == CRQ_ENTRY_OVERWRITTEN)
+ vscsi->flags |= PREP_FOR_SUSPEND_OVERWRITE;
+ ibmvscsis_post_disconnect(vscsi, WAIT_IDLE, 0);
+ break;
+ case NO_QUEUE:
+ case UNDEFINED:
+ case UNCONFIGURING:
+ case WAIT_ENABLED:
+ case ERR_DISCONNECT:
+ case ERR_DISCONNECT_RECONNECT:
+ case WAIT_IDLE:
+ pr_err("Invalid state for Prepare for Suspend Trans Event: 0x%x\n",
+ vscsi->state);
+ break;
+ }
+ break;
+
+ case RESUME_FROM_SUSP:
+ pr_debug("Resume from Suspend, crq status = 0x%x\n",
+ (int)crq->status);
+ if (vscsi->flags & PREP_FOR_SUSPEND_PENDING) {
+ vscsi->flags |= PREP_FOR_SUSPEND_ABORTED;
+ } else {
+ if ((crq->status == CRQ_ENTRY_OVERWRITTEN) ||
+ (vscsi->flags & PREP_FOR_SUSPEND_OVERWRITE)) {
+ ibmvscsis_post_disconnect(vscsi,
+ ERR_DISCONNECT_RECONNECT,
+ 0);
+ vscsi->flags &= ~PREP_FOR_SUSPEND_OVERWRITE;
+ }
+ }
+ break;
+
+ default:
+ rc = ERROR;
+ dev_err(&vscsi->dev, "trans_event: invalid format %d\n",
+ (uint)crq->format);
+ ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT,
+ RESPONSE_Q_DOWN);
+ break;
}
rc = vscsi->flags & SCHEDULE_DISCONNECT;
@@ -1201,6 +1315,7 @@ static struct ibmvscsis_cmd *ibmvscsis_get_free_cmd(struct scsi_info *vscsi)
static void ibmvscsis_adapter_idle(struct scsi_info *vscsi)
{
int free_qs = false;
+ long rc = 0;
pr_debug("adapter_idle: flags 0x%x, state 0x%hx\n", vscsi->flags,
vscsi->state);
@@ -1240,7 +1355,14 @@ static void ibmvscsis_adapter_idle(struct scsi_info *vscsi)
vscsi->rsp_q_timer.timer_pops = 0;
vscsi->debit = 0;
vscsi->credit = 0;
- if (vscsi->flags & TRANS_EVENT) {
+ if (vscsi->flags & PREP_FOR_SUSPEND_PENDING) {
+ vscsi->state = vscsi->resume_state;
+ vscsi->resume_state = 0;
+ rc = ibmvscsis_ready_for_suspend(vscsi, true);
+ vscsi->flags &= ~DISCONNECT_SCHEDULED;
+ if (rc)
+ break;
+ } else if (vscsi->flags & TRANS_EVENT) {
vscsi->state = WAIT_CONNECTION;
vscsi->flags &= PRESERVE_FLAG_FIELDS;
} else {
diff --git a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h
index b4391a8..cc96c27 100644
--- a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h
+++ b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h
@@ -262,6 +262,14 @@ struct scsi_info {
#define DISCONNECT_SCHEDULED 0x00800
/* remove function is sleeping */
#define CFG_SLEEPING 0x01000
+ /* Register for Prepare for Suspend Transport Events */
+#define PREP_FOR_SUSPEND_ENABLED 0x02000
+ /* Prepare for Suspend event sent */
+#define PREP_FOR_SUSPEND_PENDING 0x04000
+ /* Resume from Suspend event sent */
+#define PREP_FOR_SUSPEND_ABORTED 0x08000
+ /* Prepare for Suspend event overwrote another CRQ entry */
+#define PREP_FOR_SUSPEND_OVERWRITE 0x10000
u32 flags;
/* adapter lock */
spinlock_t intr_lock;
@@ -272,6 +280,7 @@ struct scsi_info {
/* used in crq, to tag what iu the response is for */
u64 empty_iu_tag;
uint new_state;
+ uint resume_state;
/* control block for the response queue timer */
struct timer_cb rsp_q_timer;
/* keep last client to enable proper accounting */
@@ -324,8 +333,13 @@ struct scsi_info {
#define TARGET_STOP(VSCSI) (long)(((VSCSI)->state & DONT_PROCESS_STATE) | \
((VSCSI)->flags & BLOCK))
+#define PREP_FOR_SUSPEND_FLAGS (PREP_FOR_SUSPEND_ENABLED | \
+ PREP_FOR_SUSPEND_PENDING | \
+ PREP_FOR_SUSPEND_ABORTED | \
+ PREP_FOR_SUSPEND_OVERWRITE)
+
/* flag bit that are not reset during disconnect */
-#define PRESERVE_FLAG_FIELDS 0
+#define PRESERVE_FLAG_FIELDS (PREP_FOR_SUSPEND_FLAGS)
#define vio_iu(IUE) ((union viosrp_iu *)((IUE)->sbuf->buf))
@@ -333,8 +347,15 @@ struct scsi_info {
#define WRITE_CMD(cdb) (((cdb)[0] & 0x1F) == 0xA)
#ifndef H_GET_PARTNER_INFO
-#define H_GET_PARTNER_INFO 0x0000000000000008LL
+#define H_GET_PARTNER_INFO 0x0000000000000008LL
#endif
+#ifndef H_ENABLE_PREPARE_FOR_SUSPEND
+#define H_ENABLE_PREPARE_FOR_SUSPEND 0x000000000000001DLL
+#endif
+#ifndef H_READY_FOR_SUSPEND
+#define H_READY_FOR_SUSPEND 0x000000000000001ELL
+#endif
+
#define h_copy_rdma(l, sa, sb, da, db) \
plpar_hcall_norets(H_COPY_RDMA, l, sa, sb, da, db)
diff --git a/drivers/scsi/ibmvscsi_tgt/libsrp.h b/drivers/scsi/ibmvscsi_tgt/libsrp.h
index 4696f33..9fec55b 100644
--- a/drivers/scsi/ibmvscsi_tgt/libsrp.h
+++ b/drivers/scsi/ibmvscsi_tgt/libsrp.h
@@ -30,10 +30,13 @@ enum srp_trans_event {
UNUSED_FORMAT = 0,
PARTNER_FAILED = 1,
PARTNER_DEREGISTER = 2,
- MIGRATED = 6
+ MIGRATED = 6,
+ PREPARE_FOR_SUSPEND = 9,
+ RESUME_FROM_SUSP = 0xA
};
enum srp_status {
+ CRQ_ENTRY_OVERWRITTEN = 0x20,
HEADER_DESCRIPTOR = 0xF1,
PING = 0xF5,
PING_RESPONSE = 0xF6
