scsi: mpi3mr: Check admin reply queue from Watchdog

Admin reply processing can be called from multiple contexts. The driver
uses an atomic flag for synchronization among multiple threads/context for
draining the admin replies.

Upon entering the admin processing routine, the driver will set the atomic
flag and start reply processing. When exiting the routine, the driver
resets the flag. However, there is a race condition when one thread (Thread
1) has processed replies and is about to reset the flag but in the meantime
few more replies are posted and another thread (Thread 2) is called to
process replies. Since the synchronization flag is still set, Thread 2 will
return without processing replies and those new replies will not be
flushed.

Make the watchdog thread monitor cases where admin ISR/poll call returns
due to another thread processing admin replies. If such an instance is
found, make driver call admin ISR to drain replies (if any).

Co-developed-by: Sumit Saxena <sumit.saxena@broadcom.com>
Signed-off-by: Sumit Saxena <sumit.saxena@broadcom.com>
Signed-off-by: Ranjan Kumar <ranjan.kumar@broadcom.com>
Link: https://lore.kernel.org/r/20250220142528.20837-4-ranjan.kumar@broadcom.com
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>

drivers/scsi/mpi3mr/mpi3mr.h

@@ -1032,6 +1032,8 @@ struct scmd_priv {
  * @admin_reply_base: Admin reply queue base virtual address
  * @admin_reply_dma: Admin reply queue base dma address
  * @admin_reply_q_in_use: Queue is handled by poll/ISR
+ * @admin_pend_isr: Count of unprocessed admin ISR/poll calls
+ * due to another thread processing replies
  * @ready_timeout: Controller ready timeout
  * @intr_info: Interrupt cookie pointer
  * @intr_info_count: Number of interrupt cookies
@@ -1206,6 +1208,7 @@ struct mpi3mr_ioc {
 	void *admin_reply_base;
 	dma_addr_t admin_reply_dma;
 	atomic_t admin_reply_q_in_use;
+	atomic_t admin_pend_isr;
 
 	u32 ready_timeout;
 

drivers/scsi/mpi3mr/mpi3mr_fw.c

@@ -446,8 +446,10 @@ int mpi3mr_process_admin_reply_q(struct mpi3mr_ioc *mrioc)
 	u16 threshold_comps = 0;
 	struct mpi3_default_reply_descriptor *reply_desc;
 
-	if (!atomic_add_unless(&mrioc->admin_reply_q_in_use, 1, 1))
+	if (!atomic_add_unless(&mrioc->admin_reply_q_in_use, 1, 1)) {
+		atomic_inc(&mrioc->admin_pend_isr);
 		return 0;
+	}
 
 	reply_desc = (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
 	    admin_reply_ci;
@@ -2757,6 +2759,12 @@ static void mpi3mr_watchdog_work(struct work_struct *work)
 		return;
 	}
 
+	if (atomic_read(&mrioc->admin_pend_isr)) {
+		ioc_err(mrioc, "Unprocessed admin ISR instance found\n"
+				"flush admin replies\n");
+		mpi3mr_process_admin_reply_q(mrioc);
+	}
+
 	if (!(mrioc->facts.ioc_capabilities &
 		MPI3_IOCFACTS_CAPABILITY_NON_SUPERVISOR_IOC) &&
 		(mrioc->ts_update_counter++ >= mrioc->ts_update_interval)) {