From d8e07efa806d6a3df853d7b1051eb755edc5a85c Mon Sep 17 00:00:00 2001
From: Dan Williams <dan.j.williams@intel.com>
Date: Wed, 3 Mar 2010 21:21:13 -0700
Subject: [PATCH] --- yaml --- r: 185766 b: refs/heads/master c:
 b9cc98697d1ca35a86bbb708acc6d93993c28f0f h: refs/heads/master v: v3

---
 [refs]                             |  2 +-
 trunk/drivers/dma/ioat/dma_v3.c    | 37 ++++++++++++++++++++++++++----
 trunk/drivers/dma/ioat/registers.h |  2 +-
 3 files changed, 35 insertions(+), 6 deletions(-)

diff --git a/[refs] b/[refs]
index 00a80be3aa4f..31ef4a7eade2 100644
--- a/[refs]
+++ b/[refs]
@@ -1,2 +1,2 @@
 ---
-refs/heads/master: aa75db0080603bae27961c0502812dfd0f522bb3
+refs/heads/master: b9cc98697d1ca35a86bbb708acc6d93993c28f0f
diff --git a/trunk/drivers/dma/ioat/dma_v3.c b/trunk/drivers/dma/ioat/dma_v3.c
index 39520f2f7da9..9988f1340186 100644
--- a/trunk/drivers/dma/ioat/dma_v3.c
+++ b/trunk/drivers/dma/ioat/dma_v3.c
@@ -295,15 +295,23 @@ static void __cleanup(struct ioat2_dma_chan *ioat, unsigned long phys_complete)
 	ioat->tail += i;
 	BUG_ON(active && !seen_current); /* no active descs have written a completion? */
 	chan->last_completion = phys_complete;
-	if (ioat->head == ioat->tail) {
+
+	active = ioat2_ring_active(ioat);
+	if (active == 0) {
 		dev_dbg(to_dev(chan), "%s: cancel completion timeout\n",
 			__func__);
 		clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
 		mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
 	}
+	/* 5 microsecond delay per pending descriptor */
+	writew(min((5 * active), IOAT_INTRDELAY_MASK),
+	       chan->device->reg_base + IOAT_INTRDELAY_OFFSET);
 }
 
-static void ioat3_cleanup(struct ioat2_dma_chan *ioat)
+/* try to cleanup, but yield (via spin_trylock) to incoming submissions
+ * with the expectation that we will immediately poll again shortly
+ */
+static void ioat3_cleanup_poll(struct ioat2_dma_chan *ioat)
 {
 	struct ioat_chan_common *chan = &ioat->base;
 	unsigned long phys_complete;
@@ -329,11 +337,32 @@ static void ioat3_cleanup(struct ioat2_dma_chan *ioat)
 	spin_unlock_bh(&chan->cleanup_lock);
 }
 
+/* run cleanup now because we already delayed the interrupt via INTRDELAY */
+static void ioat3_cleanup_sync(struct ioat2_dma_chan *ioat)
+{
+	struct ioat_chan_common *chan = &ioat->base;
+	unsigned long phys_complete;
+
+	prefetch(chan->completion);
+
+	spin_lock_bh(&chan->cleanup_lock);
+	if (!ioat_cleanup_preamble(chan, &phys_complete)) {
+		spin_unlock_bh(&chan->cleanup_lock);
+		return;
+	}
+	spin_lock_bh(&ioat->ring_lock);
+
+	__cleanup(ioat, phys_complete);
+
+	spin_unlock_bh(&ioat->ring_lock);
+	spin_unlock_bh(&chan->cleanup_lock);
+}
+
 static void ioat3_cleanup_tasklet(unsigned long data)
 {
 	struct ioat2_dma_chan *ioat = (void *) data;
 
-	ioat3_cleanup(ioat);
+	ioat3_cleanup_sync(ioat);
 	writew(IOAT_CHANCTRL_RUN, ioat->base.reg_base + IOAT_CHANCTRL_OFFSET);
 }
 
@@ -417,7 +446,7 @@ ioat3_is_complete(struct dma_chan *c, dma_cookie_t cookie,
 	if (ioat_is_complete(c, cookie, done, used) == DMA_SUCCESS)
 		return DMA_SUCCESS;
 
-	ioat3_cleanup(ioat);
+	ioat3_cleanup_poll(ioat);
 
 	return ioat_is_complete(c, cookie, done, used);
 }
diff --git a/trunk/drivers/dma/ioat/registers.h b/trunk/drivers/dma/ioat/registers.h
index e8ae63baf588..1391798542b6 100644
--- a/trunk/drivers/dma/ioat/registers.h
+++ b/trunk/drivers/dma/ioat/registers.h
@@ -60,7 +60,7 @@
 #define IOAT_PERPORTOFFSET_OFFSET		0x0A	/* 16-bit */
 
 #define IOAT_INTRDELAY_OFFSET			0x0C	/* 16-bit */
-#define IOAT_INTRDELAY_INT_DELAY_MASK		0x3FFF	/* Interrupt Delay Time */
+#define IOAT_INTRDELAY_MASK			0x3FFF	/* Interrupt Delay Time */
 #define IOAT_INTRDELAY_COALESE_SUPPORT		0x8000	/* Interrupt Coalescing Supported */
 
 #define IOAT_DEVICE_STATUS_OFFSET		0x0E	/* 16-bit */