Skip to content

Commit

Permalink
ioat: cleanup completion status reads
Browse files Browse the repository at this point in the history 
The cleanup path makes an effort to only perform an atomic read of the
64-bit completion address.  However in the 32-bit case it does not
matter if we read the upper-32 and lower-32 non-atomically because the
upper-32 will always be zero.

Signed-off-by: Maciej Sosnowski <maciej.sosnowski@intel.com>
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
  • Loading branch information
Dan Williams committed Sep 9, 2009
1 parent 6df9183 commit 4fb9b9e
Show file tree
Hide file tree
Showing 4 changed files with 46 additions and 75 deletions.
78 changes: 29 additions & 49 deletions drivers/dma/ioat/dma.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,8 +201,7 @@ static void ioat1_reset_part2(struct work_struct *work)
spin_lock_bh(&chan->cleanup_lock);
spin_lock_bh(&ioat->desc_lock);

chan->completion_virt->low = 0;
chan->completion_virt->high = 0;
*chan->completion = 0;
ioat->pending = 0;

/* count the descriptors waiting */
Expand Down Expand Up @@ -256,8 +255,7 @@ static void ioat1_reset_channel(struct ioat_dma_chan *ioat)

dev_dbg(to_dev(chan), "%s\n", __func__);
chanerr = readl(reg_base + IOAT_CHANERR_OFFSET);
chansts = (chan->completion_virt->low
& IOAT_CHANSTS_DMA_TRANSFER_STATUS);
chansts = *chan->completion & IOAT_CHANSTS_DMA_TRANSFER_STATUS;
if (chanerr) {
dev_err(to_dev(chan),
"chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
Expand Down Expand Up @@ -293,14 +291,8 @@ static void ioat1_chan_watchdog(struct work_struct *work)
struct ioat_dma_chan *ioat;
struct ioat_chan_common *chan;
int i;

union {
u64 full;
struct {
u32 low;
u32 high;
};
} completion_hw;
u64 completion;
u32 completion_low;
unsigned long compl_desc_addr_hw;

for (i = 0; i < device->common.chancnt; i++) {
Expand Down Expand Up @@ -334,25 +326,24 @@ static void ioat1_chan_watchdog(struct work_struct *work)
* try resetting the channel
*/

completion_hw.low = readl(chan->reg_base +
/* we need to read the low address first as this
* causes the chipset to latch the upper bits
* for the subsequent read
*/
completion_low = readl(chan->reg_base +
IOAT_CHANSTS_OFFSET_LOW(chan->device->version));
completion_hw.high = readl(chan->reg_base +
completion = readl(chan->reg_base +
IOAT_CHANSTS_OFFSET_HIGH(chan->device->version));
#if (BITS_PER_LONG == 64)
compl_desc_addr_hw =
completion_hw.full
& IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
#else
compl_desc_addr_hw =
completion_hw.low & IOAT_LOW_COMPLETION_MASK;
#endif
completion <<= 32;
completion |= completion_low;
compl_desc_addr_hw = completion &
IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;

if ((compl_desc_addr_hw != 0)
&& (compl_desc_addr_hw != chan->watchdog_completion)
&& (compl_desc_addr_hw != chan->last_compl_desc_addr_hw)) {
chan->last_compl_desc_addr_hw = compl_desc_addr_hw;
chan->completion_virt->low = completion_hw.low;
chan->completion_virt->high = completion_hw.high;
*chan->completion = completion;
} else {
ioat1_reset_channel(ioat);
chan->watchdog_completion = 0;
Expand Down Expand Up @@ -492,14 +483,12 @@ static int ioat1_dma_alloc_chan_resources(struct dma_chan *c)

/* allocate a completion writeback area */
/* doing 2 32bit writes to mmio since 1 64b write doesn't work */
chan->completion_virt = pci_pool_alloc(chan->device->completion_pool,
GFP_KERNEL,
&chan->completion_addr);
memset(chan->completion_virt, 0,
sizeof(*chan->completion_virt));
writel(((u64) chan->completion_addr) & 0x00000000FFFFFFFF,
chan->completion = pci_pool_alloc(chan->device->completion_pool,
GFP_KERNEL, &chan->completion_dma);
memset(chan->completion, 0, sizeof(*chan->completion));
writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF,
chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
writel(((u64) chan->completion_addr) >> 32,
writel(((u64) chan->completion_dma) >> 32,
chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);

tasklet_enable(&chan->cleanup_task);
Expand Down Expand Up @@ -558,15 +547,16 @@ static void ioat1_dma_free_chan_resources(struct dma_chan *c)
spin_unlock_bh(&ioat->desc_lock);

pci_pool_free(ioatdma_device->completion_pool,
chan->completion_virt,
chan->completion_addr);
chan->completion,
chan->completion_dma);

/* one is ok since we left it on there on purpose */
if (in_use_descs > 1)
dev_err(to_dev(chan), "Freeing %d in use descriptors!\n",
in_use_descs - 1);

chan->last_completion = chan->completion_addr = 0;
chan->last_completion = 0;
chan->completion_dma = 0;
chan->watchdog_completion = 0;
chan->last_compl_desc_addr_hw = 0;
chan->watchdog_tcp_cookie = chan->watchdog_last_tcp_cookie = 0;
Expand Down Expand Up @@ -709,25 +699,15 @@ void ioat_dma_unmap(struct ioat_chan_common *chan, enum dma_ctrl_flags flags,
unsigned long ioat_get_current_completion(struct ioat_chan_common *chan)
{
unsigned long phys_complete;
u64 completion;

/* The completion writeback can happen at any time,
so reads by the driver need to be atomic operations
The descriptor physical addresses are limited to 32-bits
when the CPU can only do a 32-bit mov */

#if (BITS_PER_LONG == 64)
phys_complete =
chan->completion_virt->full
& IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
#else
phys_complete = chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK;
#endif
completion = *chan->completion;
phys_complete = completion & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;

dev_dbg(to_dev(chan), "%s: phys_complete: %#llx\n", __func__,
(unsigned long long) phys_complete);

if ((chan->completion_virt->full
& IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
if ((completion & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
dev_err(to_dev(chan), "Channel halted, chanerr = %x\n",
readl(chan->reg_base + IOAT_CHANERR_OFFSET));
Expand All @@ -750,7 +730,7 @@ static void ioat1_cleanup(struct ioat_dma_chan *ioat)
dma_cookie_t cookie = 0;
struct dma_async_tx_descriptor *tx;

prefetch(chan->completion_virt);
prefetch(chan->completion);

if (!spin_trylock_bh(&chan->cleanup_lock))
return;
Expand Down
10 changes: 2 additions & 8 deletions drivers/dma/ioat/dma.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,14 +96,8 @@ struct ioat_chan_common {
struct ioatdma_device *device;
struct dma_chan common;

dma_addr_t completion_addr;
union {
u64 full; /* HW completion writeback */
struct {
u32 low;
u32 high;
};
} *completion_virt;
dma_addr_t completion_dma;
u64 *completion;
unsigned long last_compl_desc_addr_hw;
struct tasklet_struct cleanup_task;
};
Expand Down
25 changes: 11 additions & 14 deletions drivers/dma/ioat/dma_v2.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,8 +200,7 @@ static void ioat2_reset_channel(struct ioat2_dma_chan *ioat)
return;

chanerr = readl(chan->reg_base + IOAT_CHANERR_OFFSET);
chansts = (chan->completion_virt->low
& IOAT_CHANSTS_DMA_TRANSFER_STATUS);
chansts = *chan->completion & IOAT_CHANSTS_DMA_TRANSFER_STATUS;
if (chanerr) {
dev_err(to_dev(chan),
"chan%d, CHANSTS = 0x%08x CHANERR = 0x%04x, clearing\n",
Expand Down Expand Up @@ -281,7 +280,7 @@ static void ioat2_cleanup(struct ioat2_dma_chan *ioat)
int i;
struct dma_async_tx_descriptor *tx;

prefetch(chan->completion_virt);
prefetch(chan->completion);

spin_lock_bh(&chan->cleanup_lock);
phys_complete = ioat_get_current_completion(chan);
Expand Down Expand Up @@ -470,17 +469,15 @@ static int ioat2_alloc_chan_resources(struct dma_chan *c)

/* allocate a completion writeback area */
/* doing 2 32bit writes to mmio since 1 64b write doesn't work */
chan->completion_virt = pci_pool_alloc(chan->device->completion_pool,
GFP_KERNEL,
&chan->completion_addr);
if (!chan->completion_virt)
chan->completion = pci_pool_alloc(chan->device->completion_pool,
GFP_KERNEL, &chan->completion_dma);
if (!chan->completion)
return -ENOMEM;

memset(chan->completion_virt, 0,
sizeof(*chan->completion_virt));
writel(((u64) chan->completion_addr) & 0x00000000FFFFFFFF,
memset(chan->completion, 0, sizeof(*chan->completion));
writel(((u64) chan->completion_dma) & 0x00000000FFFFFFFF,
chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
writel(((u64) chan->completion_addr) >> 32,
writel(((u64) chan->completion_dma) >> 32,
chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);

ioat->alloc_order = ioat_get_alloc_order();
Expand Down Expand Up @@ -655,12 +652,12 @@ static void ioat2_free_chan_resources(struct dma_chan *c)
ioat->ring = NULL;
ioat->alloc_order = 0;
pci_pool_free(ioatdma_device->completion_pool,
chan->completion_virt,
chan->completion_addr);
chan->completion,
chan->completion_dma);
spin_unlock_bh(&ioat->ring_lock);

chan->last_completion = 0;
chan->completion_addr = 0;
chan->completion_dma = 0;
ioat->pending = 0;
ioat->dmacount = 0;
chan->watchdog_completion = 0;
Expand Down
8 changes: 4 additions & 4 deletions drivers/dma/ioat/registers.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,10 +94,10 @@
#define IOAT2_CHANSTS_OFFSET_HIGH 0x0C
#define IOAT_CHANSTS_OFFSET_HIGH(ver) ((ver) < IOAT_VER_2_0 \
? IOAT1_CHANSTS_OFFSET_HIGH : IOAT2_CHANSTS_OFFSET_HIGH)
#define IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR ~0x3F
#define IOAT_CHANSTS_SOFT_ERR 0x0000000000000010
#define IOAT_CHANSTS_UNAFFILIATED_ERR 0x0000000000000008
#define IOAT_CHANSTS_DMA_TRANSFER_STATUS 0x0000000000000007
#define IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR (~0x3fULL)
#define IOAT_CHANSTS_SOFT_ERR 0x10ULL
#define IOAT_CHANSTS_UNAFFILIATED_ERR 0x8ULL
#define IOAT_CHANSTS_DMA_TRANSFER_STATUS 0x7ULL
#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE 0x0
#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_DONE 0x1
#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_SUSPENDED 0x2
Expand Down

0 comments on commit 4fb9b9e

Please sign in to comment.