Skip to content

Commit

Permalink
swiotlb: refactor swiotlb_tbl_map_single
Browse files Browse the repository at this point in the history 
Split out a bunch of a self-contained helpers to make the function easier
to follow.

Signed-off-by: Christoph Hellwig <hch@lst.de>
Acked-by: Jianxiong Gao <jxgao@google.com>
Tested-by: Jianxiong Gao <jxgao@google.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
  • Loading branch information
Christoph Hellwig authored and Konrad Rzeszutek Wilk committed Feb 22, 2021
1 parent ca10d0f commit 26a7e09
Showing 1 changed file with 89 additions and 90 deletions.
179 changes: 89 additions & 90 deletions kernel/dma/swiotlb.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -468,134 +468,133 @@ static void swiotlb_bounce(phys_addr_t orig_addr, phys_addr_t tlb_addr,
}
}

phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t orig_addr,
size_t mapping_size, size_t alloc_size,
enum dma_data_direction dir, unsigned long attrs)
{
dma_addr_t tbl_dma_addr = phys_to_dma_unencrypted(hwdev, io_tlb_start);
unsigned long flags;
phys_addr_t tlb_addr;
unsigned int nslots, stride, index, wrap;
int i;
unsigned long mask;
unsigned long offset_slots;
unsigned long max_slots;
unsigned long tmp_io_tlb_used;

if (no_iotlb_memory)
panic("Can not allocate SWIOTLB buffer earlier and can't now provide you with the DMA bounce buffer");

if (mem_encrypt_active())
pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n");
#define slot_addr(start, idx) ((start) + ((idx) << IO_TLB_SHIFT))

if (mapping_size > alloc_size) {
dev_warn_once(hwdev, "Invalid sizes (mapping: %zd bytes, alloc: %zd bytes)",
mapping_size, alloc_size);
return (phys_addr_t)DMA_MAPPING_ERROR;
}

mask = dma_get_seg_boundary(hwdev);
/*
* Carefully handle integer overflow which can occur when boundary_mask == ~0UL.
*/
static inline unsigned long get_max_slots(unsigned long boundary_mask)
{
if (boundary_mask == ~0UL)
return 1UL << (BITS_PER_LONG - IO_TLB_SHIFT);
return nr_slots(boundary_mask + 1);
}

tbl_dma_addr &= mask;
static unsigned int wrap_index(unsigned int index)
{
if (index >= io_tlb_nslabs)
return 0;
return index;
}

offset_slots = nr_slots(tbl_dma_addr);
/*
* Find a suitable number of IO TLB entries size that will fit this request and
* allocate a buffer from that IO TLB pool.
*/
static int find_slots(struct device *dev, size_t alloc_size)
{
unsigned long boundary_mask = dma_get_seg_boundary(dev);
dma_addr_t tbl_dma_addr =
phys_to_dma_unencrypted(dev, io_tlb_start) & boundary_mask;
unsigned long max_slots = get_max_slots(boundary_mask);
unsigned int nslots = nr_slots(alloc_size), stride = 1;
unsigned int index, wrap, count = 0, i;
unsigned long flags;

/*
* Carefully handle integer overflow which can occur when mask == ~0UL.
*/
max_slots = mask + 1
? nr_slots(mask + 1)
: 1UL << (BITS_PER_LONG - IO_TLB_SHIFT);
BUG_ON(!nslots);

/*
* For mappings greater than or equal to a page, we limit the stride
* (and hence alignment) to a page size.
*/
nslots = nr_slots(alloc_size);
if (alloc_size >= PAGE_SIZE)
stride = (1 << (PAGE_SHIFT - IO_TLB_SHIFT));
else
stride = 1;
stride <<= (PAGE_SHIFT - IO_TLB_SHIFT);

BUG_ON(!nslots);

/*
* Find suitable number of IO TLB entries size that will fit this
* request and allocate a buffer from that IO TLB pool.
*/
spin_lock_irqsave(&io_tlb_lock, flags);

if (unlikely(nslots > io_tlb_nslabs - io_tlb_used))
goto not_found;

index = ALIGN(io_tlb_index, stride);
if (index >= io_tlb_nslabs)
index = 0;
wrap = index;

index = wrap = wrap_index(ALIGN(io_tlb_index, stride));
do {
while (iommu_is_span_boundary(index, nslots, offset_slots,
max_slots)) {
index += stride;
if (index >= io_tlb_nslabs)
index = 0;
if (index == wrap)
goto not_found;
}

/*
* If we find a slot that indicates we have 'nslots' number of
* contiguous buffers, we allocate the buffers from that slot
* and mark the entries as '0' indicating unavailable.
*/
if (io_tlb_list[index] >= nslots) {
int count = 0;

for (i = index; i < (int) (index + nslots); i++)
io_tlb_list[i] = 0;
for (i = index - 1;
io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 &&
io_tlb_list[i]; i--)
io_tlb_list[i] = ++count;
tlb_addr = io_tlb_start + (index << IO_TLB_SHIFT);

/*
* Update the indices to avoid searching in the next
* round.
*/
io_tlb_index = ((index + nslots) < io_tlb_nslabs
? (index + nslots) : 0);

goto found;
if (!iommu_is_span_boundary(index, nslots,
nr_slots(tbl_dma_addr),
max_slots)) {
if (io_tlb_list[index] >= nslots)
goto found;
}
index += stride;
if (index >= io_tlb_nslabs)
index = 0;
index = wrap_index(index + stride);
} while (index != wrap);

not_found:
tmp_io_tlb_used = io_tlb_used;

spin_unlock_irqrestore(&io_tlb_lock, flags);
if (!(attrs & DMA_ATTR_NO_WARN) && printk_ratelimit())
dev_warn(hwdev, "swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",
alloc_size, io_tlb_nslabs, tmp_io_tlb_used);
return (phys_addr_t)DMA_MAPPING_ERROR;
return -1;

found:
for (i = index; i < index + nslots; i++)
io_tlb_list[i] = 0;
for (i = index - 1;
io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 &&
io_tlb_list[i]; i--)
io_tlb_list[i] = ++count;

/*
* Update the indices to avoid searching in the next round.
*/
if (index + nslots < io_tlb_nslabs)
io_tlb_index = index + nslots;
else
io_tlb_index = 0;
io_tlb_used += nslots;

spin_unlock_irqrestore(&io_tlb_lock, flags);
return index;
}

phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
size_t mapping_size, size_t alloc_size,
enum dma_data_direction dir, unsigned long attrs)
{
unsigned int index, i;
phys_addr_t tlb_addr;

if (no_iotlb_memory)
panic("Can not allocate SWIOTLB buffer earlier and can't now provide you with the DMA bounce buffer");

if (mem_encrypt_active())
pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n");

if (mapping_size > alloc_size) {
dev_warn_once(dev, "Invalid sizes (mapping: %zd bytes, alloc: %zd bytes)",
mapping_size, alloc_size);
return (phys_addr_t)DMA_MAPPING_ERROR;
}

index = find_slots(dev, alloc_size);
if (index == -1) {
if (!(attrs & DMA_ATTR_NO_WARN))
dev_warn_ratelimited(dev,
"swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",
alloc_size, io_tlb_nslabs, io_tlb_used);
return (phys_addr_t)DMA_MAPPING_ERROR;
}

/*
* Save away the mapping from the original address to the DMA address.
* This is needed when we sync the memory. Then we sync the buffer if
* needed.
*/
for (i = 0; i < nslots; i++)
io_tlb_orig_addr[index+i] = orig_addr + (i << IO_TLB_SHIFT);
for (i = 0; i < nr_slots(alloc_size); i++)
io_tlb_orig_addr[index + i] = slot_addr(orig_addr, i);

tlb_addr = slot_addr(io_tlb_start, index);
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
swiotlb_bounce(orig_addr, tlb_addr, mapping_size, DMA_TO_DEVICE);

return tlb_addr;
}

Expand Down

0 comments on commit 26a7e09

Please sign in to comment.