Skip to content

Commit

Permalink
swiotlb: Convert io_default_tlb_mem to static allocation
Browse files Browse the repository at this point in the history 
Since commit 69031f5 ("swiotlb: Set dev->dma_io_tlb_mem to the
swiotlb pool used"), 'struct device' may hold a copy of the global
'io_default_tlb_mem' pointer if the device is using swiotlb for DMA. A
subsequent call to swiotlb_exit() will therefore leave dangling pointers
behind in these device structures, resulting in KASAN splats such as:

  |  BUG: KASAN: use-after-free in __iommu_dma_unmap_swiotlb+0x64/0xb0
  |  Read of size 8 at addr ffff8881d7830000 by task swapper/0/0
  |
  |  CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.12.0-rc3-debug #1
  |  Hardware name: HP HP Desktop M01-F1xxx/87D6, BIOS F.12 12/17/2020
  |  Call Trace:
  |   <IRQ>
  |   dump_stack+0x9c/0xcf
  |   print_address_description.constprop.0+0x18/0x130
  |   kasan_report.cold+0x7f/0x111
  |   __iommu_dma_unmap_swiotlb+0x64/0xb0
  |   nvme_pci_complete_rq+0x73/0x130
  |   blk_complete_reqs+0x6f/0x80
  |   __do_softirq+0xfc/0x3be

Convert 'io_default_tlb_mem' to a static structure, so that the
per-device pointers remain valid after swiotlb_exit() has been invoked.
All users are updated to reference the static structure directly, using
the 'nslabs' field to determine whether swiotlb has been initialised.
The 'slots' array is still allocated dynamically and referenced via a
pointer rather than a flexible array member.

Cc: Claire Chang <tientzu@chromium.org>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Robin Murphy <robin.murphy@arm.com>
Cc: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
Fixes: 69031f5 ("swiotlb: Set dev->dma_io_tlb_mem to the swiotlb pool used")
Reported-by: Nathan Chancellor <nathan@kernel.org>
Tested-by: Nathan Chancellor <nathan@kernel.org>
Tested-by: Claire Chang <tientzu@chromium.org>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Will Deacon <will@kernel.org>
Signed-off-by: Konrad Rzeszutek Wilk <konrad@kernel.org>
  • Loading branch information
Will Deacon authored and Konrad Rzeszutek Wilk committed Jul 24, 2021
1 parent 85044eb commit 463e862
Show file tree
Hide file tree
Showing 4 changed files with 41 additions and 35 deletions.
2 changes: 1 addition & 1 deletion drivers/base/core.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2848,7 +2848,7 @@ void device_initialize(struct device *dev)
dev->dma_coherent = dma_default_coherent;
#endif
#ifdef CONFIG_SWIOTLB
dev->dma_io_tlb_mem = io_tlb_default_mem;
dev->dma_io_tlb_mem = &io_tlb_default_mem;
#endif
}
EXPORT_SYMBOL_GPL(device_initialize);
Expand Down
4 changes: 2 additions & 2 deletions drivers/xen/swiotlb-xen.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ int __ref xen_swiotlb_init(void)
int rc = -ENOMEM;
char *start;

if (io_tlb_default_mem != NULL) {
if (io_tlb_default_mem.nslabs) {
pr_warn("swiotlb buffer already initialized\n");
return -EEXIST;
}
Expand Down Expand Up @@ -547,7 +547,7 @@ xen_swiotlb_sync_sg_for_device(struct device *dev, struct scatterlist *sgl,
static int
xen_swiotlb_dma_supported(struct device *hwdev, u64 mask)
{
return xen_phys_to_dma(hwdev, io_tlb_default_mem->end - 1) <= mask;
return xen_phys_to_dma(hwdev, io_tlb_default_mem.end - 1) <= mask;
}

const struct dma_map_ops xen_swiotlb_dma_ops = {
Expand Down
4 changes: 2 additions & 2 deletions include/linux/swiotlb.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,9 +103,9 @@ struct io_tlb_mem {
phys_addr_t orig_addr;
size_t alloc_size;
unsigned int list;
} slots[];
} *slots;
};
extern struct io_tlb_mem *io_tlb_default_mem;
extern struct io_tlb_mem io_tlb_default_mem;

static inline bool is_swiotlb_buffer(struct device *dev, phys_addr_t paddr)
{
Expand Down
66 changes: 36 additions & 30 deletions kernel/dma/swiotlb.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@

enum swiotlb_force swiotlb_force;

struct io_tlb_mem *io_tlb_default_mem;
struct io_tlb_mem io_tlb_default_mem;

/*
* Max segment that we can provide which (if pages are contingous) will
Expand Down Expand Up @@ -101,7 +101,7 @@ early_param("swiotlb", setup_io_tlb_npages);

unsigned int swiotlb_max_segment(void)
{
return io_tlb_default_mem ? max_segment : 0;
return io_tlb_default_mem.nslabs ? max_segment : 0;
}
EXPORT_SYMBOL_GPL(swiotlb_max_segment);

Expand Down Expand Up @@ -134,9 +134,9 @@ void __init swiotlb_adjust_size(unsigned long size)

void swiotlb_print_info(void)
{
struct io_tlb_mem *mem = io_tlb_default_mem;
struct io_tlb_mem *mem = &io_tlb_default_mem;

if (!mem) {
if (!mem->nslabs) {
pr_warn("No low mem\n");
return;
}
Expand All @@ -163,11 +163,11 @@ static inline unsigned long nr_slots(u64 val)
*/
void __init swiotlb_update_mem_attributes(void)
{
struct io_tlb_mem *mem = io_tlb_default_mem;
struct io_tlb_mem *mem = &io_tlb_default_mem;
void *vaddr;
unsigned long bytes;

if (!mem || mem->late_alloc)
if (!mem->nslabs || mem->late_alloc)
return;
vaddr = phys_to_virt(mem->start);
bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT);
Expand Down Expand Up @@ -201,25 +201,24 @@ static void swiotlb_init_io_tlb_mem(struct io_tlb_mem *mem, phys_addr_t start,

int __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose)
{
struct io_tlb_mem *mem;
struct io_tlb_mem *mem = &io_tlb_default_mem;
size_t alloc_size;

if (swiotlb_force == SWIOTLB_NO_FORCE)
return 0;

/* protect against double initialization */
if (WARN_ON_ONCE(io_tlb_default_mem))
if (WARN_ON_ONCE(mem->nslabs))
return -ENOMEM;

alloc_size = PAGE_ALIGN(struct_size(mem, slots, nslabs));
mem = memblock_alloc(alloc_size, PAGE_SIZE);
if (!mem)
alloc_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), nslabs));
mem->slots = memblock_alloc(alloc_size, PAGE_SIZE);
if (!mem->slots)
panic("%s: Failed to allocate %zu bytes align=0x%lx\n",
__func__, alloc_size, PAGE_SIZE);

swiotlb_init_io_tlb_mem(mem, __pa(tlb), nslabs, false);

io_tlb_default_mem = mem;
if (verbose)
swiotlb_print_info();
swiotlb_set_max_segment(mem->nslabs << IO_TLB_SHIFT);
Expand Down Expand Up @@ -304,45 +303,43 @@ swiotlb_late_init_with_default_size(size_t default_size)
int
swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)
{
struct io_tlb_mem *mem;
struct io_tlb_mem *mem = &io_tlb_default_mem;
unsigned long bytes = nslabs << IO_TLB_SHIFT;

if (swiotlb_force == SWIOTLB_NO_FORCE)
return 0;

/* protect against double initialization */
if (WARN_ON_ONCE(io_tlb_default_mem))
if (WARN_ON_ONCE(mem->nslabs))
return -ENOMEM;

mem = (void *)__get_free_pages(GFP_KERNEL,
get_order(struct_size(mem, slots, nslabs)));
if (!mem)
mem->slots = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
get_order(array_size(sizeof(*mem->slots), nslabs)));
if (!mem->slots)
return -ENOMEM;

memset(mem, 0, sizeof(*mem));
set_memory_decrypted((unsigned long)tlb, bytes >> PAGE_SHIFT);
swiotlb_init_io_tlb_mem(mem, virt_to_phys(tlb), nslabs, true);

io_tlb_default_mem = mem;
swiotlb_print_info();
swiotlb_set_max_segment(mem->nslabs << IO_TLB_SHIFT);
return 0;
}

void __init swiotlb_exit(void)
{
struct io_tlb_mem *mem = io_tlb_default_mem;
size_t size;
struct io_tlb_mem *mem = &io_tlb_default_mem;

if (!mem)
if (!mem->nslabs)
return;

size = struct_size(mem, slots, mem->nslabs);
size = array_size(sizeof(*mem->slots), mem->nslabs);
if (mem->late_alloc)
free_pages((unsigned long)mem, get_order(size));
free_pages((unsigned long)mem->slots, get_order(size));
else
memblock_free_late(__pa(mem), PAGE_ALIGN(size));
io_tlb_default_mem = NULL;
memblock_free_late(__pa(mem->slots), PAGE_ALIGN(size));
memset(mem, 0, sizeof(*mem));
}

/*
Expand Down Expand Up @@ -696,7 +693,9 @@ size_t swiotlb_max_mapping_size(struct device *dev)

bool is_swiotlb_active(struct device *dev)
{
return dev->dma_io_tlb_mem != NULL;
struct io_tlb_mem *mem = dev->dma_io_tlb_mem;

return mem && mem->nslabs;
}
EXPORT_SYMBOL_GPL(is_swiotlb_active);

Expand All @@ -711,10 +710,10 @@ static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem)

static int __init swiotlb_create_default_debugfs(void)
{
struct io_tlb_mem *mem = io_tlb_default_mem;
struct io_tlb_mem *mem = &io_tlb_default_mem;

debugfs_dir = debugfs_create_dir("swiotlb", NULL);
if (mem) {
if (mem->nslabs) {
mem->debugfs = debugfs_dir;
swiotlb_create_debugfs_files(mem);
}
Expand Down Expand Up @@ -783,10 +782,17 @@ static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
* to it.
*/
if (!mem) {
mem = kzalloc(struct_size(mem, slots, nslabs), GFP_KERNEL);
mem = kzalloc(sizeof(*mem), GFP_KERNEL);
if (!mem)
return -ENOMEM;

mem->slots = kzalloc(array_size(sizeof(*mem->slots), nslabs),
GFP_KERNEL);
if (!mem->slots) {
kfree(mem);
return -ENOMEM;
}

set_memory_decrypted((unsigned long)phys_to_virt(rmem->base),
rmem->size >> PAGE_SHIFT);
swiotlb_init_io_tlb_mem(mem, rmem->base, nslabs, false);
Expand All @@ -806,7 +812,7 @@ static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
static void rmem_swiotlb_device_release(struct reserved_mem *rmem,
struct device *dev)
{
dev->dma_io_tlb_mem = io_tlb_default_mem;
dev->dma_io_tlb_mem = &io_tlb_default_mem;
}

static const struct reserved_mem_ops rmem_swiotlb_ops = {
Expand Down

0 comments on commit 463e862

Please sign in to comment.