Skip to content

Commit

Permalink
RDMA/umem: Make ib_umem_odp into a sub structure of ib_umem
Browse files Browse the repository at this point in the history 
These two structures are linked together, use the container_of pattern
instead of a double allocation to make the code simpler and easier to
follow.

Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>
Signed-off-by: Leon Romanovsky <leonro@mellanox.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>
  • Loading branch information
Jason Gunthorpe authored and Doug Ledford committed Sep 21, 2018
1 parent b5231b0 commit 41b4dee
Show file tree
Hide file tree
Showing 4 changed files with 69 additions and 83 deletions.
36 changes: 22 additions & 14 deletions drivers/infiniband/core/umem.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,34 +108,39 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
if (!can_do_mlock())
return ERR_PTR(-EPERM);

umem = kzalloc(sizeof *umem, GFP_KERNEL);
if (!umem)
return ERR_PTR(-ENOMEM);
if (access & IB_ACCESS_ON_DEMAND) {
umem = kzalloc(sizeof(struct ib_umem_odp), GFP_KERNEL);
if (!umem)
return ERR_PTR(-ENOMEM);
umem->odp_data = to_ib_umem_odp(umem);
} else {
umem = kzalloc(sizeof(*umem), GFP_KERNEL);
if (!umem)
return ERR_PTR(-ENOMEM);
}

umem->context = context;
umem->length = size;
umem->address = addr;
umem->page_shift = PAGE_SHIFT;
umem->writable = ib_access_writable(access);
umem->owning_mm = mm = current->mm;
mmgrab(mm);

if (access & IB_ACCESS_ON_DEMAND) {
ret = ib_umem_odp_get(context, umem, access);
ret = ib_umem_odp_get(to_ib_umem_odp(umem), access);
if (ret)
goto umem_kfree;
return umem;
}

umem->owning_mm = mm = current->mm;
mmgrab(mm);
umem->odp_data = NULL;

/* We assume the memory is from hugetlb until proved otherwise */
umem->hugetlb = 1;

page_list = (struct page **) __get_free_page(GFP_KERNEL);
if (!page_list) {
ret = -ENOMEM;
goto umem_kfree_drop;
goto umem_kfree;
}

/*
Expand Down Expand Up @@ -226,20 +231,22 @@ struct ib_umem *ib_umem_get(struct ib_ucontext *context, unsigned long addr,
if (vma_list)
free_page((unsigned long) vma_list);
free_page((unsigned long) page_list);
umem_kfree_drop:
if (ret)
mmdrop(umem->owning_mm);
umem_kfree:
if (ret)
if (ret) {
mmdrop(umem->owning_mm);
kfree(umem);
}
return ret ? ERR_PTR(ret) : umem;
}
EXPORT_SYMBOL(ib_umem_get);

static void __ib_umem_release_tail(struct ib_umem *umem)
{
mmdrop(umem->owning_mm);
kfree(umem);
if (umem->odp_data)
kfree(to_ib_umem_odp(umem));
else
kfree(umem);
}

static void ib_umem_release_defer(struct work_struct *work)
Expand All @@ -263,6 +270,7 @@ void ib_umem_release(struct ib_umem *umem)

if (umem->odp_data) {
ib_umem_odp_release(to_ib_umem_odp(umem));
__ib_umem_release_tail(umem);
return;
}

Expand Down
79 changes: 30 additions & 49 deletions drivers/infiniband/core/umem_odp.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ static u64 node_start(struct umem_odp_node *n)
struct ib_umem_odp *umem_odp =
container_of(n, struct ib_umem_odp, interval_tree);

return ib_umem_start(umem_odp->umem);
return ib_umem_start(&umem_odp->umem);
}

/* Note that the representation of the intervals in the interval tree
Expand All @@ -71,7 +71,7 @@ static u64 node_last(struct umem_odp_node *n)
struct ib_umem_odp *umem_odp =
container_of(n, struct ib_umem_odp, interval_tree);

return ib_umem_end(umem_odp->umem) - 1;
return ib_umem_end(&umem_odp->umem) - 1;
}

INTERVAL_TREE_DEFINE(struct umem_odp_node, rb, u64, __subtree_last,
Expand Down Expand Up @@ -159,7 +159,7 @@ static void ib_ucontext_notifier_end_account(struct ib_ucontext *context)
static int ib_umem_notifier_release_trampoline(struct ib_umem_odp *umem_odp,
u64 start, u64 end, void *cookie)
{
struct ib_umem *umem = umem_odp->umem;
struct ib_umem *umem = &umem_odp->umem;

/*
* Increase the number of notifiers running, to
Expand Down Expand Up @@ -198,7 +198,7 @@ static int invalidate_page_trampoline(struct ib_umem_odp *item, u64 start,
u64 end, void *cookie)
{
ib_umem_notifier_start_account(item);
item->umem->context->invalidate_range(item, start, start + PAGE_SIZE);
item->umem.context->invalidate_range(item, start, start + PAGE_SIZE);
ib_umem_notifier_end_account(item);
return 0;
}
Expand All @@ -207,7 +207,7 @@ static int invalidate_range_start_trampoline(struct ib_umem_odp *item,
u64 start, u64 end, void *cookie)
{
ib_umem_notifier_start_account(item);
item->umem->context->invalidate_range(item, start, end);
item->umem.context->invalidate_range(item, start, end);
return 0;
}

Expand Down Expand Up @@ -277,28 +277,21 @@ static const struct mmu_notifier_ops ib_umem_notifiers = {
struct ib_umem_odp *ib_alloc_odp_umem(struct ib_ucontext *context,
unsigned long addr, size_t size)
{
struct ib_umem *umem;
struct ib_umem_odp *odp_data;
struct ib_umem *umem;
int pages = size >> PAGE_SHIFT;
int ret;

umem = kzalloc(sizeof(*umem), GFP_KERNEL);
if (!umem)
odp_data = kzalloc(sizeof(*odp_data), GFP_KERNEL);
if (!odp_data)
return ERR_PTR(-ENOMEM);

umem = &odp_data->umem;
umem->context = context;
umem->length = size;
umem->address = addr;
umem->page_shift = PAGE_SHIFT;
umem->writable = 1;

odp_data = kzalloc(sizeof(*odp_data), GFP_KERNEL);
if (!odp_data) {
ret = -ENOMEM;
goto out_umem;
}
odp_data->umem = umem;

mutex_init(&odp_data->umem_mutex);
init_completion(&odp_data->notifier_completion);

Expand Down Expand Up @@ -334,15 +327,14 @@ struct ib_umem_odp *ib_alloc_odp_umem(struct ib_ucontext *context,
vfree(odp_data->page_list);
out_odp_data:
kfree(odp_data);
out_umem:
kfree(umem);
return ERR_PTR(ret);
}
EXPORT_SYMBOL(ib_alloc_odp_umem);

int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem,
int access)
int ib_umem_odp_get(struct ib_umem_odp *umem_odp, int access)
{
struct ib_ucontext *context = umem_odp->umem.context;
struct ib_umem *umem = &umem_odp->umem;
int ret_val;
struct pid *our_pid;
struct mm_struct *mm = get_task_mm(current);
Expand Down Expand Up @@ -378,30 +370,23 @@ int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem,
goto out_mm;
}

umem->odp_data = kzalloc(sizeof(*umem->odp_data), GFP_KERNEL);
if (!umem->odp_data) {
ret_val = -ENOMEM;
goto out_mm;
}
umem->odp_data->umem = umem;

mutex_init(&umem->odp_data->umem_mutex);
mutex_init(&umem_odp->umem_mutex);

init_completion(&umem->odp_data->notifier_completion);
init_completion(&umem_odp->notifier_completion);

if (ib_umem_num_pages(umem)) {
umem->odp_data->page_list =
vzalloc(array_size(sizeof(*umem->odp_data->page_list),
umem_odp->page_list =
vzalloc(array_size(sizeof(*umem_odp->page_list),
ib_umem_num_pages(umem)));
if (!umem->odp_data->page_list) {
if (!umem_odp->page_list) {
ret_val = -ENOMEM;
goto out_odp_data;
goto out_mm;
}

umem->odp_data->dma_list =
vzalloc(array_size(sizeof(*umem->odp_data->dma_list),
umem_odp->dma_list =
vzalloc(array_size(sizeof(*umem_odp->dma_list),
ib_umem_num_pages(umem)));
if (!umem->odp_data->dma_list) {
if (!umem_odp->dma_list) {
ret_val = -ENOMEM;
goto out_page_list;
}
Expand All @@ -415,13 +400,13 @@ int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem,
down_write(&context->umem_rwsem);
context->odp_mrs_count++;
if (likely(ib_umem_start(umem) != ib_umem_end(umem)))
rbt_ib_umem_insert(&umem->odp_data->interval_tree,
rbt_ib_umem_insert(&umem_odp->interval_tree,
&context->umem_tree);
if (likely(!atomic_read(&context->notifier_count)) ||
context->odp_mrs_count == 1)
umem->odp_data->mn_counters_active = true;
umem_odp->mn_counters_active = true;
else
list_add(&umem->odp_data->no_private_counters,
list_add(&umem_odp->no_private_counters,
&context->no_private_counters);
downgrade_write(&context->umem_rwsem);

Expand Down Expand Up @@ -454,19 +439,17 @@ int ib_umem_odp_get(struct ib_ucontext *context, struct ib_umem *umem,

out_mutex:
up_read(&context->umem_rwsem);
vfree(umem->odp_data->dma_list);
vfree(umem_odp->dma_list);
out_page_list:
vfree(umem->odp_data->page_list);
out_odp_data:
kfree(umem->odp_data);
vfree(umem_odp->page_list);
out_mm:
mmput(mm);
return ret_val;
}

void ib_umem_odp_release(struct ib_umem_odp *umem_odp)
{
struct ib_umem *umem = umem_odp->umem;
struct ib_umem *umem = &umem_odp->umem;
struct ib_ucontext *context = umem->context;

/*
Expand Down Expand Up @@ -528,8 +511,6 @@ void ib_umem_odp_release(struct ib_umem_odp *umem_odp)

vfree(umem_odp->dma_list);
vfree(umem_odp->page_list);
kfree(umem_odp);
kfree(umem);
}

/*
Expand Down Expand Up @@ -557,7 +538,7 @@ static int ib_umem_odp_map_dma_single_page(
u64 access_mask,
unsigned long current_seq)
{
struct ib_umem *umem = umem_odp->umem;
struct ib_umem *umem = &umem_odp->umem;
struct ib_device *dev = umem->context->device;
dma_addr_t dma_addr;
int stored_page = 0;
Expand Down Expand Up @@ -643,7 +624,7 @@ int ib_umem_odp_map_dma_pages(struct ib_umem_odp *umem_odp, u64 user_virt,
u64 bcnt, u64 access_mask,
unsigned long current_seq)
{
struct ib_umem *umem = umem_odp->umem;
struct ib_umem *umem = &umem_odp->umem;
struct task_struct *owning_process = NULL;
struct mm_struct *owning_mm = NULL;
struct page **local_page_list = NULL;
Expand Down Expand Up @@ -759,7 +740,7 @@ EXPORT_SYMBOL(ib_umem_odp_map_dma_pages);
void ib_umem_odp_unmap_dma_pages(struct ib_umem_odp *umem_odp, u64 virt,
u64 bound)
{
struct ib_umem *umem = umem_odp->umem;
struct ib_umem *umem = &umem_odp->umem;
int idx;
u64 addr;
struct ib_device *dev = umem->context->device;
Expand Down
Loading

0 comments on commit 41b4dee

Please sign in to comment.