diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c index 0767559947b5f..cda61f7924d1e 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c @@ -272,7 +272,7 @@ static void amdgpu_mn_invalidate_node(struct amdgpu_mn_node *node, list_for_each_entry(bo, &node->bos, mn_list) { - if (!amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm, start, end)) + if (!amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm, start, end, NULL)) continue; r = dma_resv_wait_timeout_rcu(amdkcl_ttm_resvp(&bo->tbo), @@ -361,7 +361,7 @@ static int amdgpu_mn_invalidate_range_start_hsa(struct mmu_notifier *mn, if (amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm, range->start, - end)) + end, NULL)) amdgpu_amdkfd_evict_userptr(mem, range->mm); } } @@ -474,7 +474,7 @@ static void amdgpu_mn_invalidate_range_start_hsa(struct mmu_notifier *mn, struct kgd_mem *mem = bo->kfd_bo; if (amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm, - start, end)) + start, end, NULL)) amdgpu_amdkfd_evict_userptr(mem, mm); } } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c index 8bae747023516..eb732cc94016d 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c @@ -1456,7 +1456,7 @@ struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm) * */ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start, - unsigned long end) + unsigned long end, unsigned long *userptr) { struct amdgpu_ttm_tt *gtt = (void *)ttm; unsigned long size; @@ -1471,6 +1471,8 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start, if (gtt->userptr > end || gtt->userptr + size <= start) return false; + if (userptr) + *userptr = gtt->userptr; return true; } @@ -1494,7 +1496,7 @@ bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm) * */ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start, - unsigned long end) + unsigned long end, unsigned long *userptr) { struct amdgpu_ttm_tt *gtt = (void *)ttm; struct amdgpu_ttm_gup_task_list *entry; @@ -1524,6 +1526,8 @@ bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start, atomic_inc(>t->mmu_invalidations); + if (userptr) + *userptr = gtt->userptr; return true; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h index ff9989ed0d7e1..b8062f60eae68 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.h @@ -193,7 +193,7 @@ int amdgpu_ttm_tt_set_userptr(struct ttm_buffer_object *bo, bool amdgpu_ttm_tt_has_userptr(struct ttm_tt *ttm); struct mm_struct *amdgpu_ttm_tt_get_usermm(struct ttm_tt *ttm); bool amdgpu_ttm_tt_affect_userptr(struct ttm_tt *ttm, unsigned long start, - unsigned long end); + unsigned long end, unsigned long *userptr); #ifdef HAVE_AMDKCL_HMM_MIRROR_ENABLED bool amdgpu_ttm_tt_is_userptr(struct ttm_tt *ttm); #else diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c index c89eaa870bce3..4192879e67bad 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_svm.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_svm.c @@ -50,7 +50,9 @@ static bool svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni, const struct mmu_notifier_range *range, unsigned long cur_seq); - +static int +svm_range_check_vm(struct kfd_process *p, uint64_t start, uint64_t last, + uint64_t *bo_s, uint64_t *bo_l); static const struct mmu_interval_notifier_ops svm_range_mn_ops = { .invalidate = svm_range_cpu_invalidate_pagetables, }; @@ -2304,6 +2306,7 @@ svm_range_best_restore_location(struct svm_range *prange, return -1; } + static int svm_range_get_range_boundaries(struct kfd_process *p, int64_t addr, unsigned long *start, unsigned long *last) @@ -2351,8 +2354,59 @@ svm_range_get_range_boundaries(struct kfd_process *p, int64_t addr, vma->vm_end >> PAGE_SHIFT, *last); return 0; +} + +static int +svm_range_check_vm_userptr(struct kfd_process *p, uint64_t start, uint64_t last, + uint64_t *bo_s, uint64_t *bo_l) +{ + struct amdgpu_bo_va_mapping *mapping; + struct interval_tree_node *node; + struct amdgpu_bo *bo = NULL; + unsigned long userptr; + uint32_t i; + int r; + for (i = 0; i < p->n_pdds; i++) { + struct amdgpu_vm *vm; + + if (!p->pdds[i]->drm_priv) + continue; + + vm = drm_priv_to_vm(p->pdds[i]->drm_priv); + r = amdgpu_bo_reserve(vm->root.bo, false); + if (r) + return r; + + /* Check userptr by searching entire vm->va interval tree */ + node = interval_tree_iter_first(&vm->va, 0, ~0ULL); + while (node) { + mapping = container_of((struct rb_node *)node, + struct amdgpu_bo_va_mapping, rb); + bo = mapping->bo_va->base.bo; + + if (!amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm, + start << PAGE_SHIFT, + last << PAGE_SHIFT, + &userptr)) { + node = interval_tree_iter_next(node, 0, ~0ULL); + continue; + } + + pr_debug("[0x%llx 0x%llx] already userptr mapped\n", + start, last); + if (bo_s && bo_l) { + *bo_s = userptr >> PAGE_SHIFT; + *bo_l = *bo_s + bo->tbo.ttm->num_pages - 1; + } + amdgpu_bo_unreserve(vm->root.bo); + return -EADDRINUSE; + } + amdgpu_bo_unreserve(vm->root.bo); + } + return 0; } + static struct svm_range *svm_range_create_unregistered_range(struct amdgpu_device *adev, struct kfd_process *p, @@ -2362,10 +2416,26 @@ svm_range *svm_range_create_unregistered_range(struct amdgpu_device *adev, struct svm_range *prange = NULL; unsigned long start, last; uint32_t gpuid, gpuidx; + uint64_t bo_s = 0; + uint64_t bo_l = 0; + int r; if (svm_range_get_range_boundaries(p, addr, &start, &last)) return NULL; + r = svm_range_check_vm(p, start, last, &bo_s, &bo_l); + if (r != -EADDRINUSE) + r = svm_range_check_vm_userptr(p, start, last, &bo_s, &bo_l); + + if (r == -EADDRINUSE) { + if (addr >= bo_s && addr <= bo_l) + return NULL; + + /* Create one page svm range if 2MB range overlapping */ + start = addr; + last = addr; + } + prange = svm_range_new(&p->svms, start, last); if (!prange) { pr_debug("Failed to create prange in address [0x%llx]\n", addr); @@ -2668,6 +2738,8 @@ int svm_range_list_init(struct kfd_process *p) * @p: current kfd_process * @start: range start address, in pages * @last: range last address, in pages + * @bo_s: mapping start address in pages if address range already mapped + * @bo_l: mapping last address in pages if address range already mapped * * The purpose is to avoid virtual address ranges already allocated by * kfd_ioctl_alloc_memory_of_gpu ioctl. @@ -2682,8 +2754,11 @@ int svm_range_list_init(struct kfd_process *p) * a signal. Release all buffer reservations and return to user-space. */ static int -svm_range_check_vm(struct kfd_process *p, uint64_t start, uint64_t last) +svm_range_check_vm(struct kfd_process *p, uint64_t start, uint64_t last, + uint64_t *bo_s, uint64_t *bo_l) { + struct amdgpu_bo_va_mapping *mapping; + struct interval_tree_node *node; uint32_t i; int r; @@ -2697,8 +2772,17 @@ svm_range_check_vm(struct kfd_process *p, uint64_t start, uint64_t last) r = amdgpu_bo_reserve(vm->root.bo, false); if (r) return r; - if (interval_tree_iter_first(&vm->va, start, last)) { - pr_debug("Range [0x%llx 0x%llx] already mapped\n", start, last); + + node = interval_tree_iter_first(&vm->va, start, last); + if (node) { + pr_debug("range [0x%llx 0x%llx] already TTM mapped\n", + start, last); + mapping = container_of((struct rb_node *)node, + struct amdgpu_bo_va_mapping, rb); + if (bo_s && bo_l) { + *bo_s = mapping->start; + *bo_l = mapping->last; + } amdgpu_bo_unreserve(vm->root.bo); return -EADDRINUSE; } @@ -2739,7 +2823,8 @@ svm_range_is_valid(struct kfd_process *p, uint64_t start, uint64_t size) start = min(end, vma->vm_end); } while (start < end); - return svm_range_check_vm(p, start_unchg, (end - 1) >> PAGE_SHIFT); + return svm_range_check_vm(p, start_unchg, (end - 1) >> PAGE_SHIFT, NULL, + NULL); } /**