Skip to content

Commit

Permalink
drm/xe: Port Xe to GPUVA
Browse files Browse the repository at this point in the history
Rather than open coding VM binds and VMA tracking, use the GPUVA
library. GPUVA provides a common infrastructure for VM binds to use mmap
/ munmap semantics and support for VK sparse bindings.

The concepts are:

1) xe_vm inherits from drm_gpuva_manager
2) xe_vma inherits from drm_gpuva
3) xe_vma_op inherits from drm_gpuva_op
4) VM bind operations (MAP, UNMAP, PREFETCH, UNMAP_ALL) call into the
GPUVA code to generate an VMA operations list which is parsed, committed,
and executed.

v2 (CI): Add break after default in case statement.
v3: Rebase
v4: Fix some error handling
v5: Use unlocked version VMA in error paths
v6: Rebase, address some review feedback mainly Thomas H
v7: Fix compile error in xe_vma_op_unwind, address checkpatch

Signed-off-by: Matthew Brost <matthew.brost@intel.com>
Reviewed-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
Signed-off-by: Rodrigo Vivi <rodrigo.vivi@intel.com>
  • Loading branch information
Matthew Brost authored and Rodrigo Vivi committed Dec 21, 2023
1 parent 5cecdd0 commit b06d47b
Show file tree
Hide file tree
Showing 14 changed files with 1,175 additions and 1,177 deletions.
1 change: 1 addition & 0 deletions drivers/gpu/drm/xe/Kconfig
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ config DRM_XE
select VMAP_PFN
select DRM_TTM
select DRM_TTM_HELPER
select DRM_GPUVM
select DRM_SCHED
select MMU_NOTIFIER
select WANT_DEV_COREDUMP
Expand Down
2 changes: 1 addition & 1 deletion drivers/gpu/drm/xe/tests/xe_migrate.c
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,7 @@ static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test)
/* First part of the test, are we updating our pagetable bo with a new entry? */
xe_map_wr(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1), u64,
0xdeaddeadbeefbeef);
expected = xe_pte_encode(NULL, pt, 0, XE_CACHE_WB, 0, 0);
expected = xe_pte_encode(NULL, pt, 0, XE_CACHE_WB, 0);
if (m->eng->vm->flags & XE_VM_FLAGS_64K)
expected |= XE_PTE_PS64;
if (xe_bo_is_vram(pt))
Expand Down
31 changes: 21 additions & 10 deletions drivers/gpu/drm/xe/xe_bo.c
Original file line number Diff line number Diff line change
Expand Up @@ -412,7 +412,9 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
{
struct dma_resv_iter cursor;
struct dma_fence *fence;
struct xe_vma *vma;
struct drm_gpuva *gpuva;
struct drm_gem_object *obj = &bo->ttm.base;
struct drm_gpuvm_bo *vm_bo;
int ret = 0;

dma_resv_assert_held(bo->ttm.base.resv);
Expand All @@ -425,10 +427,12 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
dma_resv_iter_end(&cursor);
}

list_for_each_entry(vma, &bo->vmas, bo_link) {
struct xe_vm *vm = xe_vma_vm(vma);
drm_gem_for_each_gpuvm_bo(vm_bo, obj) {
drm_gpuvm_bo_for_each_va(gpuva, vm_bo) {
struct xe_vma *vma = gpuva_to_vma(gpuva);
struct xe_vm *vm = xe_vma_vm(vma);

trace_xe_vma_evict(vma);
trace_xe_vma_evict(vma);

if (xe_vm_in_fault_mode(vm)) {
/* Wait for pending binds / unbinds. */
Expand All @@ -454,17 +458,16 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,

} else {
bool vm_resv_locked = false;
struct xe_vm *vm = xe_vma_vm(vma);

/*
* We need to put the vma on the vm's rebind_list,
* but need the vm resv to do so. If we can't verify
* that we indeed have it locked, put the vma an the
* vm's notifier.rebind_list instead and scoop later.
*/
if (dma_resv_trylock(&vm->resv))
if (dma_resv_trylock(xe_vm_resv(vm)))
vm_resv_locked = true;
else if (ctx->resv != &vm->resv) {
else if (ctx->resv != xe_vm_resv(vm)) {
spin_lock(&vm->notifier.list_lock);
list_move_tail(&vma->notifier.rebind_link,
&vm->notifier.rebind_list);
Expand All @@ -477,7 +480,8 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo,
list_add_tail(&vma->rebind_link, &vm->rebind_list);

if (vm_resv_locked)
dma_resv_unlock(&vm->resv);
dma_resv_unlock(xe_vm_resv(vm));
}
}
}

Expand Down Expand Up @@ -1285,14 +1289,21 @@ xe_bo_create_locked_range(struct xe_device *xe,
}
}

bo = __xe_bo_create_locked(xe, bo, tile, vm ? &vm->resv : NULL,
bo = __xe_bo_create_locked(xe, bo, tile, vm ? xe_vm_resv(vm) : NULL,
vm && !xe_vm_in_fault_mode(vm) &&
flags & XE_BO_CREATE_USER_BIT ?
&vm->lru_bulk_move : NULL, size,
type, flags);
if (IS_ERR(bo))
return bo;

/*
* Note that instead of taking a reference no the drm_gpuvm_resv_bo(),
* to ensure the shared resv doesn't disappear under the bo, the bo
* will keep a reference to the vm, and avoid circular references
* by having all the vm's bo refereferences released at vm close
* time.
*/
if (vm && xe_bo_is_user(bo))
xe_vm_get(vm);
bo->vm = vm;
Expand Down Expand Up @@ -1600,7 +1611,7 @@ int xe_bo_validate(struct xe_bo *bo, struct xe_vm *vm, bool allow_res_evict)
xe_vm_assert_held(vm);

ctx.allow_res_evict = allow_res_evict;
ctx.resv = &vm->resv;
ctx.resv = xe_vm_resv(vm);
}

return ttm_bo_validate(&bo->ttm, &bo->placement, &ctx);
Expand Down
11 changes: 10 additions & 1 deletion drivers/gpu/drm/xe/xe_bo.h
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,15 @@
#include "xe_bo_types.h"
#include "xe_macros.h"
#include "xe_vm_types.h"
#include "xe_vm.h"

/**
* xe_vm_assert_held(vm) - Assert that the vm's reservation object is held.
* @vm: The vm
*/
#define xe_vm_assert_held(vm) dma_resv_assert_held(xe_vm_resv(vm))



#define XE_DEFAULT_GTT_SIZE_MB 3072ULL /* 3GB by default */

Expand Down Expand Up @@ -149,7 +158,7 @@ void xe_bo_unlock(struct xe_bo *bo, struct ww_acquire_ctx *ww);
static inline void xe_bo_unlock_vm_held(struct xe_bo *bo)
{
if (bo) {
XE_BUG_ON(bo->vm && bo->ttm.base.resv != &bo->vm->resv);
XE_BUG_ON(bo->vm && bo->ttm.base.resv != xe_vm_resv(bo->vm));
if (bo->vm)
xe_vm_assert_held(bo->vm);
else
Expand Down
2 changes: 1 addition & 1 deletion drivers/gpu/drm/xe/xe_device.c
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ static struct drm_driver driver = {
.driver_features =
DRIVER_GEM |
DRIVER_RENDER | DRIVER_SYNCOBJ |
DRIVER_SYNCOBJ_TIMELINE,
DRIVER_SYNCOBJ_TIMELINE | DRIVER_GEM_GPUVA,
.open = xe_file_open,
.postclose = xe_file_close,

Expand Down
4 changes: 2 additions & 2 deletions drivers/gpu/drm/xe/xe_exec.c
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,7 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
/* Wait behind munmap style rebinds */
if (!xe_vm_no_dma_fences(vm)) {
err = drm_sched_job_add_resv_dependencies(&job->drm,
&vm->resv,
xe_vm_resv(vm),
DMA_RESV_USAGE_KERNEL);
if (err)
goto err_put_job;
Expand Down Expand Up @@ -378,7 +378,7 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
xe_sched_job_arm(job);
if (!xe_vm_no_dma_fences(vm)) {
/* Block userptr invalidations / BO eviction */
dma_resv_add_fence(&vm->resv,
dma_resv_add_fence(xe_vm_resv(vm),
&job->drm.s_fence->finished,
DMA_RESV_USAGE_BOOKKEEP);

Expand Down
20 changes: 7 additions & 13 deletions drivers/gpu/drm/xe/xe_gt_pagefault.c
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,10 @@ static bool vma_is_valid(struct xe_tile *tile, struct xe_vma *vma)
!(BIT(tile->id) & vma->usm.tile_invalidated);
}

static bool vma_matches(struct xe_vma *vma, struct xe_vma *lookup)
static bool vma_matches(struct xe_vma *vma, u64 page_addr)
{
if (xe_vma_start(lookup) > xe_vma_end(vma) - 1 ||
xe_vma_end(lookup) - 1 < xe_vma_start(vma))
if (page_addr > xe_vma_end(vma) - 1 ||
page_addr + SZ_4K - 1 < xe_vma_start(vma))
return false;

return true;
Expand All @@ -91,16 +91,14 @@ static bool only_needs_bo_lock(struct xe_bo *bo)

static struct xe_vma *lookup_vma(struct xe_vm *vm, u64 page_addr)
{
struct xe_vma *vma = NULL, lookup;
struct xe_vma *vma = NULL;

lookup.start = page_addr;
lookup.end = lookup.start + SZ_4K - 1;
if (vm->usm.last_fault_vma) { /* Fast lookup */
if (vma_matches(vm->usm.last_fault_vma, &lookup))
if (vma_matches(vm->usm.last_fault_vma, page_addr))
vma = vm->usm.last_fault_vma;
}
if (!vma)
vma = xe_vm_find_overlapping_vma(vm, &lookup);
vma = xe_vm_find_overlapping_vma(vm, page_addr, SZ_4K);

return vma;
}
Expand Down Expand Up @@ -489,12 +487,8 @@ static struct xe_vma *get_acc_vma(struct xe_vm *vm, struct acc *acc)
{
u64 page_va = acc->va_range_base + (ffs(acc->sub_granularity) - 1) *
sub_granularity_in_byte(acc->granularity);
struct xe_vma lookup;

lookup.start = page_va;
lookup.end = lookup.start + SZ_4K - 1;

return xe_vm_find_overlapping_vma(vm, &lookup);
return xe_vm_find_overlapping_vma(vm, page_va, SZ_4K);
}

static int handle_acc(struct xe_gt *gt, struct acc *acc)
Expand Down
14 changes: 7 additions & 7 deletions drivers/gpu/drm/xe/xe_migrate.c
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,
/* Map the entire BO in our level 0 pt */
for (i = 0, level = 0; i < num_entries; level++) {
entry = xe_pte_encode(NULL, bo, i * XE_PAGE_SIZE,
XE_CACHE_WB, 0, 0);
XE_CACHE_WB, 0);

xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, entry);

Expand All @@ -216,7 +216,7 @@ static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m,
i += vm->flags & XE_VM_FLAGS_64K ? XE_64K_PAGE_SIZE :
XE_PAGE_SIZE) {
entry = xe_pte_encode(NULL, batch, i,
XE_CACHE_WB, 0, 0);
XE_CACHE_WB, 0);

xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64,
entry);
Expand Down Expand Up @@ -1068,7 +1068,7 @@ xe_migrate_update_pgtables_cpu(struct xe_migrate *m,
DMA_RESV_USAGE_KERNEL))
return ERR_PTR(-ETIME);

if (wait_vm && !dma_resv_test_signaled(&vm->resv,
if (wait_vm && !dma_resv_test_signaled(xe_vm_resv(vm),
DMA_RESV_USAGE_BOOKKEEP))
return ERR_PTR(-ETIME);

Expand Down Expand Up @@ -1159,7 +1159,8 @@ xe_migrate_update_pgtables(struct xe_migrate *m,
u64 addr;
int err = 0;
bool usm = !eng && xe->info.supports_usm;
bool first_munmap_rebind = vma && vma->first_munmap_rebind;
bool first_munmap_rebind = vma &&
vma->gpuva.flags & XE_VMA_FIRST_REBIND;
struct xe_engine *eng_override = !eng ? m->eng : eng;

/* Use the CPU if no in syncs and engine is idle */
Expand Down Expand Up @@ -1232,8 +1233,7 @@ xe_migrate_update_pgtables(struct xe_migrate *m,

BUG_ON(pt_bo->size != SZ_4K);

addr = xe_pte_encode(NULL, pt_bo, 0, XE_CACHE_WB,
0, 0);
addr = xe_pte_encode(NULL, pt_bo, 0, XE_CACHE_WB, 0);
bb->cs[bb->len++] = lower_32_bits(addr);
bb->cs[bb->len++] = upper_32_bits(addr);
}
Expand Down Expand Up @@ -1281,7 +1281,7 @@ xe_migrate_update_pgtables(struct xe_migrate *m,
* trigger preempts before moving forward
*/
if (first_munmap_rebind) {
err = job_add_deps(job, &vm->resv,
err = job_add_deps(job, xe_vm_resv(vm),
DMA_RESV_USAGE_BOOKKEEP);
if (err)
goto err_job;
Expand Down
Loading

0 comments on commit b06d47b

Please sign in to comment.