Skip to content

Commit

Permalink
drm/ttm: move the LRU into resource handling v4
Browse files Browse the repository at this point in the history 
This way we finally fix the problem that new resource are
not immediately evict-able after allocation.

That has caused numerous problems including OOM on GDS handling
and not being able to use TTM as general resource manager.

v2: stop assuming in ttm_resource_fini that res->bo is still valid.
v3: cleanup kerneldoc, add more lockdep annotation
v4: consistently use res->num_pages

Signed-off-by: Christian König <christian.koenig@amd.com>
Tested-by: Bas Nieuwenhuizen <bas@basnieuwenhuizen.nl>
Reviewed-by: Daniel Vetter <daniel.vetter@ffwll.ch>
Link: https://patchwork.freedesktop.org/patch/msgid/20220321132601.2161-1-christian.koenig@amd.com
  • Loading branch information
Christian König committed Mar 28, 2022
1 parent e795df5 commit 6a9b028
Show file tree
Hide file tree
Showing 9 changed files with 204 additions and 197 deletions.
8 changes: 3 additions & 5 deletions drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -683,12 +683,12 @@ void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,

if (vm->bulk_moveable) {
spin_lock(&adev->mman.bdev.lru_lock);
ttm_bo_bulk_move_lru_tail(&vm->lru_bulk_move);
ttm_lru_bulk_move_tail(&vm->lru_bulk_move);
spin_unlock(&adev->mman.bdev.lru_lock);
return;
}

memset(&vm->lru_bulk_move, 0, sizeof(vm->lru_bulk_move));
ttm_lru_bulk_move_init(&vm->lru_bulk_move);

spin_lock(&adev->mman.bdev.lru_lock);
list_for_each_entry(bo_base, &vm->idle, vm_status) {
Expand All @@ -698,11 +698,9 @@ void amdgpu_vm_move_to_lru_tail(struct amdgpu_device *adev,
if (!bo->parent)
continue;

ttm_bo_move_to_lru_tail(&bo->tbo, bo->tbo.resource,
&vm->lru_bulk_move);
ttm_bo_move_to_lru_tail(&bo->tbo, &vm->lru_bulk_move);
if (shadow)
ttm_bo_move_to_lru_tail(&shadow->tbo,
shadow->tbo.resource,
&vm->lru_bulk_move);
}
spin_unlock(&adev->mman.bdev.lru_lock);
Expand Down
2 changes: 1 addition & 1 deletion drivers/gpu/drm/i915/gem/i915_gem_ttm.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -849,7 +849,7 @@ void i915_ttm_adjust_lru(struct drm_i915_gem_object *obj)
bo->priority = I915_TTM_PRIO_NO_PAGES;
}

ttm_bo_move_to_lru_tail(bo, bo->resource, NULL);
ttm_bo_move_to_lru_tail(bo, NULL);
spin_unlock(&bo->bdev->lru_lock);
}

Expand Down
115 changes: 8 additions & 107 deletions drivers/gpu/drm/ttm/ttm_bo.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,108 +69,16 @@ static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo,
}
}

static inline void ttm_bo_move_to_pinned(struct ttm_buffer_object *bo)
{
struct ttm_device *bdev = bo->bdev;

list_move_tail(&bo->lru, &bdev->pinned);

if (bdev->funcs->del_from_lru_notify)
bdev->funcs->del_from_lru_notify(bo);
}

static inline void ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
{
struct ttm_device *bdev = bo->bdev;

list_del_init(&bo->lru);

if (bdev->funcs->del_from_lru_notify)
bdev->funcs->del_from_lru_notify(bo);
}

static void ttm_bo_bulk_move_set_pos(struct ttm_lru_bulk_move_pos *pos,
struct ttm_buffer_object *bo)
{
if (!pos->first)
pos->first = bo;
pos->last = bo;
}

void ttm_bo_move_to_lru_tail(struct ttm_buffer_object *bo,
struct ttm_resource *mem,
struct ttm_lru_bulk_move *bulk)
{
struct ttm_device *bdev = bo->bdev;
struct ttm_resource_manager *man;

if (!bo->deleted)
dma_resv_assert_held(bo->base.resv);

if (bo->pin_count) {
ttm_bo_move_to_pinned(bo);
return;
}

if (!mem)
return;

man = ttm_manager_type(bdev, mem->mem_type);
list_move_tail(&bo->lru, &man->lru[bo->priority]);

if (bdev->funcs->del_from_lru_notify)
bdev->funcs->del_from_lru_notify(bo);

if (bulk && !bo->pin_count) {
switch (bo->resource->mem_type) {
case TTM_PL_TT:
ttm_bo_bulk_move_set_pos(&bulk->tt[bo->priority], bo);
break;
dma_resv_assert_held(bo->base.resv);

case TTM_PL_VRAM:
ttm_bo_bulk_move_set_pos(&bulk->vram[bo->priority], bo);
break;
}
}
if (bo->resource)
ttm_resource_move_to_lru_tail(bo->resource, bulk);
}
EXPORT_SYMBOL(ttm_bo_move_to_lru_tail);

void ttm_bo_bulk_move_lru_tail(struct ttm_lru_bulk_move *bulk)
{
unsigned i;

for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
struct ttm_lru_bulk_move_pos *pos = &bulk->tt[i];
struct ttm_resource_manager *man;

if (!pos->first)
continue;

dma_resv_assert_held(pos->first->base.resv);
dma_resv_assert_held(pos->last->base.resv);

man = ttm_manager_type(pos->first->bdev, TTM_PL_TT);
list_bulk_move_tail(&man->lru[i], &pos->first->lru,
&pos->last->lru);
}

for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
struct ttm_lru_bulk_move_pos *pos = &bulk->vram[i];
struct ttm_resource_manager *man;

if (!pos->first)
continue;

dma_resv_assert_held(pos->first->base.resv);
dma_resv_assert_held(pos->last->base.resv);

man = ttm_manager_type(pos->first->bdev, TTM_PL_VRAM);
list_bulk_move_tail(&man->lru[i], &pos->first->lru,
&pos->last->lru);
}
}
EXPORT_SYMBOL(ttm_bo_bulk_move_lru_tail);

static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
struct ttm_resource *mem, bool evict,
struct ttm_operation_ctx *ctx,
Expand Down Expand Up @@ -344,7 +252,6 @@ static int ttm_bo_cleanup_refs(struct ttm_buffer_object *bo,
return ret;
}

ttm_bo_move_to_pinned(bo);
list_del_init(&bo->ddestroy);
spin_unlock(&bo->bdev->lru_lock);
ttm_bo_cleanup_memtype_use(bo);
Expand Down Expand Up @@ -445,7 +352,7 @@ static void ttm_bo_release(struct kref *kref)
*/
if (bo->pin_count) {
bo->pin_count = 0;
ttm_bo_move_to_lru_tail(bo, bo->resource, NULL);
ttm_resource_move_to_lru_tail(bo->resource, NULL);
}

kref_init(&bo->kref);
Expand All @@ -458,7 +365,6 @@ static void ttm_bo_release(struct kref *kref)
}

spin_lock(&bo->bdev->lru_lock);
ttm_bo_del_from_lru(bo);
list_del(&bo->ddestroy);
spin_unlock(&bo->bdev->lru_lock);

Expand Down Expand Up @@ -673,15 +579,17 @@ int ttm_mem_evict_first(struct ttm_device *bdev,
struct ww_acquire_ctx *ticket)
{
struct ttm_buffer_object *bo = NULL, *busy_bo = NULL;
struct ttm_resource *res;
bool locked = false;
unsigned i;
int ret;

spin_lock(&bdev->lru_lock);
for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) {
list_for_each_entry(bo, &man->lru[i], lru) {
list_for_each_entry(res, &man->lru[i], lru) {
bool busy;

bo = res->bo;
if (!ttm_bo_evict_swapout_allowable(bo, ctx, place,
&locked, &busy)) {
if (busy && !busy_bo && ticket !=
Expand All @@ -699,7 +607,7 @@ int ttm_mem_evict_first(struct ttm_device *bdev,
}

/* If the inner loop terminated early, we have our candidate */
if (&bo->lru != &man->lru[i])
if (&res->lru != &man->lru[i])
break;

bo = NULL;
Expand Down Expand Up @@ -875,9 +783,6 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo,
}

error:
if (bo->resource->mem_type == TTM_PL_SYSTEM && !bo->pin_count)
ttm_bo_move_to_lru_tail_unlocked(bo);

return ret;
}
EXPORT_SYMBOL(ttm_bo_mem_space);
Expand Down Expand Up @@ -971,7 +876,6 @@ int ttm_bo_init_reserved(struct ttm_device *bdev,
bo->destroy = destroy ? destroy : ttm_bo_default_destroy;

kref_init(&bo->kref);
INIT_LIST_HEAD(&bo->lru);
INIT_LIST_HEAD(&bo->ddestroy);
bo->bdev = bdev;
bo->type = type;
Expand Down Expand Up @@ -1021,8 +925,6 @@ int ttm_bo_init_reserved(struct ttm_device *bdev,
return ret;
}

ttm_bo_move_to_lru_tail_unlocked(bo);

return ret;
}
EXPORT_SYMBOL(ttm_bo_init_reserved);
Expand Down Expand Up @@ -1123,7 +1025,6 @@ int ttm_bo_swapout(struct ttm_buffer_object *bo, struct ttm_operation_ctx *ctx,
return ret == -EBUSY ? -ENOSPC : ret;
}

ttm_bo_move_to_pinned(bo);
/* TODO: Cleanup the locking */
spin_unlock(&bo->bdev->lru_lock);

Expand Down
1 change: 0 additions & 1 deletion drivers/gpu/drm/ttm/ttm_bo_util.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,6 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,

atomic_inc(&ttm_glob.bo_count);
INIT_LIST_HEAD(&fbo->base.ddestroy);
INIT_LIST_HEAD(&fbo->base.lru);
fbo->base.moving = NULL;
drm_vma_node_reset(&fbo->base.base.vma_node);

Expand Down
64 changes: 32 additions & 32 deletions drivers/gpu/drm/ttm/ttm_device.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,7 @@ int ttm_device_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx,
{
struct ttm_resource_manager *man;
struct ttm_buffer_object *bo;
struct ttm_resource *res;
unsigned i, j;
int ret;

Expand All @@ -154,8 +155,11 @@ int ttm_device_swapout(struct ttm_device *bdev, struct ttm_operation_ctx *ctx,
continue;

for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) {
list_for_each_entry(bo, &man->lru[j], lru) {
uint32_t num_pages = PFN_UP(bo->base.size);
list_for_each_entry(res, &man->lru[j], lru) {
uint32_t num_pages;

bo = res->bo;
num_pages = PFN_UP(bo->base.size);

ret = ttm_bo_swapout(bo, ctx, gfp_flags);
/* ttm_bo_swapout has dropped the lru_lock */
Expand Down Expand Up @@ -259,49 +263,45 @@ void ttm_device_fini(struct ttm_device *bdev)
}
EXPORT_SYMBOL(ttm_device_fini);

void ttm_device_clear_dma_mappings(struct ttm_device *bdev)
static void ttm_device_clear_lru_dma_mappings(struct ttm_device *bdev,
struct list_head *list)
{
struct ttm_resource_manager *man;
struct ttm_buffer_object *bo;
unsigned int i, j;
struct ttm_resource *res;

spin_lock(&bdev->lru_lock);
while (!list_empty(&bdev->pinned)) {
bo = list_first_entry(&bdev->pinned, struct ttm_buffer_object, lru);
while ((res = list_first_entry_or_null(list, typeof(*res), lru))) {
struct ttm_buffer_object *bo = res->bo;

/* Take ref against racing releases once lru_lock is unlocked */
if (ttm_bo_get_unless_zero(bo)) {
list_del_init(&bo->lru);
spin_unlock(&bdev->lru_lock);
if (!ttm_bo_get_unless_zero(bo))
continue;

if (bo->ttm)
ttm_tt_unpopulate(bo->bdev, bo->ttm);
list_del_init(&res->lru);
spin_unlock(&bdev->lru_lock);

ttm_bo_put(bo);
spin_lock(&bdev->lru_lock);
}
if (bo->ttm)
ttm_tt_unpopulate(bo->bdev, bo->ttm);

ttm_bo_put(bo);
spin_lock(&bdev->lru_lock);
}
spin_unlock(&bdev->lru_lock);
}

void ttm_device_clear_dma_mappings(struct ttm_device *bdev)
{
struct ttm_resource_manager *man;
unsigned int i, j;

ttm_device_clear_lru_dma_mappings(bdev, &bdev->pinned);

for (i = TTM_PL_SYSTEM; i < TTM_NUM_MEM_TYPES; ++i) {
man = ttm_manager_type(bdev, i);
if (!man || !man->use_tt)
continue;

for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) {
while (!list_empty(&man->lru[j])) {
bo = list_first_entry(&man->lru[j], struct ttm_buffer_object, lru);
if (ttm_bo_get_unless_zero(bo)) {
list_del_init(&bo->lru);
spin_unlock(&bdev->lru_lock);

if (bo->ttm)
ttm_tt_unpopulate(bo->bdev, bo->ttm);

ttm_bo_put(bo);
spin_lock(&bdev->lru_lock);
}
}
}
for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j)
ttm_device_clear_lru_dma_mappings(bdev, &man->lru[j]);
}
spin_unlock(&bdev->lru_lock);
}
EXPORT_SYMBOL(ttm_device_clear_dma_mappings);
Loading

0 comments on commit 6a9b028

Please sign in to comment.