Skip to content

Commit

Permalink
Merge branch 'for-airlied' of git://people.freedesktop.org/~mlankhors…
Browse files Browse the repository at this point in the history 
…t/linux into drm-next

TTM reservations changes, preparing for new reservation mutex system.

* 'for-airlied' of git://people.freedesktop.org/~mlankhorst/linux:
  drm/ttm: unexport ttm_bo_wait_unreserved
  drm/nouveau: use ttm_bo_reserve_slowpath in validate_init, v2
  drm/ttm: use ttm_bo_reserve_slowpath_nolru in ttm_eu_reserve_buffers, v2
  drm/ttm: add ttm_bo_reserve_slowpath
  drm/ttm: cleanup ttm_eu_reserve_buffers handling
  drm/ttm: remove lru_lock around ttm_bo_reserve
  drm/nouveau: increase reservation sequence every retry
  drm/vmwgfx: always use ttm_bo_is_reserved
  • Loading branch information
Dave Airlie committed Feb 8, 2013
2 parents 85a7ce6 + cc4c0c4 commit b9e5071
Show file tree
Hide file tree
Showing 5 changed files with 178 additions and 88 deletions.
20 changes: 16 additions & 4 deletions drivers/gpu/drm/nouveau/nouveau_gem.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,7 @@ validate_init(struct nouveau_channel *chan, struct drm_file *file_priv,
uint32_t sequence;
int trycnt = 0;
int ret, i;
struct nouveau_bo *res_bo = NULL;

sequence = atomic_add_return(1, &drm->ttm.validate_sequence);
retry:
Expand All @@ -338,6 +339,11 @@ validate_init(struct nouveau_channel *chan, struct drm_file *file_priv,
return -ENOENT;
}
nvbo = gem->driver_private;
if (nvbo == res_bo) {
res_bo = NULL;
drm_gem_object_unreference_unlocked(gem);
continue;
}

if (nvbo->reserved_by && nvbo->reserved_by == file_priv) {
NV_ERROR(drm, "multiple instances of buffer %d on "
Expand All @@ -350,15 +356,19 @@ validate_init(struct nouveau_channel *chan, struct drm_file *file_priv,
ret = ttm_bo_reserve(&nvbo->bo, true, false, true, sequence);
if (ret) {
validate_fini(op, NULL);
if (unlikely(ret == -EAGAIN))
ret = ttm_bo_wait_unreserved(&nvbo->bo, true);
drm_gem_object_unreference_unlocked(gem);
if (unlikely(ret == -EAGAIN)) {
sequence = atomic_add_return(1, &drm->ttm.validate_sequence);
ret = ttm_bo_reserve_slowpath(&nvbo->bo, true,
sequence);
if (!ret)
res_bo = nvbo;
}
if (unlikely(ret)) {
drm_gem_object_unreference_unlocked(gem);
if (ret != -ERESTARTSYS)
NV_ERROR(drm, "fail reserve\n");
return ret;
}
goto retry;
}

b->user_priv = (uint64_t)(unsigned long)nvbo;
Expand All @@ -380,6 +390,8 @@ validate_init(struct nouveau_channel *chan, struct drm_file *file_priv,
validate_fini(op, NULL);
return -EINVAL;
}
if (nvbo == res_bo)
goto retry;
}

return 0;
Expand Down
103 changes: 83 additions & 20 deletions drivers/gpu/drm/ttm/ttm_bo.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,8 @@ static void ttm_bo_release_list(struct kref *list_kref)
ttm_mem_global_free(bdev->glob->mem_glob, acc_size);
}

int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible)
static int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo,
bool interruptible)
{
if (interruptible) {
return wait_event_interruptible(bo->event_queue,
Expand All @@ -168,7 +169,6 @@ int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo, bool interruptible)
return 0;
}
}
EXPORT_SYMBOL(ttm_bo_wait_unreserved);

void ttm_bo_add_to_lru(struct ttm_buffer_object *bo)
{
Expand Down Expand Up @@ -213,14 +213,13 @@ int ttm_bo_del_from_lru(struct ttm_buffer_object *bo)
return put_count;
}

int ttm_bo_reserve_locked(struct ttm_buffer_object *bo,
int ttm_bo_reserve_nolru(struct ttm_buffer_object *bo,
bool interruptible,
bool no_wait, bool use_sequence, uint32_t sequence)
{
struct ttm_bo_global *glob = bo->glob;
int ret;

while (unlikely(atomic_read(&bo->reserved) != 0)) {
while (unlikely(atomic_xchg(&bo->reserved, 1) != 0)) {
/**
* Deadlock avoidance for multi-bo reserving.
*/
Expand All @@ -241,26 +240,36 @@ int ttm_bo_reserve_locked(struct ttm_buffer_object *bo,
if (no_wait)
return -EBUSY;

spin_unlock(&glob->lru_lock);
ret = ttm_bo_wait_unreserved(bo, interruptible);
spin_lock(&glob->lru_lock);

if (unlikely(ret))
return ret;
}

atomic_set(&bo->reserved, 1);
if (use_sequence) {
bool wake_up = false;
/**
* Wake up waiters that may need to recheck for deadlock,
* if we decreased the sequence number.
*/
if (unlikely((bo->val_seq - sequence < (1 << 31))
|| !bo->seq_valid))
wake_up_all(&bo->event_queue);
wake_up = true;

/*
* In the worst case with memory ordering these values can be
* seen in the wrong order. However since we call wake_up_all
* in that case, this will hopefully not pose a problem,
* and the worst case would only cause someone to accidentally
* hit -EAGAIN in ttm_bo_reserve when they see old value of
* val_seq. However this would only happen if seq_valid was
* written before val_seq was, and just means some slightly
* increased cpu usage
*/
bo->val_seq = sequence;
bo->seq_valid = true;
if (wake_up)
wake_up_all(&bo->event_queue);
} else {
bo->seq_valid = false;
}
Expand Down Expand Up @@ -289,17 +298,64 @@ int ttm_bo_reserve(struct ttm_buffer_object *bo,
int put_count = 0;
int ret;

spin_lock(&glob->lru_lock);
ret = ttm_bo_reserve_locked(bo, interruptible, no_wait, use_sequence,
sequence);
if (likely(ret == 0))
ret = ttm_bo_reserve_nolru(bo, interruptible, no_wait, use_sequence,
sequence);
if (likely(ret == 0)) {
spin_lock(&glob->lru_lock);
put_count = ttm_bo_del_from_lru(bo);
spin_unlock(&glob->lru_lock);
spin_unlock(&glob->lru_lock);
ttm_bo_list_ref_sub(bo, put_count, true);
}

ttm_bo_list_ref_sub(bo, put_count, true);
return ret;
}

int ttm_bo_reserve_slowpath_nolru(struct ttm_buffer_object *bo,
bool interruptible, uint32_t sequence)
{
bool wake_up = false;
int ret;

while (unlikely(atomic_xchg(&bo->reserved, 1) != 0)) {
WARN_ON(bo->seq_valid && sequence == bo->val_seq);

ret = ttm_bo_wait_unreserved(bo, interruptible);

if (unlikely(ret))
return ret;
}

if ((bo->val_seq - sequence < (1 << 31)) || !bo->seq_valid)
wake_up = true;

/**
* Wake up waiters that may need to recheck for deadlock,
* if we decreased the sequence number.
*/
bo->val_seq = sequence;
bo->seq_valid = true;
if (wake_up)
wake_up_all(&bo->event_queue);

return 0;
}

int ttm_bo_reserve_slowpath(struct ttm_buffer_object *bo,
bool interruptible, uint32_t sequence)
{
struct ttm_bo_global *glob = bo->glob;
int put_count, ret;

ret = ttm_bo_reserve_slowpath_nolru(bo, interruptible, sequence);
if (likely(!ret)) {
spin_lock(&glob->lru_lock);
put_count = ttm_bo_del_from_lru(bo);
spin_unlock(&glob->lru_lock);
ttm_bo_list_ref_sub(bo, put_count, true);
}
return ret;
}
EXPORT_SYMBOL(ttm_bo_reserve_slowpath);

void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo)
{
Expand Down Expand Up @@ -511,7 +567,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
int ret;

spin_lock(&glob->lru_lock);
ret = ttm_bo_reserve_locked(bo, false, true, false, 0);
ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);

spin_lock(&bdev->fence_lock);
(void) ttm_bo_wait(bo, false, false, true);
Expand Down Expand Up @@ -604,7 +660,7 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo,
return ret;

spin_lock(&glob->lru_lock);
ret = ttm_bo_reserve_locked(bo, false, true, false, 0);
ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);

/*
* We raced, and lost, someone else holds the reservation now,
Expand Down Expand Up @@ -668,7 +724,14 @@ static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
kref_get(&nentry->list_kref);
}

ret = ttm_bo_reserve_locked(entry, false, !remove_all, false, 0);
ret = ttm_bo_reserve_nolru(entry, false, true, false, 0);
if (remove_all && ret) {
spin_unlock(&glob->lru_lock);
ret = ttm_bo_reserve_nolru(entry, false, false,
false, 0);
spin_lock(&glob->lru_lock);
}

if (!ret)
ret = ttm_bo_cleanup_refs_and_unlock(entry, false,
!remove_all);
Expand Down Expand Up @@ -816,7 +879,7 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev,

spin_lock(&glob->lru_lock);
list_for_each_entry(bo, &man->lru, lru) {
ret = ttm_bo_reserve_locked(bo, false, true, false, 0);
ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);
if (!ret)
break;
}
Expand Down Expand Up @@ -1797,7 +1860,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)

spin_lock(&glob->lru_lock);
list_for_each_entry(bo, &glob->swap_lru, swap) {
ret = ttm_bo_reserve_locked(bo, false, true, false, 0);
ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);
if (!ret)
break;
}
Expand Down
78 changes: 43 additions & 35 deletions drivers/gpu/drm/ttm/ttm_execbuf_util.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,22 +82,6 @@ static void ttm_eu_list_ref_sub(struct list_head *list)
}
}

static int ttm_eu_wait_unreserved_locked(struct list_head *list,
struct ttm_buffer_object *bo)
{
struct ttm_bo_global *glob = bo->glob;
int ret;

ttm_eu_del_from_lru_locked(list);
spin_unlock(&glob->lru_lock);
ret = ttm_bo_wait_unreserved(bo, true);
spin_lock(&glob->lru_lock);
if (unlikely(ret != 0))
ttm_eu_backoff_reservation_locked(list);
return ret;
}


void ttm_eu_backoff_reservation(struct list_head *list)
{
struct ttm_validate_buffer *entry;
Expand Down Expand Up @@ -145,47 +129,65 @@ int ttm_eu_reserve_buffers(struct list_head *list)
entry = list_first_entry(list, struct ttm_validate_buffer, head);
glob = entry->bo->glob;

retry:
spin_lock(&glob->lru_lock);
val_seq = entry->bo->bdev->val_seq++;

retry:
list_for_each_entry(entry, list, head) {
struct ttm_buffer_object *bo = entry->bo;

retry_this_bo:
ret = ttm_bo_reserve_locked(bo, true, true, true, val_seq);
/* already slowpath reserved? */
if (entry->reserved)
continue;

ret = ttm_bo_reserve_nolru(bo, true, true, true, val_seq);
switch (ret) {
case 0:
break;
case -EBUSY:
ret = ttm_eu_wait_unreserved_locked(list, bo);
if (unlikely(ret != 0)) {
spin_unlock(&glob->lru_lock);
ttm_eu_list_ref_sub(list);
return ret;
}
goto retry_this_bo;
ttm_eu_del_from_lru_locked(list);
spin_unlock(&glob->lru_lock);
ret = ttm_bo_reserve_nolru(bo, true, false,
true, val_seq);
spin_lock(&glob->lru_lock);
if (!ret)
break;

if (unlikely(ret != -EAGAIN))
goto err;

/* fallthrough */
case -EAGAIN:
ttm_eu_backoff_reservation_locked(list);

/*
* temporarily increase sequence number every retry,
* to prevent us from seeing our old reservation
* sequence when someone else reserved the buffer,
* but hasn't updated the seq_valid/seqno members yet.
*/
val_seq = entry->bo->bdev->val_seq++;

spin_unlock(&glob->lru_lock);
ttm_eu_list_ref_sub(list);
ret = ttm_bo_wait_unreserved(bo, true);
ret = ttm_bo_reserve_slowpath_nolru(bo, true, val_seq);
if (unlikely(ret != 0))
return ret;
spin_lock(&glob->lru_lock);
entry->reserved = true;
if (unlikely(atomic_read(&bo->cpu_writers) > 0)) {
ret = -EBUSY;
goto err;
}
goto retry;
default:
ttm_eu_backoff_reservation_locked(list);
spin_unlock(&glob->lru_lock);
ttm_eu_list_ref_sub(list);
return ret;
goto err;
}

entry->reserved = true;
if (unlikely(atomic_read(&bo->cpu_writers) > 0)) {
ttm_eu_backoff_reservation_locked(list);
spin_unlock(&glob->lru_lock);
ttm_eu_list_ref_sub(list);
return -EBUSY;
ret = -EBUSY;
goto err;
}
}

Expand All @@ -194,6 +196,12 @@ int ttm_eu_reserve_buffers(struct list_head *list)
ttm_eu_list_ref_sub(list);

return 0;

err:
ttm_eu_backoff_reservation_locked(list);
spin_unlock(&glob->lru_lock);
ttm_eu_list_ref_sub(list);
return ret;
}
EXPORT_SYMBOL(ttm_eu_reserve_buffers);

Expand Down
4 changes: 2 additions & 2 deletions drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -959,13 +959,13 @@ void vmw_resource_unreserve(struct vmw_resource *res,
if (new_backup && new_backup != res->backup) {

if (res->backup) {
BUG_ON(atomic_read(&res->backup->base.reserved) == 0);
BUG_ON(!ttm_bo_is_reserved(&res->backup->base));
list_del_init(&res->mob_head);
vmw_dmabuf_unreference(&res->backup);
}

res->backup = vmw_dmabuf_reference(new_backup);
BUG_ON(atomic_read(&new_backup->base.reserved) == 0);
BUG_ON(!ttm_bo_is_reserved(&new_backup->base));
list_add_tail(&res->mob_head, &new_backup->res_list);
}
if (new_backup)
Expand Down
Loading

0 comments on commit b9e5071

Please sign in to comment.