Skip to content

Commit

Permalink
futex: reduce mmap_sem usage
Browse files Browse the repository at this point in the history 
now that we rely on get_user_pages() for the shared key handling
move all the mmap_sem stuff closely around the slow paths.

Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
Acked-by: Nick Piggin <nickpiggin@yahoo.com.au>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
  • Loading branch information
Peter Zijlstra authored and Ingo Molnar committed Sep 30, 2008
1 parent 38d47c1 commit 6127070
Showing 1 changed file with 4 additions and 79 deletions.
83 changes: 4 additions & 79 deletions kernel/futex.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,24 +122,6 @@ struct futex_hash_bucket {

static struct futex_hash_bucket futex_queues[1<<FUTEX_HASHBITS];

/*
* Take mm->mmap_sem, when futex is shared
*/
static inline void futex_lock_mm(struct rw_semaphore *fshared)
{
if (fshared)
down_read(fshared);
}

/*
* Release mm->mmap_sem, when the futex is shared
*/
static inline void futex_unlock_mm(struct rw_semaphore *fshared)
{
if (fshared)
up_read(fshared);
}

/*
* We hash on the keys returned from get_futex_key (see below).
*/
Expand Down Expand Up @@ -250,7 +232,9 @@ static int get_futex_key(u32 __user *uaddr, struct rw_semaphore *fshared,
}

again:
down_read(&mm->mmap_sem);
err = get_user_pages(current, mm, address, 1, 0, 0, &page, NULL);
up_read(&mm->mmap_sem);
if (err < 0)
return err;

Expand Down Expand Up @@ -327,8 +311,7 @@ static int futex_handle_fault(unsigned long address,
if (attempt > 2)
return ret;

if (!fshared)
down_read(&mm->mmap_sem);
down_read(&mm->mmap_sem);
vma = find_vma(mm, address);
if (vma && address >= vma->vm_start &&
(vma->vm_flags & VM_WRITE)) {
Expand All @@ -348,8 +331,7 @@ static int futex_handle_fault(unsigned long address,
current->min_flt++;
}
}
if (!fshared)
up_read(&mm->mmap_sem);
up_read(&mm->mmap_sem);
return ret;
}

Expand Down Expand Up @@ -719,8 +701,6 @@ static int futex_wake(u32 __user *uaddr, struct rw_semaphore *fshared,
if (!bitset)
return -EINVAL;

futex_lock_mm(fshared);

ret = get_futex_key(uaddr, fshared, &key);
if (unlikely(ret != 0))
goto out;
Expand Down Expand Up @@ -749,7 +729,6 @@ static int futex_wake(u32 __user *uaddr, struct rw_semaphore *fshared,
spin_unlock(&hb->lock);
out:
put_futex_key(fshared, &key);
futex_unlock_mm(fshared);
return ret;
}

Expand All @@ -769,8 +748,6 @@ futex_wake_op(u32 __user *uaddr1, struct rw_semaphore *fshared,
int ret, op_ret, attempt = 0;

retryfull:
futex_lock_mm(fshared);

ret = get_futex_key(uaddr1, fshared, &key1);
if (unlikely(ret != 0))
goto out;
Expand Down Expand Up @@ -821,12 +798,6 @@ futex_wake_op(u32 __user *uaddr1, struct rw_semaphore *fshared,
goto retry;
}

/*
* If we would have faulted, release mmap_sem,
* fault it in and start all over again.
*/
futex_unlock_mm(fshared);

ret = get_user(dummy, uaddr2);
if (ret)
return ret;
Expand Down Expand Up @@ -864,7 +835,6 @@ futex_wake_op(u32 __user *uaddr1, struct rw_semaphore *fshared,
out:
put_futex_key(fshared, &key2);
put_futex_key(fshared, &key1);
futex_unlock_mm(fshared);

return ret;
}
Expand All @@ -884,8 +854,6 @@ static int futex_requeue(u32 __user *uaddr1, struct rw_semaphore *fshared,
int ret, drop_count = 0;

retry:
futex_lock_mm(fshared);

ret = get_futex_key(uaddr1, fshared, &key1);
if (unlikely(ret != 0))
goto out;
Expand All @@ -908,12 +876,6 @@ static int futex_requeue(u32 __user *uaddr1, struct rw_semaphore *fshared,
if (hb1 != hb2)
spin_unlock(&hb2->lock);

/*
* If we would have faulted, release mmap_sem, fault
* it in and start all over again.
*/
futex_unlock_mm(fshared);

ret = get_user(curval, uaddr1);

if (!ret)
Expand Down Expand Up @@ -967,7 +929,6 @@ static int futex_requeue(u32 __user *uaddr1, struct rw_semaphore *fshared,
out:
put_futex_key(fshared, &key2);
put_futex_key(fshared, &key1);
futex_unlock_mm(fshared);
return ret;
}

Expand Down Expand Up @@ -1211,8 +1172,6 @@ static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared,
q.pi_state = NULL;
q.bitset = bitset;
retry:
futex_lock_mm(fshared);

q.key = FUTEX_KEY_INIT;
ret = get_futex_key(uaddr, fshared, &q.key);
if (unlikely(ret != 0))
Expand Down Expand Up @@ -1245,12 +1204,6 @@ static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared,
if (unlikely(ret)) {
queue_unlock(&q, hb);

/*
* If we would have faulted, release mmap_sem, fault it in and
* start all over again.
*/
futex_unlock_mm(fshared);

ret = get_user(uval, uaddr);

if (!ret)
Expand All @@ -1264,12 +1217,6 @@ static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared,
/* Only actually queue if *uaddr contained val. */
queue_me(&q, hb);

/*
* Now the futex is queued and we have checked the data, we
* don't want to hold mmap_sem while we sleep.
*/
futex_unlock_mm(fshared);

/*
* There might have been scheduling since the queue_me(), as we
* cannot hold a spinlock across the get_user() in case it
Expand Down Expand Up @@ -1355,7 +1302,6 @@ static int futex_wait(u32 __user *uaddr, struct rw_semaphore *fshared,

out_release_sem:
put_futex_key(fshared, &q.key);
futex_unlock_mm(fshared);
return ret;
}

Expand Down Expand Up @@ -1404,8 +1350,6 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared,

q.pi_state = NULL;
retry:
futex_lock_mm(fshared);

q.key = FUTEX_KEY_INIT;
ret = get_futex_key(uaddr, fshared, &q.key);
if (unlikely(ret != 0))
Expand Down Expand Up @@ -1495,7 +1439,6 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared,
* exit to complete.
*/
queue_unlock(&q, hb);
futex_unlock_mm(fshared);
cond_resched();
goto retry;

Expand Down Expand Up @@ -1527,12 +1470,6 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared,
*/
queue_me(&q, hb);

/*
* Now the futex is queued and we have checked the data, we
* don't want to hold mmap_sem while we sleep.
*/
futex_unlock_mm(fshared);

WARN_ON(!q.pi_state);
/*
* Block on the PI mutex:
Expand All @@ -1545,7 +1482,6 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared,
ret = ret ? 0 : -EWOULDBLOCK;
}

futex_lock_mm(fshared);
spin_lock(q.lock_ptr);

if (!ret) {
Expand Down Expand Up @@ -1611,7 +1547,6 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared,

/* Unqueue and drop the lock */
unqueue_me_pi(&q);
futex_unlock_mm(fshared);

if (to)
destroy_hrtimer_on_stack(&to->timer);
Expand All @@ -1622,7 +1557,6 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared,

out_release_sem:
put_futex_key(fshared, &q.key);
futex_unlock_mm(fshared);
if (to)
destroy_hrtimer_on_stack(&to->timer);
return ret;
Expand All @@ -1646,8 +1580,6 @@ static int futex_lock_pi(u32 __user *uaddr, struct rw_semaphore *fshared,
goto retry_unlocked;
}

futex_unlock_mm(fshared);

ret = get_user(uval, uaddr);
if (!ret && (uval != -EFAULT))
goto retry;
Expand Down Expand Up @@ -1679,10 +1611,6 @@ static int futex_unlock_pi(u32 __user *uaddr, struct rw_semaphore *fshared)
*/
if ((uval & FUTEX_TID_MASK) != task_pid_vnr(current))
return -EPERM;
/*
* First take all the futex related locks:
*/
futex_lock_mm(fshared);

ret = get_futex_key(uaddr, fshared, &key);
if (unlikely(ret != 0))
Expand Down Expand Up @@ -1742,7 +1670,6 @@ static int futex_unlock_pi(u32 __user *uaddr, struct rw_semaphore *fshared)
spin_unlock(&hb->lock);
out:
put_futex_key(fshared, &key);
futex_unlock_mm(fshared);

return ret;

Expand All @@ -1766,8 +1693,6 @@ static int futex_unlock_pi(u32 __user *uaddr, struct rw_semaphore *fshared)
goto retry_unlocked;
}

futex_unlock_mm(fshared);

ret = get_user(uval, uaddr);
if (!ret && (uval != -EFAULT))
goto retry;
Expand Down

0 comments on commit 6127070

Please sign in to comment.