Skip to content

Commit

Permalink
locking/lockdep, sched/core: Implement a better lock pinning scheme
Browse files Browse the repository at this point in the history
The problem with the existing lock pinning is that each pin is of
value 1; this mean you can simply unpin if you know its pinned,
without having any extra information.

This scheme generates a random (16 bit) cookie for each pin and
requires this same cookie to unpin. This means you have to keep the
cookie in context.

No objsize difference for !LOCKDEP kernels.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
  • Loading branch information
Peter Zijlstra authored and Ingo Molnar committed May 5, 2016
1 parent eb58075 commit e7904a2
Show file tree
Hide file tree
Showing 9 changed files with 140 additions and 68 deletions.
23 changes: 17 additions & 6 deletions include/linux/lockdep.h
Original file line number Diff line number Diff line change
Expand Up @@ -354,8 +354,13 @@ extern void lockdep_set_current_reclaim_state(gfp_t gfp_mask);
extern void lockdep_clear_current_reclaim_state(void);
extern void lockdep_trace_alloc(gfp_t mask);

extern void lock_pin_lock(struct lockdep_map *lock);
extern void lock_unpin_lock(struct lockdep_map *lock);
struct pin_cookie { unsigned int val; };

#define NIL_COOKIE (struct pin_cookie){ .val = 0U, }

extern struct pin_cookie lock_pin_lock(struct lockdep_map *lock);
extern void lock_repin_lock(struct lockdep_map *lock, struct pin_cookie);
extern void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie);

# define INIT_LOCKDEP .lockdep_recursion = 0, .lockdep_reclaim_gfp = 0,

Expand All @@ -371,8 +376,9 @@ extern void lock_unpin_lock(struct lockdep_map *lock);

#define lockdep_recursing(tsk) ((tsk)->lockdep_recursion)

#define lockdep_pin_lock(l) lock_pin_lock(&(l)->dep_map)
#define lockdep_unpin_lock(l) lock_unpin_lock(&(l)->dep_map)
#define lockdep_pin_lock(l) lock_pin_lock(&(l)->dep_map)
#define lockdep_repin_lock(l,c) lock_repin_lock(&(l)->dep_map, (c))
#define lockdep_unpin_lock(l,c) lock_unpin_lock(&(l)->dep_map, (c))

#else /* !CONFIG_LOCKDEP */

Expand Down Expand Up @@ -425,8 +431,13 @@ struct lock_class_key { };

#define lockdep_recursing(tsk) (0)

#define lockdep_pin_lock(l) do { (void)(l); } while (0)
#define lockdep_unpin_lock(l) do { (void)(l); } while (0)
struct pin_cookie { };

#define NIL_COOKIE (struct pin_cookie){ }

#define lockdep_pin_lock(l) ({ struct pin_cookie cookie; cookie; })
#define lockdep_repin_lock(l, c) do { (void)(l); (void)(c); } while (0)
#define lockdep_unpin_lock(l, c) do { (void)(l); (void)(c); } while (0)

#endif /* !LOCKDEP */

Expand Down
71 changes: 62 additions & 9 deletions kernel/locking/lockdep.c
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
#include <linux/bitops.h>
#include <linux/gfp.h>
#include <linux/kmemcheck.h>
#include <linux/random.h>

#include <asm/sections.h>

Expand Down Expand Up @@ -3554,7 +3555,35 @@ static int __lock_is_held(struct lockdep_map *lock)
return 0;
}

static void __lock_pin_lock(struct lockdep_map *lock)
static struct pin_cookie __lock_pin_lock(struct lockdep_map *lock)
{
struct pin_cookie cookie = NIL_COOKIE;
struct task_struct *curr = current;
int i;

if (unlikely(!debug_locks))
return cookie;

for (i = 0; i < curr->lockdep_depth; i++) {
struct held_lock *hlock = curr->held_locks + i;

if (match_held_lock(hlock, lock)) {
/*
* Grab 16bits of randomness; this is sufficient to not
* be guessable and still allows some pin nesting in
* our u32 pin_count.
*/
cookie.val = 1 + (prandom_u32() >> 16);
hlock->pin_count += cookie.val;
return cookie;
}
}

WARN(1, "pinning an unheld lock\n");
return cookie;
}

static void __lock_repin_lock(struct lockdep_map *lock, struct pin_cookie cookie)
{
struct task_struct *curr = current;
int i;
Expand All @@ -3566,15 +3595,15 @@ static void __lock_pin_lock(struct lockdep_map *lock)
struct held_lock *hlock = curr->held_locks + i;

if (match_held_lock(hlock, lock)) {
hlock->pin_count++;
hlock->pin_count += cookie.val;
return;
}
}

WARN(1, "pinning an unheld lock\n");
}

static void __lock_unpin_lock(struct lockdep_map *lock)
static void __lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie)
{
struct task_struct *curr = current;
int i;
Expand All @@ -3589,7 +3618,11 @@ static void __lock_unpin_lock(struct lockdep_map *lock)
if (WARN(!hlock->pin_count, "unpinning an unpinned lock\n"))
return;

hlock->pin_count--;
hlock->pin_count -= cookie.val;

if (WARN((int)hlock->pin_count < 0, "pin count corrupted\n"))
hlock->pin_count = 0;

return;
}
}
Expand Down Expand Up @@ -3720,24 +3753,44 @@ int lock_is_held(struct lockdep_map *lock)
}
EXPORT_SYMBOL_GPL(lock_is_held);

void lock_pin_lock(struct lockdep_map *lock)
struct pin_cookie lock_pin_lock(struct lockdep_map *lock)
{
struct pin_cookie cookie = NIL_COOKIE;
unsigned long flags;

if (unlikely(current->lockdep_recursion))
return;
return cookie;

raw_local_irq_save(flags);
check_flags(flags);

current->lockdep_recursion = 1;
__lock_pin_lock(lock);
cookie = __lock_pin_lock(lock);
current->lockdep_recursion = 0;
raw_local_irq_restore(flags);

return cookie;
}
EXPORT_SYMBOL_GPL(lock_pin_lock);

void lock_unpin_lock(struct lockdep_map *lock)
void lock_repin_lock(struct lockdep_map *lock, struct pin_cookie cookie)
{
unsigned long flags;

if (unlikely(current->lockdep_recursion))
return;

raw_local_irq_save(flags);
check_flags(flags);

current->lockdep_recursion = 1;
__lock_repin_lock(lock, cookie);
current->lockdep_recursion = 0;
raw_local_irq_restore(flags);
}
EXPORT_SYMBOL_GPL(lock_repin_lock);

void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie cookie)
{
unsigned long flags;

Expand All @@ -3748,7 +3801,7 @@ void lock_unpin_lock(struct lockdep_map *lock)
check_flags(flags);

current->lockdep_recursion = 1;
__lock_unpin_lock(lock);
__lock_unpin_lock(lock, cookie);
current->lockdep_recursion = 0;
raw_local_irq_restore(flags);
}
Expand Down
Loading

0 comments on commit e7904a2

Please sign in to comment.