Skip to content

Commit

Permalink
locking/rtmutex: Split out the inner parts of 'struct rtmutex'
Browse files Browse the repository at this point in the history 
RT builds substitutions for rwsem, mutex, spinlock and rwlock around
rtmutexes. Split the inner working out so each lock substitution can use
them with the appropriate lockdep annotations. This avoids having an extra
unused lockdep map in the wrapped rtmutex.

No functional change.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/20210815211302.784739994@linutronix.de
  • Loading branch information
Peter Zijlstra authored and Ingo Molnar committed Aug 17, 2021
1 parent 531ae4b commit 830e6ac
Show file tree
Hide file tree
Showing 6 changed files with 97 additions and 79 deletions.
23 changes: 17 additions & 6 deletions include/linux/rtmutex.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,21 @@

extern int max_lock_depth; /* for sysctl */

struct rt_mutex_base {
raw_spinlock_t wait_lock;
struct rb_root_cached waiters;
struct task_struct *owner;
};

#define __RT_MUTEX_BASE_INITIALIZER(rtbasename) \
{ \
.wait_lock = __RAW_SPIN_LOCK_UNLOCKED(rtbasename.wait_lock), \
.waiters = RB_ROOT_CACHED, \
.owner = NULL \
}

extern void rt_mutex_base_init(struct rt_mutex_base *rtb);

/**
* The rt_mutex structure
*
Expand All @@ -28,9 +43,7 @@ extern int max_lock_depth; /* for sysctl */
* @owner: the mutex owner
*/
struct rt_mutex {
raw_spinlock_t wait_lock;
struct rb_root_cached waiters;
struct task_struct *owner;
struct rt_mutex_base rtmutex;
#ifdef CONFIG_DEBUG_LOCK_ALLOC
struct lockdep_map dep_map;
#endif
Expand Down Expand Up @@ -63,9 +76,7 @@ do { \

#define __RT_MUTEX_INITIALIZER(mutexname) \
{ \
.wait_lock = __RAW_SPIN_LOCK_UNLOCKED(mutexname.wait_lock), \
.waiters = RB_ROOT_CACHED, \
.owner = NULL, \
.rtmutex = __RT_MUTEX_BASE_INITIALIZER(mutexname.rtmutex), \
__DEP_MAP_RT_MUTEX_INITIALIZER(mutexname) \
}

Expand Down
4 changes: 2 additions & 2 deletions kernel/futex.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ struct futex_pi_state {
/*
* The PI object:
*/
struct rt_mutex pi_mutex;
struct rt_mutex_base pi_mutex;

struct task_struct *owner;
refcount_t refcount;
Expand Down Expand Up @@ -3254,7 +3254,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags,
ret = ret < 0 ? ret : 0;
}
} else {
struct rt_mutex *pi_mutex;
struct rt_mutex_base *pi_mutex;

/*
* We have been woken up by futex_unlock_pi(), a timeout, or a
Expand Down
64 changes: 33 additions & 31 deletions kernel/locking/rtmutex.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
*/

static __always_inline void
rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner)
rt_mutex_set_owner(struct rt_mutex_base *lock, struct task_struct *owner)
{
unsigned long val = (unsigned long)owner;

Expand All @@ -58,13 +58,13 @@ rt_mutex_set_owner(struct rt_mutex *lock, struct task_struct *owner)
WRITE_ONCE(lock->owner, (struct task_struct *)val);
}

static __always_inline void clear_rt_mutex_waiters(struct rt_mutex *lock)
static __always_inline void clear_rt_mutex_waiters(struct rt_mutex_base *lock)
{
lock->owner = (struct task_struct *)
((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);
}

static __always_inline void fixup_rt_mutex_waiters(struct rt_mutex *lock)
static __always_inline void fixup_rt_mutex_waiters(struct rt_mutex_base *lock)
{
unsigned long owner, *p = (unsigned long *) &lock->owner;

Expand Down Expand Up @@ -139,14 +139,14 @@ static __always_inline void fixup_rt_mutex_waiters(struct rt_mutex *lock)
* set up.
*/
#ifndef CONFIG_DEBUG_RT_MUTEXES
static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex *lock,
static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex_base *lock,
struct task_struct *old,
struct task_struct *new)
{
return try_cmpxchg_acquire(&lock->owner, &old, new);
}

static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex *lock,
static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,
struct task_struct *old,
struct task_struct *new)
{
Expand All @@ -158,7 +158,7 @@ static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex *lock,
* all future threads that attempt to [Rmw] the lock to the slowpath. As such
* relaxed semantics suffice.
*/
static __always_inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
static __always_inline void mark_rt_mutex_waiters(struct rt_mutex_base *lock)
{
unsigned long owner, *p = (unsigned long *) &lock->owner;

Expand All @@ -174,7 +174,7 @@ static __always_inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
* 2) Drop lock->wait_lock
* 3) Try to unlock the lock with cmpxchg
*/
static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,
static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex_base *lock,
unsigned long flags)
__releases(lock->wait_lock)
{
Expand Down Expand Up @@ -210,22 +210,22 @@ static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,
}

#else
static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex *lock,
static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex_base *lock,
struct task_struct *old,
struct task_struct *new)
{
return false;

}

static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex *lock,
static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock,
struct task_struct *old,
struct task_struct *new)
{
return false;
}

static __always_inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
static __always_inline void mark_rt_mutex_waiters(struct rt_mutex_base *lock)
{
lock->owner = (struct task_struct *)
((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);
Expand All @@ -234,7 +234,7 @@ static __always_inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
/*
* Simple slow path only version: lock->owner is protected by lock->wait_lock.
*/
static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,
static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex_base *lock,
unsigned long flags)
__releases(lock->wait_lock)
{
Expand Down Expand Up @@ -295,13 +295,13 @@ static __always_inline bool __waiter_less(struct rb_node *a, const struct rb_nod
}

static __always_inline void
rt_mutex_enqueue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)
rt_mutex_enqueue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)
{
rb_add_cached(&waiter->tree_entry, &lock->waiters, __waiter_less);
}

static __always_inline void
rt_mutex_dequeue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)
rt_mutex_dequeue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter)
{
if (RB_EMPTY_NODE(&waiter->tree_entry))
return;
Expand Down Expand Up @@ -369,7 +369,7 @@ rt_mutex_cond_detect_deadlock(struct rt_mutex_waiter *waiter,
return chwalk == RT_MUTEX_FULL_CHAINWALK;
}

static __always_inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p)
static __always_inline struct rt_mutex_base *task_blocked_on_lock(struct task_struct *p)
{
return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL;
}
Expand Down Expand Up @@ -439,15 +439,15 @@ static __always_inline struct rt_mutex *task_blocked_on_lock(struct task_struct
*/
static int __sched rt_mutex_adjust_prio_chain(struct task_struct *task,
enum rtmutex_chainwalk chwalk,
struct rt_mutex *orig_lock,
struct rt_mutex *next_lock,
struct rt_mutex_base *orig_lock,
struct rt_mutex_base *next_lock,
struct rt_mutex_waiter *orig_waiter,
struct task_struct *top_task)
{
struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;
struct rt_mutex_waiter *prerequeue_top_waiter;
int ret = 0, depth = 0;
struct rt_mutex *lock;
struct rt_mutex_base *lock;
bool detect_deadlock;
bool requeue = true;

Expand Down Expand Up @@ -795,7 +795,7 @@ static int __sched rt_mutex_adjust_prio_chain(struct task_struct *task,
* callsite called task_blocked_on_lock(), otherwise NULL
*/
static int __sched
try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
try_to_take_rt_mutex(struct rt_mutex_base *lock, struct task_struct *task,
struct rt_mutex_waiter *waiter)
{
lockdep_assert_held(&lock->wait_lock);
Expand Down Expand Up @@ -913,14 +913,14 @@ try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
*
* This must be called with lock->wait_lock held and interrupts disabled
*/
static int __sched task_blocks_on_rt_mutex(struct rt_mutex *lock,
static int __sched task_blocks_on_rt_mutex(struct rt_mutex_base *lock,
struct rt_mutex_waiter *waiter,
struct task_struct *task,
enum rtmutex_chainwalk chwalk)
{
struct task_struct *owner = rt_mutex_owner(lock);
struct rt_mutex_waiter *top_waiter = waiter;
struct rt_mutex *next_lock;
struct rt_mutex_base *next_lock;
int chain_walk = 0, res;

lockdep_assert_held(&lock->wait_lock);
Expand Down Expand Up @@ -1003,7 +1003,7 @@ static int __sched task_blocks_on_rt_mutex(struct rt_mutex *lock,
* Called with lock->wait_lock held and interrupts disabled.
*/
static void __sched mark_wakeup_next_waiter(struct wake_q_head *wake_q,
struct rt_mutex *lock)
struct rt_mutex_base *lock)
{
struct rt_mutex_waiter *waiter;

Expand Down Expand Up @@ -1052,12 +1052,12 @@ static void __sched mark_wakeup_next_waiter(struct wake_q_head *wake_q,
* Must be called with lock->wait_lock held and interrupts disabled. I must
* have just failed to try_to_take_rt_mutex().
*/
static void __sched remove_waiter(struct rt_mutex *lock,
static void __sched remove_waiter(struct rt_mutex_base *lock,
struct rt_mutex_waiter *waiter)
{
bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));
struct task_struct *owner = rt_mutex_owner(lock);
struct rt_mutex *next_lock;
struct rt_mutex_base *next_lock;

lockdep_assert_held(&lock->wait_lock);

Expand Down Expand Up @@ -1115,7 +1115,8 @@ static void __sched remove_waiter(struct rt_mutex *lock,
*
* Must be called with lock->wait_lock held and interrupts disabled
*/
static int __sched __rt_mutex_slowlock(struct rt_mutex *lock, unsigned int state,
static int __sched __rt_mutex_slowlock(struct rt_mutex_base *lock,
unsigned int state,
struct hrtimer_sleeper *timeout,
struct rt_mutex_waiter *waiter)
{
Expand Down Expand Up @@ -1170,7 +1171,8 @@ static void __sched rt_mutex_handle_deadlock(int res, int detect_deadlock,
/*
* Slow path lock function:
*/
static int __sched rt_mutex_slowlock(struct rt_mutex *lock, unsigned int state,
static int __sched rt_mutex_slowlock(struct rt_mutex_base *lock,
unsigned int state,
struct hrtimer_sleeper *timeout,
enum rtmutex_chainwalk chwalk)
{
Expand Down Expand Up @@ -1231,7 +1233,7 @@ static int __sched rt_mutex_slowlock(struct rt_mutex *lock, unsigned int state,
return ret;
}

static __always_inline int __rt_mutex_lock(struct rt_mutex *lock,
static __always_inline int __rt_mutex_lock(struct rt_mutex_base *lock,
unsigned int state)
{
if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
Expand All @@ -1240,7 +1242,7 @@ static __always_inline int __rt_mutex_lock(struct rt_mutex *lock,
return rt_mutex_slowlock(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK);
}

static int __sched __rt_mutex_slowtrylock(struct rt_mutex *lock)
static int __sched __rt_mutex_slowtrylock(struct rt_mutex_base *lock)
{
int ret = try_to_take_rt_mutex(lock, current, NULL);

Expand All @@ -1256,7 +1258,7 @@ static int __sched __rt_mutex_slowtrylock(struct rt_mutex *lock)
/*
* Slow path try-lock function:
*/
static int __sched rt_mutex_slowtrylock(struct rt_mutex *lock)
static int __sched rt_mutex_slowtrylock(struct rt_mutex_base *lock)
{
unsigned long flags;
int ret;
Expand All @@ -1282,7 +1284,7 @@ static int __sched rt_mutex_slowtrylock(struct rt_mutex *lock)
return ret;
}

static __always_inline int __rt_mutex_trylock(struct rt_mutex *lock)
static __always_inline int __rt_mutex_trylock(struct rt_mutex_base *lock)
{
if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current)))
return 1;
Expand All @@ -1293,7 +1295,7 @@ static __always_inline int __rt_mutex_trylock(struct rt_mutex *lock)
/*
* Slow path to release a rt-mutex.
*/
static void __sched rt_mutex_slowunlock(struct rt_mutex *lock)
static void __sched rt_mutex_slowunlock(struct rt_mutex_base *lock)
{
DEFINE_WAKE_Q(wake_q);
unsigned long flags;
Expand Down Expand Up @@ -1354,7 +1356,7 @@ static void __sched rt_mutex_slowunlock(struct rt_mutex *lock)
rt_mutex_postunlock(&wake_q);
}

static __always_inline void __rt_mutex_unlock(struct rt_mutex *lock)
static __always_inline void __rt_mutex_unlock(struct rt_mutex_base *lock)
{
if (likely(rt_mutex_cmpxchg_release(lock, current, NULL)))
return;
Expand Down
Loading

0 comments on commit 830e6ac

Please sign in to comment.