Skip to content

Commit

Permalink
sched: Introduce affinity_context
Browse files Browse the repository at this point in the history 
In order to prepare for passing through additional data through the
affinity call-chains, convert the mask and flags argument into a
structure.

Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Waiman Long <longman@redhat.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20220922180041.1768141-5-longman@redhat.com
  • Loading branch information
Waiman Long authored and Peter Zijlstra committed Oct 27, 2022
1 parent 5584e8a commit 713a2e2
Show file tree
Hide file tree
Showing 3 changed files with 85 additions and 47 deletions.
114 changes: 75 additions & 39 deletions kernel/sched/core.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2189,14 +2189,18 @@ void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags)
#ifdef CONFIG_SMP

static void
__do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32 flags);
__do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx);

static int __set_cpus_allowed_ptr(struct task_struct *p,
const struct cpumask *new_mask,
u32 flags);
struct affinity_context *ctx);

static void migrate_disable_switch(struct rq *rq, struct task_struct *p)
{
struct affinity_context ac = {
.new_mask = cpumask_of(rq->cpu),
.flags = SCA_MIGRATE_DISABLE,
};

if (likely(!p->migration_disabled))
return;

Expand All @@ -2206,7 +2210,7 @@ static void migrate_disable_switch(struct rq *rq, struct task_struct *p)
/*
* Violates locking rules! see comment in __do_set_cpus_allowed().
*/
__do_set_cpus_allowed(p, cpumask_of(rq->cpu), SCA_MIGRATE_DISABLE);
__do_set_cpus_allowed(p, &ac);
}

void migrate_disable(void)
Expand All @@ -2228,6 +2232,10 @@ EXPORT_SYMBOL_GPL(migrate_disable);
void migrate_enable(void)
{
struct task_struct *p = current;
struct affinity_context ac = {
.new_mask = &p->cpus_mask,
.flags = SCA_MIGRATE_ENABLE,
};

if (p->migration_disabled > 1) {
p->migration_disabled--;
Expand All @@ -2243,7 +2251,7 @@ void migrate_enable(void)
*/
preempt_disable();
if (p->cpus_ptr != &p->cpus_mask)
__set_cpus_allowed_ptr(p, &p->cpus_mask, SCA_MIGRATE_ENABLE);
__set_cpus_allowed_ptr(p, &ac);
/*
* Mustn't clear migration_disabled() until cpus_ptr points back at the
* regular cpus_mask, otherwise things that race (eg.
Expand Down Expand Up @@ -2523,19 +2531,19 @@ int push_cpu_stop(void *arg)
* sched_class::set_cpus_allowed must do the below, but is not required to
* actually call this function.
*/
void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask, u32 flags)
void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx)
{
if (flags & (SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) {
p->cpus_ptr = new_mask;
if (ctx->flags & (SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) {
p->cpus_ptr = ctx->new_mask;
return;
}

cpumask_copy(&p->cpus_mask, new_mask);
p->nr_cpus_allowed = cpumask_weight(new_mask);
cpumask_copy(&p->cpus_mask, ctx->new_mask);
p->nr_cpus_allowed = cpumask_weight(ctx->new_mask);
}

static void
__do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32 flags)
__do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx)
{
struct rq *rq = task_rq(p);
bool queued, running;
Expand All @@ -2552,7 +2560,7 @@ __do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32
*
* XXX do further audits, this smells like something putrid.
*/
if (flags & SCA_MIGRATE_DISABLE)
if (ctx->flags & SCA_MIGRATE_DISABLE)
SCHED_WARN_ON(!p->on_cpu);
else
lockdep_assert_held(&p->pi_lock);
Expand All @@ -2571,7 +2579,7 @@ __do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32
if (running)
put_prev_task(rq, p);

p->sched_class->set_cpus_allowed(p, new_mask, flags);
p->sched_class->set_cpus_allowed(p, ctx);

if (queued)
enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
Expand All @@ -2581,7 +2589,12 @@ __do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask, u32

void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
{
__do_set_cpus_allowed(p, new_mask, 0);
struct affinity_context ac = {
.new_mask = new_mask,
.flags = 0,
};

__do_set_cpus_allowed(p, &ac);
}

int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src,
Expand Down Expand Up @@ -2834,8 +2847,7 @@ static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flag
* Called with both p->pi_lock and rq->lock held; drops both before returning.
*/
static int __set_cpus_allowed_ptr_locked(struct task_struct *p,
const struct cpumask *new_mask,
u32 flags,
struct affinity_context *ctx,
struct rq *rq,
struct rq_flags *rf)
__releases(rq->lock)
Expand Down Expand Up @@ -2864,7 +2876,7 @@ static int __set_cpus_allowed_ptr_locked(struct task_struct *p,
cpu_valid_mask = cpu_online_mask;
}

if (!kthread && !cpumask_subset(new_mask, cpu_allowed_mask)) {
if (!kthread && !cpumask_subset(ctx->new_mask, cpu_allowed_mask)) {
ret = -EINVAL;
goto out;
}
Expand All @@ -2873,18 +2885,18 @@ static int __set_cpus_allowed_ptr_locked(struct task_struct *p,
* Must re-check here, to close a race against __kthread_bind(),
* sched_setaffinity() is not guaranteed to observe the flag.
*/
if ((flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) {
if ((ctx->flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) {
ret = -EINVAL;
goto out;
}

if (!(flags & SCA_MIGRATE_ENABLE)) {
if (cpumask_equal(&p->cpus_mask, new_mask))
if (!(ctx->flags & SCA_MIGRATE_ENABLE)) {
if (cpumask_equal(&p->cpus_mask, ctx->new_mask))
goto out;

if (WARN_ON_ONCE(p == current &&
is_migration_disabled(p) &&
!cpumask_test_cpu(task_cpu(p), new_mask))) {
!cpumask_test_cpu(task_cpu(p), ctx->new_mask))) {
ret = -EBUSY;
goto out;
}
Expand All @@ -2895,18 +2907,18 @@ static int __set_cpus_allowed_ptr_locked(struct task_struct *p,
* for groups of tasks (ie. cpuset), so that load balancing is not
* immediately required to distribute the tasks within their new mask.
*/
dest_cpu = cpumask_any_and_distribute(cpu_valid_mask, new_mask);
dest_cpu = cpumask_any_and_distribute(cpu_valid_mask, ctx->new_mask);
if (dest_cpu >= nr_cpu_ids) {
ret = -EINVAL;
goto out;
}

__do_set_cpus_allowed(p, new_mask, flags);
__do_set_cpus_allowed(p, ctx);

if (flags & SCA_USER)
if (ctx->flags & SCA_USER)
user_mask = clear_user_cpus_ptr(p);

ret = affine_move_task(rq, p, rf, dest_cpu, flags);
ret = affine_move_task(rq, p, rf, dest_cpu, ctx->flags);

kfree(user_mask);

Expand All @@ -2928,18 +2940,23 @@ static int __set_cpus_allowed_ptr_locked(struct task_struct *p,
* call is not atomic; no spinlocks may be held.
*/
static int __set_cpus_allowed_ptr(struct task_struct *p,
const struct cpumask *new_mask, u32 flags)
struct affinity_context *ctx)
{
struct rq_flags rf;
struct rq *rq;

rq = task_rq_lock(p, &rf);
return __set_cpus_allowed_ptr_locked(p, new_mask, flags, rq, &rf);
return __set_cpus_allowed_ptr_locked(p, ctx, rq, &rf);
}

int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
{
return __set_cpus_allowed_ptr(p, new_mask, 0);
struct affinity_context ac = {
.new_mask = new_mask,
.flags = 0,
};

return __set_cpus_allowed_ptr(p, &ac);
}
EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);

Expand All @@ -2955,6 +2972,7 @@ static int restrict_cpus_allowed_ptr(struct task_struct *p,
const struct cpumask *subset_mask)
{
struct cpumask *user_mask = NULL;
struct affinity_context ac;
struct rq_flags rf;
struct rq *rq;
int err;
Expand Down Expand Up @@ -2991,7 +3009,11 @@ static int restrict_cpus_allowed_ptr(struct task_struct *p,
p->user_cpus_ptr = user_mask;
}

return __set_cpus_allowed_ptr_locked(p, new_mask, 0, rq, &rf);
ac = (struct affinity_context){
.new_mask = new_mask,
};

return __set_cpus_allowed_ptr_locked(p, &ac, rq, &rf);

err_unlock:
task_rq_unlock(rq, p, &rf);
Expand Down Expand Up @@ -3045,7 +3067,7 @@ void force_compatible_cpus_allowed_ptr(struct task_struct *p)
}

static int
__sched_setaffinity(struct task_struct *p, const struct cpumask *mask);
__sched_setaffinity(struct task_struct *p, struct affinity_context *ctx);

/*
* Restore the affinity of a task @p which was previously restricted by a
Expand All @@ -3058,14 +3080,17 @@ __sched_setaffinity(struct task_struct *p, const struct cpumask *mask);
void relax_compatible_cpus_allowed_ptr(struct task_struct *p)
{
struct cpumask *user_mask = p->user_cpus_ptr;
struct affinity_context ac = {
.new_mask = user_mask,
};
unsigned long flags;

/*
* Try to restore the old affinity mask. If this fails, then
* we free the mask explicitly to avoid it being inherited across
* a subsequent fork().
*/
if (!user_mask || !__sched_setaffinity(p, user_mask))
if (!user_mask || !__sched_setaffinity(p, &ac))
return;

raw_spin_lock_irqsave(&p->pi_lock, flags);
Expand Down Expand Up @@ -3550,10 +3575,9 @@ void sched_set_stop_task(int cpu, struct task_struct *stop)
#else /* CONFIG_SMP */

static inline int __set_cpus_allowed_ptr(struct task_struct *p,
const struct cpumask *new_mask,
u32 flags)
struct affinity_context *ctx)
{
return set_cpus_allowed_ptr(p, new_mask);
return set_cpus_allowed_ptr(p, ctx->new_mask);
}

static inline void migrate_disable_switch(struct rq *rq, struct task_struct *p) { }
Expand Down Expand Up @@ -8090,7 +8114,7 @@ int dl_task_check_affinity(struct task_struct *p, const struct cpumask *mask)
#endif

static int
__sched_setaffinity(struct task_struct *p, const struct cpumask *mask)
__sched_setaffinity(struct task_struct *p, struct affinity_context *ctx)
{
int retval;
cpumask_var_t cpus_allowed, new_mask;
Expand All @@ -8104,13 +8128,16 @@ __sched_setaffinity(struct task_struct *p, const struct cpumask *mask)
}

cpuset_cpus_allowed(p, cpus_allowed);
cpumask_and(new_mask, mask, cpus_allowed);
cpumask_and(new_mask, ctx->new_mask, cpus_allowed);

ctx->new_mask = new_mask;
ctx->flags |= SCA_CHECK;

retval = dl_task_check_affinity(p, new_mask);
if (retval)
goto out_free_new_mask;
again:
retval = __set_cpus_allowed_ptr(p, new_mask, SCA_CHECK | SCA_USER);
retval = __set_cpus_allowed_ptr(p, ctx);
if (retval)
goto out_free_new_mask;

Expand All @@ -8133,6 +8160,9 @@ __sched_setaffinity(struct task_struct *p, const struct cpumask *mask)

long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
{
struct affinity_context ac = {
.new_mask = in_mask,
};
struct task_struct *p;
int retval;

Expand Down Expand Up @@ -8167,7 +8197,7 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask)
if (retval)
goto out_put_task;

retval = __sched_setaffinity(p, in_mask);
retval = __sched_setaffinity(p, &ac);
out_put_task:
put_task_struct(p);
return retval;
Expand Down Expand Up @@ -8948,6 +8978,12 @@ void show_state_filter(unsigned int state_filter)
*/
void __init init_idle(struct task_struct *idle, int cpu)
{
#ifdef CONFIG_SMP
struct affinity_context ac = (struct affinity_context) {
.new_mask = cpumask_of(cpu),
.flags = 0,
};
#endif
struct rq *rq = cpu_rq(cpu);
unsigned long flags;

Expand All @@ -8972,7 +9008,7 @@ void __init init_idle(struct task_struct *idle, int cpu)
*
* And since this is boot we can forgo the serialization.
*/
set_cpus_allowed_common(idle, cpumask_of(cpu), 0);
set_cpus_allowed_common(idle, &ac);
#endif
/*
* We're having a chicken and egg problem, even though we are
Expand Down
7 changes: 3 additions & 4 deletions kernel/sched/deadline.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2485,8 +2485,7 @@ static void task_woken_dl(struct rq *rq, struct task_struct *p)
}

static void set_cpus_allowed_dl(struct task_struct *p,
const struct cpumask *new_mask,
u32 flags)
struct affinity_context *ctx)
{
struct root_domain *src_rd;
struct rq *rq;
Expand All @@ -2501,7 +2500,7 @@ static void set_cpus_allowed_dl(struct task_struct *p,
* update. We already made space for us in the destination
* domain (see cpuset_can_attach()).
*/
if (!cpumask_intersects(src_rd->span, new_mask)) {
if (!cpumask_intersects(src_rd->span, ctx->new_mask)) {
struct dl_bw *src_dl_b;

src_dl_b = dl_bw_of(cpu_of(rq));
Expand All @@ -2515,7 +2514,7 @@ static void set_cpus_allowed_dl(struct task_struct *p,
raw_spin_unlock(&src_dl_b->lock);
}

set_cpus_allowed_common(p, new_mask, flags);
set_cpus_allowed_common(p, ctx);
}

/* Assumes rq->lock is held */
Expand Down
11 changes: 7 additions & 4 deletions kernel/sched/sched.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2145,6 +2145,11 @@ extern const u32 sched_prio_to_wmult[40];

#define RETRY_TASK ((void *)-1UL)

struct affinity_context {
const struct cpumask *new_mask;
unsigned int flags;
};

struct sched_class {

#ifdef CONFIG_UCLAMP_TASK
Expand Down Expand Up @@ -2173,9 +2178,7 @@ struct sched_class {

void (*task_woken)(struct rq *this_rq, struct task_struct *task);

void (*set_cpus_allowed)(struct task_struct *p,
const struct cpumask *newmask,
u32 flags);
void (*set_cpus_allowed)(struct task_struct *p, struct affinity_context *ctx);

void (*rq_online)(struct rq *rq);
void (*rq_offline)(struct rq *rq);
Expand Down Expand Up @@ -2286,7 +2289,7 @@ extern void update_group_capacity(struct sched_domain *sd, int cpu);

extern void trigger_load_balance(struct rq *rq);

extern void set_cpus_allowed_common(struct task_struct *p, const struct cpumask *new_mask, u32 flags);
extern void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx);

static inline struct task_struct *get_push_task(struct rq *rq)
{
Expand Down

0 comments on commit 713a2e2

Please sign in to comment.