diff --git a/include/linux/topology.h b/include/linux/topology.h index 52f5850730b3e..da92fea385858 100644 --- a/include/linux/topology.h +++ b/include/linux/topology.h @@ -245,8 +245,13 @@ static inline const struct cpumask *cpu_cpu_mask(int cpu) return cpumask_of_node(cpu_to_node(cpu)); } +/* + * sched_numa_find_*_cpu() functions family traverses only accessible CPUs, + * i.e. those listed in cpu_online_mask. + */ #ifdef CONFIG_NUMA int sched_numa_find_nth_cpu(const struct cpumask *cpus, int cpu, int node); +int sched_numa_find_next_cpu(const struct cpumask *cpus, int cpu, int node, unsigned int *hop); extern const struct cpumask *sched_numa_hop_mask(unsigned int node, unsigned int hops); #else static __always_inline int sched_numa_find_nth_cpu(const struct cpumask *cpus, int cpu, int node) @@ -254,6 +259,13 @@ static __always_inline int sched_numa_find_nth_cpu(const struct cpumask *cpus, i return cpumask_nth_and(cpu, cpus, cpu_online_mask); } +static __always_inline +int sched_numa_find_next_cpu(const struct cpumask *cpus, int cpu, int node, unsigned int *hop) +{ + return find_next_and_bit(cpumask_bits(cpus), cpumask_bits(cpu_online_mask), + small_cpumask_bits, cpu); +} + static inline const struct cpumask * sched_numa_hop_mask(unsigned int node, unsigned int hops) { diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c index d3a3b2646ec4f..3b5751a3445de 100644 --- a/kernel/sched/topology.c +++ b/kernel/sched/topology.c @@ -2135,6 +2135,45 @@ int sched_numa_find_nth_cpu(const struct cpumask *cpus, int cpu, int node) } EXPORT_SYMBOL_GPL(sched_numa_find_nth_cpu); +/* + * sched_numa_find_next_cpu() - given the NUMA topology, find the next cpu + * cpumask: cpumask to find a CPU from + * cpu: current CPU + * node: local node + * hop: (in/out) indicates distance order of current CPU to a local node + * + * The function searches for a next CPU at a given NUMA distance, indicated + * by hop, and if nothing found, tries to find CPUs at a greater distance, + * starting from the beginning. + * + * Return: cpu, or >= nr_cpu_ids when nothing found. + */ +int sched_numa_find_next_cpu(const struct cpumask *cpus, int cpu, int node, unsigned int *hop) +{ + unsigned long *cur, *prev; + struct cpumask ***masks; + unsigned int ret; + + if (*hop >= sched_domains_numa_levels) + return nr_cpu_ids; + + masks = rcu_dereference(sched_domains_numa_masks); + cur = cpumask_bits(masks[*hop][node]); + if (*hop == 0) + ret = find_next_and_bit(cpumask_bits(cpus), cur, nr_cpu_ids, cpu); + else { + prev = cpumask_bits(masks[*hop - 1][node]); + ret = find_next_and_andnot_bit(cpumask_bits(cpus), cur, prev, nr_cpu_ids, cpu); + } + + if (ret < nr_cpu_ids) + return ret; + + *hop += 1; + return sched_numa_find_next_cpu(cpus, 0, node, hop); +} +EXPORT_SYMBOL_GPL(sched_numa_find_next_cpu); + /** * sched_numa_hop_mask() - Get the cpumask of CPUs at most @hops hops away from * @node