Skip to content

Commit

Permalink
x86/mm/tlb: Unify flush_tlb_func_local() and flush_tlb_func_remote()
Browse files Browse the repository at this point in the history
The unification of these two functions allows to use them in the updated
SMP infrastrucutre.

To do so, remove the reason argument from flush_tlb_func_local(), add
a member to struct tlb_flush_info that says which CPU initiated the
flush and act accordingly. Optimize the size of flush_tlb_info while we
are at it.

Unfortunately, this prevents us from using a constant tlb_flush_info for
arch_tlbbatch_flush(), but in a later stage we may be able to inline
tlb_flush_info into the IPI data, so it should not have an impact
eventually.

Signed-off-by: Nadav Amit <namit@vmware.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Reviewed-by: Dave Hansen <dave.hansen@linux.intel.com>
Link: https://lore.kernel.org/r/20210220231712.2475218-3-namit@vmware.com
  • Loading branch information
Nadav Amit authored and Ingo Molnar committed Mar 6, 2021
1 parent a32a4d8 commit 4c1ba39
Show file tree
Hide file tree
Showing 2 changed files with 39 additions and 47 deletions.
5 changes: 3 additions & 2 deletions arch/x86/include/asm/tlbflush.h
Original file line number Diff line number Diff line change
Expand Up @@ -201,8 +201,9 @@ struct flush_tlb_info {
unsigned long start;
unsigned long end;
u64 new_tlb_gen;
unsigned int stride_shift;
bool freed_tables;
unsigned int initiating_cpu;
u8 stride_shift;
u8 freed_tables;
};

void flush_tlb_local(void);
Expand Down
81 changes: 36 additions & 45 deletions arch/x86/mm/tlb.c
Original file line number Diff line number Diff line change
Expand Up @@ -439,7 +439,7 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
* NB: leave_mm() calls us with prev == NULL and tsk == NULL.
*/

/* We don't want flush_tlb_func_* to run concurrently with us. */
/* We don't want flush_tlb_func() to run concurrently with us. */
if (IS_ENABLED(CONFIG_PROVE_LOCKING))
WARN_ON_ONCE(!irqs_disabled());

Expand Down Expand Up @@ -647,14 +647,13 @@ void initialize_tlbstate_and_flush(void)
}

/*
* flush_tlb_func_common()'s memory ordering requirement is that any
* flush_tlb_func()'s memory ordering requirement is that any
* TLB fills that happen after we flush the TLB are ordered after we
* read active_mm's tlb_gen. We don't need any explicit barriers
* because all x86 flush operations are serializing and the
* atomic64_read operation won't be reordered by the compiler.
*/
static void flush_tlb_func_common(const struct flush_tlb_info *f,
bool local, enum tlb_flush_reason reason)
static void flush_tlb_func(void *info)
{
/*
* We have three different tlb_gen values in here. They are:
Expand All @@ -665,14 +664,26 @@ static void flush_tlb_func_common(const struct flush_tlb_info *f,
* - f->new_tlb_gen: the generation that the requester of the flush
* wants us to catch up to.
*/
const struct flush_tlb_info *f = info;
struct mm_struct *loaded_mm = this_cpu_read(cpu_tlbstate.loaded_mm);
u32 loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
u64 mm_tlb_gen = atomic64_read(&loaded_mm->context.tlb_gen);
u64 local_tlb_gen = this_cpu_read(cpu_tlbstate.ctxs[loaded_mm_asid].tlb_gen);
bool local = smp_processor_id() == f->initiating_cpu;
unsigned long nr_invalidate = 0;

/* This code cannot presently handle being reentered. */
VM_WARN_ON(!irqs_disabled());

if (!local) {
inc_irq_stat(irq_tlb_count);
count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);

/* Can only happen on remote CPUs */
if (f->mm && f->mm != loaded_mm)
return;
}

if (unlikely(loaded_mm == &init_mm))
return;

Expand Down Expand Up @@ -700,8 +711,7 @@ static void flush_tlb_func_common(const struct flush_tlb_info *f,
* be handled can catch us all the way up, leaving no work for
* the second flush.
*/
trace_tlb_flush(reason, 0);
return;
goto done;
}

WARN_ON_ONCE(local_tlb_gen > mm_tlb_gen);
Expand Down Expand Up @@ -748,46 +758,34 @@ static void flush_tlb_func_common(const struct flush_tlb_info *f,
f->new_tlb_gen == local_tlb_gen + 1 &&
f->new_tlb_gen == mm_tlb_gen) {
/* Partial flush */
unsigned long nr_invalidate = (f->end - f->start) >> f->stride_shift;
unsigned long addr = f->start;

nr_invalidate = (f->end - f->start) >> f->stride_shift;

while (addr < f->end) {
flush_tlb_one_user(addr);
addr += 1UL << f->stride_shift;
}
if (local)
count_vm_tlb_events(NR_TLB_LOCAL_FLUSH_ONE, nr_invalidate);
trace_tlb_flush(reason, nr_invalidate);
} else {
/* Full flush. */
nr_invalidate = TLB_FLUSH_ALL;

flush_tlb_local();
if (local)
count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
trace_tlb_flush(reason, TLB_FLUSH_ALL);
}

/* Both paths above update our state to mm_tlb_gen. */
this_cpu_write(cpu_tlbstate.ctxs[loaded_mm_asid].tlb_gen, mm_tlb_gen);
}

static void flush_tlb_func_local(const void *info, enum tlb_flush_reason reason)
{
const struct flush_tlb_info *f = info;

flush_tlb_func_common(f, true, reason);
}

static void flush_tlb_func_remote(void *info)
{
const struct flush_tlb_info *f = info;

inc_irq_stat(irq_tlb_count);

if (f->mm && f->mm != this_cpu_read(cpu_tlbstate.loaded_mm))
return;

count_vm_tlb_event(NR_TLB_REMOTE_FLUSH_RECEIVED);
flush_tlb_func_common(f, false, TLB_REMOTE_SHOOTDOWN);
/* Tracing is done in a unified manner to reduce the code size */
done:
trace_tlb_flush(!local ? TLB_REMOTE_SHOOTDOWN :
(f->mm == NULL) ? TLB_LOCAL_SHOOTDOWN :
TLB_LOCAL_MM_SHOOTDOWN,
nr_invalidate);
}

static bool tlb_is_not_lazy(int cpu, void *data)
Expand Down Expand Up @@ -816,10 +814,10 @@ STATIC_NOPV void native_flush_tlb_others(const struct cpumask *cpumask,
* doing a speculative memory access.
*/
if (info->freed_tables)
smp_call_function_many(cpumask, flush_tlb_func_remote,
smp_call_function_many(cpumask, flush_tlb_func,
(void *)info, 1);
else
on_each_cpu_cond_mask(tlb_is_not_lazy, flush_tlb_func_remote,
on_each_cpu_cond_mask(tlb_is_not_lazy, flush_tlb_func,
(void *)info, 1, cpumask);
}

Expand Down Expand Up @@ -869,6 +867,7 @@ static inline struct flush_tlb_info *get_flush_tlb_info(struct mm_struct *mm,
info->stride_shift = stride_shift;
info->freed_tables = freed_tables;
info->new_tlb_gen = new_tlb_gen;
info->initiating_cpu = smp_processor_id();

return info;
}
Expand Down Expand Up @@ -908,7 +907,7 @@ void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
if (mm == this_cpu_read(cpu_tlbstate.loaded_mm)) {
lockdep_assert_irqs_enabled();
local_irq_disable();
flush_tlb_func_local(info, TLB_LOCAL_MM_SHOOTDOWN);
flush_tlb_func(info);
local_irq_enable();
}

Expand Down Expand Up @@ -1119,34 +1118,26 @@ void __flush_tlb_all(void)
}
EXPORT_SYMBOL_GPL(__flush_tlb_all);

/*
* arch_tlbbatch_flush() performs a full TLB flush regardless of the active mm.
* This means that the 'struct flush_tlb_info' that describes which mappings to
* flush is actually fixed. We therefore set a single fixed struct and use it in
* arch_tlbbatch_flush().
*/
static const struct flush_tlb_info full_flush_tlb_info = {
.mm = NULL,
.start = 0,
.end = TLB_FLUSH_ALL,
};

void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
{
struct flush_tlb_info *info;

int cpu = get_cpu();

info = get_flush_tlb_info(NULL, 0, TLB_FLUSH_ALL, 0, false, 0);
if (cpumask_test_cpu(cpu, &batch->cpumask)) {
lockdep_assert_irqs_enabled();
local_irq_disable();
flush_tlb_func_local(&full_flush_tlb_info, TLB_LOCAL_SHOOTDOWN);
flush_tlb_func(info);
local_irq_enable();
}

if (cpumask_any_but(&batch->cpumask, cpu) < nr_cpu_ids)
flush_tlb_others(&batch->cpumask, &full_flush_tlb_info);
flush_tlb_others(&batch->cpumask, info);

cpumask_clear(&batch->cpumask);

put_flush_tlb_info();
put_cpu();
}

Expand Down

0 comments on commit 4c1ba39

Please sign in to comment.