Skip to content

Commit

Permalink
powerpc/64s: Improve local TLB flush for boot and MCE on POWER9
Browse files Browse the repository at this point in the history
There are several cases outside the normal address space management
where a CPU's entire local TLB is to be flushed:

  1. Booting the kernel, in case something has left stale entries in
     the TLB (e.g., kexec).

  2. Machine check, to clean corrupted TLB entries.

One other place where the TLB is flushed, is waking from deep idle
states. The flush is a side-effect of calling ->cpu_restore with the
intention of re-setting various SPRs. The flush itself is unnecessary
because in the first case, the TLB should not acquire new corrupted
TLB entries as part of sleep/wake (though they may be lost).

This type of TLB flush is coded inflexibly, several times for each CPU
type, and they have a number of problems with ISA v3.0B:

- The current radix mode of the MMU is not taken into account, it is
  always done as a hash flushn For IS=2 (LPID-matching flush from host)
  and IS=3 with HV=0 (guest kernel flush), tlbie(l) is undefined if
  the R field does not match the current radix mode.

- ISA v3.0B hash must flush the partition and process table caches as
  well.

- ISA v3.0B radix must flush partition and process scoped translations,
  partition and process table caches, and also the page walk cache.

So consolidate the flushing code and implement it in C and inline asm
under the mm/ directory with the rest of the flush code. Add ISA v3.0B
cases for radix and hash, and use the radix flush in radix environment.

Provide a way for IS=2 (LPID flush) to specify the radix mode of the
partition. Have KVM pass in the radix mode of the guest.

Take out the flushes from early cputable/dt_cpu_ftrs detection hooks,
and move it later in the boot process after, the MMU registers are set
up and before relocation is first turned on.

The TLB flush is no longer called when restoring from deep idle states.
This was not be done as a separate step because booting secondaries
uses the same cpu_restore as idle restore, which needs the TLB flush.

Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
  • Loading branch information
Nicholas Piggin authored and Michael Ellerman committed Jan 17, 2018
1 parent 4552d12 commit d474827
Show file tree
Hide file tree
Showing 13 changed files with 219 additions and 224 deletions.
1 change: 1 addition & 0 deletions arch/powerpc/include/asm/book3s/64/tlbflush-hash.h
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ static inline void arch_leave_lazy_mmu_mode(void)

#define arch_flush_lazy_mmu_mode() do {} while (0)

extern void hash__tlbiel_all(unsigned int action);

extern void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize,
int ssize, unsigned long flags);
Expand Down
3 changes: 3 additions & 0 deletions arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ static inline int mmu_get_ap(int psize)
return mmu_psize_defs[psize].ap;
}

extern void radix__tlbiel_all(unsigned int action);

extern void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end);
extern void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
Expand Down Expand Up @@ -47,4 +49,5 @@ extern void radix__flush_tlb_lpid(unsigned long lpid);
extern void radix__flush_tlb_all(void);
extern void radix__flush_tlb_pte_p9_dd1(unsigned long old_pte, struct mm_struct *mm,
unsigned long address);

#endif
34 changes: 34 additions & 0 deletions arch/powerpc/include/asm/book3s/64/tlbflush.h
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,40 @@
#include <asm/book3s/64/tlbflush-hash.h>
#include <asm/book3s/64/tlbflush-radix.h>

/* TLB flush actions. Used as argument to tlbiel_all() */
enum {
TLB_INVAL_SCOPE_GLOBAL = 0, /* invalidate all TLBs */
TLB_INVAL_SCOPE_LPID = 1, /* invalidate TLBs for current LPID */
};

static inline void tlbiel_all(void)
{
/*
* This is used for host machine check and bootup.
*
* This uses early_radix_enabled and implementations use
* early_cpu_has_feature etc because that works early in boot
* and this is the machine check path which is not performance
* critical.
*/
if (early_radix_enabled())
radix__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL);
else
hash__tlbiel_all(TLB_INVAL_SCOPE_GLOBAL);
}

static inline void tlbiel_all_lpid(bool radix)
{
/*
* This is used for guest machine check.
*/
if (radix)
radix__tlbiel_all(TLB_INVAL_SCOPE_LPID);
else
hash__tlbiel_all(TLB_INVAL_SCOPE_LPID);
}


#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
static inline void flush_pmd_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
Expand Down
12 changes: 0 additions & 12 deletions arch/powerpc/include/asm/cputable.h
Original file line number Diff line number Diff line change
Expand Up @@ -107,12 +107,6 @@ struct cpu_spec {
* called in real mode to handle SLB and TLB errors.
*/
long (*machine_check_early)(struct pt_regs *regs);

/*
* Processor specific routine to flush tlbs.
*/
void (*flush_tlb)(unsigned int action);

};

extern struct cpu_spec *cur_cpu_spec;
Expand All @@ -133,12 +127,6 @@ extern void cpu_feature_keys_init(void);
static inline void cpu_feature_keys_init(void) { }
#endif

/* TLB flush actions. Used as argument to cpu_spec.flush_tlb() hook */
enum {
TLB_INVAL_SCOPE_GLOBAL = 0, /* invalidate all TLBs */
TLB_INVAL_SCOPE_LPID = 1, /* invalidate TLBs for current LPID */
};

#endif /* __ASSEMBLY__ */

/* CPU kernel features */
Expand Down
50 changes: 0 additions & 50 deletions arch/powerpc/kernel/cpu_setup_power.S
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ _GLOBAL(__setup_cpu_power7)
mfspr r3,SPRN_LPCR
li r4,(LPCR_LPES1 >> LPCR_LPES_SH)
bl __init_LPCR_ISA206
bl __init_tlb_power7
mtlr r11
blr

Expand All @@ -45,7 +44,6 @@ _GLOBAL(__restore_cpu_power7)
mfspr r3,SPRN_LPCR
li r4,(LPCR_LPES1 >> LPCR_LPES_SH)
bl __init_LPCR_ISA206
bl __init_tlb_power7
mtlr r11
blr

Expand All @@ -64,7 +62,6 @@ _GLOBAL(__setup_cpu_power8)
li r4,0 /* LPES = 0 */
bl __init_LPCR_ISA206
bl __init_HFSCR
bl __init_tlb_power8
bl __init_PMU_HV
bl __init_PMU_HV_ISA207
mtlr r11
Expand All @@ -86,7 +83,6 @@ _GLOBAL(__restore_cpu_power8)
li r4,0 /* LPES = 0 */
bl __init_LPCR_ISA206
bl __init_HFSCR
bl __init_tlb_power8
bl __init_PMU_HV
bl __init_PMU_HV_ISA207
mtlr r11
Expand All @@ -110,7 +106,6 @@ _GLOBAL(__setup_cpu_power9)
li r4,0 /* LPES = 0 */
bl __init_LPCR_ISA300
bl __init_HFSCR
bl __init_tlb_power9
bl __init_PMU_HV
mtlr r11
blr
Expand All @@ -134,7 +129,6 @@ _GLOBAL(__restore_cpu_power9)
li r4,0 /* LPES = 0 */
bl __init_LPCR_ISA300
bl __init_HFSCR
bl __init_tlb_power9
bl __init_PMU_HV
mtlr r11
blr
Expand Down Expand Up @@ -192,50 +186,6 @@ __init_HFSCR:
mtspr SPRN_HFSCR,r3
blr

/*
* Clear the TLB using the specified IS form of tlbiel instruction
* (invalidate by congruence class). P7 has 128 CCs., P8 has 512.
*/
__init_tlb_power7:
li r6,POWER7_TLB_SETS
mtctr r6
li r7,0xc00 /* IS field = 0b11 */
ptesync
2: tlbiel r7
addi r7,r7,0x1000
bdnz 2b
ptesync
1: blr

__init_tlb_power8:
li r6,POWER8_TLB_SETS
mtctr r6
li r7,0xc00 /* IS field = 0b11 */
ptesync
2: tlbiel r7
addi r7,r7,0x1000
bdnz 2b
ptesync
1: blr

/*
* Flush the TLB in hash mode. Hash must flush with RIC=2 once for process
* and one for partition scope to clear process and partition table entries.
*/
__init_tlb_power9:
li r6,POWER9_TLB_SETS_HASH - 1
mtctr r6
li r7,0xc00 /* IS field = 0b11 */
li r8,0
ptesync
PPC_TLBIEL(7, 8, 2, 1, 0)
PPC_TLBIEL(7, 8, 2, 0, 0)
2: addi r7,r7,0x1000
PPC_TLBIEL(7, 8, 0, 0, 0)
bdnz 2b
ptesync
1: blr

__init_PMU_HV:
li r5,0
mtspr SPRN_MMCRC,r5
Expand Down
15 changes: 0 additions & 15 deletions arch/powerpc/kernel/cputable.c
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,6 @@ extern void __setup_cpu_power8(unsigned long offset, struct cpu_spec* spec);
extern void __restore_cpu_power8(void);
extern void __setup_cpu_power9(unsigned long offset, struct cpu_spec* spec);
extern void __restore_cpu_power9(void);
extern void __flush_tlb_power7(unsigned int action);
extern void __flush_tlb_power8(unsigned int action);
extern void __flush_tlb_power9(unsigned int action);
extern long __machine_check_early_realmode_p7(struct pt_regs *regs);
extern long __machine_check_early_realmode_p8(struct pt_regs *regs);
extern long __machine_check_early_realmode_p9(struct pt_regs *regs);
Expand Down Expand Up @@ -368,7 +365,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
.oprofile_cpu_type = "ppc64/ibm-compat-v1",
.cpu_setup = __setup_cpu_power7,
.cpu_restore = __restore_cpu_power7,
.flush_tlb = __flush_tlb_power7,
.machine_check_early = __machine_check_early_realmode_p7,
.platform = "power7",
},
Expand All @@ -386,7 +382,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
.oprofile_cpu_type = "ppc64/ibm-compat-v1",
.cpu_setup = __setup_cpu_power8,
.cpu_restore = __restore_cpu_power8,
.flush_tlb = __flush_tlb_power8,
.machine_check_early = __machine_check_early_realmode_p8,
.platform = "power8",
},
Expand All @@ -404,7 +399,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
.oprofile_cpu_type = "ppc64/ibm-compat-v1",
.cpu_setup = __setup_cpu_power9,
.cpu_restore = __restore_cpu_power9,
.flush_tlb = __flush_tlb_power9,
.platform = "power9",
},
{ /* Power7 */
Expand All @@ -423,7 +417,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
.oprofile_type = PPC_OPROFILE_POWER4,
.cpu_setup = __setup_cpu_power7,
.cpu_restore = __restore_cpu_power7,
.flush_tlb = __flush_tlb_power7,
.machine_check_early = __machine_check_early_realmode_p7,
.platform = "power7",
},
Expand All @@ -443,7 +436,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
.oprofile_type = PPC_OPROFILE_POWER4,
.cpu_setup = __setup_cpu_power7,
.cpu_restore = __restore_cpu_power7,
.flush_tlb = __flush_tlb_power7,
.machine_check_early = __machine_check_early_realmode_p7,
.platform = "power7+",
},
Expand All @@ -463,7 +455,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
.oprofile_type = PPC_OPROFILE_INVALID,
.cpu_setup = __setup_cpu_power8,
.cpu_restore = __restore_cpu_power8,
.flush_tlb = __flush_tlb_power8,
.machine_check_early = __machine_check_early_realmode_p8,
.platform = "power8",
},
Expand All @@ -483,7 +474,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
.oprofile_type = PPC_OPROFILE_INVALID,
.cpu_setup = __setup_cpu_power8,
.cpu_restore = __restore_cpu_power8,
.flush_tlb = __flush_tlb_power8,
.machine_check_early = __machine_check_early_realmode_p8,
.platform = "power8",
},
Expand All @@ -503,7 +493,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
.oprofile_type = PPC_OPROFILE_INVALID,
.cpu_setup = __setup_cpu_power8,
.cpu_restore = __restore_cpu_power8,
.flush_tlb = __flush_tlb_power8,
.machine_check_early = __machine_check_early_realmode_p8,
.platform = "power8",
},
Expand All @@ -523,7 +512,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
.oprofile_type = PPC_OPROFILE_INVALID,
.cpu_setup = __setup_cpu_power8,
.cpu_restore = __restore_cpu_power8,
.flush_tlb = __flush_tlb_power8,
.machine_check_early = __machine_check_early_realmode_p8,
.platform = "power8",
},
Expand All @@ -543,7 +531,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
.oprofile_type = PPC_OPROFILE_INVALID,
.cpu_setup = __setup_cpu_power9,
.cpu_restore = __restore_cpu_power9,
.flush_tlb = __flush_tlb_power9,
.machine_check_early = __machine_check_early_realmode_p9,
.platform = "power9",
},
Expand All @@ -563,7 +550,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
.oprofile_type = PPC_OPROFILE_INVALID,
.cpu_setup = __setup_cpu_power9,
.cpu_restore = __restore_cpu_power9,
.flush_tlb = __flush_tlb_power9,
.machine_check_early = __machine_check_early_realmode_p9,
.platform = "power9",
},
Expand All @@ -583,7 +569,6 @@ static struct cpu_spec __initdata cpu_specs[] = {
.oprofile_type = PPC_OPROFILE_INVALID,
.cpu_setup = __setup_cpu_power9,
.cpu_restore = __restore_cpu_power9,
.flush_tlb = __flush_tlb_power9,
.machine_check_early = __machine_check_early_realmode_p9,
.platform = "power9",
},
Expand Down
30 changes: 0 additions & 30 deletions arch/powerpc/kernel/dt_cpu_ftrs.c
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,6 @@ struct dt_cpu_feature {
* Set up the base CPU
*/

extern void __flush_tlb_power8(unsigned int action);
extern void __flush_tlb_power9(unsigned int action);
extern long __machine_check_early_realmode_p8(struct pt_regs *regs);
extern long __machine_check_early_realmode_p9(struct pt_regs *regs);

Expand All @@ -92,27 +90,6 @@ static struct {

static void (*init_pmu_registers)(void);

static void cpufeatures_flush_tlb(void)
{
/*
* This is a temporary measure to keep equivalent TLB flush as the
* cputable based setup code.
*/
switch (PVR_VER(mfspr(SPRN_PVR))) {
case PVR_POWER8:
case PVR_POWER8E:
case PVR_POWER8NVL:
__flush_tlb_power8(TLB_INVAL_SCOPE_GLOBAL);
break;
case PVR_POWER9:
__flush_tlb_power9(TLB_INVAL_SCOPE_GLOBAL);
break;
default:
pr_err("unknown CPU version for boot TLB flush\n");
break;
}
}

static void __restore_cpu_cpufeatures(void)
{
/*
Expand All @@ -137,8 +114,6 @@ static void __restore_cpu_cpufeatures(void)

if (init_pmu_registers)
init_pmu_registers();

cpufeatures_flush_tlb();
}

static char dt_cpu_name[64];
Expand All @@ -157,7 +132,6 @@ static struct cpu_spec __initdata base_cpu_spec = {
.oprofile_type = PPC_OPROFILE_INVALID,
.cpu_setup = NULL,
.cpu_restore = __restore_cpu_cpufeatures,
.flush_tlb = NULL,
.machine_check_early = NULL,
.platform = NULL,
};
Expand Down Expand Up @@ -412,7 +386,6 @@ static void init_pmu_power8(void)
static int __init feat_enable_mce_power8(struct dt_cpu_feature *f)
{
cur_cpu_spec->platform = "power8";
cur_cpu_spec->flush_tlb = __flush_tlb_power8;
cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p8;

return 1;
Expand Down Expand Up @@ -451,7 +424,6 @@ static void init_pmu_power9(void)
static int __init feat_enable_mce_power9(struct dt_cpu_feature *f)
{
cur_cpu_spec->platform = "power9";
cur_cpu_spec->flush_tlb = __flush_tlb_power9;
cur_cpu_spec->machine_check_early = __machine_check_early_realmode_p9;

return 1;
Expand Down Expand Up @@ -752,8 +724,6 @@ static void __init cpufeatures_setup_finished(void)
system_registers.hfscr = mfspr(SPRN_HFSCR);
system_registers.fscr = mfspr(SPRN_FSCR);

cpufeatures_flush_tlb();

pr_info("final cpu/mmu features = 0x%016lx 0x%08x\n",
cur_cpu_spec->cpu_features, cur_cpu_spec->mmu_features);
}
Expand Down
Loading

0 comments on commit d474827

Please sign in to comment.