Skip to content

Commit

Permalink
powerpc/mm: handle hugepage size correctly when invalidating hpte ent…
Browse files Browse the repository at this point in the history 
…ries

If a hash bucket gets full, we "evict" a more/less random entry from it.
When we do that we don't invalidate the TLB (hpte_remove) because we assume
the old translation is still technically "valid". This implies that when
we are invalidating or updating pte, even if HPTE entry is not valid
we should do a tlb invalidate. With hugepages, we need to pass the correct
actual page size value for tlb invalidation.

This change update the patch 0608d69
"powerpc/mm: Always invalidate tlb on hpte invalidate and update" to handle
transparent hugepages correctly.

Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
  • Loading branch information
Aneesh Kumar K.V authored and Benjamin Herrenschmidt committed Jun 21, 2013
1 parent 8998897 commit db3d853
Show file tree
Hide file tree
Showing 9 changed files with 95 additions and 107 deletions.
8 changes: 4 additions & 4 deletions arch/powerpc/include/asm/machdep.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,13 @@ struct machdep_calls {
#ifdef CONFIG_PPC64
void (*hpte_invalidate)(unsigned long slot,
unsigned long vpn,
int psize, int ssize,
int local);
int bpsize, int apsize,
int ssize, int local);
long (*hpte_updatepp)(unsigned long slot,
unsigned long newpp,
unsigned long vpn,
int psize, int ssize,
int local);
int bpsize, int apsize,
int ssize, int local);
void (*hpte_updateboltedpp)(unsigned long newpp,
unsigned long ea,
int psize, int ssize);
Expand Down
2 changes: 1 addition & 1 deletion arch/powerpc/kvm/book3s_64_mmu_host.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
void kvmppc_mmu_invalidate_pte(struct kvm_vcpu *vcpu, struct hpte_cache *pte)
{
ppc_md.hpte_invalidate(pte->slot, pte->host_vpn,
MMU_PAGE_4K, MMU_SEGSIZE_256M,
MMU_PAGE_4K, MMU_PAGE_4K, MMU_SEGSIZE_256M,
false);
}

Expand Down
21 changes: 12 additions & 9 deletions arch/powerpc/mm/hash_low_64.S
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,9 +289,10 @@ htab_modify_pte:

/* Call ppc_md.hpte_updatepp */
mr r5,r29 /* vpn */
li r6,MMU_PAGE_4K /* page size */
ld r7,STK_PARAM(R9)(r1) /* segment size */
ld r8,STK_PARAM(R8)(r1) /* get "local" param */
li r6,MMU_PAGE_4K /* base page size */
li r7,MMU_PAGE_4K /* actual page size */
ld r8,STK_PARAM(R9)(r1) /* segment size */
ld r9,STK_PARAM(R8)(r1) /* get "local" param */
_GLOBAL(htab_call_hpte_updatepp)
bl . /* Patched by htab_finish_init() */

Expand Down Expand Up @@ -649,9 +650,10 @@ htab_modify_pte:

/* Call ppc_md.hpte_updatepp */
mr r5,r29 /* vpn */
li r6,MMU_PAGE_4K /* page size */
ld r7,STK_PARAM(R9)(r1) /* segment size */
ld r8,STK_PARAM(R8)(r1) /* get "local" param */
li r6,MMU_PAGE_4K /* base page size */
li r7,MMU_PAGE_4K /* actual page size */
ld r8,STK_PARAM(R9)(r1) /* segment size */
ld r9,STK_PARAM(R8)(r1) /* get "local" param */
_GLOBAL(htab_call_hpte_updatepp)
bl . /* patched by htab_finish_init() */

Expand Down Expand Up @@ -937,9 +939,10 @@ ht64_modify_pte:

/* Call ppc_md.hpte_updatepp */
mr r5,r29 /* vpn */
li r6,MMU_PAGE_64K
ld r7,STK_PARAM(R9)(r1) /* segment size */
ld r8,STK_PARAM(R8)(r1) /* get "local" param */
li r6,MMU_PAGE_64K /* base page size */
li r7,MMU_PAGE_64K /* actual page size */
ld r8,STK_PARAM(R9)(r1) /* segment size */
ld r9,STK_PARAM(R8)(r1) /* get "local" param */
_GLOBAL(ht64_call_hpte_updatepp)
bl . /* patched by htab_finish_init() */

Expand Down
122 changes: 45 additions & 77 deletions arch/powerpc/mm/hash_native_64.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,82 +273,30 @@ static long native_hpte_remove(unsigned long hpte_group)
return i;
}

static inline int __hpte_actual_psize(unsigned int lp, int psize)
{
int i, shift;
unsigned int mask;

/* start from 1 ignoring MMU_PAGE_4K */
for (i = 1; i < MMU_PAGE_COUNT; i++) {

/* invalid penc */
if (mmu_psize_defs[psize].penc[i] == -1)
continue;
/*
* encoding bits per actual page size
* PTE LP actual page size
* rrrr rrrz >=8KB
* rrrr rrzz >=16KB
* rrrr rzzz >=32KB
* rrrr zzzz >=64KB
* .......
*/
shift = mmu_psize_defs[i].shift - LP_SHIFT;
if (shift > LP_BITS)
shift = LP_BITS;
mask = (1 << shift) - 1;
if ((lp & mask) == mmu_psize_defs[psize].penc[i])
return i;
}
return -1;
}

static inline int hpte_actual_psize(struct hash_pte *hptep, int psize)
{
/* Look at the 8 bit LP value */
unsigned int lp = (hptep->r >> LP_SHIFT) & ((1 << LP_BITS) - 1);

if (!(hptep->v & HPTE_V_VALID))
return -1;

/* First check if it is large page */
if (!(hptep->v & HPTE_V_LARGE))
return MMU_PAGE_4K;

return __hpte_actual_psize(lp, psize);
}

static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
unsigned long vpn, int psize, int ssize,
int local)
unsigned long vpn, int bpsize,
int apsize, int ssize, int local)
{
struct hash_pte *hptep = htab_address + slot;
unsigned long hpte_v, want_v;
int ret = 0;
int actual_psize;

want_v = hpte_encode_avpn(vpn, psize, ssize);
want_v = hpte_encode_avpn(vpn, bpsize, ssize);

DBG_LOW(" update(vpn=%016lx, avpnv=%016lx, group=%lx, newpp=%lx)",
vpn, want_v & HPTE_V_AVPN, slot, newpp);

native_lock_hpte(hptep);

hpte_v = hptep->v;
actual_psize = hpte_actual_psize(hptep, psize);
/*
* We need to invalidate the TLB always because hpte_remove doesn't do
* a tlb invalidate. If a hash bucket gets full, we "evict" a more/less
* random entry from it. When we do that we don't invalidate the TLB
* (hpte_remove) because we assume the old translation is still
* technically "valid".
*/
if (actual_psize < 0) {
actual_psize = psize;
ret = -1;
goto err_out;
}
if (!HPTE_V_COMPARE(hpte_v, want_v)) {
if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) {
DBG_LOW(" -> miss\n");
ret = -1;
} else {
Expand All @@ -357,11 +305,10 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp,
hptep->r = (hptep->r & ~(HPTE_R_PP | HPTE_R_N)) |
(newpp & (HPTE_R_PP | HPTE_R_N | HPTE_R_C));
}
err_out:
native_unlock_hpte(hptep);

/* Ensure it is out of the tlb too. */
tlbie(vpn, psize, actual_psize, ssize, local);
tlbie(vpn, bpsize, apsize, ssize, local);

return ret;
}
Expand Down Expand Up @@ -402,7 +349,6 @@ static long native_hpte_find(unsigned long vpn, int psize, int ssize)
static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
int psize, int ssize)
{
int actual_psize;
unsigned long vpn;
unsigned long vsid;
long slot;
Expand All @@ -415,60 +361,82 @@ static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
if (slot == -1)
panic("could not find page to bolt\n");
hptep = htab_address + slot;
actual_psize = hpte_actual_psize(hptep, psize);
if (actual_psize < 0)
actual_psize = psize;

/* Update the HPTE */
hptep->r = (hptep->r & ~(HPTE_R_PP | HPTE_R_N)) |
(newpp & (HPTE_R_PP | HPTE_R_N));

/* Ensure it is out of the tlb too. */
tlbie(vpn, psize, actual_psize, ssize, 0);
/*
* Ensure it is out of the tlb too. Bolted entries base and
* actual page size will be same.
*/
tlbie(vpn, psize, psize, ssize, 0);
}

static void native_hpte_invalidate(unsigned long slot, unsigned long vpn,
int psize, int ssize, int local)
int bpsize, int apsize, int ssize, int local)
{
struct hash_pte *hptep = htab_address + slot;
unsigned long hpte_v;
unsigned long want_v;
unsigned long flags;
int actual_psize;

local_irq_save(flags);

DBG_LOW(" invalidate(vpn=%016lx, hash: %lx)\n", vpn, slot);

want_v = hpte_encode_avpn(vpn, psize, ssize);
want_v = hpte_encode_avpn(vpn, bpsize, ssize);
native_lock_hpte(hptep);
hpte_v = hptep->v;

actual_psize = hpte_actual_psize(hptep, psize);
/*
* We need to invalidate the TLB always because hpte_remove doesn't do
* a tlb invalidate. If a hash bucket gets full, we "evict" a more/less
* random entry from it. When we do that we don't invalidate the TLB
* (hpte_remove) because we assume the old translation is still
* technically "valid".
*/
if (actual_psize < 0) {
actual_psize = psize;
native_unlock_hpte(hptep);
goto err_out;
}
if (!HPTE_V_COMPARE(hpte_v, want_v))
if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))
native_unlock_hpte(hptep);
else
/* Invalidate the hpte. NOTE: this also unlocks it */
hptep->v = 0;

err_out:
/* Invalidate the TLB */
tlbie(vpn, psize, actual_psize, ssize, local);
tlbie(vpn, bpsize, apsize, ssize, local);

local_irq_restore(flags);
}

static inline int __hpte_actual_psize(unsigned int lp, int psize)
{
int i, shift;
unsigned int mask;

/* start from 1 ignoring MMU_PAGE_4K */
for (i = 1; i < MMU_PAGE_COUNT; i++) {

/* invalid penc */
if (mmu_psize_defs[psize].penc[i] == -1)
continue;
/*
* encoding bits per actual page size
* PTE LP actual page size
* rrrr rrrz >=8KB
* rrrr rrzz >=16KB
* rrrr rzzz >=32KB
* rrrr zzzz >=64KB
* .......
*/
shift = mmu_psize_defs[i].shift - LP_SHIFT;
if (shift > LP_BITS)
shift = LP_BITS;
mask = (1 << shift) - 1;
if ((lp & mask) == mmu_psize_defs[psize].penc[i])
return i;
}
return -1;
}

static void hpte_decode(struct hash_pte *hpte, unsigned long slot,
int *psize, int *apsize, int *ssize, unsigned long *vpn)
{
Expand Down
9 changes: 7 additions & 2 deletions arch/powerpc/mm/hash_utils_64.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1232,7 +1232,11 @@ void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, int ssize,
slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
slot += hidx & _PTEIDX_GROUP_IX;
DBG_LOW(" sub %ld: hash=%lx, hidx=%lx\n", index, slot, hidx);
ppc_md.hpte_invalidate(slot, vpn, psize, ssize, local);
/*
* We use same base page size and actual psize, because we don't
* use these functions for hugepage
*/
ppc_md.hpte_invalidate(slot, vpn, psize, psize, ssize, local);
} pte_iterate_hashed_end();

#ifdef CONFIG_PPC_TRANSACTIONAL_MEM
Expand Down Expand Up @@ -1365,7 +1369,8 @@ static void kernel_unmap_linear_page(unsigned long vaddr, unsigned long lmi)
hash = ~hash;
slot = (hash & htab_hash_mask) * HPTES_PER_GROUP;
slot += hidx & _PTEIDX_GROUP_IX;
ppc_md.hpte_invalidate(slot, vpn, mmu_linear_psize, mmu_kernel_ssize, 0);
ppc_md.hpte_invalidate(slot, vpn, mmu_linear_psize, mmu_linear_psize,
mmu_kernel_ssize, 0);
}

void kernel_map_pages(struct page *page, int numpages, int enable)
Expand Down
2 changes: 1 addition & 1 deletion arch/powerpc/mm/hugetlbpage-hash64.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
slot += (old_pte & _PAGE_F_GIX) >> 12;

if (ppc_md.hpte_updatepp(slot, rflags, vpn, mmu_psize,
ssize, local) == -1)
mmu_psize, ssize, local) == -1)
old_pte &= ~_PAGE_HPTEFLAGS;
}

Expand Down
16 changes: 10 additions & 6 deletions arch/powerpc/platforms/cell/beat_htab.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,8 @@ static void beat_lpar_hptab_clear(void)
static long beat_lpar_hpte_updatepp(unsigned long slot,
unsigned long newpp,
unsigned long vpn,
int psize, int ssize, int local)
int psize, int apsize,
int ssize, int local)
{
unsigned long lpar_rc;
u64 dummy0, dummy1;
Expand Down Expand Up @@ -274,7 +275,8 @@ static void beat_lpar_hpte_updateboltedpp(unsigned long newpp,
}

static void beat_lpar_hpte_invalidate(unsigned long slot, unsigned long vpn,
int psize, int ssize, int local)
int psize, int apsize,
int ssize, int local)
{
unsigned long want_v;
unsigned long lpar_rc;
Expand Down Expand Up @@ -364,9 +366,10 @@ static long beat_lpar_hpte_insert_v3(unsigned long hpte_group,
* already zero. For now I am paranoid.
*/
static long beat_lpar_hpte_updatepp_v3(unsigned long slot,
unsigned long newpp,
unsigned long vpn,
int psize, int ssize, int local)
unsigned long newpp,
unsigned long vpn,
int psize, int apsize,
int ssize, int local)
{
unsigned long lpar_rc;
unsigned long want_v;
Expand Down Expand Up @@ -394,7 +397,8 @@ static long beat_lpar_hpte_updatepp_v3(unsigned long slot,
}

static void beat_lpar_hpte_invalidate_v3(unsigned long slot, unsigned long vpn,
int psize, int ssize, int local)
int psize, int apsize,
int ssize, int local)
{
unsigned long want_v;
unsigned long lpar_rc;
Expand Down
5 changes: 3 additions & 2 deletions arch/powerpc/platforms/ps3/htab.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,8 @@ static long ps3_hpte_remove(unsigned long hpte_group)
}

static long ps3_hpte_updatepp(unsigned long slot, unsigned long newpp,
unsigned long vpn, int psize, int ssize, int local)
unsigned long vpn, int psize, int apsize,
int ssize, int local)
{
int result;
u64 hpte_v, want_v, hpte_rs;
Expand Down Expand Up @@ -162,7 +163,7 @@ static void ps3_hpte_updateboltedpp(unsigned long newpp, unsigned long ea,
}

static void ps3_hpte_invalidate(unsigned long slot, unsigned long vpn,
int psize, int ssize, int local)
int psize, int apsize, int ssize, int local)
{
unsigned long flags;
int result;
Expand Down
Loading

0 comments on commit db3d853

Please sign in to comment.