Skip to content

Commit

Permalink
powerpc/mm/hash64: Store the slot information at the right offset for…
Browse files Browse the repository at this point in the history 
… hugetlb

The hugetlb pte entries are at the PMD and PUD level, so we can't use
PTRS_PER_PTE to find the second half of the page table. Use the right
offset for PUD/PMD to get to the second half of the table.

Fixes: bf9a95f ("powerpc: Free up four 64K PTE bits in 64K backed HPTE pages")
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Reviewed-by: Ram Pai <linuxram@us.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
  • Loading branch information
Aneesh Kumar K.V authored and Michael Ellerman committed Feb 13, 2018
1 parent 4a7aa4f commit ff31e10
Show file tree
Hide file tree
Showing 7 changed files with 28 additions and 17 deletions.
3 changes: 2 additions & 1 deletion arch/powerpc/include/asm/book3s/64/hash-4k.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ static inline int hash__hugepd_ok(hugepd_t hpd)
* keeping the prototype consistent across the two formats.
*/
static inline unsigned long pte_set_hidx(pte_t *ptep, real_pte_t rpte,
unsigned int subpg_index, unsigned long hidx)
unsigned int subpg_index, unsigned long hidx,
int offset)
{
return (hidx << H_PAGE_F_GIX_SHIFT) &
(H_PAGE_F_SECOND | H_PAGE_F_GIX);
Expand Down
9 changes: 5 additions & 4 deletions arch/powerpc/include/asm/book3s/64/hash-64k.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
* generic accessors and iterators here
*/
#define __real_pte __real_pte
static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep)
static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep, int offset)
{
real_pte_t rpte;
unsigned long *hidxp;
Expand All @@ -59,7 +59,7 @@ static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep)
*/
smp_rmb();

hidxp = (unsigned long *)(ptep + PTRS_PER_PTE);
hidxp = (unsigned long *)(ptep + offset);
rpte.hidx = *hidxp;
return rpte;
}
Expand All @@ -86,9 +86,10 @@ static inline unsigned long __rpte_to_hidx(real_pte_t rpte, unsigned long index)
* expected to modify the PTE bits accordingly and commit the PTE to memory.
*/
static inline unsigned long pte_set_hidx(pte_t *ptep, real_pte_t rpte,
unsigned int subpg_index, unsigned long hidx)
unsigned int subpg_index,
unsigned long hidx, int offset)
{
unsigned long *hidxp = (unsigned long *)(ptep + PTRS_PER_PTE);
unsigned long *hidxp = (unsigned long *)(ptep + offset);

rpte.hidx &= ~HIDX_BITS(0xfUL, subpg_index);
*hidxp = rpte.hidx | HIDX_BITS(HIDX_SHIFT_BY_ONE(hidx), subpg_index);
Expand Down
2 changes: 1 addition & 1 deletion arch/powerpc/include/asm/book3s/64/pgtable.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -350,7 +350,7 @@ extern unsigned long pci_io_base;
*/
#ifndef __real_pte

#define __real_pte(e,p) ((real_pte_t){(e)})
#define __real_pte(e, p, o) ((real_pte_t){(e)})
#define __rpte_to_pte(r) ((r).pte)
#define __rpte_to_hidx(r,index) (pte_val(__rpte_to_pte(r)) >> H_PAGE_F_GIX_SHIFT)

Expand Down
4 changes: 2 additions & 2 deletions arch/powerpc/mm/hash64_4k.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
* need to add in 0x1 if it's a read-only user page
*/
rflags = htab_convert_pte_flags(new_pte);
rpte = __real_pte(__pte(old_pte), ptep);
rpte = __real_pte(__pte(old_pte), ptep, PTRS_PER_PTE);

if (cpu_has_feature(CPU_FTR_NOEXECUTE) &&
!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
Expand Down Expand Up @@ -117,7 +117,7 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
return -1;
}
new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | H_PAGE_HASHPTE;
new_pte |= pte_set_hidx(ptep, rpte, 0, slot);
new_pte |= pte_set_hidx(ptep, rpte, 0, slot, PTRS_PER_PTE);
}
*ptep = __pte(new_pte & ~H_PAGE_BUSY);
return 0;
Expand Down
8 changes: 4 additions & 4 deletions arch/powerpc/mm/hash64_64k.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,

subpg_index = (ea & (PAGE_SIZE - 1)) >> shift;
vpn = hpt_vpn(ea, vsid, ssize);
rpte = __real_pte(__pte(old_pte), ptep);
rpte = __real_pte(__pte(old_pte), ptep, PTRS_PER_PTE);
/*
*None of the sub 4k page is hashed
*/
Expand Down Expand Up @@ -214,7 +214,7 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
return -1;
}

new_pte |= pte_set_hidx(ptep, rpte, subpg_index, slot);
new_pte |= pte_set_hidx(ptep, rpte, subpg_index, slot, PTRS_PER_PTE);
new_pte |= H_PAGE_HASHPTE;

*ptep = __pte(new_pte & ~H_PAGE_BUSY);
Expand Down Expand Up @@ -262,7 +262,7 @@ int __hash_page_64K(unsigned long ea, unsigned long access,
} while (!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));

rflags = htab_convert_pte_flags(new_pte);
rpte = __real_pte(__pte(old_pte), ptep);
rpte = __real_pte(__pte(old_pte), ptep, PTRS_PER_PTE);

if (cpu_has_feature(CPU_FTR_NOEXECUTE) &&
!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
Expand Down Expand Up @@ -327,7 +327,7 @@ int __hash_page_64K(unsigned long ea, unsigned long access,
}

new_pte = (new_pte & ~_PAGE_HPTEFLAGS) | H_PAGE_HASHPTE;
new_pte |= pte_set_hidx(ptep, rpte, 0, slot);
new_pte |= pte_set_hidx(ptep, rpte, 0, slot, PTRS_PER_PTE);
}
*ptep = __pte(new_pte & ~H_PAGE_BUSY);
return 0;
Expand Down
10 changes: 7 additions & 3 deletions arch/powerpc/mm/hugetlbpage-hash64.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
unsigned long vpn;
unsigned long old_pte, new_pte;
unsigned long rflags, pa, sz;
long slot;
long slot, offset;

BUG_ON(shift != mmu_psize_defs[mmu_psize].shift);

Expand Down Expand Up @@ -63,7 +63,11 @@ int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
} while(!pte_xchg(ptep, __pte(old_pte), __pte(new_pte)));

rflags = htab_convert_pte_flags(new_pte);
rpte = __real_pte(__pte(old_pte), ptep);
if (unlikely(mmu_psize == MMU_PAGE_16G))
offset = PTRS_PER_PUD;
else
offset = PTRS_PER_PMD;
rpte = __real_pte(__pte(old_pte), ptep, offset);

sz = ((1UL) << shift);
if (!cpu_has_feature(CPU_FTR_COHERENT_ICACHE))
Expand Down Expand Up @@ -104,7 +108,7 @@ int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid,
return -1;
}

new_pte |= pte_set_hidx(ptep, rpte, 0, slot);
new_pte |= pte_set_hidx(ptep, rpte, 0, slot, offset);
}

/*
Expand Down
9 changes: 7 additions & 2 deletions arch/powerpc/mm/tlb_hash64.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
unsigned int psize;
int ssize;
real_pte_t rpte;
int i;
int i, offset;

i = batch->index;

Expand All @@ -67,6 +67,10 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
psize = get_slice_psize(mm, addr);
/* Mask the address for the correct page size */
addr &= ~((1UL << mmu_psize_defs[psize].shift) - 1);
if (unlikely(psize == MMU_PAGE_16G))
offset = PTRS_PER_PUD;
else
offset = PTRS_PER_PMD;
#else
BUG();
psize = pte_pagesize_index(mm, addr, pte); /* shutup gcc */
Expand All @@ -78,6 +82,7 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
* support 64k pages, this might be different from the
* hardware page size encoded in the slice table. */
addr &= PAGE_MASK;
offset = PTRS_PER_PTE;
}


Expand All @@ -91,7 +96,7 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr,
}
WARN_ON(vsid == 0);
vpn = hpt_vpn(addr, vsid, ssize);
rpte = __real_pte(__pte(pte), ptep);
rpte = __real_pte(__pte(pte), ptep, offset);

/*
* Check if we have an active batch on this CPU. If not, just
Expand Down

0 comments on commit ff31e10

Please sign in to comment.