Skip to content

Commit

Permalink
arm64: handle 52-bit physical addresses in page table entries
Browse files Browse the repository at this point in the history 
The top 4 bits of a 52-bit physical address are positioned at bits
12..15 of a page table entry. Introduce macros to convert between a
physical address and its placement in a table entry, and change all
macros/functions that access PTEs to use them.

Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Tested-by: Suzuki K Poulose <suzuki.poulose@arm.com>
Reviewed-by: Suzuki K Poulose <suzuki.poulose@arm.com>
Tested-by: Bob Picco <bob.picco@oracle.com>
Reviewed-by: Bob Picco <bob.picco@oracle.com>
Signed-off-by: Kristina Martsenko <kristina.martsenko@arm.com>
[catalin.marinas@arm.com: some long lines wrapped]
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
  • Loading branch information
Kristina Martsenko authored and Catalin Marinas committed Dec 22, 2017
1 parent 1933830 commit 75387b9
Show file tree
Hide file tree
Showing 5 changed files with 51 additions and 20 deletions.
7 changes: 5 additions & 2 deletions arch/arm64/include/asm/kvm_mmu.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,14 +287,16 @@ static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd,
unsigned long hyp_idmap_start)
{
int idmap_idx;
u64 pgd_addr;

/*
* Use the first entry to access the HYP mappings. It is
* guaranteed to be free, otherwise we wouldn't use an
* extended idmap.
*/
VM_BUG_ON(pgd_val(merged_hyp_pgd[0]));
merged_hyp_pgd[0] = __pgd(__pa(hyp_pgd) | PMD_TYPE_TABLE);
pgd_addr = __phys_to_pgd_val(__pa(hyp_pgd));
merged_hyp_pgd[0] = __pgd(pgd_addr | PMD_TYPE_TABLE);

/*
* Create another extended level entry that points to the boot HYP map,
Expand All @@ -304,7 +306,8 @@ static inline void __kvm_extend_hypmap(pgd_t *boot_hyp_pgd,
*/
idmap_idx = hyp_idmap_start >> VA_BITS;
VM_BUG_ON(pgd_val(merged_hyp_pgd[idmap_idx]));
merged_hyp_pgd[idmap_idx] = __pgd(__pa(boot_hyp_pgd) | PMD_TYPE_TABLE);
pgd_addr = __phys_to_pgd_val(__pa(boot_hyp_pgd));
merged_hyp_pgd[idmap_idx] = __pgd(pgd_addr | PMD_TYPE_TABLE);
}

static inline unsigned int kvm_get_vmid_bits(void)
Expand Down
6 changes: 3 additions & 3 deletions arch/arm64/include/asm/pgalloc.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)

static inline void __pud_populate(pud_t *pud, phys_addr_t pmd, pudval_t prot)
{
set_pud(pud, __pud(pmd | prot));
set_pud(pud, __pud(__phys_to_pud_val(pmd) | prot));
}

static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
Expand Down Expand Up @@ -73,7 +73,7 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud)

static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t pud, pgdval_t prot)
{
set_pgd(pgdp, __pgd(pud | prot));
set_pgd(pgdp, __pgd(__phys_to_pgd_val(pud) | prot));
}

static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)
Expand Down Expand Up @@ -129,7 +129,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t pte,
pmdval_t prot)
{
set_pmd(pmdp, __pmd(pte | prot));
set_pmd(pmdp, __pmd(__phys_to_pmd_val(pte) | prot));
}

/*
Expand Down
6 changes: 4 additions & 2 deletions arch/arm64/include/asm/pgtable-hwdef.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,10 +168,12 @@
#define PTE_UXN (_AT(pteval_t, 1) << 54) /* User XN */
#define PTE_HYP_XN (_AT(pteval_t, 1) << 54) /* HYP XN */

#ifdef CONFIG_ARM64_PA_BITS_52
#define PTE_ADDR_LOW (((_AT(pteval_t, 1) << (48 - PAGE_SHIFT)) - 1) << PAGE_SHIFT)
#ifdef CONFIG_ARM64_PA_BITS_52
#define PTE_ADDR_HIGH (_AT(pteval_t, 0xf) << 12)
#define PTE_ADDR_MASK_52 (PTE_ADDR_LOW | PTE_ADDR_HIGH)
#define PTE_ADDR_MASK (PTE_ADDR_LOW | PTE_ADDR_HIGH)
#else
#define PTE_ADDR_MASK PTE_ADDR_LOW
#endif

/*
Expand Down
50 changes: 38 additions & 12 deletions arch/arm64/include/asm/pgtable.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,22 @@ extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];

#define pte_ERROR(pte) __pte_error(__FILE__, __LINE__, pte_val(pte))

#define pte_pfn(pte) ((pte_val(pte) & PHYS_MASK) >> PAGE_SHIFT)
/*
* Macros to convert between a physical address and its placement in a
* page table entry, taking care of 52-bit addresses.
*/
#ifdef CONFIG_ARM64_PA_BITS_52
#define __pte_to_phys(pte) \
((pte_val(pte) & PTE_ADDR_LOW) | ((pte_val(pte) & PTE_ADDR_HIGH) << 36))
#define __phys_to_pte_val(phys) (((phys) | ((phys) >> 36)) & PTE_ADDR_MASK)
#else
#define __pte_to_phys(pte) (pte_val(pte) & PTE_ADDR_MASK)
#define __phys_to_pte_val(phys) (phys)
#endif

#define pfn_pte(pfn,prot) (__pte(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)))
#define pte_pfn(pte) (__pte_to_phys(pte) >> PAGE_SHIFT)
#define pfn_pte(pfn,prot) \
__pte(__phys_to_pte_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))

#define pte_none(pte) (!pte_val(pte))
#define pte_clear(mm,addr,ptep) set_pte(ptep, __pte(0))
Expand Down Expand Up @@ -284,6 +297,11 @@ static inline int pte_same(pte_t pte_a, pte_t pte_b)

#define __HAVE_ARCH_PTE_SPECIAL

static inline pte_t pgd_pte(pgd_t pgd)
{
return __pte(pgd_val(pgd));
}

static inline pte_t pud_pte(pud_t pud)
{
return __pte(pud_val(pud));
Expand Down Expand Up @@ -349,16 +367,24 @@ static inline int pmd_protnone(pmd_t pmd)

#define pmd_mkhuge(pmd) (__pmd(pmd_val(pmd) & ~PMD_TABLE_BIT))

#define pmd_pfn(pmd) (((pmd_val(pmd) & PMD_MASK) & PHYS_MASK) >> PAGE_SHIFT)
#define pfn_pmd(pfn,prot) (__pmd(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)))
#define __pmd_to_phys(pmd) __pte_to_phys(pmd_pte(pmd))
#define __phys_to_pmd_val(phys) __phys_to_pte_val(phys)
#define pmd_pfn(pmd) ((__pmd_to_phys(pmd) & PMD_MASK) >> PAGE_SHIFT)
#define pfn_pmd(pfn,prot) __pmd(__phys_to_pmd_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))
#define mk_pmd(page,prot) pfn_pmd(page_to_pfn(page),prot)

#define pud_write(pud) pte_write(pud_pte(pud))
#define pud_pfn(pud) (((pud_val(pud) & PUD_MASK) & PHYS_MASK) >> PAGE_SHIFT)
#define pfn_pud(pfn,prot) (__pud(((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot)))

#define __pud_to_phys(pud) __pte_to_phys(pud_pte(pud))
#define __phys_to_pud_val(phys) __phys_to_pte_val(phys)
#define pud_pfn(pud) ((__pud_to_phys(pud) & PUD_MASK) >> PAGE_SHIFT)
#define pfn_pud(pfn,prot) __pud(__phys_to_pud_val((phys_addr_t)(pfn) << PAGE_SHIFT) | pgprot_val(prot))

#define set_pmd_at(mm, addr, pmdp, pmd) set_pte_at(mm, addr, (pte_t *)pmdp, pmd_pte(pmd))

#define __pgd_to_phys(pgd) __pte_to_phys(pgd_pte(pgd))
#define __phys_to_pgd_val(phys) __phys_to_pte_val(phys)

#define __pgprot_modify(prot,mask,bits) \
__pgprot((pgprot_val(prot) & ~(mask)) | (bits))

Expand Down Expand Up @@ -409,7 +435,7 @@ static inline void pmd_clear(pmd_t *pmdp)

static inline phys_addr_t pmd_page_paddr(pmd_t pmd)
{
return pmd_val(pmd) & PHYS_MASK & (s32)PAGE_MASK;
return __pmd_to_phys(pmd);
}

/* Find an entry in the third-level page table. */
Expand All @@ -427,7 +453,7 @@ static inline phys_addr_t pmd_page_paddr(pmd_t pmd)
#define pte_set_fixmap_offset(pmd, addr) pte_set_fixmap(pte_offset_phys(pmd, addr))
#define pte_clear_fixmap() clear_fixmap(FIX_PTE)

#define pmd_page(pmd) pfn_to_page(__phys_to_pfn(pmd_val(pmd) & PHYS_MASK))
#define pmd_page(pmd) pfn_to_page(__phys_to_pfn(__pmd_to_phys(pmd)))

/* use ONLY for statically allocated translation tables */
#define pte_offset_kimg(dir,addr) ((pte_t *)__phys_to_kimg(pte_offset_phys((dir), (addr))))
Expand Down Expand Up @@ -460,7 +486,7 @@ static inline void pud_clear(pud_t *pudp)

static inline phys_addr_t pud_page_paddr(pud_t pud)
{
return pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK;
return __pud_to_phys(pud);
}

/* Find an entry in the second-level page table. */
Expand All @@ -473,7 +499,7 @@ static inline phys_addr_t pud_page_paddr(pud_t pud)
#define pmd_set_fixmap_offset(pud, addr) pmd_set_fixmap(pmd_offset_phys(pud, addr))
#define pmd_clear_fixmap() clear_fixmap(FIX_PMD)

#define pud_page(pud) pfn_to_page(__phys_to_pfn(pud_val(pud) & PHYS_MASK))
#define pud_page(pud) pfn_to_page(__phys_to_pfn(__pud_to_phys(pud)))

/* use ONLY for statically allocated translation tables */
#define pmd_offset_kimg(dir,addr) ((pmd_t *)__phys_to_kimg(pmd_offset_phys((dir), (addr))))
Expand Down Expand Up @@ -512,7 +538,7 @@ static inline void pgd_clear(pgd_t *pgdp)

static inline phys_addr_t pgd_page_paddr(pgd_t pgd)
{
return pgd_val(pgd) & PHYS_MASK & (s32)PAGE_MASK;
return __pgd_to_phys(pgd);
}

/* Find an entry in the frst-level page table. */
Expand All @@ -525,7 +551,7 @@ static inline phys_addr_t pgd_page_paddr(pgd_t pgd)
#define pud_set_fixmap_offset(pgd, addr) pud_set_fixmap(pud_offset_phys(pgd, addr))
#define pud_clear_fixmap() clear_fixmap(FIX_PUD)

#define pgd_page(pgd) pfn_to_page(__phys_to_pfn(pgd_val(pgd) & PHYS_MASK))
#define pgd_page(pgd) pfn_to_page(__phys_to_pfn(__pgd_to_phys(pgd)))

/* use ONLY for statically allocated translation tables */
#define pud_offset_kimg(dir,addr) ((pud_t *)__phys_to_kimg(pud_offset_phys((dir), (addr))))
Expand Down
2 changes: 1 addition & 1 deletion arch/arm64/kernel/head.S
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ ENDPROC(preserve_boot_args)
* supporting this configuration with 64K pages.
*/
orr \pte, \phys, \phys, lsr #36
and \pte, \pte, #PTE_ADDR_MASK_52
and \pte, \pte, #PTE_ADDR_MASK
#else
mov \pte, \phys
#endif
Expand Down

0 comments on commit 75387b9

Please sign in to comment.