Skip to content

Commit

Permalink
riscv: mm: Prepare pt_ops helper functions for sv57
Browse files Browse the repository at this point in the history 
This patch prepare some pt_ops helper functions which will be used in
creating sv57 mappings during boot time.

Signed-off-by: Qinglin Pan <panqinglin2020@iscas.ac.cn>
Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>
  • Loading branch information
Qinglin Pan authored and Palmer Dabbelt committed Feb 15, 2022
1 parent d10efa2 commit 677b9eb
Show file tree
Hide file tree
Showing 3 changed files with 119 additions and 15 deletions.
1 change: 1 addition & 0 deletions arch/riscv/include/asm/fixmap.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ enum fixed_addresses {
FIX_PTE,
FIX_PMD,
FIX_PUD,
FIX_P4D,
FIX_TEXT_POKE1,
FIX_TEXT_POKE0,
FIX_EARLYCON_MEM_BASE,
Expand Down
2 changes: 2 additions & 0 deletions arch/riscv/include/asm/pgtable.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,8 @@ struct pt_alloc_ops {
phys_addr_t (*alloc_pmd)(uintptr_t va);
pud_t *(*get_pud_virt)(phys_addr_t pa);
phys_addr_t (*alloc_pud)(uintptr_t va);
p4d_t *(*get_p4d_virt)(phys_addr_t pa);
phys_addr_t (*alloc_p4d)(uintptr_t va);
#endif
};

Expand Down
131 changes: 116 additions & 15 deletions arch/riscv/mm/init.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,7 @@ pgd_t trampoline_pg_dir[PTRS_PER_PGD] __page_aligned_bss;
static pte_t fixmap_pte[PTRS_PER_PTE] __page_aligned_bss;

pgd_t early_pg_dir[PTRS_PER_PGD] __initdata __aligned(PAGE_SIZE);
static p4d_t __maybe_unused early_dtb_p4d[PTRS_PER_P4D] __initdata __aligned(PAGE_SIZE);
static pud_t __maybe_unused early_dtb_pud[PTRS_PER_PUD] __initdata __aligned(PAGE_SIZE);
static pmd_t __maybe_unused early_dtb_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE);

Expand Down Expand Up @@ -319,6 +320,16 @@ static pmd_t early_pmd[PTRS_PER_PMD] __initdata __aligned(PAGE_SIZE);
#define early_pmd ((pmd_t *)XIP_FIXUP(early_pmd))
#endif /* CONFIG_XIP_KERNEL */

static p4d_t trampoline_p4d[PTRS_PER_P4D] __page_aligned_bss;
static p4d_t fixmap_p4d[PTRS_PER_P4D] __page_aligned_bss;
static p4d_t early_p4d[PTRS_PER_P4D] __initdata __aligned(PAGE_SIZE);

#ifdef CONFIG_XIP_KERNEL
#define trampoline_p4d ((p4d_t *)XIP_FIXUP(trampoline_p4d))
#define fixmap_p4d ((p4d_t *)XIP_FIXUP(fixmap_p4d))
#define early_p4d ((p4d_t *)XIP_FIXUP(early_p4d))
#endif /* CONFIG_XIP_KERNEL */

static pud_t trampoline_pud[PTRS_PER_PUD] __page_aligned_bss;
static pud_t fixmap_pud[PTRS_PER_PUD] __page_aligned_bss;
static pud_t early_pud[PTRS_PER_PUD] __initdata __aligned(PAGE_SIZE);
Expand Down Expand Up @@ -433,6 +444,44 @@ static phys_addr_t alloc_pud_late(uintptr_t va)
return __pa(vaddr);
}

static p4d_t *__init get_p4d_virt_early(phys_addr_t pa)
{
return (p4d_t *)((uintptr_t)pa);
}

static p4d_t *__init get_p4d_virt_fixmap(phys_addr_t pa)
{
clear_fixmap(FIX_P4D);
return (p4d_t *)set_fixmap_offset(FIX_P4D, pa);
}

static p4d_t *__init get_p4d_virt_late(phys_addr_t pa)
{
return (p4d_t *)__va(pa);
}

static phys_addr_t __init alloc_p4d_early(uintptr_t va)
{
/* Only one P4D is available for early mapping */
BUG_ON((va - kernel_map.virt_addr) >> PGDIR_SHIFT);

return (uintptr_t)early_p4d;
}

static phys_addr_t __init alloc_p4d_fixmap(uintptr_t va)
{
return memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE);
}

static phys_addr_t alloc_p4d_late(uintptr_t va)
{
unsigned long vaddr;

vaddr = __get_free_page(GFP_KERNEL);
BUG_ON(!vaddr);
return __pa(vaddr);
}

static void __init create_pud_mapping(pud_t *pudp,
uintptr_t va, phys_addr_t pa,
phys_addr_t sz, pgprot_t prot)
Expand Down Expand Up @@ -460,21 +509,55 @@ static void __init create_pud_mapping(pud_t *pudp,
create_pmd_mapping(nextp, va, pa, sz, prot);
}

#define pgd_next_t pud_t
#define alloc_pgd_next(__va) (pgtable_l4_enabled ? \
pt_ops.alloc_pud(__va) : pt_ops.alloc_pmd(__va))
#define get_pgd_next_virt(__pa) (pgtable_l4_enabled ? \
pt_ops.get_pud_virt(__pa) : (pgd_next_t *)pt_ops.get_pmd_virt(__pa))
static void __init create_p4d_mapping(p4d_t *p4dp,
uintptr_t va, phys_addr_t pa,
phys_addr_t sz, pgprot_t prot)
{
pud_t *nextp;
phys_addr_t next_phys;
uintptr_t p4d_index = p4d_index(va);

if (sz == P4D_SIZE) {
if (p4d_val(p4dp[p4d_index]) == 0)
p4dp[p4d_index] = pfn_p4d(PFN_DOWN(pa), prot);
return;
}

if (p4d_val(p4dp[p4d_index]) == 0) {
next_phys = pt_ops.alloc_pud(va);
p4dp[p4d_index] = pfn_p4d(PFN_DOWN(next_phys), PAGE_TABLE);
nextp = pt_ops.get_pud_virt(next_phys);
memset(nextp, 0, PAGE_SIZE);
} else {
next_phys = PFN_PHYS(_p4d_pfn(p4dp[p4d_index]));
nextp = pt_ops.get_pud_virt(next_phys);
}

create_pud_mapping(nextp, va, pa, sz, prot);
}

#define pgd_next_t p4d_t
#define alloc_pgd_next(__va) (pgtable_l5_enabled ? \
pt_ops.alloc_p4d(__va) : (pgtable_l4_enabled ? \
pt_ops.alloc_pud(__va) : pt_ops.alloc_pmd(__va)))
#define get_pgd_next_virt(__pa) (pgtable_l5_enabled ? \
pt_ops.get_p4d_virt(__pa) : (pgd_next_t *)(pgtable_l4_enabled ? \
pt_ops.get_pud_virt(__pa) : (pud_t *)pt_ops.get_pmd_virt(__pa)))
#define create_pgd_next_mapping(__nextp, __va, __pa, __sz, __prot) \
(pgtable_l5_enabled ? \
create_p4d_mapping(__nextp, __va, __pa, __sz, __prot) : \
(pgtable_l4_enabled ? \
create_pud_mapping(__nextp, __va, __pa, __sz, __prot) : \
create_pmd_mapping((pmd_t *)__nextp, __va, __pa, __sz, __prot))
#define fixmap_pgd_next (pgtable_l4_enabled ? \
(uintptr_t)fixmap_pud : (uintptr_t)fixmap_pmd)
#define trampoline_pgd_next (pgtable_l4_enabled ? \
(uintptr_t)trampoline_pud : (uintptr_t)trampoline_pmd)
#define early_dtb_pgd_next (pgtable_l4_enabled ? \
(uintptr_t)early_dtb_pud : (uintptr_t)early_dtb_pmd)
create_pud_mapping((pud_t *)__nextp, __va, __pa, __sz, __prot) : \
create_pmd_mapping((pmd_t *)__nextp, __va, __pa, __sz, __prot)))
#define fixmap_pgd_next (pgtable_l5_enabled ? \
(uintptr_t)fixmap_p4d : (pgtable_l4_enabled ? \
(uintptr_t)fixmap_pud : (uintptr_t)fixmap_pmd))
#define trampoline_pgd_next (pgtable_l5_enabled ? \
(uintptr_t)trampoline_p4d : (pgtable_l4_enabled ? \
(uintptr_t)trampoline_pud : (uintptr_t)trampoline_pmd))
#define early_dtb_pgd_next (pgtable_l5_enabled ? \
(uintptr_t)early_dtb_p4d : (pgtable_l4_enabled ? \
(uintptr_t)early_dtb_pud : (uintptr_t)early_dtb_pmd))
#else
#define pgd_next_t pte_t
#define alloc_pgd_next(__va) pt_ops.alloc_pte(__va)
Expand All @@ -483,6 +566,7 @@ static void __init create_pud_mapping(pud_t *pudp,
create_pte_mapping(__nextp, __va, __pa, __sz, __prot)
#define fixmap_pgd_next ((uintptr_t)fixmap_pte)
#define early_dtb_pgd_next ((uintptr_t)early_dtb_pmd)
#define create_p4d_mapping(__pmdp, __va, __pa, __sz, __prot)
#define create_pud_mapping(__pmdp, __va, __pa, __sz, __prot)
#define create_pmd_mapping(__pmdp, __va, __pa, __sz, __prot)
#endif /* __PAGETABLE_PMD_FOLDED */
Expand Down Expand Up @@ -693,10 +777,13 @@ static void __init create_fdt_early_page_table(pgd_t *pgdir, uintptr_t dtb_pa)
PGDIR_SIZE,
IS_ENABLED(CONFIG_64BIT) ? PAGE_TABLE : PAGE_KERNEL);

if (pgtable_l4_enabled) {
if (pgtable_l5_enabled)
create_p4d_mapping(early_dtb_p4d, DTB_EARLY_BASE_VA,
(uintptr_t)early_dtb_pud, P4D_SIZE, PAGE_TABLE);

if (pgtable_l4_enabled)
create_pud_mapping(early_dtb_pud, DTB_EARLY_BASE_VA,
(uintptr_t)early_dtb_pmd, PUD_SIZE, PAGE_TABLE);
}

if (IS_ENABLED(CONFIG_64BIT)) {
create_pmd_mapping(early_dtb_pmd, DTB_EARLY_BASE_VA,
Expand Down Expand Up @@ -732,6 +819,8 @@ void __init pt_ops_set_early(void)
pt_ops.get_pmd_virt = get_pmd_virt_early;
pt_ops.alloc_pud = alloc_pud_early;
pt_ops.get_pud_virt = get_pud_virt_early;
pt_ops.alloc_p4d = alloc_p4d_early;
pt_ops.get_p4d_virt = get_p4d_virt_early;
#endif
}

Expand All @@ -752,6 +841,8 @@ void __init pt_ops_set_fixmap(void)
pt_ops.get_pmd_virt = kernel_mapping_pa_to_va((uintptr_t)get_pmd_virt_fixmap);
pt_ops.alloc_pud = kernel_mapping_pa_to_va((uintptr_t)alloc_pud_fixmap);
pt_ops.get_pud_virt = kernel_mapping_pa_to_va((uintptr_t)get_pud_virt_fixmap);
pt_ops.alloc_p4d = kernel_mapping_pa_to_va((uintptr_t)alloc_p4d_fixmap);
pt_ops.get_p4d_virt = kernel_mapping_pa_to_va((uintptr_t)get_p4d_virt_fixmap);
#endif
}

Expand All @@ -768,6 +859,8 @@ void __init pt_ops_set_late(void)
pt_ops.get_pmd_virt = get_pmd_virt_late;
pt_ops.alloc_pud = alloc_pud_late;
pt_ops.get_pud_virt = get_pud_virt_late;
pt_ops.alloc_p4d = alloc_p4d_late;
pt_ops.get_p4d_virt = get_p4d_virt_late;
#endif
}

Expand Down Expand Up @@ -828,6 +921,10 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)
fixmap_pgd_next, PGDIR_SIZE, PAGE_TABLE);

#ifndef __PAGETABLE_PMD_FOLDED
/* Setup fixmap P4D and PUD */
if (pgtable_l5_enabled)
create_p4d_mapping(fixmap_p4d, FIXADDR_START,
(uintptr_t)fixmap_pud, P4D_SIZE, PAGE_TABLE);
/* Setup fixmap PUD and PMD */
if (pgtable_l4_enabled)
create_pud_mapping(fixmap_pud, FIXADDR_START,
Expand All @@ -837,6 +934,9 @@ asmlinkage void __init setup_vm(uintptr_t dtb_pa)
/* Setup trampoline PGD and PMD */
create_pgd_mapping(trampoline_pg_dir, kernel_map.virt_addr,
trampoline_pgd_next, PGDIR_SIZE, PAGE_TABLE);
if (pgtable_l5_enabled)
create_p4d_mapping(trampoline_p4d, kernel_map.virt_addr,
(uintptr_t)trampoline_pud, P4D_SIZE, PAGE_TABLE);
if (pgtable_l4_enabled)
create_pud_mapping(trampoline_pud, kernel_map.virt_addr,
(uintptr_t)trampoline_pmd, PUD_SIZE, PAGE_TABLE);
Expand Down Expand Up @@ -938,6 +1038,7 @@ static void __init setup_vm_final(void)
clear_fixmap(FIX_PTE);
clear_fixmap(FIX_PMD);
clear_fixmap(FIX_PUD);
clear_fixmap(FIX_P4D);

/* Move to swapper page table */
csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | satp_mode);
Expand Down

0 comments on commit 677b9eb

Please sign in to comment.