Skip to content

Commit

Permalink
riscv: mm: Support kasan for sv57
Browse files Browse the repository at this point in the history 
This patchset add kasan_populate and kasan_shallow_populate for sv57,
and is tested on both qemu and unmatched with CONFIG_KASAN and
CONFIG_KASAN_VMALLOC on.

Signed-off-by: Qinglin Pan <panqinglin2020@iscas.ac.cn>
Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>
  • Loading branch information
Qinglin Pan authored and Palmer Dabbelt committed Feb 15, 2022
1 parent 011f09d commit 8fbdccd
Showing 1 changed file with 141 additions and 14 deletions.
155 changes: 141 additions & 14 deletions arch/riscv/mm/kasan_init.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,8 @@ static void __init kasan_populate_pud(pgd_t *pgd,
* pt_ops facility.
*/
base_pud = pt_ops.get_pud_virt(pfn_to_phys(_pgd_pfn(*pgd)));
} else if (pgd_none(*pgd)) {
base_pud = memblock_alloc(PTRS_PER_PUD * sizeof(pud_t), PAGE_SIZE);
} else {
base_pud = (pud_t *)pgd_page_vaddr(*pgd);
if (base_pud == lm_alias(kasan_early_shadow_pud))
Expand Down Expand Up @@ -149,13 +151,72 @@ static void __init kasan_populate_pud(pgd_t *pgd,
set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(base_pud)), PAGE_TABLE));
}

#define kasan_early_shadow_pgd_next (pgtable_l4_enabled ? \
static void __init kasan_populate_p4d(pgd_t *pgd,
unsigned long vaddr, unsigned long end,
bool early)
{
phys_addr_t phys_addr;
p4d_t *p4dp, *base_p4d;
unsigned long next;

if (early) {
/*
* We can't use pgd_page_vaddr here as it would return a linear
* mapping address but it is not mapped yet, but when populating
* early_pg_dir, we need the physical address and when populating
* swapper_pg_dir, we need the kernel virtual address so use
* pt_ops facility.
*/
base_p4d = pt_ops.get_p4d_virt(pfn_to_phys(_pgd_pfn(*pgd)));
} else {
base_p4d = (p4d_t *)pgd_page_vaddr(*pgd);
if (base_p4d == lm_alias(kasan_early_shadow_p4d))
base_p4d = memblock_alloc(PTRS_PER_PUD * sizeof(p4d_t), PAGE_SIZE);
}

p4dp = base_p4d + p4d_index(vaddr);

do {
next = p4d_addr_end(vaddr, end);

if (p4d_none(*p4dp) && IS_ALIGNED(vaddr, P4D_SIZE) && (next - vaddr) >= P4D_SIZE) {
if (early) {
phys_addr = __pa(((uintptr_t)kasan_early_shadow_pud));
set_p4d(p4dp, pfn_p4d(PFN_DOWN(phys_addr), PAGE_TABLE));
continue;
} else {
phys_addr = memblock_phys_alloc(P4D_SIZE, P4D_SIZE);
if (phys_addr) {
set_p4d(p4dp, pfn_p4d(PFN_DOWN(phys_addr), PAGE_KERNEL));
continue;
}
}
}

kasan_populate_pud((pgd_t *)p4dp, vaddr, next, early);
} while (p4dp++, vaddr = next, vaddr != end);

/*
* Wait for the whole P4D to be populated before setting the P4D in
* the page table, otherwise, if we did set the P4D before populating
* it entirely, memblock could allocate a page at a physical address
* where KASAN is not populated yet and then we'd get a page fault.
*/
if (!early)
set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(base_p4d)), PAGE_TABLE));
}

#define kasan_early_shadow_pgd_next (pgtable_l5_enabled ? \
(uintptr_t)kasan_early_shadow_p4d : \
(pgtable_l4_enabled ? \
(uintptr_t)kasan_early_shadow_pud : \
(uintptr_t)kasan_early_shadow_pmd)
(uintptr_t)kasan_early_shadow_pmd))
#define kasan_populate_pgd_next(pgdp, vaddr, next, early) \
(pgtable_l5_enabled ? \
kasan_populate_p4d(pgdp, vaddr, next, early) : \
(pgtable_l4_enabled ? \
kasan_populate_pud(pgdp, vaddr, next, early) : \
kasan_populate_pmd((pud_t *)pgdp, vaddr, next))
kasan_populate_pmd((pud_t *)pgdp, vaddr, next)))

static void __init kasan_populate_pgd(pgd_t *pgdp,
unsigned long vaddr, unsigned long end,
Expand Down Expand Up @@ -219,6 +280,14 @@ asmlinkage void __init kasan_early_init(void)
PAGE_TABLE));
}

if (pgtable_l5_enabled) {
for (i = 0; i < PTRS_PER_P4D; ++i)
set_p4d(kasan_early_shadow_p4d + i,
pfn_p4d(PFN_DOWN
(__pa(((uintptr_t)kasan_early_shadow_pud))),
PAGE_TABLE));
}

kasan_populate_pgd(early_pg_dir + pgd_index(KASAN_SHADOW_START),
KASAN_SHADOW_START, KASAN_SHADOW_END, true);

Expand All @@ -244,9 +313,27 @@ static void __init kasan_populate(void *start, void *end)
memset(start, KASAN_SHADOW_INIT, end - start);
}

static void __init kasan_shallow_populate_pmd(pgd_t *pgdp,
unsigned long vaddr, unsigned long end)
{
unsigned long next;
pmd_t *pmdp, *base_pmd;
bool is_kasan_pte;

base_pmd = (pmd_t *)pgd_page_vaddr(*pgdp);
pmdp = base_pmd + pmd_index(vaddr);

do {
next = pmd_addr_end(vaddr, end);
is_kasan_pte = (pmd_pgtable(*pmdp) == lm_alias(kasan_early_shadow_pte));

if (is_kasan_pte)
pmd_clear(pmdp);
} while (pmdp++, vaddr = next, vaddr != end);
}

static void __init kasan_shallow_populate_pud(pgd_t *pgdp,
unsigned long vaddr, unsigned long end,
bool kasan_populate)
unsigned long vaddr, unsigned long end)
{
unsigned long next;
pud_t *pudp, *base_pud;
Expand All @@ -256,21 +343,60 @@ static void __init kasan_shallow_populate_pud(pgd_t *pgdp,
base_pud = (pud_t *)pgd_page_vaddr(*pgdp);
pudp = base_pud + pud_index(vaddr);

if (kasan_populate)
memcpy(base_pud, (void *)kasan_early_shadow_pgd_next,
sizeof(pud_t) * PTRS_PER_PUD);

do {
next = pud_addr_end(vaddr, end);
is_kasan_pmd = (pud_pgtable(*pudp) == lm_alias(kasan_early_shadow_pmd));

if (is_kasan_pmd) {
base_pmd = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
set_pud(pudp, pfn_pud(PFN_DOWN(__pa(base_pmd)), PAGE_TABLE));
}
if (!is_kasan_pmd)
continue;

base_pmd = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
set_pud(pudp, pfn_pud(PFN_DOWN(__pa(base_pmd)), PAGE_TABLE));

if (IS_ALIGNED(vaddr, PUD_SIZE) && (next - vaddr) >= PUD_SIZE)
continue;

memcpy(base_pmd, (void *)kasan_early_shadow_pmd, PAGE_SIZE);
kasan_shallow_populate_pmd((pgd_t *)pudp, vaddr, next);
} while (pudp++, vaddr = next, vaddr != end);
}

static void __init kasan_shallow_populate_p4d(pgd_t *pgdp,
unsigned long vaddr, unsigned long end)
{
unsigned long next;
p4d_t *p4dp, *base_p4d;
pud_t *base_pud;
bool is_kasan_pud;

base_p4d = (p4d_t *)pgd_page_vaddr(*pgdp);
p4dp = base_p4d + p4d_index(vaddr);

do {
next = p4d_addr_end(vaddr, end);
is_kasan_pud = (p4d_pgtable(*p4dp) == lm_alias(kasan_early_shadow_pud));

if (!is_kasan_pud)
continue;

base_pud = memblock_alloc(PAGE_SIZE, PAGE_SIZE);
set_p4d(p4dp, pfn_p4d(PFN_DOWN(__pa(base_pud)), PAGE_TABLE));

if (IS_ALIGNED(vaddr, P4D_SIZE) && (next - vaddr) >= P4D_SIZE)
continue;

memcpy(base_pud, (void *)kasan_early_shadow_pud, PAGE_SIZE);
kasan_shallow_populate_pud((pgd_t *)p4dp, vaddr, next);
} while (p4dp++, vaddr = next, vaddr != end);
}

#define kasan_shallow_populate_pgd_next(pgdp, vaddr, next) \
(pgtable_l5_enabled ? \
kasan_shallow_populate_p4d(pgdp, vaddr, next) : \
(pgtable_l4_enabled ? \
kasan_shallow_populate_pud(pgdp, vaddr, next) : \
kasan_shallow_populate_pmd(pgdp, vaddr, next)))

static void __init kasan_shallow_populate_pgd(unsigned long vaddr, unsigned long end)
{
unsigned long next;
Expand All @@ -291,7 +417,8 @@ static void __init kasan_shallow_populate_pgd(unsigned long vaddr, unsigned long
if (IS_ALIGNED(vaddr, PGDIR_SIZE) && (next - vaddr) >= PGDIR_SIZE)
continue;

kasan_shallow_populate_pud(pgd_k, vaddr, next, is_kasan_pgd_next);
memcpy(p, (void *)kasan_early_shadow_pgd_next, PAGE_SIZE);
kasan_shallow_populate_pgd_next(pgd_k, vaddr, next);
} while (pgd_k++, vaddr = next, vaddr != end);
}

Expand Down

0 comments on commit 8fbdccd

Please sign in to comment.